00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022 #include <cstdio>
00023 #include "ChartManager.h"
00024 #include "ChartCell.h"
00025 #include "ChartHypothesis.h"
00026 #include "ChartKBestExtractor.h"
00027 #include "ChartTranslationOptions.h"
00028 #include "HypergraphOutput.h"
00029 #include "StaticData.h"
00030 #include "DecodeStep.h"
00031 #include "TreeInput.h"
00032 #include "moses/FF/StatefulFeatureFunction.h"
00033 #include "moses/FF/WordPenaltyProducer.h"
00034 #include "moses/OutputCollector.h"
00035 #include "moses/ChartKBestExtractor.h"
00036 #include "moses/HypergraphOutput.h"
00037 #include "moses/TranslationTask.h"
00038
00039 using namespace std;
00040
00041 namespace Moses
00042 {
00043
00044
00045
00046
00047
00048 ChartManager::ChartManager(ttasksptr const& ttask)
00049 : BaseManager(ttask)
00050 , m_hypoStackColl(m_source, *this)
00051 , m_start(clock())
00052 , m_hypothesisId(0)
00053 , m_parser(ttask, m_hypoStackColl)
00054 , m_translationOptionList(ttask->options()->syntax.rule_limit, m_source)
00055 { }
00056
00057 ChartManager::~ChartManager()
00058 {
00059 clock_t end = clock();
00060 float et = (end - m_start);
00061 et /= (float)CLOCKS_PER_SEC;
00062 VERBOSE(1, "Translation took " << et << " seconds" << endl);
00063
00064 }
00065
00067 void ChartManager::Decode()
00068 {
00069
00070 VERBOSE(1,"Translating: " << m_source << endl);
00071
00072 ResetSentenceStats(m_source);
00073
00074 VERBOSE(2,"Decoding: " << endl);
00075
00076
00077 AddXmlChartOptions();
00078
00079
00080 size_t size = m_source.GetSize();
00081 for (int startPos = size-1; startPos >= 0; --startPos) {
00082 for (size_t width = 1; width <= size-startPos; ++width) {
00083 size_t endPos = startPos + width - 1;
00084 Range range(startPos, endPos);
00085
00086
00087 m_translationOptionList.Clear();
00088 m_parser.Create(range, m_translationOptionList);
00089 m_translationOptionList.ApplyThreshold(options()->search.trans_opt_threshold);
00090
00091 const InputPath &inputPath = m_parser.GetInputPath(range);
00092 m_translationOptionList.EvaluateWithSourceContext(m_source, inputPath);
00093
00094
00095 ChartCell &cell = m_hypoStackColl.Get(range);
00096 cell.Decode(m_translationOptionList, m_hypoStackColl);
00097
00098 m_translationOptionList.Clear();
00099 cell.PruneToSize();
00100 cell.CleanupArcList();
00101 cell.SortHypotheses();
00102 }
00103 }
00104
00105 IFVERBOSE(1) {
00106
00107 for (size_t startPos = 0; startPos < size; ++startPos) {
00108 cerr.width(3);
00109 cerr << startPos << " ";
00110 }
00111 cerr << endl;
00112 for (size_t width = 1; width <= size; width++) {
00113 for( size_t space = 0; space < width-1; space++ ) {
00114 cerr << " ";
00115 }
00116 for (size_t startPos = 0; startPos <= size-width; ++startPos) {
00117 Range range(startPos, startPos+width-1);
00118 cerr.width(3);
00119 cerr << m_hypoStackColl.Get(range).GetSize() << " ";
00120 }
00121 cerr << endl;
00122 }
00123 }
00124 }
00125
00130 void ChartManager::AddXmlChartOptions()
00131 {
00132 const std::vector <ChartTranslationOptions*> xmlChartOptionsList
00133 = m_source.GetXmlChartTranslationOptions();
00134 IFVERBOSE(2) {
00135 cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl;
00136 }
00137 if (xmlChartOptionsList.size() == 0) return;
00138
00139 typedef std::vector<ChartTranslationOptions*>::const_iterator citer;
00140 for(citer i = xmlChartOptionsList.begin(); i != xmlChartOptionsList.end(); ++i) {
00141 ChartTranslationOptions* opt = *i;
00142
00143 const Range &range = opt->GetSourceWordsRange();
00144
00145 RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
00146 ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
00147 hypo->EvaluateWhenApplied();
00148
00149
00150 ChartCell &cell = m_hypoStackColl.Get(range);
00151 cell.AddHypothesis(hypo);
00152 }
00153 }
00154
00156 const ChartHypothesis *ChartManager::GetBestHypothesis() const
00157 {
00158 size_t size = m_source.GetSize();
00159
00160 if (size == 0)
00161 return NULL;
00162 else {
00163 Range range(0, size-1);
00164 const ChartCell &lastCell = m_hypoStackColl.Get(range);
00165 return lastCell.GetBestHypothesis();
00166 }
00167 }
00168
00175 void ChartManager::CalcNBest(
00176 std::size_t n,
00177 std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > &nBestList,
00178 bool onlyDistinct) const
00179 {
00180 nBestList.clear();
00181 if (n == 0 || m_source.GetSize() == 0) {
00182 return;
00183 }
00184
00185
00186 Range range(0, m_source.GetSize()-1);
00187 const ChartCell &lastCell = m_hypoStackColl.Get(range);
00188 boost::scoped_ptr<const std::vector<const ChartHypothesis*> > topLevelHypos(
00189 lastCell.GetAllSortedHypotheses());
00190 if (!topLevelHypos) {
00191 return;
00192 }
00193
00194 ChartKBestExtractor extractor;
00195
00196 if (!onlyDistinct) {
00197
00198 extractor.Extract(*topLevelHypos, n, nBestList);
00199 return;
00200 }
00201
00202
00203
00204
00205
00206
00207 const std::size_t nBestFactor = options()->nbest.factor;
00208 std::size_t numDerivations = (nBestFactor == 0) ? n*1000 : n*nBestFactor;
00209
00210
00211 ChartKBestExtractor::KBestVec bigList;
00212 bigList.reserve(numDerivations);
00213 extractor.Extract(*topLevelHypos, numDerivations, bigList);
00214
00215
00216 std::set<Phrase> distinct;
00217 for (ChartKBestExtractor::KBestVec::const_iterator p = bigList.begin();
00218 nBestList.size() < n && p != bigList.end(); ++p) {
00219 boost::shared_ptr<ChartKBestExtractor::Derivation> derivation = *p;
00220 Phrase translation = ChartKBestExtractor::GetOutputPhrase(*derivation);
00221 if (distinct.insert(translation).second) {
00222 nBestList.push_back(derivation);
00223 }
00224 }
00225 }
00226
00227 void ChartManager::WriteSearchGraph(const ChartSearchGraphWriter& writer) const
00228 {
00229
00230 size_t size = m_source.GetSize();
00231
00232
00233 std::map<unsigned,bool> reachable;
00234 Range fullRange(0, size-1);
00235 const ChartCell &lastCell = m_hypoStackColl.Get(fullRange);
00236 const ChartHypothesis *hypo = lastCell.GetBestHypothesis();
00237
00238 if (hypo == NULL) {
00239
00240 return;
00241 }
00242 size_t winners = 0;
00243 size_t losers = 0;
00244
00245 FindReachableHypotheses( hypo, reachable, &winners, &losers);
00246 writer.WriteHeader(winners, losers);
00247
00248 for (size_t width = 1; width <= size; ++width) {
00249 for (size_t startPos = 0; startPos <= size-width; ++startPos) {
00250 size_t endPos = startPos + width - 1;
00251 Range range(startPos, endPos);
00252 TRACE_ERR(" " << range << "=");
00253
00254 const ChartCell &cell = m_hypoStackColl.Get(range);
00255 cell.WriteSearchGraph(writer, reachable);
00256 }
00257 }
00258 }
00259
00260 void ChartManager::FindReachableHypotheses(
00261 const ChartHypothesis *hypo, std::map<unsigned,bool> &reachable, size_t* winners, size_t* losers) const
00262 {
00263
00264 if (reachable.find(hypo->GetId()) != reachable.end()) {
00265 return;
00266 }
00267
00268
00269 reachable[ hypo->GetId() ] = true;
00270 if (hypo->GetWinningHypothesis() == hypo) {
00271 (*winners)++;
00272 } else {
00273 (*losers)++;
00274 }
00275 const std::vector<const ChartHypothesis*> &previous = hypo->GetPrevHypos();
00276 for(std::vector<const ChartHypothesis*>::const_iterator i = previous.begin(); i != previous.end(); ++i) {
00277 FindReachableHypotheses( *i, reachable, winners, losers );
00278 }
00279
00280
00281 const ChartArcList *arcList = hypo->GetArcList();
00282 if (arcList) {
00283 ChartArcList::const_iterator iterArc;
00284 for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc) {
00285 const ChartHypothesis &arc = **iterArc;
00286 FindReachableHypotheses( &arc, reachable, winners, losers );
00287 }
00288 }
00289 }
00290
00291 void
00292 ChartManager::
00293 OutputSearchGraphAsHypergraph(std::ostream& out) const
00294 {
00295 ChartSearchGraphWriterHypergraph writer(options(), &out);
00296 WriteSearchGraph(writer);
00297 }
00298
00299 void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const
00300 {
00301 ChartSearchGraphWriterMoses writer(options(), &outputSearchGraphStream,
00302 m_source.GetTranslationId());
00303 WriteSearchGraph(writer);
00304 }
00305
00306 void ChartManager::OutputBest(OutputCollector *collector) const
00307 {
00308 const ChartHypothesis *bestHypo = GetBestHypothesis();
00309 if (collector && bestHypo) {
00310 const size_t translationId = m_source.GetTranslationId();
00311 const ChartHypothesis *bestHypo = GetBestHypothesis();
00312 OutputBestHypo(collector, bestHypo, translationId);
00313 }
00314 }
00315
00316 void ChartManager::OutputNBest(OutputCollector *collector) const
00317 {
00318 size_t nBestSize = options()->nbest.nbest_size;
00319 if (nBestSize > 0) {
00320 const size_t translationId = m_source.GetTranslationId();
00321
00322 VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO "
00323 << options()->nbest.output_file_path << endl);
00324 std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > nBestList;
00325 CalcNBest(nBestSize, nBestList, options()->nbest.only_distinct);
00326 OutputNBestList(collector, nBestList, translationId);
00327 IFVERBOSE(2) {
00328 PrintUserTime("N-Best Hypotheses Generation Time:");
00329 }
00330 }
00331
00332 }
00333
00334 void ChartManager::OutputNBestList(OutputCollector *collector,
00335 const ChartKBestExtractor::KBestVec &nBestList,
00336 long translationId) const
00337 {
00338 std::ostringstream out;
00339
00340 if (collector->OutputIsCout()) {
00341
00342
00343 FixPrecision(out);
00344 }
00345
00346 NBestOptions const& nbo = options()->nbest;
00347 bool includeWordAlignment = nbo.include_alignment_info;
00348 bool PrintNBestTrees = nbo.print_trees;
00349
00350 for (ChartKBestExtractor::KBestVec::const_iterator p = nBestList.begin();
00351 p != nBestList.end(); ++p) {
00352 const ChartKBestExtractor::Derivation &derivation = **p;
00353
00354
00355 Phrase outputPhrase = ChartKBestExtractor::GetOutputPhrase(derivation);
00356
00357
00358 UTIL_THROW_IF2(outputPhrase.GetSize() < 2,
00359 "Output phrase should have contained at least 2 words (beginning and end-of-sentence)");
00360 outputPhrase.RemoveWord(0);
00361 outputPhrase.RemoveWord(outputPhrase.GetSize() - 1);
00362
00363
00364 out << translationId << " ||| ";
00365 OutputSurface(out, outputPhrase);
00366 out << " ||| ";
00367 boost::shared_ptr<ScoreComponentCollection> scoreBreakdown = ChartKBestExtractor::GetOutputScoreBreakdown(derivation);
00368 bool with_labels = options()->nbest.include_feature_labels;
00369 scoreBreakdown->OutputAllFeatureScores(out, with_labels);
00370 out << " ||| " << derivation.score;
00371
00372
00373 if (includeWordAlignment) {
00374 out << " ||| ";
00375 Alignments align;
00376 OutputAlignmentNBest(align, derivation, 0);
00377 for (Alignments::const_iterator q = align.begin(); q != align.end();
00378 ++q) {
00379 out << q->first << "-" << q->second << " ";
00380 }
00381 }
00382
00383
00384 if (PrintNBestTrees) {
00385 TreePointer tree = ChartKBestExtractor::GetOutputTree(derivation);
00386 out << " ||| " << tree->GetString();
00387 }
00388
00389 out << std::endl;
00390 }
00391
00392 assert(collector);
00393 collector->Write(translationId, out.str());
00394 }
00395
00396 size_t ChartManager::CalcSourceSize(const Moses::ChartHypothesis *hypo) const
00397 {
00398 size_t ret = hypo->GetCurrSourceRange().GetNumWordsCovered();
00399 const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
00400 for (size_t i = 0; i < prevHypos.size(); ++i) {
00401 size_t childSize = prevHypos[i]->GetCurrSourceRange().GetNumWordsCovered();
00402 ret -= (childSize - 1);
00403 }
00404 return ret;
00405 }
00406
00407 size_t ChartManager::OutputAlignmentNBest(
00408 Alignments &retAlign,
00409 const Moses::ChartKBestExtractor::Derivation &derivation,
00410 size_t startTarget) const
00411 {
00412 const ChartHypothesis &hypo = derivation.edge.head->hypothesis;
00413
00414 size_t totalTargetSize = 0;
00415 size_t startSource = hypo.GetCurrSourceRange().GetStartPos();
00416
00417 const TargetPhrase &tp = hypo.GetCurrTargetPhrase();
00418
00419 size_t thisSourceSize = CalcSourceSize(&hypo);
00420
00421
00422
00423 vector<size_t> sourceOffsets(thisSourceSize, 0);
00424 vector<size_t> targetOffsets(tp.GetSize(), 0);
00425
00426 const AlignmentInfo &aiNonTerm = hypo.GetCurrTargetPhrase().GetAlignNonTerm();
00427 vector<size_t> sourceInd2pos = aiNonTerm.GetSourceIndex2PosMap();
00428 const AlignmentInfo::NonTermIndexMap &targetPos2SourceInd = aiNonTerm.GetNonTermIndexMap();
00429
00430 UTIL_THROW_IF2(sourceInd2pos.size() != derivation.subderivations.size(),
00431 "Error");
00432
00433 size_t targetInd = 0;
00434 for (size_t targetPos = 0; targetPos < tp.GetSize(); ++targetPos) {
00435 if (tp.GetWord(targetPos).IsNonTerminal()) {
00436 UTIL_THROW_IF2(targetPos >= targetPos2SourceInd.size(), "Error");
00437 size_t sourceInd = targetPos2SourceInd[targetPos];
00438 size_t sourcePos = sourceInd2pos[sourceInd];
00439
00440 const Moses::ChartKBestExtractor::Derivation &subderivation =
00441 *derivation.subderivations[sourceInd];
00442
00443
00444 size_t sourceSize = subderivation.edge.head->hypothesis.GetCurrSourceRange().GetNumWordsCovered();
00445 sourceOffsets[sourcePos] = sourceSize;
00446
00447
00448
00449 size_t currStartTarget = startTarget + totalTargetSize;
00450 size_t targetSize = OutputAlignmentNBest(retAlign, subderivation,
00451 currStartTarget);
00452 targetOffsets[targetPos] = targetSize;
00453
00454 totalTargetSize += targetSize;
00455 ++targetInd;
00456 } else {
00457 ++totalTargetSize;
00458 }
00459 }
00460
00461
00462
00463 ShiftOffsets(sourceOffsets, startSource);
00464 ShiftOffsets(targetOffsets, startTarget);
00465
00466
00467 const AlignmentInfo &aiTerm = hypo.GetCurrTargetPhrase().GetAlignTerm();
00468
00469
00470 AlignmentInfo::const_iterator iter;
00471 for (iter = aiTerm.begin(); iter != aiTerm.end(); ++iter) {
00472 const std::pair<size_t,size_t> &align = *iter;
00473 size_t relSource = align.first;
00474 size_t relTarget = align.second;
00475 size_t absSource = sourceOffsets[relSource];
00476 size_t absTarget = targetOffsets[relTarget];
00477
00478 pair<size_t, size_t> alignPoint(absSource, absTarget);
00479 pair<Alignments::iterator, bool> ret = retAlign.insert(alignPoint);
00480 UTIL_THROW_IF2(!ret.second, "Error");
00481 }
00482
00483 return totalTargetSize;
00484 }
00485
00486 void ChartManager::OutputAlignment(OutputCollector *collector) const
00487 {
00488 if (collector == NULL) {
00489 return;
00490 }
00491
00492 ostringstream out;
00493
00494 const ChartHypothesis *hypo = GetBestHypothesis();
00495 if (hypo) {
00496 Alignments retAlign;
00497 OutputAlignment(retAlign, hypo, 0);
00498
00499
00500 Alignments::const_iterator iter;
00501 for (iter = retAlign.begin(); iter != retAlign.end(); ++iter) {
00502 const pair<size_t, size_t> &alignPoint = *iter;
00503 out << alignPoint.first << "-" << alignPoint.second << " ";
00504 }
00505 }
00506 out << endl;
00507
00508 collector->Write(m_source.GetTranslationId(), out.str());
00509
00510 }
00511
00512 size_t ChartManager::OutputAlignment(Alignments &retAlign,
00513 const Moses::ChartHypothesis *hypo,
00514 size_t startTarget) const
00515 {
00516 size_t totalTargetSize = 0;
00517 size_t startSource = hypo->GetCurrSourceRange().GetStartPos();
00518
00519 const TargetPhrase &tp = hypo->GetCurrTargetPhrase();
00520
00521 size_t thisSourceSize = CalcSourceSize(hypo);
00522
00523
00524
00525 vector<size_t> sourceOffsets(thisSourceSize, 0);
00526 vector<size_t> targetOffsets(tp.GetSize(), 0);
00527
00528 const vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
00529
00530 const AlignmentInfo &aiNonTerm = hypo->GetCurrTargetPhrase().GetAlignNonTerm();
00531 vector<size_t> sourceInd2pos = aiNonTerm.GetSourceIndex2PosMap();
00532 const AlignmentInfo::NonTermIndexMap &targetPos2SourceInd = aiNonTerm.GetNonTermIndexMap();
00533
00534 UTIL_THROW_IF2(sourceInd2pos.size() != prevHypos.size(), "Error");
00535
00536 size_t targetInd = 0;
00537 for (size_t targetPos = 0; targetPos < tp.GetSize(); ++targetPos) {
00538 if (tp.GetWord(targetPos).IsNonTerminal()) {
00539 UTIL_THROW_IF2(targetPos >= targetPos2SourceInd.size(), "Error");
00540 size_t sourceInd = targetPos2SourceInd[targetPos];
00541 size_t sourcePos = sourceInd2pos[sourceInd];
00542
00543 const ChartHypothesis *prevHypo = prevHypos[sourceInd];
00544
00545
00546 size_t sourceSize = prevHypo->GetCurrSourceRange().GetNumWordsCovered();
00547 sourceOffsets[sourcePos] = sourceSize;
00548
00549
00550
00551 size_t currStartTarget = startTarget + totalTargetSize;
00552 size_t targetSize = OutputAlignment(retAlign, prevHypo, currStartTarget);
00553 targetOffsets[targetPos] = targetSize;
00554
00555 totalTargetSize += targetSize;
00556 ++targetInd;
00557 } else {
00558 ++totalTargetSize;
00559 }
00560 }
00561
00562
00563
00564 ShiftOffsets(sourceOffsets, startSource);
00565 ShiftOffsets(targetOffsets, startTarget);
00566
00567
00568 const AlignmentInfo &aiTerm = hypo->GetCurrTargetPhrase().GetAlignTerm();
00569
00570
00571 AlignmentInfo::const_iterator iter;
00572 for (iter = aiTerm.begin(); iter != aiTerm.end(); ++iter) {
00573 const std::pair<size_t,size_t> &align = *iter;
00574 size_t relSource = align.first;
00575 size_t relTarget = align.second;
00576 size_t absSource = sourceOffsets[relSource];
00577 size_t absTarget = targetOffsets[relTarget];
00578
00579 pair<size_t, size_t> alignPoint(absSource, absTarget);
00580 pair<Alignments::iterator, bool> ret = retAlign.insert(alignPoint);
00581 UTIL_THROW_IF2(!ret.second, "Error");
00582
00583 }
00584
00585 return totalTargetSize;
00586 }
00587
00588 void ChartManager::OutputDetailedTranslationReport(OutputCollector *collector) const
00589 {
00590 if (collector) {
00591 OutputDetailedTranslationReport(collector,
00592 GetBestHypothesis(),
00593 static_cast<const Sentence&>(m_source),
00594 m_source.GetTranslationId());
00595 }
00596 }
00597
00598 void ChartManager::OutputDetailedTranslationReport(
00599 OutputCollector *collector,
00600 const ChartHypothesis *hypo,
00601 const Sentence &sentence,
00602 long translationId) const
00603 {
00604 if (hypo == NULL) {
00605 return;
00606 }
00607 std::ostringstream out;
00608 ApplicationContext applicationContext;
00609
00610 OutputTranslationOptions(out, applicationContext, hypo, sentence, translationId);
00611 collector->Write(translationId, out.str());
00612
00613
00614 if (options()->output.detailed_all_transrep_filepath.size()) {
00615 const Sentence &sentence = static_cast<const Sentence &>(m_source);
00616 size_t nBestSize = options()->nbest.nbest_size;
00617 std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > nBestList;
00618 CalcNBest(nBestSize, nBestList, options()->nbest.only_distinct);
00619 OutputDetailedAllTranslationReport(collector, nBestList, sentence, translationId);
00620 }
00621
00622 }
00623
00624 void ChartManager::OutputTranslationOptions(std::ostream &out,
00625 ApplicationContext &applicationContext,
00626 const ChartHypothesis *hypo,
00627 const Sentence &sentence,
00628 long translationId) const
00629 {
00630 if (hypo != NULL) {
00631 OutputTranslationOption(out, applicationContext, hypo, sentence, translationId);
00632 out << std::endl;
00633 }
00634
00635
00636 const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
00637 std::vector<const ChartHypothesis*>::const_iterator iter;
00638 for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
00639 const ChartHypothesis *prevHypo = *iter;
00640 OutputTranslationOptions(out, applicationContext, prevHypo, sentence, translationId);
00641 }
00642 }
00643
00644 void ChartManager::OutputTranslationOption(std::ostream &out,
00645 ApplicationContext &applicationContext,
00646 const ChartHypothesis *hypo,
00647 const Sentence &sentence,
00648 long translationId) const
00649 {
00650 ReconstructApplicationContext(*hypo, sentence, applicationContext);
00651 out << "Trans Opt " << translationId
00652 << " " << hypo->GetCurrSourceRange()
00653 << ": ";
00654 WriteApplicationContext(out, applicationContext);
00655 out << ": " << hypo->GetCurrTargetPhrase().GetTargetLHS()
00656 << "->" << hypo->GetCurrTargetPhrase()
00657 << " " << hypo->GetFutureScore() << hypo->GetScoreBreakdown();
00658 }
00659
00660
00661
00662 void ChartManager::ReconstructApplicationContext(const ChartHypothesis &hypo,
00663 const Sentence &sentence,
00664 ApplicationContext &context) const
00665 {
00666 context.clear();
00667 const std::vector<const ChartHypothesis*> &prevHypos = hypo.GetPrevHypos();
00668 std::vector<const ChartHypothesis*>::const_iterator p = prevHypos.begin();
00669 std::vector<const ChartHypothesis*>::const_iterator end = prevHypos.end();
00670 const Range &span = hypo.GetCurrSourceRange();
00671 size_t i = span.GetStartPos();
00672 while (i <= span.GetEndPos()) {
00673 if (p == end || i < (*p)->GetCurrSourceRange().GetStartPos()) {
00674
00675 const Word &symbol = sentence.GetWord(i);
00676 context.push_back(std::make_pair(symbol, Range(i, i)));
00677 ++i;
00678 } else {
00679
00680 const Word &symbol = (*p)->GetTargetLHS();
00681 const Range &range = (*p)->GetCurrSourceRange();
00682 context.push_back(std::make_pair(symbol, range));
00683 i = range.GetEndPos()+1;
00684 ++p;
00685 }
00686 }
00687 }
00688
00689 void ChartManager::OutputUnknowns(OutputCollector *collector) const
00690 {
00691 if (collector) {
00692 long translationId = m_source.GetTranslationId();
00693 const std::vector<Phrase*> &oovs = GetParser().GetUnknownSources();
00694
00695 std::ostringstream out;
00696 for (std::vector<Phrase*>::const_iterator p = oovs.begin();
00697 p != oovs.end(); ++p) {
00698 out << **p;
00699 }
00700 out << std::endl;
00701 collector->Write(translationId, out.str());
00702 }
00703
00704 }
00705
00706 void ChartManager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const
00707 {
00708 const ChartHypothesis *hypo = GetBestHypothesis();
00709 if (collector == NULL || hypo == NULL) {
00710 return;
00711 }
00712
00713 std::ostringstream out;
00714 ApplicationContext applicationContext;
00715
00716 const Sentence &sentence = static_cast<const Sentence &>(m_source);
00717 const size_t translationId = m_source.GetTranslationId();
00718
00719 OutputTreeFragmentsTranslationOptions(out, applicationContext, hypo, sentence, translationId);
00720
00721
00722 const StatefulFeatureFunction* treeStructure;
00723 treeStructure = StaticData::Instance().GetTreeStructure();
00724 if (treeStructure != NULL) {
00725 const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
00726 for( size_t i=0; i<sff.size(); i++ ) {
00727 if (sff[i] == treeStructure) {
00728 const TreeState* tree = static_cast<const TreeState*>(hypo->GetFFState(i));
00729 out << "Full Tree " << translationId << ": " << tree->GetTree()->GetString() << "\n";
00730 break;
00731 }
00732 }
00733 }
00734
00735 collector->Write(translationId, out.str());
00736
00737 }
00738
00739 void ChartManager::OutputTreeFragmentsTranslationOptions(std::ostream &out,
00740 ApplicationContext &applicationContext,
00741 const ChartHypothesis *hypo,
00742 const Sentence &sentence,
00743 long translationId) const
00744 {
00745
00746 if (hypo != NULL) {
00747 OutputTranslationOption(out, applicationContext, hypo, sentence, translationId);
00748
00749 const TargetPhrase &currTarPhr = hypo->GetCurrTargetPhrase();
00750
00751 out << " ||| ";
00752 if (const PhraseProperty *property = currTarPhr.GetProperty("Tree")) {
00753 out << " " << *property->GetValueString();
00754 } else {
00755 out << " " << "noTreeInfo";
00756 }
00757 out << std::endl;
00758 }
00759
00760
00761 const std::vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
00762 std::vector<const ChartHypothesis*>::const_iterator iter;
00763 for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
00764 const ChartHypothesis *prevHypo = *iter;
00765 OutputTreeFragmentsTranslationOptions(out, applicationContext, prevHypo, sentence, translationId);
00766 }
00767 }
00768
00769 void ChartManager::OutputSearchGraph(OutputCollector *collector) const
00770 {
00771 if (collector) {
00772 long translationId = m_source.GetTranslationId();
00773 std::ostringstream out;
00774 OutputSearchGraphMoses( out);
00775 collector->Write(translationId, out.str());
00776 }
00777 }
00778
00779
00780 void ChartManager::OutputDetailedAllTranslationReport(
00781 OutputCollector *collector,
00782 const std::vector<boost::shared_ptr<Moses::ChartKBestExtractor::Derivation> > &nBestList,
00783 const Sentence &sentence,
00784 long translationId) const
00785 {
00786 std::ostringstream out;
00787 ApplicationContext applicationContext;
00788
00789 const ChartCellCollection& cells = GetChartCellCollection();
00790 size_t size = GetSource().GetSize();
00791 for (size_t width = 1; width <= size; ++width) {
00792 for (size_t startPos = 0; startPos <= size-width; ++startPos) {
00793 size_t endPos = startPos + width - 1;
00794 Range range(startPos, endPos);
00795 const ChartCell& cell = cells.Get(range);
00796 const HypoList* hyps = cell.GetAllSortedHypotheses();
00797 out << "Chart Cell [" << startPos << ".." << endPos << "]" << endl;
00798 HypoList::const_iterator iter;
00799 size_t c = 1;
00800 for (iter = hyps->begin(); iter != hyps->end(); ++iter) {
00801 out << "----------------Item " << c++ << " ---------------------"
00802 << endl;
00803 OutputTranslationOptions(out, applicationContext, *iter,
00804 sentence, translationId);
00805 }
00806 }
00807 }
00808 collector->Write(translationId, out.str());
00809 }
00810
00811 void ChartManager::OutputBestHypo(OutputCollector *collector, const ChartHypothesis *hypo, long translationId) const
00812 {
00813 if (!collector)
00814 return;
00815 std::ostringstream out;
00816 FixPrecision(out);
00817 if (hypo != NULL) {
00818 VERBOSE(1,"BEST TRANSLATION: " << *hypo << endl);
00819 VERBOSE(3,"Best path: ");
00820 Backtrack(hypo);
00821 VERBOSE(3,"0" << std::endl);
00822
00823 if (options()->output.ReportHypoScore) {
00824 out << hypo->GetFutureScore() << " ";
00825 }
00826
00827 if (options()->output.RecoverPath) {
00828 out << "||| ";
00829 }
00830 Phrase outPhrase(ARRAY_SIZE_INCR);
00831 hypo->GetOutputPhrase(outPhrase);
00832
00833
00834 UTIL_THROW_IF2(outPhrase.GetSize() < 2,
00835 "Output phrase should have contained at least 2 words (beginning and end-of-sentence)");
00836
00837 outPhrase.RemoveWord(0);
00838 outPhrase.RemoveWord(outPhrase.GetSize() - 1);
00839
00840 string output = outPhrase.GetStringRep(options()->output.factor_order);
00841 out << output << endl;
00842 } else {
00843 VERBOSE(1, "NO BEST TRANSLATION" << endl);
00844
00845 if (options()->output.ReportHypoScore) {
00846 out << "0 ";
00847 }
00848
00849 out << endl;
00850 }
00851 collector->Write(translationId, out.str());
00852 }
00853
00854 void ChartManager::Backtrack(const ChartHypothesis *hypo) const
00855 {
00856 const vector<const ChartHypothesis*> &prevHypos = hypo->GetPrevHypos();
00857
00858 vector<const ChartHypothesis*>::const_iterator iter;
00859 for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
00860 const ChartHypothesis *prevHypo = *iter;
00861
00862 VERBOSE(3,prevHypo->GetId() << " <= ");
00863 Backtrack(prevHypo);
00864 }
00865 }
00866
00867 }