00001 #include "BleuScoreFeature.h"
00002
00003 #include "moses/StaticData.h"
00004 #include "moses/Hypothesis.h"
00005 #include "moses/FactorCollection.h"
00006 #include "util/exception.hh"
00007
00008 using namespace std;
00009
00010 namespace Moses
00011 {
00012
00013 size_t BleuScoreState::bleu_order = 4;
00014 std::vector<BleuScoreFeature*> BleuScoreFeature::s_staticColl;
00015
00016 BleuScoreState::BleuScoreState(bool is_syntax)
00017 : m_words(1),
00018 m_source_length(0),
00019 m_target_length(0),
00020 m_is_syntax(false),
00021 m_scaled_ref_length(0),
00022 m_ngram_counts(bleu_order),
00023 m_ngram_matches(bleu_order)
00024 { }
00025
00026 size_t BleuScoreState::hash() const
00027 {
00028 if (m_is_syntax)
00029 return 0;
00030
00031 size_t ret = hash_value(m_words);
00032 return ret;
00033 }
00034
00035 bool BleuScoreState::operator==(const FFState& o) const
00036 {
00037 if (&o == this)
00038 return true;
00039
00040 if (m_is_syntax)
00041 return true;
00042
00043 const BleuScoreState& other = static_cast<const BleuScoreState&>(o);
00044 return m_words == other.m_words;
00045 }
00046
00047 std::ostream& operator<<(std::ostream& out, const BleuScoreState& state)
00048 {
00049 state.print(out);
00050 return out;
00051 }
00052
00053 void BleuScoreState::print(std::ostream& out) const
00054 {
00055 out << "ref=" << m_scaled_ref_length
00056 << ";source=" << m_source_length
00057 << ";target=" << m_target_length << ";counts=";
00058 for (size_t i = 0; i < bleu_order; ++i) {
00059 out << m_ngram_matches[i] << "/" << m_ngram_counts[i] << ",";
00060 }
00061 out << "ctxt=" << m_words;
00062
00063 }
00064
00065 void BleuScoreState::AddNgramCountAndMatches(std::vector< size_t >& counts,
00066 std::vector< size_t >& matches)
00067 {
00068 for (size_t order = 0; order < BleuScoreState::bleu_order; ++order) {
00069 m_ngram_counts[order] += counts[order];
00070 m_ngram_matches[order] += matches[order];
00071 }
00072 }
00073
00074
00075 BleuScoreFeature::BleuScoreFeature(const std::string &line)
00076 :StatefulFeatureFunction(1, line),
00077 m_enabled(true),
00078 m_sentence_bleu(true),
00079 m_simple_history_bleu(false),
00080 m_count_history(BleuScoreState::bleu_order),
00081 m_match_history(BleuScoreState::bleu_order),
00082 m_source_length_history(0),
00083 m_target_length_history(0),
00084 m_ref_length_history(0),
00085 m_scale_by_input_length(true),
00086 m_scale_by_avg_input_length(false),
00087 m_scale_by_inverse_length(false),
00088 m_scale_by_avg_inverse_length(false),
00089 m_scale_by_x(1),
00090 m_historySmoothing(0.9),
00091 m_smoothing_scheme(PLUS_POINT_ONE)
00092 {
00093 std::cerr << "Initializing BleuScoreFeature." << std::endl;
00094 s_staticColl.push_back(this);
00095
00096 m_tuneable = false;
00097
00098 ReadParameters();
00099 std::cerr << "Finished initializing BleuScoreFeature." << std::endl;
00100 }
00101
00102 void BleuScoreFeature::SetParameter(const std::string& key, const std::string& value)
00103 {
00104 if (key == "references") {
00105 vector<string> referenceFiles = Tokenize(value, ",");
00106 UTIL_THROW_IF2(referenceFiles.size() == 0, "No reference file");
00107 vector<vector<string> > references(referenceFiles.size());
00108
00109 for (size_t i =0; i < referenceFiles.size(); ++i) {
00110 ifstream in(referenceFiles[i].c_str());
00111 if (!in) {
00112 UTIL_THROW2("Unable to load references from " << referenceFiles[i]);
00113 }
00114 string line;
00115 while (getline(in,line)) {
00116
00117
00118
00119
00120
00121
00122 references[i].push_back(line);
00123 }
00124 if (i > 0) {
00125 if (references[i].size() != references[i-1].size()) {
00126 UTIL_THROW2("Reference files are of different lengths");
00127 }
00128 }
00129 in.close();
00130 }
00131
00132
00133 LoadReferences(references);
00134
00135 } else {
00136 StatefulFeatureFunction::SetParameter(key, value);
00137 }
00138
00139 }
00140
00141 std::vector<float> BleuScoreFeature::DefaultWeights() const
00142 {
00143 std::vector<float> ret(m_numScoreComponents, 1);
00144 return ret;
00145 }
00146
00147 void BleuScoreFeature::PrintHistory(std::ostream& out) const
00148 {
00149 out << "source length history=" << m_source_length_history << endl;
00150 out << "target length history=" << m_target_length_history << endl;
00151 out << "ref length history=" << m_ref_length_history << endl;
00152
00153 for (size_t i = 0; i < BleuScoreState::bleu_order; ++i) {
00154 out << "match history/count history (" << i << "):" << m_match_history[i] << "/" << m_count_history[i] << endl;
00155 }
00156 }
00157
00158 void BleuScoreFeature::SetBleuParameters(bool disable, bool sentenceBleu, bool scaleByInputLength, bool scaleByAvgInputLength,
00159 bool scaleByInverseLength, bool scaleByAvgInverseLength,
00160 float scaleByX, float historySmoothing, size_t scheme, bool simpleHistoryBleu)
00161 {
00162 m_enabled = !disable;
00163 m_sentence_bleu = sentenceBleu;
00164 m_simple_history_bleu = simpleHistoryBleu;
00165 m_scale_by_input_length = scaleByInputLength;
00166 m_scale_by_avg_input_length = scaleByAvgInputLength;
00167 m_scale_by_inverse_length = scaleByInverseLength;
00168 m_scale_by_avg_inverse_length = scaleByAvgInverseLength;
00169 m_scale_by_x = scaleByX;
00170 m_historySmoothing = historySmoothing;
00171 m_smoothing_scheme = (SmoothingScheme)scheme;
00172 }
00173
00174
00175
00176 void BleuScoreFeature::LoadReferences(const std::vector< std::vector< std::string > >& refs)
00177 {
00178 m_refs.clear();
00179 FactorCollection& fc = FactorCollection::Instance();
00180 for (size_t file_id = 0; file_id < refs.size(); file_id++) {
00181 for (size_t sent_id = 0; sent_id < refs[file_id].size(); sent_id++) {
00182 const string& ref = refs[file_id][sent_id];
00183 vector<string> refTokens = Tokenize(ref);
00184 if (file_id == 0)
00185 m_refs[sent_id] = RefValue();
00186 pair<vector<size_t>,NGrams>& ref_pair = m_refs[sent_id];
00187 (ref_pair.first).push_back(refTokens.size());
00188 for (size_t order = 1; order <= BleuScoreState::bleu_order; order++) {
00189 for (size_t end_idx = order; end_idx <= refTokens.size(); end_idx++) {
00190 Phrase ngram(1);
00191 for (size_t s_idx = end_idx - order; s_idx < end_idx; s_idx++) {
00192 const Factor* f = fc.AddFactor(Output, 0, refTokens[s_idx]);
00193 Word w;
00194 w.SetFactor(0, f);
00195 ngram.AddWord(w);
00196 }
00197 ref_pair.second[ngram] += 1;
00198 }
00199 }
00200 }
00201 }
00202
00203
00204
00205
00206
00207 }
00208
00209 void BleuScoreFeature::SetCurrSourceLength(size_t source_length)
00210 {
00211 m_cur_source_length = source_length;
00212 }
00213 void BleuScoreFeature::SetCurrNormSourceLength(size_t source_length)
00214 {
00215 m_cur_norm_source_length = source_length;
00216 }
00217
00218
00219 void BleuScoreFeature::SetCurrShortestRefLength(size_t sent_id)
00220 {
00221
00222 int shortestRef = -1;
00223 for (size_t i = 0; i < (m_refs[sent_id].first).size(); ++i) {
00224 if (shortestRef == -1 || (m_refs[sent_id].first)[i] < shortestRef)
00225 shortestRef = (m_refs[sent_id].first)[i];
00226 }
00227 m_cur_ref_length = shortestRef;
00228
00229 }
00230
00231 void BleuScoreFeature::SetCurrAvgRefLength(size_t sent_id)
00232 {
00233
00234 size_t sum = 0;
00235 size_t numberRefs = (m_refs[sent_id].first).size();
00236 for (size_t i = 0; i < numberRefs; ++i) {
00237 sum += (m_refs[sent_id].first)[i];
00238 }
00239 m_cur_ref_length = (float)sum/numberRefs;
00240
00241 }
00242
00243 void BleuScoreFeature::SetCurrReferenceNgrams(size_t sent_id)
00244 {
00245 m_cur_ref_ngrams = m_refs[sent_id].second;
00246 }
00247
00248 size_t BleuScoreFeature::GetShortestRefIndex(size_t ref_id)
00249 {
00250
00251 int shortestRef = -1;
00252 size_t shortestRefIndex = 0;
00253 for (size_t i = 0; i < (m_refs[ref_id].first).size(); ++i) {
00254 if (shortestRef == -1 || (m_refs[ref_id].first)[i] < shortestRef) {
00255 shortestRef = (m_refs[ref_id].first)[i];
00256 shortestRefIndex = i;
00257 }
00258 }
00259 return shortestRefIndex;
00260 }
00261
00262
00263
00264
00265
00266
00267
00268 void BleuScoreFeature::UpdateHistory(const vector< const Word* >& hypo)
00269 {
00270 Phrase phrase(hypo);
00271 std::vector< size_t > ngram_counts(BleuScoreState::bleu_order);
00272 std::vector< size_t > ngram_matches(BleuScoreState::bleu_order);
00273
00274
00275
00276 GetNgramMatchCounts(phrase, m_cur_ref_ngrams, ngram_counts, ngram_matches, 0);
00277
00278
00279 for (size_t i = 0; i < BleuScoreState::bleu_order; i++) {
00280 m_count_history[i] = m_historySmoothing * (m_count_history[i] + ngram_counts[i]);
00281 m_match_history[i] = m_historySmoothing * (m_match_history[i] + ngram_matches[i]);
00282 }
00283
00284
00285 m_source_length_history = m_historySmoothing * (m_source_length_history + m_cur_source_length);
00286 m_target_length_history = m_historySmoothing * (m_target_length_history + hypo.size());
00287 m_ref_length_history = m_historySmoothing * (m_ref_length_history + m_cur_ref_length);
00288 }
00289
00290
00291
00292
00293 void BleuScoreFeature::UpdateHistory(const vector< vector< const Word* > >& hypos, vector<size_t>& sourceLengths, vector<size_t>& ref_ids, size_t rank, size_t epoch)
00294 {
00295 for (size_t ref_id = 0; ref_id < hypos.size(); ++ref_id) {
00296 Phrase phrase(hypos[ref_id]);
00297 std::vector< size_t > ngram_counts(BleuScoreState::bleu_order);
00298 std::vector< size_t > ngram_matches(BleuScoreState::bleu_order);
00299
00300
00301 size_t cur_source_length = sourceLengths[ref_id];
00302 size_t hypo_length = hypos[ref_id].size();
00303 size_t cur_ref_length = GetClosestRefLength(ref_ids[ref_id], hypo_length);
00304 NGrams cur_ref_ngrams = m_refs[ref_ids[ref_id]].second;
00305 cerr << "reference length: " << cur_ref_length << endl;
00306
00307
00308
00309 GetNgramMatchCounts(phrase, cur_ref_ngrams, ngram_counts, ngram_matches, 0);
00310
00311
00312 for (size_t i = 0; i < BleuScoreState::bleu_order; i++) {
00313 m_count_history[i] += ngram_counts[i];
00314 m_match_history[i] += ngram_matches[i];
00315
00316
00317 if (ref_id == hypos.size() - 1) {
00318 m_count_history[i] *= m_historySmoothing;
00319 m_match_history[i] *= m_historySmoothing;
00320 }
00321 }
00322
00323
00324 m_source_length_history += cur_source_length;
00325 m_target_length_history += hypos[ref_id].size();
00326 m_ref_length_history += cur_ref_length;
00327
00328
00329 if (ref_id == hypos.size() - 1) {
00330 cerr << "Rank " << rank << ", epoch " << epoch << " ,source length history: " << m_source_length_history << " --> " << m_source_length_history * m_historySmoothing << endl;
00331 cerr << "Rank " << rank << ", epoch " << epoch << " ,target length history: " << m_target_length_history << " --> " << m_target_length_history * m_historySmoothing << endl;
00332 m_source_length_history *= m_historySmoothing;
00333 m_target_length_history *= m_historySmoothing;
00334 m_ref_length_history *= m_historySmoothing;
00335 }
00336 }
00337 }
00338
00339
00340
00341
00342
00343
00344
00345
00346
00347
00348
00349 size_t BleuScoreFeature::GetClosestRefLength(size_t ref_id, int hypoLength)
00350 {
00351
00352 int currentDist = -1;
00353 int closestRefLength = -1;
00354 for (size_t i = 0; i < (m_refs[ref_id].first).size(); ++i) {
00355 if (closestRefLength == -1 || abs(hypoLength - (int)(m_refs[ref_id].first)[i]) < currentDist) {
00356 closestRefLength = (m_refs[ref_id].first)[i];
00357 currentDist = abs(hypoLength - (int)(m_refs[ref_id].first)[i]);
00358 }
00359 }
00360 return (size_t)closestRefLength;
00361 }
00362
00363
00364
00365
00366
00367 void BleuScoreFeature::GetNgramMatchCounts(Phrase& phrase,
00368 const NGrams& ref_ngram_counts,
00369 std::vector< size_t >& ret_counts,
00370 std::vector< size_t >& ret_matches,
00371 size_t skip_first) const
00372 {
00373 NGrams::const_iterator ref_ngram_counts_iter;
00374 size_t ngram_start_idx, ngram_end_idx;
00375
00376
00377 for (size_t end_idx = skip_first; end_idx < phrase.GetSize(); end_idx++) {
00378 for (size_t order = 0; order < BleuScoreState::bleu_order; order++) {
00379 if (order > end_idx) break;
00380
00381 ngram_end_idx = end_idx;
00382 ngram_start_idx = end_idx - order;
00383
00384 Phrase ngram = phrase.GetSubString(Range(ngram_start_idx, ngram_end_idx), 0);
00385 ret_counts[order]++;
00386
00387 ref_ngram_counts_iter = ref_ngram_counts.find(ngram);
00388 if (ref_ngram_counts_iter != ref_ngram_counts.end())
00389 ret_matches[order]++;
00390 }
00391 }
00392 }
00393
00394
00395 void BleuScoreFeature::GetNgramMatchCounts_prefix(Phrase& phrase,
00396 const NGrams& ref_ngram_counts,
00397 std::vector< size_t >& ret_counts,
00398 std::vector< size_t >& ret_matches,
00399 size_t new_start_indices,
00400 size_t last_end_index) const
00401 {
00402 NGrams::const_iterator ref_ngram_counts_iter;
00403 size_t ngram_start_idx, ngram_end_idx;
00404
00405
00406 for (size_t start_idx = 0; start_idx < new_start_indices; start_idx++) {
00407 for (size_t order = 0; order < BleuScoreState::bleu_order; order++) {
00408 ngram_start_idx = start_idx;
00409 ngram_end_idx = start_idx + order;
00410 if (order > ngram_end_idx) break;
00411 if (ngram_end_idx > last_end_index) break;
00412
00413 Phrase ngram = phrase.GetSubString(Range(ngram_start_idx, ngram_end_idx), 0);
00414 ret_counts[order]++;
00415
00416 ref_ngram_counts_iter = ref_ngram_counts.find(ngram);
00417 if (ref_ngram_counts_iter != ref_ngram_counts.end())
00418 ret_matches[order]++;
00419 }
00420 }
00421 }
00422
00423
00424 void BleuScoreFeature::GetNgramMatchCounts_overlap(Phrase& phrase,
00425 const NGrams& ref_ngram_counts,
00426 std::vector< size_t >& ret_counts,
00427 std::vector< size_t >& ret_matches,
00428 size_t overlap_index) const
00429 {
00430 NGrams::const_iterator ref_ngram_counts_iter;
00431 size_t ngram_start_idx, ngram_end_idx;
00432
00433
00434 for (size_t end_idx = overlap_index; end_idx < phrase.GetSize(); end_idx++) {
00435 if (end_idx >= (overlap_index+BleuScoreState::bleu_order-1)) break;
00436 for (size_t order = 0; order < BleuScoreState::bleu_order; order++) {
00437 if (order > end_idx) break;
00438
00439 ngram_end_idx = end_idx;
00440 ngram_start_idx = end_idx - order;
00441 if (ngram_start_idx >= overlap_index) continue;
00442
00443 Phrase ngram = phrase.GetSubString(Range(ngram_start_idx, ngram_end_idx), 0);
00444 ret_counts[order]++;
00445
00446 ref_ngram_counts_iter = ref_ngram_counts.find(ngram);
00447 if (ref_ngram_counts_iter != ref_ngram_counts.end())
00448 ret_matches[order]++;
00449 }
00450 }
00451 }
00452
00453 void BleuScoreFeature::GetClippedNgramMatchesAndCounts(Phrase& phrase,
00454 const NGrams& ref_ngram_counts,
00455 std::vector< size_t >& ret_counts,
00456 std::vector< size_t >& ret_matches,
00457 size_t skip_first) const
00458 {
00459 NGrams::const_iterator ref_ngram_counts_iter;
00460 size_t ngram_start_idx, ngram_end_idx;
00461
00462 Matches ngram_matches;
00463 for (size_t end_idx = skip_first; end_idx < phrase.GetSize(); end_idx++) {
00464 for (size_t order = 0; order < BleuScoreState::bleu_order; order++) {
00465 if (order > end_idx) break;
00466
00467 ngram_end_idx = end_idx;
00468 ngram_start_idx = end_idx - order;
00469
00470 Phrase ngram = phrase.GetSubString(Range(ngram_start_idx, ngram_end_idx), 0);
00471 ret_counts[order]++;
00472
00473 ref_ngram_counts_iter = ref_ngram_counts.find(ngram);
00474 if (ref_ngram_counts_iter != ref_ngram_counts.end()) {
00475 ngram_matches[order][ngram]++;
00476 }
00477 }
00478 }
00479
00480
00481 for (size_t order = 0; order < BleuScoreState::bleu_order; order++) {
00482 NGrams::const_iterator iter;
00483
00484
00485 for (iter=ngram_matches[order].begin(); iter != ngram_matches[order].end(); ++iter) {
00486 ref_ngram_counts_iter = ref_ngram_counts.find(iter->first);
00487 if (iter->second > ref_ngram_counts_iter->second) {
00488 ret_matches[order] += ref_ngram_counts_iter->second;
00489 } else {
00490 ret_matches[order] += iter->second;
00491 }
00492 }
00493 }
00494 }
00495
00496
00497
00498
00499
00500 FFState* BleuScoreFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
00501 const FFState* prev_state,
00502 ScoreComponentCollection* accumulator) const
00503 {
00504 if (!m_enabled) return new BleuScoreState(m_is_syntax);
00505
00506 NGrams::const_iterator reference_ngrams_iter;
00507 const BleuScoreState& ps = static_cast<const BleuScoreState&>(*prev_state);
00508 BleuScoreState* new_state = new BleuScoreState(ps);
00509
00510 float old_bleu, new_bleu;
00511 size_t num_new_words, ctx_start_idx, ctx_end_idx;
00512
00513
00514 old_bleu = CalculateBleu(new_state);
00515
00516
00517 num_new_words = cur_hypo.GetCurrTargetLength();
00518 if (num_new_words == 0) {
00519 return new_state;
00520 }
00521
00522 Phrase new_words = ps.m_words;
00523 new_words.Append(cur_hypo.GetCurrTargetPhrase());
00524
00525
00526
00527 GetNgramMatchCounts(new_words,
00528 m_cur_ref_ngrams,
00529 new_state->m_ngram_counts,
00530 new_state->m_ngram_matches,
00531 new_state->m_words.GetSize());
00532
00533
00534 ctx_end_idx = new_words.GetSize()-1;
00535 size_t bleu_context_length = BleuScoreState::bleu_order -1;
00536 if (ctx_end_idx > bleu_context_length) {
00537 ctx_start_idx = ctx_end_idx - bleu_context_length;
00538 } else {
00539 ctx_start_idx = 0;
00540 }
00541
00542 const Bitmap &coverageVector = cur_hypo.GetWordsBitmap();
00543 new_state->m_source_length = coverageVector.GetNumWordsCovered();
00544
00545 new_state->m_words = new_words.GetSubString(Range(ctx_start_idx,
00546 ctx_end_idx));
00547 new_state->m_target_length += cur_hypo.GetCurrTargetLength();
00548
00549
00550 new_state->m_scaled_ref_length = m_cur_ref_length *
00551 ((float)coverageVector.GetNumWordsCovered()/coverageVector.GetSize());
00552
00553
00554 new_bleu = CalculateBleu(new_state);
00555
00556
00557 accumulator->PlusEquals(this, new_bleu - old_bleu);
00558 return new_state;
00559 }
00560
00561 FFState* BleuScoreFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID,
00562 ScoreComponentCollection* accumulator ) const
00563 {
00564 if (!m_enabled) return new BleuScoreState(m_is_syntax);
00565
00566 NGrams::const_iterator reference_ngrams_iter;
00567
00568 const Phrase& curr_target_phrase = static_cast<const Phrase&>(cur_hypo.GetCurrTargetPhrase());
00569
00570
00571
00572 float old_bleu = 0, new_bleu = 0;
00573 size_t num_old_words = 0, num_words_first_prev = 0;
00574 size_t num_words_added_left = 0, num_words_added_right = 0;
00575
00576
00577 assert(cur_hypo.GetPrevHypos().size() <= 2);
00578 BleuScoreState* new_state;
00579 if (cur_hypo.GetPrevHypos().size() == 0)
00580 new_state = new BleuScoreState(m_is_syntax);
00581 else {
00582 const FFState* prev_state_zero = cur_hypo.GetPrevHypo(0)->GetFFState(featureID);
00583 const BleuScoreState& ps_zero = static_cast<const BleuScoreState&>(*prev_state_zero);
00584 new_state = new BleuScoreState(ps_zero);
00585 num_words_first_prev = ps_zero.m_target_length;
00586
00587 for (size_t i = 0; i < cur_hypo.GetPrevHypos().size(); ++i) {
00588 const FFState* prev_state = cur_hypo.GetPrevHypo(i)->GetFFState(featureID);
00589 const BleuScoreState* ps = static_cast<const BleuScoreState*>(prev_state);
00590 BleuScoreState* ps_nonConst = const_cast<BleuScoreState*>(ps);
00591
00592
00593
00594 old_bleu += CalculateBleu(ps_nonConst);
00595 num_old_words += ps->m_target_length;
00596
00597 if (i > 0)
00598
00599 new_state->AddNgramCountAndMatches(ps_nonConst->m_ngram_counts, ps_nonConst->m_ngram_matches);
00600 }
00601 }
00602
00603
00604 size_t numWordsCovered = cur_hypo.GetCurrSourceRange().GetNumWordsCovered();
00605 if (numWordsCovered == m_cur_source_length) {
00606
00607
00608 return new_state;
00609 }
00610
00611
00612 Phrase new_words = cur_hypo.GetOutputPhrase();
00613 new_state->m_words = new_words;
00614 size_t num_curr_words = new_words.GetSize();
00615
00616
00617 if (num_old_words == 0) {
00618
00619 GetNgramMatchCounts(new_words,
00620 m_cur_ref_ngrams,
00621 new_state->m_ngram_counts,
00622 new_state->m_ngram_matches,
00623 0);
00624 } else if (new_words.GetSize() == num_old_words) {
00625
00626 num_words_added_right = num_curr_words - num_words_first_prev;
00627
00628
00629 GetNgramMatchCounts_overlap(new_words,
00630 m_cur_ref_ngrams,
00631 new_state->m_ngram_counts,
00632 new_state->m_ngram_matches,
00633 num_words_first_prev);
00634 } else if (num_old_words + curr_target_phrase.GetNumTerminals() == num_curr_words) {
00635 assert(curr_target_phrase.GetSize() == curr_target_phrase.GetNumTerminals()+1);
00636
00637 for (size_t i = 0; i < curr_target_phrase.GetSize(); ++i)
00638 if (curr_target_phrase.GetWord(i).IsNonTerminal()) {
00639 num_words_added_left = i;
00640 num_words_added_right = curr_target_phrase.GetSize() - (i+1);
00641 break;
00642 }
00643
00644
00645
00646 if (num_words_added_left > 0)
00647 GetNgramMatchCounts_prefix(new_words,
00648 m_cur_ref_ngrams,
00649 new_state->m_ngram_counts,
00650 new_state->m_ngram_matches,
00651 num_words_added_left,
00652 num_curr_words - num_words_added_right - 1);
00653
00654
00655
00656 if (num_words_added_right > 0)
00657 GetNgramMatchCounts(new_words,
00658 m_cur_ref_ngrams,
00659 new_state->m_ngram_counts,
00660 new_state->m_ngram_matches,
00661 num_words_added_left + num_old_words);
00662 } else {
00663 cerr << "undefined state.. " << endl;
00664 exit(1);
00665 }
00666
00667
00668 size_t ctx_start_idx = 0;
00669 size_t ctx_end_idx = new_words.GetSize()-1;
00670 size_t bleu_context_length = BleuScoreState::bleu_order -1;
00671 if (ctx_end_idx > bleu_context_length) {
00672 ctx_start_idx = ctx_end_idx - bleu_context_length;
00673 }
00674
00675 new_state->m_source_length = cur_hypo.GetCurrSourceRange().GetNumWordsCovered();
00676 new_state->m_words = new_words.GetSubString(Range(ctx_start_idx, ctx_end_idx));
00677 new_state->m_target_length = cur_hypo.GetOutputPhrase().GetSize();
00678
00679
00680
00681 size_t cur_source_length = m_cur_source_length;
00682 new_state->m_scaled_ref_length = m_cur_ref_length * (float(new_state->m_source_length)/cur_source_length);
00683
00684
00685 new_bleu = CalculateBleu(new_state);
00686
00687
00688 accumulator->PlusEquals(this, new_bleu - old_bleu);
00689 return new_state;
00690 }
00691
00695 float BleuScoreFeature::CalculateBleu(Phrase translation) const
00696 {
00697 if (translation.GetSize() == 0)
00698 return 0.0;
00699
00700 Phrase normTranslation = translation;
00701
00702 if (m_cur_source_length != m_cur_norm_source_length) {
00703 Range* range = new Range(1, translation.GetSize()-2);
00704 normTranslation = translation.GetSubString(*range);
00705 }
00706
00707
00708 BleuScoreState* state = new BleuScoreState(m_is_syntax);
00709 GetClippedNgramMatchesAndCounts(normTranslation,
00710 m_cur_ref_ngrams,
00711 state->m_ngram_counts,
00712 state->m_ngram_matches,
00713 0);
00714
00715
00716 state->m_words = normTranslation;
00717 state->m_source_length = m_cur_norm_source_length;
00718 state->m_target_length = normTranslation.GetSize();
00719 state->m_scaled_ref_length = m_cur_ref_length;
00720
00721
00722 return CalculateBleu(state);
00723 }
00724
00725
00726
00727
00728 float BleuScoreFeature::CalculateBleu(BleuScoreState* state) const
00729 {
00730 if (!state->m_ngram_counts[0]) return 0;
00731 if (!state->m_ngram_matches[0]) return 0;
00732
00733 float precision = 1.0;
00734 float smooth = 1;
00735 float smoothed_count, smoothed_matches;
00736
00737 if (m_sentence_bleu || m_simple_history_bleu) {
00738
00739
00740
00741 for (size_t i = 0; i < BleuScoreState::bleu_order; i++) {
00742 if (state->m_ngram_counts[i]) {
00743 smoothed_matches = state->m_ngram_matches[i];
00744 smoothed_count = state->m_ngram_counts[i];
00745
00746 switch (m_smoothing_scheme) {
00747 case PLUS_ONE:
00748 default:
00749 if (i > 0) {
00750
00751 smoothed_matches += 1;
00752 smoothed_count += 1;
00753 }
00754 break;
00755 case PLUS_POINT_ONE:
00756 if (i > 0) {
00757
00758 smoothed_matches += 0.1;
00759 smoothed_count += 0.1;
00760 }
00761 break;
00762 case PAPINENI:
00763 if (state->m_ngram_matches[i] == 0) {
00764 smooth *= 0.5;
00765 smoothed_matches += smooth;
00766 smoothed_count += smooth;
00767 }
00768 break;
00769 }
00770
00771 if (m_simple_history_bleu) {
00772 smoothed_matches += m_match_history[i];
00773 smoothed_count += m_count_history[i];
00774 }
00775
00776 precision *= smoothed_matches/smoothed_count;
00777 }
00778 }
00779
00780
00781 precision = pow(precision, (float)1/4);
00782
00783
00784
00785
00786
00787
00788
00789 if (m_simple_history_bleu) {
00790 if ((m_target_length_history + state->m_target_length) < (m_ref_length_history + state->m_scaled_ref_length)) {
00791 float smoothed_target_length = m_target_length_history + state->m_target_length;
00792 float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length;
00793 precision *= exp(1 - (smoothed_ref_length/smoothed_target_length));
00794 }
00795 } else {
00796 if (state->m_target_length < state->m_scaled_ref_length) {
00797 float target_length = state->m_target_length;
00798 float ref_length = state->m_scaled_ref_length;
00799 precision *= exp(1 - (ref_length/target_length));
00800 }
00801 }
00802
00803
00804
00805
00806
00807
00808 if (m_scale_by_input_length) {
00809 precision *= m_cur_norm_source_length;
00810 } else if (m_scale_by_avg_input_length) {
00811 precision *= m_avg_input_length;
00812 } else if (m_scale_by_inverse_length) {
00813 precision *= (100/m_cur_norm_source_length);
00814 } else if (m_scale_by_avg_inverse_length) {
00815 precision *= (100/m_avg_input_length);
00816 }
00817
00818 return precision * m_scale_by_x;
00819 } else {
00820
00821
00822
00823
00824
00825 for (size_t i = 0; i < BleuScoreState::bleu_order; i++) {
00826 if (state->m_ngram_counts[i]) {
00827 smoothed_matches = m_match_history[i] + state->m_ngram_matches[i] + 0.1;
00828 smoothed_count = m_count_history[i] + state->m_ngram_counts[i] + 0.1;
00829 precision *= smoothed_matches/smoothed_count;
00830 }
00831 }
00832
00833
00834 precision = pow(precision, (float)1/4);
00835
00836
00837 if ((m_target_length_history + state->m_target_length) < (m_ref_length_history + state->m_scaled_ref_length))
00838 precision *= exp(1 - ((m_ref_length_history + state->m_scaled_ref_length)/(m_target_length_history + state->m_target_length)));
00839
00840 cerr << "precision: " << precision << endl;
00841
00842
00843 float precision_pd = 1.0;
00844 if (m_target_length_history > 0) {
00845 for (size_t i = 0; i < BleuScoreState::bleu_order; i++)
00846 if (m_count_history[i] != 0)
00847 precision_pd *= (m_match_history[i] + 0.1)/(m_count_history[i] + 0.1);
00848
00849
00850 precision_pd = pow(precision_pd, (float)1/4);
00851
00852
00853 if (m_target_length_history < m_ref_length_history)
00854 precision_pd *= exp(1 - (m_ref_length_history/m_target_length_history));
00855 } else
00856 precision_pd = 0;
00857
00858
00859 cerr << "precision pd: " << precision_pd << endl;
00860
00861 float sentence_impact;
00862 if (m_target_length_history > 0)
00863 sentence_impact = m_target_length_history * (precision - precision_pd);
00864 else
00865 sentence_impact = precision;
00866
00867 cerr << "sentence impact: " << sentence_impact << endl;
00868 return sentence_impact * m_scale_by_x;
00869 }
00870 }
00871
00872 const FFState* BleuScoreFeature::EmptyHypothesisState(const InputType& input) const
00873 {
00874 return new BleuScoreState(m_is_syntax);
00875 }
00876
00877 bool BleuScoreFeature::IsUseable(const FactorMask &mask) const
00878 {
00879
00880 bool ret = mask[0];
00881 return 0;
00882 }
00883
00884 void
00885 BleuScoreFeature::
00886 Load(AllOptions::ptr const& opts)
00887 {
00888 m_is_syntax = is_syntax(opts->search.algo);
00889 }
00890
00891 }
00892