#include <BleuScoreFeature.h>
Public Types | |
typedef boost::unordered_map < size_t, RefValue > | RefCounts |
typedef boost::unordered_map < size_t, NGrams > | Matches |
Public Member Functions | |
BleuScoreFeature (const std::string &line) | |
void | SetParameter (const std::string &key, const std::string &value) |
std::vector< float > | DefaultWeights () const |
void | PrintHistory (std::ostream &out) const |
void | LoadReferences (const std::vector< std::vector< std::string > > &) |
void | SetCurrSourceLength (size_t) |
void | SetCurrNormSourceLength (size_t) |
void | SetCurrShortestRefLength (size_t) |
void | SetCurrAvgRefLength (size_t sent_id) |
void | SetAvgInputLength (float l) |
void | SetCurrReferenceNgrams (size_t sent_id) |
size_t | GetShortestRefIndex (size_t ref_id) |
size_t | GetClosestRefLength (size_t ref_id, int hypoLength) |
void | UpdateHistory (const std::vector< const Word * > &) |
void | UpdateHistory (const std::vector< std::vector< const Word * > > &hypos, std::vector< size_t > &sourceLengths, std::vector< size_t > &ref_ids, size_t rank, size_t epoch) |
void | PrintRefLength (const std::vector< size_t > &ref_ids) |
void | SetBleuParameters (bool disable, bool sentenceBleu, bool scaleByInputLength, bool scaleByAvgInputLength, bool scaleByInverseLength, bool scaleByAvgInverseLength, float scaleByX, float historySmoothing, size_t scheme, bool simpleHistoryBleu) |
void | GetNgramMatchCounts (Phrase &, const NGrams &, std::vector< size_t > &, std::vector< size_t > &, size_t skip=0) const |
void | GetNgramMatchCounts_prefix (Phrase &, const NGrams &, std::vector< size_t > &, std::vector< size_t > &, size_t new_start_indices, size_t last_end_index) const |
void | GetNgramMatchCounts_overlap (Phrase &phrase, const NGrams &ref_ngram_counts, std::vector< size_t > &ret_counts, std::vector< size_t > &ret_matches, size_t overlap_index) const |
void | GetClippedNgramMatchesAndCounts (Phrase &, const NGrams &, std::vector< size_t > &, std::vector< size_t > &, size_t skip=0) const |
FFState * | EvaluateWhenApplied (const Hypothesis &cur_hypo, const FFState *prev_state, ScoreComponentCollection *accumulator) const |
This interface should be implemented. Notes: When evaluating the value of this feature function, you should avoid calling hypo.GetPrevHypo(). If you need something from the "previous" hypothesis, you should store it in an FFState object which will be passed in as prev_state. If you don't do this, you will get in trouble. | |
FFState * | EvaluateWhenApplied (const ChartHypothesis &cur_hypo, int featureID, ScoreComponentCollection *accumulator) const |
bool | Enabled () const |
bool | IsUseable (const FactorMask &mask) const |
float | CalculateBleu (BleuScoreState *) const |
float | CalculateBleu (Phrase translation) const |
const FFState * | EmptyHypothesisState (const InputType &) const |
return the state associated with the empty hypothesis for a given sentence | |
float | GetSourceLengthHistory () |
float | GetTargetLengthHistory () |
float | GetAverageInputLength () |
void | Load (AllOptions::ptr const &opts) |
override to load model files | |
Static Public Member Functions | |
static const std::vector < BleuScoreFeature * > & | GetColl () |
Definition at line 63 of file BleuScoreFeature.h.
typedef boost::unordered_map<size_t, NGrams> Moses::BleuScoreFeature::Matches |
Definition at line 71 of file BleuScoreFeature.h.
typedef boost::unordered_map<size_t, RefValue > Moses::BleuScoreFeature::RefCounts |
Definition at line 70 of file BleuScoreFeature.h.
Moses::BleuScoreFeature::BleuScoreFeature | ( | const std::string & | line | ) |
Definition at line 75 of file BleuScoreFeature.cpp.
References Moses::FeatureFunction::m_tuneable, and Moses::FeatureFunction::ReadParameters().
float Moses::BleuScoreFeature::CalculateBleu | ( | Phrase | translation | ) | const |
Calculate real sentence Bleu score of complete translation
Definition at line 695 of file BleuScoreFeature.cpp.
References CalculateBleu(), GetClippedNgramMatchesAndCounts(), Moses::Phrase::GetSize(), Moses::Phrase::GetSubString(), range, and state.
float Moses::BleuScoreFeature::CalculateBleu | ( | BleuScoreState * | state | ) | const |
Definition at line 728 of file BleuScoreFeature.cpp.
References smooth.
Referenced by CalculateBleu(), and EvaluateWhenApplied().
std::vector< float > Moses::BleuScoreFeature::DefaultWeights | ( | ) | const [virtual] |
Reimplemented from Moses::FeatureFunction.
Definition at line 141 of file BleuScoreFeature.cpp.
References Moses::FeatureFunction::m_numScoreComponents.
const FFState * Moses::BleuScoreFeature::EmptyHypothesisState | ( | const InputType & | input | ) | const [virtual] |
return the state associated with the empty hypothesis for a given sentence
Implements Moses::StatefulFeatureFunction.
Definition at line 872 of file BleuScoreFeature.cpp.
bool Moses::BleuScoreFeature::Enabled | ( | ) | const [inline] |
Definition at line 127 of file BleuScoreFeature.h.
FFState * Moses::BleuScoreFeature::EvaluateWhenApplied | ( | const ChartHypothesis & | cur_hypo, | |
int | featureID, | |||
ScoreComponentCollection * | accumulator | |||
) | const [virtual] |
Implements Moses::StatefulFeatureFunction.
Definition at line 561 of file BleuScoreFeature.cpp.
References CalculateBleu(), Moses::ChartHypothesis::GetCurrSourceRange(), Moses::ChartHypothesis::GetCurrTargetPhrase(), Moses::ChartHypothesis::GetFFState(), GetNgramMatchCounts(), GetNgramMatchCounts_overlap(), GetNgramMatchCounts_prefix(), Moses::Phrase::GetNumTerminals(), Moses::Range::GetNumWordsCovered(), Moses::ChartHypothesis::GetOutputPhrase(), Moses::ChartHypothesis::GetPrevHypo(), Moses::ChartHypothesis::GetPrevHypos(), Moses::Phrase::GetSize(), Moses::Phrase::GetSubString(), Moses::Phrase::GetWord(), Moses::Word::IsNonTerminal(), and Moses::ScoreComponentCollection::PlusEquals().
FFState * Moses::BleuScoreFeature::EvaluateWhenApplied | ( | const Hypothesis & | cur_hypo, | |
const FFState * | prev_state, | |||
ScoreComponentCollection * | accumulator | |||
) | const [virtual] |
This interface should be implemented. Notes: When evaluating the value of this feature function, you should avoid calling hypo.GetPrevHypo(). If you need something from the "previous" hypothesis, you should store it in an FFState object which will be passed in as prev_state. If you don't do this, you will get in trouble.
Implements Moses::StatefulFeatureFunction.
Definition at line 500 of file BleuScoreFeature.cpp.
References Moses::Phrase::Append(), CalculateBleu(), Moses::Hypothesis::GetCurrTargetLength(), Moses::Hypothesis::GetCurrTargetPhrase(), GetNgramMatchCounts(), Moses::Bitmap::GetNumWordsCovered(), Moses::Bitmap::GetSize(), Moses::Phrase::GetSize(), Moses::Phrase::GetSubString(), Moses::Hypothesis::GetWordsBitmap(), and Moses::ScoreComponentCollection::PlusEquals().
float Moses::BleuScoreFeature::GetAverageInputLength | ( | ) | [inline] |
Definition at line 143 of file BleuScoreFeature.h.
void Moses::BleuScoreFeature::GetClippedNgramMatchesAndCounts | ( | Phrase & | phrase, | |
const NGrams & | ref_ngram_counts, | |||
std::vector< size_t > & | ret_counts, | |||
std::vector< size_t > & | ret_matches, | |||
size_t | skip = 0 | |||
) | const |
Definition at line 453 of file BleuScoreFeature.cpp.
References begin, Moses::Phrase::GetSize(), and Moses::Phrase::GetSubString().
Referenced by CalculateBleu().
size_t Moses::BleuScoreFeature::GetClosestRefLength | ( | size_t | ref_id, | |
int | hypoLength | |||
) |
Definition at line 349 of file BleuScoreFeature.cpp.
static const std::vector<BleuScoreFeature*>& Moses::BleuScoreFeature::GetColl | ( | ) | [inline, static] |
Definition at line 66 of file BleuScoreFeature.h.
void Moses::BleuScoreFeature::GetNgramMatchCounts | ( | Phrase & | phrase, | |
const NGrams & | ref_ngram_counts, | |||
std::vector< size_t > & | ret_counts, | |||
std::vector< size_t > & | ret_matches, | |||
size_t | skip = 0 | |||
) | const |
Definition at line 367 of file BleuScoreFeature.cpp.
References Moses::Phrase::GetSize(), and Moses::Phrase::GetSubString().
Referenced by EvaluateWhenApplied().
void Moses::BleuScoreFeature::GetNgramMatchCounts_overlap | ( | Phrase & | phrase, | |
const NGrams & | ref_ngram_counts, | |||
std::vector< size_t > & | ret_counts, | |||
std::vector< size_t > & | ret_matches, | |||
size_t | overlap_index | |||
) | const |
Definition at line 424 of file BleuScoreFeature.cpp.
References Moses::Phrase::GetSize(), and Moses::Phrase::GetSubString().
Referenced by EvaluateWhenApplied().
void Moses::BleuScoreFeature::GetNgramMatchCounts_prefix | ( | Phrase & | phrase, | |
const NGrams & | ref_ngram_counts, | |||
std::vector< size_t > & | ret_counts, | |||
std::vector< size_t > & | ret_matches, | |||
size_t | new_start_indices, | |||
size_t | last_end_index | |||
) | const |
Definition at line 395 of file BleuScoreFeature.cpp.
References Moses::Phrase::GetSubString().
Referenced by EvaluateWhenApplied().
size_t Moses::BleuScoreFeature::GetShortestRefIndex | ( | size_t | ref_id | ) |
Definition at line 248 of file BleuScoreFeature.cpp.
float Moses::BleuScoreFeature::GetSourceLengthHistory | ( | ) | [inline] |
Definition at line 137 of file BleuScoreFeature.h.
float Moses::BleuScoreFeature::GetTargetLengthHistory | ( | ) | [inline] |
Definition at line 140 of file BleuScoreFeature.h.
bool Moses::BleuScoreFeature::IsUseable | ( | const FactorMask & | mask | ) | const [virtual] |
void Moses::BleuScoreFeature::Load | ( | AllOptions::ptr const & | opts | ) | [virtual] |
override to load model files
Reimplemented from Moses::FeatureFunction.
Definition at line 886 of file BleuScoreFeature.cpp.
References Moses::is_syntax().
void Moses::BleuScoreFeature::LoadReferences | ( | const std::vector< std::vector< std::string > > & | refs | ) |
Definition at line 176 of file BleuScoreFeature.cpp.
References Moses::FactorCollection::AddFactor(), Moses::Phrase::AddWord(), Moses::BleuScoreState::bleu_order, Moses::FactorCollection::Instance(), Moses::Output, Moses::Word::SetFactor(), and Moses::Tokenize().
Referenced by SetParameter().
void Moses::BleuScoreFeature::PrintHistory | ( | std::ostream & | out | ) | const |
void Moses::BleuScoreFeature::PrintRefLength | ( | const std::vector< size_t > & | ref_ids | ) |
void Moses::BleuScoreFeature::SetAvgInputLength | ( | float | l | ) | [inline] |
Definition at line 85 of file BleuScoreFeature.h.
void Moses::BleuScoreFeature::SetBleuParameters | ( | bool | disable, | |
bool | sentenceBleu, | |||
bool | scaleByInputLength, | |||
bool | scaleByAvgInputLength, | |||
bool | scaleByInverseLength, | |||
bool | scaleByAvgInverseLength, | |||
float | scaleByX, | |||
float | historySmoothing, | |||
size_t | scheme, | |||
bool | simpleHistoryBleu | |||
) |
Definition at line 158 of file BleuScoreFeature.cpp.
void Moses::BleuScoreFeature::SetCurrAvgRefLength | ( | size_t | sent_id | ) |
Definition at line 231 of file BleuScoreFeature.cpp.
void Moses::BleuScoreFeature::SetCurrNormSourceLength | ( | size_t | source_length | ) |
Definition at line 213 of file BleuScoreFeature.cpp.
void Moses::BleuScoreFeature::SetCurrReferenceNgrams | ( | size_t | sent_id | ) |
Definition at line 243 of file BleuScoreFeature.cpp.
void Moses::BleuScoreFeature::SetCurrShortestRefLength | ( | size_t | sent_id | ) |
Definition at line 219 of file BleuScoreFeature.cpp.
void Moses::BleuScoreFeature::SetCurrSourceLength | ( | size_t | source_length | ) |
Definition at line 209 of file BleuScoreFeature.cpp.
void Moses::BleuScoreFeature::SetParameter | ( | const std::string & | key, | |
const std::string & | value | |||
) | [virtual] |
Reimplemented from Moses::FeatureFunction.
Definition at line 102 of file BleuScoreFeature.cpp.
References LoadReferences(), Moses::Tokenize(), UTIL_THROW2, and UTIL_THROW_IF2.
void Moses::BleuScoreFeature::UpdateHistory | ( | const std::vector< std::vector< const Word * > > & | hypos, | |
std::vector< size_t > & | sourceLengths, | |||
std::vector< size_t > & | ref_ids, | |||
size_t | rank, | |||
size_t | epoch | |||
) |
void Moses::BleuScoreFeature::UpdateHistory | ( | const std::vector< const Word * > & | ) |