#include <Ken.h>
Public Member Functions | |
LanguageModelKen (const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method) | |
virtual const FFState * | EmptyHypothesisState (const InputType &) const |
return the state associated with the empty hypothesis for a given sentence | |
virtual void | CalcScore (const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const |
virtual FFState * | EvaluateWhenApplied (const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const |
This interface should be implemented. Notes: When evaluating the value of this feature function, you should avoid calling hypo.GetPrevHypo(). If you need something from the "previous" hypothesis, you should store it in an FFState object which will be passed in as prev_state. If you don't do this, you will get in trouble. | |
virtual FFState * | EvaluateWhenApplied (const ChartHypothesis &cur_hypo, int featureID, ScoreComponentCollection *accumulator) const |
virtual FFState * | EvaluateWhenApplied (const Syntax::SHyperedge &hyperedge, int featureID, ScoreComponentCollection *accumulator) const |
virtual void | IncrementalCallback (Incremental::Manager &manager) const |
virtual void | ReportHistoryOrder (std::ostream &out, const Phrase &phrase) const |
virtual bool | IsUseable (const FactorMask &mask) const |
Protected Member Functions | |
void | LoadModel (const std::string &file, util::LoadMethod load_method) |
lm::WordIndex | TranslateID (const Word &word) const |
Protected Attributes | |
boost::shared_ptr< Model > | m_ngram |
const Factor * | m_beginSentenceFactor |
FactorType | m_factorType |
std::vector< lm::WordIndex > | m_lmIdLookup |
Friends | |
class | InMemoryPerSentenceOnDemandLM |
Definition at line 53 of file Ken.h.
Moses::LanguageModelKen< Model >::LanguageModelKen | ( | const std::string & | line, | |
const std::string & | file, | |||
FactorType | factorType, | |||
util::LoadMethod | load_method | |||
) | [inline] |
Definition at line 111 of file Ken.cpp.
References Moses::LanguageModelKen< Model >::LoadModel(), and Moses::FeatureFunction::ReadParameters().
void Moses::LanguageModelKen< Model >::CalcScore | ( | const Phrase & | phrase, | |
float & | fullScore, | |||
float & | ngramScore, | |||
size_t & | oovCount | |||
) | const [inline, virtual] |
Reimplemented in Moses::BackwardLanguageModel< Model >, and Moses::BackwardLanguageModel< lm::ngram::ProbingModel >.
Definition at line 146 of file Ken.cpp.
References lm::ngram::RuleScore< M >::BeginSentence(), lm::ngram::RuleScore< M >::Finish(), Moses::Word::GetFactor(), Moses::Phrase::GetSize(), Moses::Phrase::GetWord(), Moses::Word::IsNonTerminal(), Moses::LanguageModelKen< Model >::m_beginSentenceFactor, Moses::LanguageModelKen< Model >::m_factorType, Moses::LanguageModelKen< Model >::m_ngram, lm::ngram::RuleScore< M >::Reset(), lm::ngram::RuleScore< M >::Terminal(), Moses::TransformLMScore(), and Moses::LanguageModelKen< Model >::TranslateID().
Referenced by Moses::InMemoryPerSentenceOnDemandLM::CalcScore().
const FFState * Moses::LanguageModelKen< Model >::EmptyHypothesisState | ( | const InputType & | input | ) | const [inline, virtual] |
return the state associated with the empty hypothesis for a given sentence
Implements Moses::LanguageModel.
Reimplemented in Moses::BackwardLanguageModel< Model >, and Moses::BackwardLanguageModel< lm::ngram::ProbingModel >.
Definition at line 139 of file Ken.cpp.
Referenced by Moses::InMemoryPerSentenceOnDemandLM::EmptyHypothesisState().
FFState * Moses::LanguageModelKen< Model >::EvaluateWhenApplied | ( | const Syntax::SHyperedge & | hyperedge, | |
int | featureID, | |||
ScoreComponentCollection * | accumulator | |||
) | const [inline, virtual] |
Reimplemented from Moses::StatefulFeatureFunction.
Definition at line 331 of file Ken.cpp.
References lm::ngram::RuleScore< M >::BeginSentence(), Moses::LanguageModelChartStateKenLM::GetChartState(), Moses::Word::GetFactor(), Moses::Word::IsNonTerminal(), Moses::Syntax::SHyperedge::label, Moses::ScoreComponentCollection::PlusEquals(), Moses::Syntax::SVertex::states, Moses::Syntax::SHyperedge::tail, and Moses::TransformLMScore().
FFState * Moses::LanguageModelKen< Model >::EvaluateWhenApplied | ( | const ChartHypothesis & | cur_hypo, | |
int | featureID, | |||
ScoreComponentCollection * | accumulator | |||
) | const [inline, virtual] |
Implements Moses::StatefulFeatureFunction.
Definition at line 279 of file Ken.cpp.
References Moses::LanguageModelChartStateKenLM::GetChartState(), Moses::ChartHypothesis::GetCurrTargetPhrase(), Moses::Word::GetFactor(), Moses::ChartHypothesis::GetFFState(), Moses::ChartHypothesis::GetPrevHypo(), Moses::ChartTranslationOption::GetScores(), Moses::ScoreComponentCollection::GetScoresForProducer(), Moses::Phrase::GetSize(), Moses::ChartHypothesis::GetTranslationOption(), Moses::Phrase::GetWord(), Moses::Word::IsNonTerminal(), Moses::LanguageModelKen< Model >::m_beginSentenceFactor, Moses::LanguageModelKen< Model >::m_factorType, Moses::LanguageModelKen< Model >::m_ngram, Moses::LanguageModel::OOVFeatureEnabled(), Moses::ScoreComponentCollection::PlusEquals(), Moses::TransformLMScore(), and Moses::LanguageModelKen< Model >::TranslateID().
FFState * Moses::LanguageModelKen< Model >::EvaluateWhenApplied | ( | const Hypothesis & | cur_hypo, | |
const FFState * | prev_state, | |||
ScoreComponentCollection * | accumulator | |||
) | const [inline, virtual] |
This interface should be implemented. Notes: When evaluating the value of this feature function, you should avoid calling hypo.GetPrevHypo(). If you need something from the "previous" hypothesis, you should store it in an FFState object which will be passed in as prev_state. If you don't do this, you will get in trouble.
Implements Moses::StatefulFeatureFunction.
Definition at line 197 of file Ken.cpp.
References begin, end, Moses::Hypothesis::GetCurrTargetLength(), Moses::Hypothesis::GetCurrTargetWordsRange(), Moses::Range::GetEndPos(), Moses::Range::GetStartPos(), Moses::Hypothesis::GetWord(), Moses::Hypothesis::IsSourceCompleted(), Moses::ScoreComponentCollection::PlusEquals(), prob(), swap(), and Moses::TransformLMScore().
Referenced by Moses::InMemoryPerSentenceOnDemandLM::EvaluateWhenApplied().
void Moses::LanguageModelKen< Model >::IncrementalCallback | ( | Incremental::Manager & | manager | ) | const [inline, virtual] |
Reimplemented from Moses::LanguageModel.
Definition at line 383 of file Ken.cpp.
References Moses::Incremental::Manager::LMCallback(), Moses::LanguageModelKen< Model >::m_lmIdLookup, and Moses::LanguageModelKen< Model >::m_ngram.
Referenced by Moses::InMemoryPerSentenceOnDemandLM::IncrementalCallback().
bool Moses::LanguageModelKen< Model >::IsUseable | ( | const FactorMask & | mask | ) | const [inline, virtual] |
Implements Moses::FeatureFunction.
Definition at line 410 of file Ken.cpp.
References Moses::LanguageModelKen< Model >::m_factorType.
void Moses::LanguageModelKen< Model >::LoadModel | ( | const std::string & | file, | |
util::LoadMethod | load_method | |||
) | [inline, protected] |
Definition at line 92 of file Ken.cpp.
References lm::ngram::Config::enumerate_vocab, lm::ngram::Config::load_method, lm::ngram::Config::messages, NULL, and VERBOSE.
Referenced by Moses::InMemoryPerSentenceOnDemandLM::GetPerThreadLM(), and Moses::LanguageModelKen< Model >::LanguageModelKen().
void Moses::LanguageModelKen< Model >::ReportHistoryOrder | ( | std::ostream & | out, | |
const Phrase & | phrase | |||
) | const [inline, virtual] |
Reimplemented from Moses::LanguageModel.
Definition at line 388 of file Ken.cpp.
References Moses::Phrase::GetSize(), Moses::Phrase::GetWord(), swap(), and Moses::TransformLMScore().
Referenced by Moses::InMemoryPerSentenceOnDemandLM::ReportHistoryOrder().
lm::WordIndex Moses::LanguageModelKen< Model >::TranslateID | ( | const Word & | word | ) | const [inline, protected] |
Definition at line 84 of file Ken.h.
Referenced by Moses::LanguageModelKen< Model >::CalcScore(), and Moses::LanguageModelKen< Model >::EvaluateWhenApplied().
friend class InMemoryPerSentenceOnDemandLM [friend] |
const Factor* Moses::LanguageModelKen< Model >::m_beginSentenceFactor [protected] |
Definition at line 78 of file Ken.h.
Referenced by Moses::LanguageModelKen< Model >::CalcScore(), and Moses::LanguageModelKen< Model >::EvaluateWhenApplied().
FactorType Moses::LanguageModelKen< Model >::m_factorType [protected] |
Definition at line 80 of file Ken.h.
Referenced by Moses::LanguageModelKen< Model >::CalcScore(), Moses::LanguageModelKen< Model >::EvaluateWhenApplied(), Moses::LanguageModelKen< Model >::IsUseable(), and Moses::LanguageModelKen< lm::ngram::ProbingModel >::TranslateID().
std::vector<lm::WordIndex> Moses::LanguageModelKen< Model >::m_lmIdLookup [protected] |
Definition at line 89 of file Ken.h.
Referenced by Moses::LanguageModelKen< Model >::IncrementalCallback(), and Moses::LanguageModelKen< lm::ngram::ProbingModel >::TranslateID().
boost::shared_ptr<Model> Moses::LanguageModelKen< Model >::m_ngram [protected] |
Definition at line 76 of file Ken.h.
Referenced by Moses::LanguageModelKen< Model >::CalcScore(), Moses::LanguageModelKen< Model >::EvaluateWhenApplied(), and Moses::LanguageModelKen< Model >::IncrementalCallback().