00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022 #ifndef moses_LanguageModelKen_h
00023 #define moses_LanguageModelKen_h
00024
00025 #include <string>
00026 #include <boost/shared_ptr.hpp>
00027
00028 #include "lm/word_index.hh"
00029 #include "util/mmap.hh"
00030
00031 #include "moses/LM/Base.h"
00032 #include "moses/Hypothesis.h"
00033 #include "moses/TypeDef.h"
00034 #include "moses/Word.h"
00035
00036
00037
00038 namespace Moses
00039 {
00040
00041
00042 class FFState;
00043 class InMemoryPerSentenceOnDemandLM;
00044
00045 LanguageModel *ConstructKenLM(const std::string &line);
00046
00048 LanguageModel *ConstructKenLM(const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method);
00049
00050
00051
00052
00053 template <class Model> class LanguageModelKen : public LanguageModel
00054 {
00055 public:
00056 LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method);
00057
00058 virtual const FFState *EmptyHypothesisState(const InputType &) const;
00059
00060 virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
00061
00062 virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
00063
00064 virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
00065
00066 virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const;
00067
00068 virtual void IncrementalCallback(Incremental::Manager &manager) const;
00069 virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const;
00070
00071 virtual bool IsUseable(const FactorMask &mask) const;
00072
00073 friend class InMemoryPerSentenceOnDemandLM;
00074
00075 protected:
00076 boost::shared_ptr<Model> m_ngram;
00077
00078 const Factor *m_beginSentenceFactor;
00079
00080 FactorType m_factorType;
00081
00082 void LoadModel(const std::string &file, util::LoadMethod load_method);
00083
00084 lm::WordIndex TranslateID(const Word &word) const {
00085 std::size_t factor = word.GetFactor(m_factorType)->GetId();
00086 return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
00087 }
00088
00089 std::vector<lm::WordIndex> m_lmIdLookup;
00090
00091 private:
00092 LanguageModelKen();
00093 LanguageModelKen(const LanguageModelKen<Model> ©_from);
00094
00095
00096 lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
00097 lm::WordIndex *index = indices;
00098 lm::WordIndex *end = indices + m_ngram->Order() - 1;
00099 int position = hypo.GetCurrTargetWordsRange().GetEndPos();
00100 for (; ; ++index, --position) {
00101 if (index == end) return index;
00102 if (position == -1) {
00103 *index = m_ngram->GetVocabulary().BeginSentence();
00104 return index + 1;
00105 }
00106 *index = TranslateID(hypo.GetWord(position));
00107 }
00108 }
00109
00110
00111 protected:
00112
00113 };
00114
00115 }
00116
00117 #endif