00001
00002 #pragma once
00003
00004 #include <string>
00005 #include <vector>
00006 #include "SingleFactor.h"
00007 #include <boost/thread/tss.hpp>
00008 #include "lm/model.hh"
00009 #include "moses/LM/Ken.h"
00010 #include "moses/FF/FFState.h"
00011
00012 namespace Moses
00013 {
00014
00015 struct InMemoryPerSentenceOnDemandLMState : public FFState {
00016 lm::ngram::State state;
00017 virtual size_t hash() const {
00018 size_t ret = hash_value(state);
00019 return ret;
00020 }
00021 virtual bool operator==(const FFState& o) const {
00022 const InMemoryPerSentenceOnDemandLMState &other = static_cast<const InMemoryPerSentenceOnDemandLMState &>(o);
00023 bool ret = state == other.state;
00024 return ret;
00025 }
00026
00027 };
00028
00029 class InMemoryPerSentenceOnDemandLM : public LanguageModel
00030 {
00031 public:
00032 InMemoryPerSentenceOnDemandLM(const std::string &line);
00033 ~InMemoryPerSentenceOnDemandLM();
00034
00035 void InitializeForInput(ttasksptr const& ttask);
00036
00037 virtual void SetParameter(const std::string& key, const std::string& value) {
00038 GetPerThreadLM().SetParameter(key, value);
00039 }
00040
00041 virtual const FFState* EmptyHypothesisState(const InputType &input) const {
00042 if (isInitialized()) {
00043 return GetPerThreadLM().EmptyHypothesisState(input);
00044 } else {
00045 return new InMemoryPerSentenceOnDemandLMState();
00046 }
00047 }
00048
00049 virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const {
00050 if (isInitialized()) {
00051 return GetPerThreadLM().EvaluateWhenApplied(hypo, ps, out);
00052 } else {
00053 UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
00054 }
00055 }
00056
00057 virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const {
00058 if (isInitialized()) {
00059 return GetPerThreadLM().EvaluateWhenApplied(cur_hypo, featureID, accumulator);
00060 } else {
00061 UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
00062 }
00063 }
00064
00065 virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const {
00066 if (isInitialized()) {
00067 return GetPerThreadLM().EvaluateWhenApplied(hyperedge, featureID, accumulator);
00068 } else {
00069 UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
00070 }
00071 }
00072
00073
00074 virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
00075 if (isInitialized()) {
00076 GetPerThreadLM().CalcScore(phrase, fullScore, ngramScore, oovCount);
00077 } else {
00078 UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScore called prior to being initialized");
00079 }
00080 }
00081
00082 virtual void CalcScoreFromCache(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
00083 if (isInitialized()) {
00084 GetPerThreadLM().CalcScoreFromCache(phrase, fullScore, ngramScore, oovCount);
00085 } else {
00086 UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScoreFromCache called prior to being initialized");
00087 }
00088 }
00089
00090 virtual void IssueRequestsFor(Hypothesis& hypo, const FFState* input_state) {
00091 if (isInitialized()) {
00092 GetPerThreadLM().IssueRequestsFor(hypo, input_state);
00093 } else {
00094 UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IssueRequestsFor called prior to being initialized");
00095 }
00096 }
00097
00098 virtual void sync() {
00099 if (isInitialized()) {
00100 GetPerThreadLM().sync();
00101 } else {
00102 UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::sync called prior to being initialized");
00103 }
00104 }
00105
00106 virtual void SetFFStateIdx(int state_idx) {
00107 if (isInitialized()) {
00108 GetPerThreadLM().SetFFStateIdx(state_idx);
00109 } else {
00110 UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::SetFFStateIdx called prior to being initialized");
00111 }
00112 }
00113
00114 virtual void IncrementalCallback(Incremental::Manager &manager) const {
00115 if (isInitialized()) {
00116 GetPerThreadLM().IncrementalCallback(manager);
00117 } else {
00118 UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IncrementalCallback called prior to being initialized");
00119 }
00120 }
00121
00122 virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const {
00123 if (isInitialized()) {
00124 GetPerThreadLM().ReportHistoryOrder(out, phrase);
00125 } else {
00126 UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::ReportHistoryOrder called prior to being initialized");
00127 }
00128 }
00129
00130 virtual void EvaluateInIsolation(const Phrase &source
00131 , const TargetPhrase &targetPhrase
00132 , ScoreComponentCollection &scoreBreakdown
00133 , ScoreComponentCollection &estimatedScores) const {
00134 if (isInitialized()) {
00135 GetPerThreadLM().EvaluateInIsolation(source, targetPhrase, scoreBreakdown, estimatedScores);
00136 } else {
00137
00138 }
00139 }
00140
00141 bool IsUseable(const FactorMask &mask) const {
00142 bool ret = mask[m_factorType];
00143 return ret;
00144 }
00145
00146
00147 protected:
00148 LanguageModelKen<lm::ngram::ProbingModel> & GetPerThreadLM() const;
00149
00150 mutable boost::thread_specific_ptr<LanguageModelKen<lm::ngram::ProbingModel> > m_perThreadLM;
00151 mutable boost::thread_specific_ptr<std::string> m_tmpFilename;
00152
00153 FactorType m_factorType;
00154
00155 bool isInitialized() const {
00156 if (m_tmpFilename.get() == NULL) {
00157 return false;
00158 } else {
00159 return true;
00160 }
00161 }
00162
00163 };
00164
00165
00166 }