00001 // $Id$ 00002 00003 /*********************************************************************** 00004 Moses - factored phrase-based language decoder 00005 Copyright (C) 2006 University of Edinburgh 00006 00007 This library is free software; you can redistribute it and/or 00008 modify it under the terms of the GNU Lesser General Public 00009 License as published by the Free Software Foundation; either 00010 version 2.1 of the License, or (at your option) any later version. 00011 00012 This library is distributed in the hope that it will be useful, 00013 but WITHOUT ANY WARRANTY; without even the implied warranty of 00014 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00015 Lesser General Public License for more details. 00016 00017 You should have received a copy of the GNU Lesser General Public 00018 License along with this library; if not, write to the Free Software 00019 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 00020 ***********************************************************************/ 00021 00022 #include "lm/binary_format.hh" 00023 #include "lm/enumerate_vocab.hh" 00024 #include "lm/left.hh" 00025 #include "lm/model.hh" 00026 00027 #include "moses/FF/FFState.h" 00028 #include "moses/Hypothesis.h" 00029 #include "moses/Phrase.h" 00030 00031 #include "moses/LM/Ken.h" 00032 #include "moses/LM/Reloading.h" 00033 #include "util/exception.hh" 00034 00035 //#include "moses/Util.h" 00036 //#include "moses/StaticData.h" 00037 //#include <iostream> 00038 /* 00039 namespace Moses 00040 { 00041 namespace 00042 { 00043 00044 struct ReloadingLMState : public FFState { 00045 lm::ngram::State state; 00046 virtual size_t hash() const { 00047 return 0; 00048 } 00049 virtual bool operator==(const FFState& o) const { 00050 return true; 00051 } 00052 00053 }; 00054 } // namespace 00055 00056 00057 template <class Model> ReloadingLanguageModel<Model>::ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy) : LanguageModelKen<Model>(line,file,factorType,lazy) 00058 { 00059 // 00060 // This space intentionally left blank 00061 // 00062 } 00063 template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypothesisState(const InputType &input) const 00064 { 00065 ReloadingLMState *ret = new ReloadingLMState(); 00066 ret->state = m_ngram->BeginSentenceState(); 00067 return ret; 00068 } 00069 00070 00071 template <class Model> FFState *ReloadingLanguageModel<Model>::EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const 00072 { 00073 00074 std::auto_ptr<FFState> kenlmState(LanguageModelKen<Model>::EvaluateWhenApplied(hypo, ps, out)); 00075 const lm::ngram::State &out_state = static_cast<const ReloadingLMState&>(*kenlmState).state; 00076 00077 00078 std::auto_ptr<ReloadingLMState> ret(new ReloadingLMState()); 00079 ret->state = out_state; 00080 00081 kenlmState.release(); 00082 return ret.release(); 00083 } 00084 00085 00086 LanguageModel *ConstructReloadingLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy) 00087 { 00088 lm::ngram::ModelType model_type; 00089 if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) { 00090 switch(model_type) { 00091 case lm::ngram::PROBING: 00092 return new ReloadingLanguageModel<lm::ngram::ProbingModel>(line, file, factorType, lazy); 00093 case lm::ngram::REST_PROBING: 00094 return new ReloadingLanguageModel<lm::ngram::RestProbingModel>(line, file, factorType, lazy); 00095 case lm::ngram::TRIE: 00096 return new ReloadingLanguageModel<lm::ngram::TrieModel>(line, file, factorType, lazy); 00097 case lm::ngram::QUANT_TRIE: 00098 return new ReloadingLanguageModel<lm::ngram::QuantTrieModel>(line, file, factorType, lazy); 00099 case lm::ngram::ARRAY_TRIE: 00100 return new ReloadingLanguageModel<lm::ngram::ArrayTrieModel>(line, file, factorType, lazy); 00101 case lm::ngram::QUANT_ARRAY_TRIE: 00102 return new ReloadingLanguageModel<lm::ngram::QuantArrayTrieModel>(line, file, factorType, lazy); 00103 default: 00104 UTIL_THROW2("Unrecognized kenlm model type " << model_type); 00105 } 00106 } else { 00107 return new ReloadingLanguageModel<lm::ngram::ProbingModel>(line, file, factorType, lazy); 00108 } 00109 } 00110 00111 } // namespace Moses 00112 */