00001 #include "lm/value_build.hh"
00002
00003 #include "lm/model.hh"
00004 #include "lm/read_arpa.hh"
00005
00006 namespace lm {
00007 namespace ngram {
00008
00009 template <class Model> LowerRestBuild<Model>::LowerRestBuild(const Config &config, unsigned int order, const typename Model::Vocabulary &vocab) {
00010 UTIL_THROW_IF(config.rest_lower_files.size() != order - 1, ConfigException, "This model has order " << order << " so there should be " << (order - 1) << " lower-order models for rest cost purposes.");
00011 Config for_lower = config;
00012 for_lower.write_mmap = NULL;
00013 for_lower.rest_lower_files.clear();
00014
00015
00016
00017 {
00018 util::FilePiece uni(config.rest_lower_files[0].c_str());
00019 std::vector<uint64_t> number;
00020 ReadARPACounts(uni, number);
00021 UTIL_THROW_IF(number.size() != 1, FormatLoadException, "Expected the unigram model to have order 1, not " << number.size());
00022 ReadNGramHeader(uni, 1);
00023 unigrams_.resize(number[0]);
00024 unigrams_[0] = config.unknown_missing_logprob;
00025 PositiveProbWarn warn;
00026 for (uint64_t i = 0; i < number[0]; ++i) {
00027 WordIndex w;
00028 Prob entry;
00029 ReadNGram(uni, 1, vocab, &w, entry, warn);
00030 unigrams_[w] = entry.prob;
00031 }
00032 }
00033
00034 try {
00035 for (unsigned int i = 2; i < order; ++i) {
00036 models_.push_back(new Model(config.rest_lower_files[i - 1].c_str(), for_lower));
00037 UTIL_THROW_IF(models_.back()->Order() != i, FormatLoadException, "Lower order file " << config.rest_lower_files[i-1] << " should have order " << i);
00038 }
00039 } catch (...) {
00040 for (typename std::vector<const Model*>::const_iterator i = models_.begin(); i != models_.end(); ++i) {
00041 delete *i;
00042 }
00043 models_.clear();
00044 throw;
00045 }
00046
00047
00048 }
00049
00050 template <class Model> LowerRestBuild<Model>::~LowerRestBuild() {
00051 for (typename std::vector<const Model*>::const_iterator i = models_.begin(); i != models_.end(); ++i) {
00052 delete *i;
00053 }
00054 }
00055
00056 template class LowerRestBuild<ProbingModel>;
00057
00058 }
00059 }