00001 #ifndef LM_MODEL_H
00002 #define LM_MODEL_H
00003
00004 #include "lm/bhiksha.hh"
00005 #include "lm/binary_format.hh"
00006 #include "lm/config.hh"
00007 #include "lm/facade.hh"
00008 #include "lm/quantize.hh"
00009 #include "lm/search_hashed.hh"
00010 #include "lm/search_trie.hh"
00011 #include "lm/state.hh"
00012 #include "lm/value.hh"
00013 #include "lm/vocab.hh"
00014 #include "lm/weights.hh"
00015
00016 #include "util/murmur_hash.hh"
00017
00018 #include <algorithm>
00019 #include <vector>
00020 #include <cstring>
00021
00022 namespace util { class FilePiece; }
00023
00024 namespace lm {
00025 namespace ngram {
00026 namespace detail {
00027
00028
00029
00030 template <class Search, class VocabularyT> class GenericModel : public base::ModelFacade<GenericModel<Search, VocabularyT>, State, VocabularyT> {
00031 private:
00032 typedef base::ModelFacade<GenericModel<Search, VocabularyT>, State, VocabularyT> P;
00033 public:
00034
00035 static const ModelType kModelType;
00036
00037 static const unsigned int kVersion = Search::kVersion;
00038
00039
00040
00041
00042
00043 static uint64_t Size(const std::vector<uint64_t> &counts, const Config &config = Config());
00044
00045
00046
00047
00048
00049
00050
00051 explicit GenericModel(const char *file, const Config &config = Config());
00052
00053
00054
00055
00056
00057 FullScoreReturn FullScore(const State &in_state, const WordIndex new_word, State &out_state) const;
00058
00059
00060
00061
00062
00063
00064
00065
00066 FullScoreReturn FullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, State &out_state) const;
00067
00068
00069
00070
00071
00072
00073
00074
00075
00076 void GetState(const WordIndex *context_rbegin, const WordIndex *context_rend, State &out_state) const;
00077
00078
00079
00080
00081
00082 FullScoreReturn ExtendLeft(
00083
00084 const WordIndex *add_rbegin, const WordIndex *add_rend,
00085
00086 const float *backoff_in,
00087
00088 uint64_t extend_pointer,
00089
00090 unsigned char extend_length,
00091
00092 float *backoff_out,
00093
00094 unsigned char &next_use) const;
00095
00096
00097
00098
00099
00100 float UnRest(const uint64_t *pointers_begin, const uint64_t *pointers_end, unsigned char first_length) const {
00101
00102 return Search::kDifferentRest ? InternalUnRest(pointers_begin, pointers_end, first_length) : 0.0;
00103 }
00104
00105 private:
00106 FullScoreReturn ScoreExceptBackoff(const WordIndex *const context_rbegin, const WordIndex *const context_rend, const WordIndex new_word, State &out_state) const;
00107
00108
00109 void ResumeScore(const WordIndex *context_rbegin, const WordIndex *const context_rend, unsigned char starting_order_minus_2, typename Search::Node &node, float *backoff_out, unsigned char &next_use, FullScoreReturn &ret) const;
00110
00111
00112 void SetupMemory(void *start, const std::vector<uint64_t> &counts, const Config &config);
00113
00114 void InitializeFromARPA(int fd, const char *file, const Config &config);
00115
00116 float InternalUnRest(const uint64_t *pointers_begin, const uint64_t *pointers_end, unsigned char first_length) const;
00117
00118 BinaryFormat backing_;
00119
00120 VocabularyT vocab_;
00121
00122 Search search_;
00123 };
00124
00125 }
00126
00127
00128
00129 #define LM_COMMA() ,
00130 #define LM_NAME_MODEL(name, from)\
00131 class name : public from {\
00132 public:\
00133 name(const char *file, const Config &config = Config()) : from(file, config) {}\
00134 };
00135
00136 LM_NAME_MODEL(ProbingModel, detail::GenericModel<detail::HashedSearch<BackoffValue> LM_COMMA() ProbingVocabulary>);
00137 LM_NAME_MODEL(RestProbingModel, detail::GenericModel<detail::HashedSearch<RestValue> LM_COMMA() ProbingVocabulary>);
00138 LM_NAME_MODEL(TrieModel, detail::GenericModel<trie::TrieSearch<DontQuantize LM_COMMA() trie::DontBhiksha> LM_COMMA() SortedVocabulary>);
00139 LM_NAME_MODEL(ArrayTrieModel, detail::GenericModel<trie::TrieSearch<DontQuantize LM_COMMA() trie::ArrayBhiksha> LM_COMMA() SortedVocabulary>);
00140 LM_NAME_MODEL(QuantTrieModel, detail::GenericModel<trie::TrieSearch<SeparatelyQuantize LM_COMMA() trie::DontBhiksha> LM_COMMA() SortedVocabulary>);
00141 LM_NAME_MODEL(QuantArrayTrieModel, detail::GenericModel<trie::TrieSearch<SeparatelyQuantize LM_COMMA() trie::ArrayBhiksha> LM_COMMA() SortedVocabulary>);
00142
00143
00144 typedef ::lm::ngram::ProbingVocabulary Vocabulary;
00145 typedef ProbingModel Model;
00146
00147
00148
00149
00150 base::Model *LoadVirtual(const char *file_name, const Config &config = Config(), ModelType if_arpa = PROBING);
00151
00152 }
00153 }
00154
00155 #endif // LM_MODEL_H