00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023 #ifndef MF_LMINTERPOLATION_H
00024 #define MF_LMINTERPOLATION_H
00025
00026 #include <stdio.h>
00027 #include <cstdlib>
00028 #include <stdlib.h>
00029 #include <string>
00030 #include <math.h>
00031 #include <vector>
00032 #include "util.h"
00033 #include "dictionary.h"
00034 #include "n_gram.h"
00035 #include "lmContainer.h"
00036
00037
00038
00039
00040
00041
00042 #define MAX_TOKEN 2
00043
00044 class lmInterpolation: public lmContainer
00045 {
00046 static const bool debug=true;
00047 int m_number_lm;
00048 int order;
00049 int dictionary_upperbound;
00050 double logOOVpenalty;
00051 bool isInverted;
00052 int memmap;
00053
00054 std::vector<double> m_weight;
00055 std::vector<std::string> m_file;
00056 std::vector<bool> m_isinverted;
00057 std::vector<lmContainer*> m_lm;
00058
00059 int maxlev;
00060
00061 float ngramcache_load_factor;
00062 float dictionary_load_factor;
00063
00064 dictionary *dict;
00065
00066 public:
00067
00068 lmInterpolation(float nlf=0.0, float dlfi=0.0);
00069 virtual ~lmInterpolation() {};
00070
00071 void load(const std::string filename,int mmap=0);
00072 lmContainer* load_lm(int i, int memmap, float nlf, float dlf);
00073
00074 virtual double clprob(ngram ng, double* bow=NULL,int* bol=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL);
00075 virtual double clprob(int* ng, int ngsize, double* bow=NULL,int* bol=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL);
00076
00077 int maxlevel() const {
00078 return maxlev;
00079 };
00080
00081 virtual inline void setDict(dictionary* d) {
00082 if (dict) delete dict;
00083 dict=d;
00084 };
00085
00086 virtual inline dictionary* getDict() const {
00087 return dict;
00088 };
00089
00090
00091 virtual inline double getlogOOVpenalty() const {
00092 return logOOVpenalty;
00093 }
00094
00095 virtual double setlogOOVpenalty(int dub);
00096
00097 double inline setlogOOVpenalty(double oovp) {
00098 return logOOVpenalty=oovp;
00099 }
00100
00101
00102 inline bool is_inverted(const bool flag) {
00103 return isInverted = flag;
00104 }
00105
00106
00107
00108 inline bool is_inverted() {
00109 for (int i=0; i<m_number_lm; i++) {
00110 if (m_isinverted[i] == false) return false;
00111 }
00112 return true;
00113 }
00114
00115 inline virtual void dictionary_incflag(const bool flag) {
00116 dict->incflag(flag);
00117 };
00118
00119 inline virtual bool is_OOV(int code) {
00120 for (int i=0; i<m_number_lm; i++) {
00121 int _code=m_lm[i]->getDict()->encode(getDict()->decode(code));
00122 if (m_lm[i]->is_OOV(_code) == false) return false;
00123 }
00124 return true;
00125 }
00126 };
00127
00128 #endif
00129