00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024 #ifndef MF_LMCLASS_H
00025 #define MF_LMCLASS_H
00026
00027 #ifndef WIN32
00028 #include <sys/types.h>
00029 #include <sys/mman.h>
00030 #endif
00031
00032 #include "util.h"
00033 #include "ngramcache.h"
00034 #include "dictionary.h"
00035 #include "n_gram.h"
00036 #include "lmtable.h"
00037
00038 #define MAX_TOKEN 2
00039
00040 class lmclass: public lmtable
00041 {
00042 dictionary *dict;
00043 double *MapScore;
00044 int MapScoreN;
00045 int MaxMapSize;
00046
00047 protected:
00048 void loadMap(std::istream& inp);
00049 void loadMapElement(const char* in, const char* out, double sc);
00050 void mapping(ngram &in, ngram &out);
00051
00052 inline double getMapScore(int wcode) {
00053
00054 if (wcode >= MapScoreN) {
00055 wcode = getDict()->oovcode();
00056 }
00057 return MapScore[wcode];
00058 };
00059
00060 inline size_t getMap(int wcode) {
00061
00062 if (wcode >= MapScoreN) {
00063 wcode = getDict()->oovcode();
00064 }
00065 return dict->freq(wcode);
00066 };
00067
00068 void checkMap();
00069
00070 public:
00071 lmclass(float nlf=0.0, float dlfi=0.0);
00072
00073 ~lmclass();
00074
00075 void load(const std::string filename,int mmap=0);
00076
00077 double lprob(ngram ng, double* bow=NULL,int* bol=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL);
00078 inline double clprob(ngram ng,double* bow=NULL,int* bol=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL) {
00079 return lprob(ng,bow,bol,maxsuffptr,statesize,extendible);
00080 };
00081 inline double clprob(int* ng, int ngsize, double* bow=NULL,int* bol=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL) {
00082 ngram ong(getDict());
00083 ong.pushc(ng,ngsize);
00084 return lprob(ong,bow,bol,maxsuffptr,statesize,extendible);
00085 };
00086
00087 inline dictionary* getDict() const {
00088 return dict;
00089 }
00090 inline virtual void dictionary_incflag(const bool flag) {
00091 dict->incflag(flag);
00092 };
00093 };
00094
00095
00096
00097 #endif
00098