#include <mdiadapt.h>
Public Member Functions | |
mdiadaptlm (char *ngtfile, int depth=0, TABLETYPE tt=FULL) | |
normcache * | get_zetacache () |
NGRAMCACHE_t * | get_probcache (int level) |
NGRAMCACHE_t * | get_backoffcache (int level) |
void | create_caches (int mcl) |
void | init_caches () |
void | init_caches (int level) |
void | delete_caches () |
void | delete_caches (int level) |
void | check_cache_levels () |
void | check_cache_levels (int level) |
void | reset_caches () |
void | reset_caches (int level) |
void | caches_stat () |
double | zeta (ngram ng, int size) |
int | discount (ngram ng, int size, double &fstar, double &lambda, int cv=0) |
int | bodiscount (ngram ng, int size, double &fstar, double &lambda, double &bo) |
int | compute_backoff () |
int | compute_backoff_per_level () |
int | compute_backoff_per_word () |
double | backunig (ngram ng) |
double | foreunig (ngram ng) |
int | adapt (char *ngtfile, int alev=1, double gis_step=0.4) |
int | scalefact (char *ngtfile) |
int | savescalefactor (char *filename) |
double | scalefact (ngram ng) |
double | prob (ngram ng, int size) |
double | prob (ngram ng, int size, double &fstar, double &lambda, double &bo) |
double | prob2 (ngram ng, int size, double &fstar) |
double | txclprob (ngram ng, int size) |
int | saveASR (char *filename, int backoff, char *subdictfile=NULL) |
int | saveMT (char *filename, int backoff, char *subdictfile=NULL, int resolution=10000000, double decay=0.999900) |
int | saveARPA (char *filename, int backoff=0, char *subdictfile=NULL) |
int | saveBIN (char *filename, int backoff=0, char *subdictfile=NULL, int mmap=0) |
void | save_per_level (bool value) |
bool | save_per_level () |
int | netsize () |
~mdiadaptlm () | |
double | myround (double x) |
bool | is_train_cache_enabled () |
Public Attributes | |
double | gis_step |
Protected Member Functions | |
int | saveARPA_per_word (char *filename, int backoff=0, char *subdictfile=NULL) |
int | saveARPA_per_level (char *filename, int backoff=0, char *subdictfile=NULL) |
int | saveBIN_per_word (char *filename, int backoff=0, char *subdictfile=NULL, int mmap=0) |
int | saveBIN_per_level (char *filename, int backoff=0, char *subdictfile=NULL, int mmap=0) |
Protected Attributes | |
normcache * | cache |
NGRAMCACHE_t ** | probcache |
NGRAMCACHE_t ** | backoffcache |
int | max_caching_level |
Definition at line 30 of file mdiadapt.h.
mdiadaptlm::mdiadaptlm | ( | char * | ngtfile, | |
int | depth = 0 , |
|||
TABLETYPE | tt = FULL | |||
) |
mdiadaptlm::~mdiadaptlm | ( | ) |
Definition at line 53 of file mdiadapt.cpp.
References cache, and delete_caches().
int mdiadaptlm::adapt | ( | char * | ngtfile, | |
int | alev = 1 , |
|||
double | gis_step = 0.4 | |||
) |
Definition at line 250 of file mdiadapt.cpp.
References backunig(), ngramtable::dict, gis_step, interplm::lmsize(), NULL, scalefact(), dictionary::size(), ngram::wordp(), and zeta().
Referenced by main().
double mdiadaptlm::backunig | ( | ngram | ng | ) |
Definition at line 238 of file mdiadapt.cpp.
References discount().
Referenced by adapt(), and scalefact().
int mdiadaptlm::bodiscount | ( | ngram | ng, | |
int | size, | |||
double & | fstar, | |||
double & | lambda, | |||
double & | bo | |||
) |
Definition at line 538 of file mdiadapt.cpp.
References interplm::backoff, ngramtable::boff(), ngramtable::dict, discount(), ngram::link, and ngram::trans().
Referenced by prob(), saveARPA_per_level(), saveARPA_per_word(), saveASR(), saveBIN_per_level(), saveBIN_per_word(), and saveMT().
void mdiadaptlm::caches_stat | ( | ) |
Definition at line 75 of file mdiadapt.cpp.
References backoffcache, interplm::lmsize(), max_caching_level, and probcache.
void mdiadaptlm::check_cache_levels | ( | int | level | ) |
void mdiadaptlm::check_cache_levels | ( | ) |
int mdiadaptlm::compute_backoff | ( | ) | [inline] |
Definition at line 82 of file mdiadapt.h.
References compute_backoff_per_level(), and compute_backoff_per_word().
Referenced by main().
int mdiadaptlm::compute_backoff_per_level | ( | ) |
Definition at line 409 of file mdiadapt.cpp.
References interplm::backoff, ngramtable::boff(), CONT, ngramtable::dict, discount(), INIT, ngram::link, interplm::lmsize(), prob(), ngram::pushc(), ngramtable::scan(), ngram::size, and ngramtable::succscan().
Referenced by compute_backoff().
int mdiadaptlm::compute_backoff_per_word | ( | ) |
Definition at line 453 of file mdiadapt.cpp.
Referenced by compute_backoff().
void mdiadaptlm::create_caches | ( | int | mcl | ) |
Definition at line 92 of file mdiadapt.cpp.
References backoffcache, init_caches(), interplm::lmsize(), max_caching_level, NGRAMCACHE_t, NULL, and probcache.
Referenced by main().
void mdiadaptlm::delete_caches | ( | int | level | ) |
void mdiadaptlm::delete_caches | ( | ) |
Definition at line 65 of file mdiadapt.cpp.
References backoffcache, max_caching_level, and probcache.
Referenced by ~mdiadaptlm().
int mdiadaptlm::discount | ( | ngram | ng, | |
int | size, | |||
double & | fstar, | |||
double & | lambda, | |||
int | cv = 0 | |||
) | [virtual] |
Reimplemented from interplm.
Reimplemented in linearwb, mixture, shiftone, shiftbeta, symshiftbeta, and mshiftbeta.
Definition at line 349 of file mdiadapt.cpp.
References backoffcache, ngramtable::dict, max_caching_level, scalefact(), ngram::shift(), ngram::size, ngram::trans(), ngram::wordp(), and zeta().
Referenced by backunig(), bodiscount(), compute_backoff_per_level(), netsize(), prob2(), saveASR(), saveMT(), txclprob(), and zeta().
double mdiadaptlm::foreunig | ( | ngram | ng | ) |
Definition at line 228 of file mdiadapt.cpp.
References interplm::discount().
Referenced by scalefact().
NGRAMCACHE_t * mdiadaptlm::get_backoffcache | ( | int | level | ) | [inline] |
NGRAMCACHE_t * mdiadaptlm::get_probcache | ( | int | level | ) | [inline] |
normcache* mdiadaptlm::get_zetacache | ( | ) | [inline] |
void mdiadaptlm::init_caches | ( | int | level | ) |
Definition at line 107 of file mdiadapt.cpp.
References backoffcache, NGRAMCACHE_t, NULL, and probcache.
void mdiadaptlm::init_caches | ( | ) |
Definition at line 115 of file mdiadapt.cpp.
References max_caching_level.
Referenced by create_caches().
bool mdiadaptlm::is_train_cache_enabled | ( | ) | [inline] |
Definition at line 150 of file mdiadapt.h.
Referenced by main().
double mdiadaptlm::myround | ( | double | x | ) | [inline] |
int mdiadaptlm::netsize | ( | ) | [virtual] |
Reimplemented from interplm.
Definition at line 591 of file mdiadapt.cpp.
References CONT, ngramtable::dict, discount(), INIT, ngramtable::maxlevel(), ngramtable::scan(), and dictionary::size().
Referenced by main().
double mdiadaptlm::prob | ( | ngram | ng, | |
int | size, | |||
double & | fstar, | |||
double & | lambda, | |||
double & | bo | |||
) |
Definition at line 483 of file mdiadapt.cpp.
References interplm::backoff, bodiscount(), max_caching_level, prob(), probcache, ngram::size, UPPER_SINGLE_PRECISION_OF_1, and ngram::wordp().
double mdiadaptlm::prob | ( | ngram | ng, | |
int | size | |||
) | [virtual] |
Reimplemented from interplm.
Definition at line 477 of file mdiadapt.cpp.
Referenced by compute_backoff_per_level(), main(), prob(), prob2(), saveARPA_per_level(), saveARPA_per_word(), saveASR(), saveBIN_per_level(), saveBIN_per_word(), saveMT(), and mixture::train().
double mdiadaptlm::prob2 | ( | ngram | ng, | |
int | size, | |||
double & | fstar | |||
) |
Definition at line 462 of file mdiadapt.cpp.
References discount(), and prob().
void mdiadaptlm::reset_caches | ( | int | level | ) |
void mdiadaptlm::reset_caches | ( | ) |
bool mdiadaptlm::save_per_level | ( | ) | [inline] |
Definition at line 139 of file mdiadapt.h.
void mdiadaptlm::save_per_level | ( | bool | value | ) | [inline] |
Definition at line 138 of file mdiadapt.h.
Referenced by main().
int mdiadaptlm::saveARPA | ( | char * | filename, | |
int | backoff = 0 , |
|||
char * | subdictfile = NULL | |||
) | [inline] |
Definition at line 119 of file mdiadapt.h.
References interplm::backoff, saveARPA_per_level(), and saveARPA_per_word().
Referenced by main().
int mdiadaptlm::saveARPA_per_level | ( | char * | filename, | |
int | backoff = 0 , |
|||
char * | subdictfile = NULL | |||
) | [protected] |
Definition at line 1787 of file mdiadapt.cpp.
References bodiscount(), CONT, ngram::containsWord(), dictionary::decode(), ngramtable::dict, dictionary::EoS(), INIT, interplm::lmsize(), LOWER_SINGLE_PRECISION_OF_1, dictionary::OOV(), dictionary::oovcode(), prob(), ngram::pushc(), ngramtable::scan(), dictionary::size(), ngram::trans(), UPPER_SINGLE_PRECISION_OF_0, UPPER_SINGLE_PRECISION_OF_1, VERBOSE, and ngram::wordp().
Referenced by saveARPA().
int mdiadaptlm::saveARPA_per_word | ( | char * | filename, | |
int | backoff = 0 , |
|||
char * | subdictfile = NULL | |||
) | [protected] |
Definition at line 1597 of file mdiadapt.cpp.
References bodiscount(), CONT, ngram::containsWord(), dictionary::decode(), ngramtable::dict, dictionary::EoS(), ngram::info, INIT, ngram::link, interplm::lmsize(), LOWER_SINGLE_PRECISION_OF_1, MAX_NGRAM, dictionary::OOV(), dictionary::oovcode(), prob(), ngram::pushc(), removefile(), ngramtable::scan(), ngram::shift(), dictionary::size(), ngram::trans(), UPPER_SINGLE_PRECISION_OF_0, UPPER_SINGLE_PRECISION_OF_1, VERBOSE, and ngram::wordp().
Referenced by saveARPA().
int mdiadaptlm::saveASR | ( | char * | filename, | |
int | backoff, | |||
char * | subdictfile = NULL | |||
) |
Definition at line 723 of file mdiadapt.cpp.
References bodiscount(), ngramtable::boff(), CONT, ngram::containsWord(), dictionary::decode(), ngramtable::dict, discount(), fseek(), ftell(), fwritex(), ifwrite(), INIT, interplm::lmsize(), dictionary::OOV(), dictionary::oovcode(), prob(), ngramtable::scan(), ngram::size, dictionary::size(), stop, ngramtable::succ1(), ngramtable::succ2(), ngramtable::succscan(), ngram::trans(), ngram::wordp(), and writeNull().
Referenced by main().
int mdiadaptlm::saveBIN | ( | char * | filename, | |
int | backoff = 0 , |
|||
char * | subdictfile = NULL , |
|||
int | mmap = 0 | |||
) | [inline] |
Definition at line 128 of file mdiadapt.h.
References interplm::backoff, saveBIN_per_level(), and saveBIN_per_word().
Referenced by main().
int mdiadaptlm::saveBIN_per_level | ( | char * | filename, | |
int | backoff = 0 , |
|||
char * | subdictfile = NULL , |
|||
int | mmap = 0 | |||
) | [protected] |
Definition at line 1403 of file mdiadapt.cpp.
References lmtable::add(), bodiscount(), lmtable::checkbounds(), lmtable::compact_all_levels(), lmtable::configure(), CONT, ngram::containsWord(), ngramtable::dict, ngramtable::entries(), dictionary::EoS(), lmtable::expand_level(), lmtable::getCurrentSize(), lmtable::getDict(), INIT, interplm::lmsize(), LOWER_SINGLE_PRECISION_OF_1, dictionary::OOV(), dictionary::oovcode(), prob(), ngram::pushc(), lmtable::resize_level(), dictionary::save(), lmtable::savebin_level(), ngramtable::scan(), lmtable::setDict(), dictionary::size(), ngram::trans(), UPPER_SINGLE_PRECISION_OF_0, UPPER_SINGLE_PRECISION_OF_1, VERBOSE, and ngram::wordp().
Referenced by saveBIN().
int mdiadaptlm::saveBIN_per_word | ( | char * | filename, | |
int | backoff = 0 , |
|||
char * | subdictfile = NULL , |
|||
int | mmap = 0 | |||
) | [protected] |
Definition at line 1150 of file mdiadapt.cpp.
References lmtable::addwithoffset(), lmtable::appendbin_level(), bodiscount(), lmtable::checkbounds(), mfstream::close(), lmtable::compact_all_levels(), lmtable::configure(), CONT, ngram::containsWord(), lmtable::delete_level(), ngramtable::dict, ngramtable::entries(), dictionary::EoS(), lmtable::expand_level(), ngram::info, INIT, ngram::link, interplm::lmsize(), LOWER_SINGLE_PRECISION_OF_1, MAX_NGRAM, dictionary::OOV(), prob(), ngram::pushc(), lmtable::resize_level(), dictionary::save(), lmtable::savebin_level(), ngramtable::scan(), mfstream::seekp(), lmtable::setDict(), ngram::shift(), dictionary::size(), mfstream::tellp(), ngram::trans(), lmtable::update_offset(), UPPER_SINGLE_PRECISION_OF_0, UPPER_SINGLE_PRECISION_OF_1, VERBOSE, and ngram::wordp().
Referenced by saveBIN().
int mdiadaptlm::saveMT | ( | char * | filename, | |
int | backoff, | |||
char * | subdictfile = NULL , |
|||
int | resolution = 10000000 , |
|||
double | decay = 0.999900 | |||
) |
write backoff of higher order!!
Definition at line 1017 of file mdiadapt.cpp.
References BACKOFF_, bodiscount(), CONT, ngramtable::dict, discount(), DUMMY_, dictionary::encode(), dictionary::freq(), ngram::freq, dictionary::incflag(), INIT, interplm::lmsize(), ngramtable::maxlevel(), dictionary::OOV(), OOV_, dictionary::oovcode(), prob(), ngram::pushc(), ngramtable::put(), dictionary::save(), ngramtable::scan(), ngram::size, ngram::trans(), and ngram::wordp().
int mdiadaptlm::savescalefactor | ( | char * | filename | ) |
Definition at line 190 of file mdiadapt.cpp.
References dictionary::decode(), ngramtable::dict, dictionary::oovcode(), scalefact(), dictionary::size(), and ngram::wordp().
Referenced by main().
double mdiadaptlm::scalefact | ( | ngram | ng | ) |
Definition at line 214 of file mdiadapt.cpp.
References backunig(), ngramtable::dict, foreunig(), gis_step, dictionary::oovcode(), ngram::trans(), and ngram::wordp().
int mdiadaptlm::scalefact | ( | char * | ngtfile | ) |
Definition at line 159 of file mdiadapt.cpp.
References backunig(), cache, ngramtable::dict, foreunig(), NULL, dictionary::oovcode(), dictionary::size(), interplm::train(), ngram::trans(), and ngram::wordp().
Referenced by adapt(), discount(), savescalefactor(), and zeta().
double mdiadaptlm::txclprob | ( | ngram | ng, | |
int | size | |||
) |
Definition at line 572 of file mdiadapt.cpp.
References ngramtable::dict, discount(), dictionary::dub(), ngram::freq, ngramtable::freq(), dictionary::oovcode(), dictionary::size(), ngramtable::totfreq(), and ngram::wordp().
double mdiadaptlm::zeta | ( | ngram | ng, | |
int | size | |||
) |
Definition at line 300 of file mdiadapt.cpp.
References cache, CONT, discount(), INIT, normcache::put(), scalefact(), ngram::size, and ngramtable::succscan().
Referenced by adapt(), and discount().
NGRAMCACHE_t** mdiadaptlm::backoffcache [protected] |
Definition at line 44 of file mdiadapt.h.
Referenced by caches_stat(), check_cache_levels(), create_caches(), delete_caches(), discount(), get_backoffcache(), init_caches(), and reset_caches().
normcache* mdiadaptlm::cache [protected] |
Definition at line 40 of file mdiadapt.h.
Referenced by get_zetacache(), mdiadaptlm(), scalefact(), zeta(), and ~mdiadaptlm().
double mdiadaptlm::gis_step |
int mdiadaptlm::max_caching_level [protected] |
Definition at line 45 of file mdiadapt.h.
Referenced by caches_stat(), check_cache_levels(), create_caches(), delete_caches(), discount(), init_caches(), prob(), and reset_caches().
NGRAMCACHE_t** mdiadaptlm::probcache [protected] |
Definition at line 43 of file mdiadapt.h.
Referenced by caches_stat(), check_cache_levels(), create_caches(), delete_caches(), get_probcache(), init_caches(), prob(), and reset_caches().