lmtable Class Reference

#include <lmtable.h>

Inheritance diagram for lmtable:

Collaboration diagram for lmtable:

List of all members.

Public Member Functions

lmtable (float nlf=0.0, float dlfi=0.0)

virtual ~lmtable ()

table_entry_pos_t wdprune (float *thr, int aflag=0)

table_entry_pos_t wdprune (float *thr, int aflag, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos, double lk=0, double bo=0, double *ts=0, double *tbs=0)

double lprobx (ngram ong, double *lkp=0, double *bop=0, int *bol=0)

table_entry_pos_t ngcnt (table_entry_pos_t *cnt)

table_entry_pos_t ngcnt (table_entry_pos_t *cnt, ngram ng, int l, table_entry_pos_t ipos, table_entry_pos_t epos)

int pscale (int lev, table_entry_pos_t ipos, table_entry_pos_t epos, double s)

void init_prob_and_state_cache ()

void init_probcache ()

void init_statecache ()

void init_lmtcaches (int uptolev)

void init_caches (int uptolev)

void used_prob_and_state_cache ()

void used_lmtcaches ()

void used_caches ()

void delete_prob_and_state_cache ()

void delete_probcache ()

void delete_statecache ()

void delete_lmtcaches ()

void delete_caches ()

void check_prob_and_state_cache_levels ()

void check_probcache_levels ()

void check_statecache_levels ()

void check_lmtcaches_levels ()

void check_caches_levels ()

void reset_prob_and_state_cache ()

void reset_probcache ()

void reset_statecache ()

void reset_lmtcaches ()

void reset_caches ()

bool are_prob_and_state_cache_active ()

bool is_probcache_active ()

bool is_statecache_active ()

bool are_lmtcaches_active ()

bool are_caches_active ()

void reset_mmap ()

bool is_inverted (const bool flag)

bool is_inverted ()

void configure (int n, bool quantized)

double getlogOOVpenalty () const

double setlogOOVpenalty (int dub)

double setlogOOVpenalty (double oovp)

virtual int maxlevel () const

bool isQuantized () const

void savetxt (const char *filename)

void savebin (const char *filename)

void appendbin_level (int level, fstream &out, int mmap)

void appendbin_level_nommap (int level, fstream &out)

void appendbin_level_mmap (int level, fstream &out)

void savebin_level (int level, const char *filename, int mmap)

void savebin_level_nommap (int level, const char *filename)

void savebin_level_mmap (int level, const char *filename)

void savebin_dict (std::fstream &out)

void compact_all_levels (const char *filename)

void compact_single_level (int level, const char *filename)

void concatenate_all_levels (const char *fromfilename, const char *tofilename)

void concatenate_single_level (int level, const char *fromfilename, const char *tofilename)

void remove_all_levels (const char *filename)

void remove_single_level (int level, const char *filename)

void print_table_stat ()

void print_table_stat (int level)

void dumplm (std::fstream &out, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos)

void delete_level (int level, const char *outfilename, int mmap)

void delete_level_nommap (int level)

void delete_level_mmap (int level, const char *filename)

void resize_level (int level, const char *outfilename, int mmap)

void resize_level_nommap (int level)

void resize_level_mmap (int level, const char *filename)

void update_offset (int level, table_entry_pos_t value)

void load (const std::string filename, int mmap=0)

void load (std::istream &inp, const char *filename=NULL, const char *outfilename=NULL, int mmap=0, OUTFILE_TYPE outtype=NONE)

void load_centers (std::istream &inp, int l)

void expand_level (int level, table_entry_pos_t size, const char *outfilename, int mmap)

void expand_level_nommap (int level, table_entry_pos_t size)

void expand_level_mmap (int level, table_entry_pos_t size, const char *outfilename)

void cpsublm (lmtable *sublmt, dictionary *subdict, bool keepunigr=true)

int reload (std::set< string > words)

void filter (const char *)

virtual double lprob (ngram ng, double *bow=NULL, int *bol=NULL, char **maxsuffptr=NULL, unsigned int *statesize=NULL, bool *extendible=NULL, double *lastbow=NULL)

virtual double clprob (ngram ng, double *bow=NULL, int *bol=NULL, char **maxsuffptr=NULL, unsigned int *statesize=NULL, bool *extendible=NULL)

virtual double clprob (int *ng, int ngsize, double *bow=NULL, int *bol=NULL, char **maxsuffptr=NULL, unsigned int *statesize=NULL, bool *extendible=NULL)

void * search (int lev, table_entry_pos_t offs, table_entry_pos_t n, int sz, int *w, LMT_ACTION action, char **found=(char **) NULL)

int mybsearch (char *ar, table_entry_pos_t n, int size, char *key, table_entry_pos_t *idx)

int add (ngram &ng, float prob, float bow)

int addwithoffset (ngram &ng, float prob, float bow)

void checkbounds (int level)

int get (ngram &ng)

int get (ngram &ng, int n, int lev)

int succscan (ngram &h, ngram &ng, LMT_ACTION action, int lev)

virtual const char * maxsuffptr (ngram ong, unsigned int *size=NULL)

virtual const char * cmaxsuffptr (ngram ong, unsigned int *size=NULL)

void putmem (char *ptr, int value, int offs, int size)

void getmem (char *ptr, int *value, int offs, int size)

template<typename T >

void putmem (char *ptr, T value, int offs)

template<typename T >

void getmem (char *ptr, T *value, int offs)

int nodesize (LMT_TYPE ndt)

int word (node nd, int value=-1)

int codecmp (node a, node b)

int codediff (node a, node b)

float prob (node nd, LMT_TYPE ndt)

template<typename T >

T prob (node nd, LMT_TYPE ndt, T value)

float bow (node nd, LMT_TYPE ndt)

template<typename T >

T bow (node nd, LMT_TYPE ndt, T value)

table_entry_pos_t boundwithoffset (node nd, LMT_TYPE ndt, int level)

table_entry_pos_t boundwithoffset (node nd, LMT_TYPE ndt, table_entry_pos_t value, int level)

table_entry_pos_t bound (node nd, LMT_TYPE ndt)

table_entry_pos_t bound (node nd, LMT_TYPE ndt, table_entry_pos_t value)

int succrange (node ndp, int level, table_entry_pos_t *isucc=NULL, table_entry_pos_t *esucc=NULL)

void stat (int lev=0)

void printTable (int level)

virtual void setDict (dictionary *d)

virtual dictionary * getDict () const

table_entry_pos_t getCurrentSize (int l) const

void setOrderQuery (bool v)

bool isOrderQuery () const

float GetNgramcacheLoadFactor ()

float GetDictioanryLoadFactor ()

virtual void dictionary_incflag (const bool flag)

virtual bool filter (const string sfilter, lmtable *sublmt, const string skeepunigrams)

virtual bool is_OOV (int code)

Public Attributes

dictionary * dict

Protected Attributes

char * table [LMTMAXLEV+1]

LMT_TYPE tbltype [LMTMAXLEV+1]

table_entry_pos_t cursize [LMTMAXLEV+1]

table_entry_pos_t tb_offset [LMTMAXLEV+1]

table_entry_pos_t maxsize [LMTMAXLEV+1]

table_entry_pos_t * startpos [LMTMAXLEV+1]

char info [100]

int totget [LMTMAXLEV+1]

int totbsearch [LMTMAXLEV+1]

bool isQtable

bool isItable

bool isInverted

bool isPruned

int NumCenters [LMTMAXLEV+1]

float * Pcenters [LMTMAXLEV+1]

float * Bcenters [LMTMAXLEV+1]

double logOOVpenalty

int dictionary_upperbound

int backoff_state

int max_cache_lev

NGRAMCACHE_t * prob_and_state_cache

NGRAMCACHE_t * lmtcache [LMTMAXLEV+1]

float ngramcache_load_factor

float dictionary_load_factor

int memmap

int diskid

off_t tableOffs [LMTMAXLEV+1]

off_t tableGaps [LMTMAXLEV+1]

bool orderQuery

bool delete_dict

Detailed Description

Definition at line 84 of file lmtable.h.

Constructor & Destructor Documentation

lmtable::lmtable	(	float	nlf = `0.0`,
		float	dlfi = `0.0`
	)

Definition at line 72 of file lmtable.cpp.

References configure(), cursize, delete_dict, dict, dictionary_load_factor, info, isInverted, isPruned, lmtcache, LMTMAXLEV, logOOVpenalty, max_cache_lev, maxsize, memmap, ngramcache_load_factor, NULL, NumCenters, prob_and_state_cache, lmContainer::requiredMaxlev, setOrderQuery(), table, tableGaps, tb_offset, tbltype, totbsearch, and totget.

Here is the call graph for this function:

lmtable::~lmtable ( ) [virtual]

Definition at line 116 of file lmtable.cpp.

References Bcenters, cursize, delete_caches(), delete_dict, dict, isQtable, lmContainer::maxlev, memmap, Munmap(), nodesize(), Pcenters, table, tableGaps, and tbltype.

Here is the call graph for this function:

Member Function Documentation

int lmtable::add	(	ngram &	ng,
		float	prob,
		float	bow
	)

Definition at line 1063 of file lmtable.cpp.

References bound(), BOUND_EMPTY1, bow(), cursize, end, LMT_FIND, lmContainer::maxlev, nodesize(), prob(), search(), ngram::size, startpos, table, tbltype, word(), and ngram::wordp().

Referenced by mdiadaptlm::saveBIN_per_level().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::addwithoffset	(	ngram &	ng,
		float	prob,
		float	bow
	)

Definition at line 975 of file lmtable.cpp.

References BOUND_EMPTY1, boundwithoffset(), bow(), cursize, end, LMT_FIND, lmContainer::maxlev, nodesize(), prob(), search(), ngram::size, startpos, table, tbltype, word(), and ngram::wordp().

Referenced by mdiadaptlm::saveBIN_per_word().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::appendbin_level	(	int	level,
		fstream &	out,
		int	mmap
	)

Definition at line 1508 of file lmtable.cpp.

References appendbin_level_mmap(), appendbin_level_nommap(), and getCurrentSize().

Referenced by mdiadaptlm::saveBIN_per_word().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::appendbin_level_mmap	(	int	level,
		fstream &	out
	)

Definition at line 1557 of file lmtable.cpp.

References UNUSED.

Referenced by appendbin_level().

Here is the caller graph for this function:

void lmtable::appendbin_level_nommap	(	int	level,
		fstream &	out
	)

Definition at line 1519 of file lmtable.cpp.

References cursize, lmContainer::maxlev, nodesize(), table, tbltype, and VERBOSE.

Referenced by appendbin_level().

Here is the call graph for this function:

Here is the caller graph for this function:

bool lmtable::are_caches_active ( )

Definition at line 287 of file lmtable.cpp.

References are_lmtcaches_active(), and are_prob_and_state_cache_active().

Here is the call graph for this function:

bool lmtable::are_lmtcaches_active ( )

Definition at line 274 of file lmtable.cpp.

References lmtcache, max_cache_lev, and NULL.

Referenced by are_caches_active().

Here is the caller graph for this function:

bool lmtable::are_prob_and_state_cache_active ( )

Definition at line 265 of file lmtable.cpp.

References NULL, and prob_and_state_cache.

Referenced by are_caches_active(), is_probcache_active(), and is_statecache_active().

Here is the caller graph for this function:

table_entry_pos_t lmtable::bound	(	node	nd,
		LMT_TYPE	ndt,
		table_entry_pos_t	value
	)			`[inline]`

Definition at line 535 of file lmtable.h.

References LMTCODESIZE, PROBSIZE, putmem(), QINTERNAL, and QPROBSIZE.

Here is the call graph for this function:

table_entry_pos_t lmtable::bound	(	node	nd,
		LMT_TYPE	ndt
	)			`[inline]`

Definition at line 520 of file lmtable.h.

References getmem(), LMTCODESIZE, PROBSIZE, QINTERNAL, and QPROBSIZE.

Referenced by add(), boundwithoffset(), cpsublm(), get(), printTable(), succrange(), and succscan().

Here is the call graph for this function:

Here is the caller graph for this function:

table_entry_pos_t lmtable::boundwithoffset	(	node	nd,
		LMT_TYPE	ndt,
		table_entry_pos_t	value,
		int	level
	)			`[inline]`

Definition at line 517 of file lmtable.h.

References bound(), and tb_offset.

Here is the call graph for this function:

table_entry_pos_t lmtable::boundwithoffset	(	node	nd,
		LMT_TYPE	ndt,
		int	level
	)			`[inline]`

Definition at line 515 of file lmtable.h.

References bound(), and tb_offset.

Referenced by addwithoffset(), and checkbounds().

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

T lmtable::bow	(	node	nd,
		LMT_TYPE	ndt,
		T	value
	)			`[inline]`

Definition at line 490 of file lmtable.h.

References INTERNAL, LEAF, LMTCODESIZE, PROBSIZE, putmem(), QINTERNAL, QLEAF, and QPROBSIZE.

Here is the call graph for this function:

float lmtable::bow	(	node	nd,
		LMT_TYPE	ndt
	)			`[inline]`

Definition at line 465 of file lmtable.h.

References getmem(), INTERNAL, LEAF, LMTCODESIZE, PROBSIZE, QINTERNAL, QLEAF, and QPROBSIZE.

Referenced by add(), addwithoffset(), lmclass::clprob(), get(), lprob(), printTable(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::check_caches_levels ( ) [virtual]

Reimplemented from lmContainer.

Definition at line 237 of file lmtable.cpp.

References check_lmtcaches_levels(), and check_prob_and_state_cache_levels().

Here is the call graph for this function:

void lmtable::check_lmtcaches_levels ( )

Definition at line 229 of file lmtable.cpp.

References cursize, lmtcache, and max_cache_lev.

Referenced by check_caches_levels().

Here is the caller graph for this function:

void lmtable::check_prob_and_state_cache_levels ( )

Definition at line 221 of file lmtable.cpp.

References prob_and_state_cache.

Referenced by check_caches_levels(), and check_probcache_levels().

Here is the caller graph for this function:

void lmtable::check_probcache_levels ( ) [inline]

Definition at line 201 of file lmtable.h.

References check_prob_and_state_cache_levels().

Here is the call graph for this function:

void lmtable::check_statecache_levels ( ) [inline]

Definition at line 204 of file lmtable.h.

void lmtable::checkbounds ( int level )

Definition at line 874 of file lmtable.cpp.

References BOUND_EMPTY1, boundwithoffset(), mfstream::close(), createtempfile(), cursize, end, getCurrentSize(), nodesize(), removefile(), startpos, table, tbltype, and VERBOSE.

Referenced by mdiadaptlm::saveBIN_per_level(), and mdiadaptlm::saveBIN_per_word().

Here is the call graph for this function:

Here is the caller graph for this function:

double lmtable::clprob	(	int *	ng,
		int	ngsize,
		double *	bow = `NULL`,
		int *	bol = `NULL`,
		char **	maxsuffptr = `NULL`,
		unsigned int *	statesize = `NULL`,
		bool *	extendible = `NULL`
	)			`[virtual]`

Reimplemented from lmContainer.

Reimplemented in lmclass, and lmmacro.

Definition at line 2410 of file lmtable.cpp.

References PROB_AND_STATE_ENTRY::bol, PROB_AND_STATE_ENTRY::bow, dict, PROB_AND_STATE_ENTRY::extendible, PROB_AND_STATE_ENTRY::logpr, lprob(), NULL, prob_and_state_cache, ngram::pushc(), ngram::size, PROB_AND_STATE_ENTRY::state, PROB_AND_STATE_ENTRY::statesize, VERBOSE, and ngram::wordp().

Here is the call graph for this function:

double lmtable::clprob	(	ngram	ng,
		double *	bow = `NULL`,
		int *	bol = `NULL`,
		char **	maxsuffptr = `NULL`,
		unsigned int *	statesize = `NULL`,
		bool *	extendible = `NULL`
	)			`[virtual]`

Reimplemented from lmContainer.

Reimplemented in lmclass, and lmmacro.

Definition at line 2350 of file lmtable.cpp.

References PROB_AND_STATE_ENTRY::bol, PROB_AND_STATE_ENTRY::bow, PROB_AND_STATE_ENTRY::extendible, PROB_AND_STATE_ENTRY::logpr, lprob(), NULL, prob_and_state_cache, ngram::size, PROB_AND_STATE_ENTRY::state, PROB_AND_STATE_ENTRY::statesize, VERBOSE, and ngram::wordp().

Here is the call graph for this function:

const char * lmtable::cmaxsuffptr	(	ngram	ong,
		unsigned int *	size = `NULL`
	)			`[virtual]`

Reimplemented in lmmacro.

Definition at line 2190 of file lmtable.cpp.

References maxsuffptr(), NULL, prob_and_state_cache, ngram::size, PROB_AND_STATE_ENTRY::state, PROB_AND_STATE_ENTRY::statesize, and ngram::wordp().

Here is the call graph for this function:

int lmtable::codecmp	(	node	a,
		node	b
	)			`[inline]`

Definition at line 401 of file lmtable.h.

References LMTCODESIZE.

Referenced by mybsearch().

Here is the caller graph for this function:

int lmtable::codediff	(	node	a,
		node	b
	)			`[inline]`

Definition at line 410 of file lmtable.h.

References word().

Referenced by mybsearch().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::compact_all_levels ( const char * filename )

Definition at line 1732 of file lmtable.cpp.

References compact_single_level(), and maxlevel().

Referenced by mdiadaptlm::saveBIN_per_level(), and mdiadaptlm::saveBIN_per_word().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::compact_single_level	(	int	level,
		const char *	filename
	)

Definition at line 1739 of file lmtable.cpp.

References removefile(), and VERBOSE.

Referenced by compact_all_levels().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::concatenate_all_levels	(	const char *	fromfilename,
		const char *	tofilename
	)

Definition at line 1657 of file lmtable.cpp.

References concatenate_single_level(), and maxlevel().

Here is the call graph for this function:

void lmtable::concatenate_single_level	(	int	level,
		const char *	fromfilename,
		const char *	tofilename
	)

Definition at line 1666 of file lmtable.cpp.

References VERBOSE.

Referenced by concatenate_all_levels().

Here is the caller graph for this function:

void lmtable::configure	(	int	n,
		bool	quantized
	)

Definition at line 292 of file lmtable.cpp.

References INTERNAL, LEAF, lmContainer::maxlev, maxlevel(), QINTERNAL, QLEAF, tbltype, and VERBOSE.

Referenced by cpsublm(), lmtable(), mdiadaptlm::saveBIN_per_level(), and mdiadaptlm::saveBIN_per_word().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::cpsublm	(	lmtable *	sublmt,
		dictionary *	subdict,
		bool	keepunigr = `true`
	)

Definition at line 1269 of file lmtable.cpp.

References Bcenters, bound(), configure(), cursize, dictionary::decode(), dict, end, dictionary::genoovcode(), lmContainer::maxlev, nodesize(), NULL, NumCenters, dictionary::oovcode(), Pcenters, dictionary::size(), table, tbltype, and word().

Referenced by filter().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::delete_caches ( )

Definition at line 189 of file lmtable.cpp.

References delete_lmtcaches(), and delete_prob_and_state_cache().

Referenced by ~lmtable().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::delete_level	(	int	level,
		const char *	outfilename,
		int	mmap
	)

Definition at line 1703 of file lmtable.cpp.

References delete_level_mmap(), and delete_level_nommap().

Referenced by mdiadaptlm::saveBIN_per_word().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::delete_level_mmap	(	int	level,
		const char *	filename
	)

Definition at line 1711 of file lmtable.cpp.

References cursize, Munmap(), nodesize(), table, tableGaps, and tbltype.

Referenced by delete_level().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::delete_level_nommap ( int level )

Definition at line 1726 of file lmtable.cpp.

References cursize, and table.

Referenced by delete_level().

Here is the caller graph for this function:

void lmtable::delete_lmtcaches ( )

Definition at line 178 of file lmtable.cpp.

References lmtcache, max_cache_lev, and NULL.

Referenced by delete_caches().

Here is the caller graph for this function:

void lmtable::delete_prob_and_state_cache ( )

Definition at line 169 of file lmtable.cpp.

References NULL, and prob_and_state_cache.

Referenced by delete_caches(), and delete_probcache().

Here is the caller graph for this function:

void lmtable::delete_probcache ( ) [inline]

Definition at line 193 of file lmtable.h.

References delete_prob_and_state_cache().

Here is the call graph for this function:

void lmtable::delete_statecache ( ) [inline]

Definition at line 196 of file lmtable.h.

virtual void lmtable::dictionary_incflag ( const bool flag ) [inline, virtual]

Reimplemented from lmContainer.

Reimplemented in lmclass, and lmmacro.

Definition at line 629 of file lmtable.h.

References UNUSED.

void lmtable::dumplm	(	std::fstream &	out,
		ngram	ng,
		int	ilev,
		int	elev,
		table_entry_pos_t	ipos,
		table_entry_pos_t	epos
	)

Referenced by savetxt().

Here is the caller graph for this function:

void lmtable::expand_level	(	int	level,
		table_entry_pos_t	size,
		const char *	outfilename,
		int	mmap
	)

Definition at line 762 of file lmtable.cpp.

References expand_level_mmap(), and expand_level_nommap().

Referenced by mdiadaptlm::saveBIN_per_level(), and mdiadaptlm::saveBIN_per_word().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::expand_level_mmap	(	int	level,
		table_entry_pos_t	size,
		const char *	outfilename
	)

Definition at line 772 of file lmtable.cpp.

References BOUND_EMPTY1, lmContainer::maxlev, MMap(), nodesize(), NULL, startpos, table, tableGaps, and tbltype.

Referenced by expand_level().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::expand_level_nommap	(	int	level,
		table_entry_pos_t	size
	)

Definition at line 815 of file lmtable.cpp.

References BOUND_EMPTY1, lmContainer::maxlev, nodesize(), startpos, table, tbltype, and VERBOSE.

Referenced by expand_level().

Here is the call graph for this function:

Here is the caller graph for this function:

virtual bool lmtable::filter	(	const string	sfilter,
		lmtable *	sublmt,
		const string	skeepunigrams
	)			`[inline, virtual]`

Definition at line 633 of file lmtable.h.

References cpsublm(), and dict.

Here is the call graph for this function:

void lmtable::filter ( const char * ) [inline]

Definition at line 315 of file lmtable.h.

int lmtable::get	(	ngram &	ng,
		int	n,
		int	lev
	)

Definition at line 1936 of file lmtable.cpp.

References bound(), BOUND_EMPTY1, bow(), ngram::bow, cursize, error(), ngram::freq, ngram::info, ngram::lev, ngram::link, LMT_FIND, lmtcache, nodesize(), NOPROB, NULL, ngram::path, ngram::prob, prob(), search(), ngram::size, ngram::succ, table, tbltype, totget, and ngram::wordp().

Here is the call graph for this function:

int lmtable::get ( ngram & ng ) [inline]

Definition at line 336 of file lmtable.h.

table_entry_pos_t lmtable::getCurrentSize ( int l ) const [inline]

Definition at line 610 of file lmtable.h.

References cursize.

Referenced by appendbin_level(), checkbounds(), printTable(), resize_level(), and mdiadaptlm::saveBIN_per_level().

Here is the caller graph for this function:

virtual dictionary* lmtable::getDict ( ) const [inline, virtual]

Reimplemented from lmContainer.

Reimplemented in lmclass, and lmmacro.

Definition at line 606 of file lmtable.h.

References dict.

Referenced by lmmacro::clprob(), lmmacro::cmaxsuffptr(), load(), lmmacro::lprob(), lmclass::lprob(), lmmacro::maxsuffptr(), lmmacro::Micro2MacroMapping(), ngcnt(), lmmacro::One2OneMapping(), savebin(), savebin_dict(), mdiadaptlm::saveBIN_per_level(), savetxt(), stat(), and wdprune().

Here is the caller graph for this function:

float lmtable::GetDictioanryLoadFactor ( ) [inline]

Definition at line 624 of file lmtable.h.

References ngramcache_load_factor.

double lmtable::getlogOOVpenalty ( ) const [inline, virtual]

Reimplemented from lmContainer.

Definition at line 242 of file lmtable.h.

References logOOVpenalty.

template<typename T >

void lmtable::getmem	(	char *	ptr,
		T *	value,
		int	offs
	)			`[inline]`

Definition at line 367 of file lmtable.h.

References NULL, and T.

void lmtable::getmem	(	char *	ptr,
		int *	value,
		int	offs,
		int	size
	)			`[inline]`

Definition at line 352 of file lmtable.h.

References NULL.

Referenced by bound(), bow(), prob(), and word().

Here is the caller graph for this function:

float lmtable::GetNgramcacheLoadFactor ( ) [inline]

Definition at line 621 of file lmtable.h.

References ngramcache_load_factor.

void lmtable::init_caches ( int uptolev ) [virtual]

Reimplemented from lmContainer.

Definition at line 163 of file lmtable.cpp.

References init_lmtcaches(), and init_prob_and_state_cache().

Here is the call graph for this function:

void lmtable::init_lmtcaches ( int uptolev )

Definition at line 151 of file lmtable.cpp.

References lmtcache, max_cache_lev, ngramcache_load_factor, NGRAMCACHE_t, and NULL.

Referenced by init_caches().

Here is the caller graph for this function:

void lmtable::init_prob_and_state_cache ( )

Definition at line 142 of file lmtable.cpp.

References lmContainer::maxlev, ngramcache_load_factor, NGRAMCACHE_t, NULL, and prob_and_state_cache.

Referenced by init_caches(), and init_probcache().

Here is the caller graph for this function:

void lmtable::init_probcache ( ) [inline]

Definition at line 180 of file lmtable.h.

References init_prob_and_state_cache().

Here is the call graph for this function:

void lmtable::init_statecache ( ) [inline]

Definition at line 183 of file lmtable.h.

bool lmtable::is_inverted ( ) [inline, virtual]

Reimplemented from lmContainer.

Definition at line 235 of file lmtable.h.

References isInverted.

bool lmtable::is_inverted ( const bool flag ) [inline, virtual]

Reimplemented from lmContainer.

Definition at line 232 of file lmtable.h.

References isInverted.

virtual bool lmtable::is_OOV ( int code ) [inline, virtual]

Reimplemented from lmContainer.

Definition at line 643 of file lmtable.h.

References dict, and dictionary::oovcode().

Here is the call graph for this function:

bool lmtable::is_probcache_active ( ) [inline]

Definition at line 218 of file lmtable.h.

References are_prob_and_state_cache_active().

Here is the call graph for this function:

bool lmtable::is_statecache_active ( ) [inline]

Definition at line 221 of file lmtable.h.

References are_prob_and_state_cache_active().

Here is the call graph for this function:

bool lmtable::isOrderQuery ( ) const [inline]

Definition at line 617 of file lmtable.h.

References orderQuery.

bool lmtable::isQuantized ( ) const [inline]

Definition at line 259 of file lmtable.h.

References isQtable.

void lmtable::load	(	std::istream &	inp,
		const char *	filename = `NULL`,
		const char *	outfilename = `NULL`,
		int	mmap = `0`,
		OUTFILE_TYPE	outtype = `NONE`
	)

void lmtable::load	(	const std::string	filename,
		int	mmap = `0`
	)			`[virtual]`

Reimplemented from lmContainer.

Reimplemented in lmclass, and lmmacro.

Definition at line 313 of file lmtable.cpp.

References getDict(), dictionary::incflag(), NONE, NULL, lmContainer::requiredMaxlev, lmContainer::setMaxLoadedLevel(), and VERBOSE.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::load_centers	(	std::istream &	inp,
		int	l
	)

double lmtable::lprob	(	ngram	ng,
		double *	bow = `NULL`,
		int *	bol = `NULL`,
		char **	maxsuffptr = `NULL`,
		unsigned int *	statesize = `NULL`,
		bool *	extendible = `NULL`,
		double *	lastbow = `NULL`
	)			`[virtual]`

Definition at line 2247 of file lmtable.cpp.

References Bcenters, bow(), ngram::bow, dict, ngram::invert(), isInverted, ngram::lev, ngram::link, logOOVpenalty, MIN, NULL, dictionary::oovcode(), ngram::path, ngram::prob, ngram::size, succrange(), tbltype, UNIGRAM_RESOLUTION, VERBOSE, and ngram::wordp().

Referenced by clprob(), main(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

double lmtable::lprobx	(	ngram	ong,
		double *	lkp = `0`,
		double *	bop = `0`,
		int *	bol = `0`
	)

Definition at line 2550 of file lmtable.cpp.

References Bcenters, ngram::bow, dict, ngram::prob, ngram::shift(), ngram::size, and UNIGRAM_RESOLUTION.

Here is the call graph for this function:

virtual int lmtable::maxlevel ( ) const [inline, virtual]

Reimplemented from lmContainer.

Reimplemented in lmmacro.

Definition at line 256 of file lmtable.h.

References lmContainer::maxlev.

Referenced by compact_all_levels(), concatenate_all_levels(), configure(), main(), and remove_all_levels().

Here is the caller graph for this function:

const char * lmtable::maxsuffptr	(	ngram	ong,
		unsigned int *	size = `NULL`
	)			`[virtual]`

Reimplemented in lmmacro.

Definition at line 2144 of file lmtable.cpp.

References ngram::invert(), isInverted, ngram::lev, ngram::link, MIN, NULL, ngram::path, ngram::size, and ngram::succ.

Referenced by lmclass::clprob(), and cmaxsuffptr().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::mybsearch	(	char *	ar,
		table_entry_pos_t	n,
		int	size,
		char *	key,
		table_entry_pos_t *	idx
	)

Definition at line 1199 of file lmtable.cpp.

References codecmp(), codediff(), and NULL.

Referenced by search().

Here is the call graph for this function:

Here is the caller graph for this function:

table_entry_pos_t lmtable::ngcnt	(	table_entry_pos_t *	cnt,
		ngram	ng,
		int	l,
		table_entry_pos_t	ipos,
		table_entry_pos_t	epos
	)

Definition at line 2741 of file lmtable.cpp.

References ngcnt(), nodesize(), NOPROB, prob(), ngram::pushc(), succrange(), table, tbltype, word(), and ngram::wordp().

Here is the call graph for this function:

table_entry_pos_t lmtable::ngcnt ( table_entry_pos_t * cnt )

Definition at line 2732 of file lmtable.cpp.

References cursize, and getDict().

Referenced by ngcnt(), and savetxt().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::nodesize ( LMT_TYPE ndt ) [inline]

Definition at line 373 of file lmtable.h.

References BOUNDSIZE, INTERNAL, LEAF, LMTCODESIZE, PROBSIZE, QINTERNAL, QLEAF, and QPROBSIZE.

Referenced by add(), addwithoffset(), appendbin_level_nommap(), checkbounds(), cpsublm(), delete_level_mmap(), expand_level_mmap(), expand_level_nommap(), get(), ngcnt(), printTable(), pscale(), reset_mmap(), resize_level_mmap(), resize_level_nommap(), savebin(), savebin_level_nommap(), stat(), succrange(), succscan(), wdprune(), and ~lmtable().

Here is the caller graph for this function:

void lmtable::print_table_stat ( int level )

Definition at line 1644 of file lmtable.cpp.

References cursize, table, tableGaps, and tb_offset.

void lmtable::print_table_stat ( )

Definition at line 1637 of file lmtable.cpp.

References lmContainer::maxlev.

void lmtable::printTable ( int level )

Definition at line 835 of file lmtable.cpp.

References bound(), bow(), dictionary::decode(), dict, getCurrentSize(), lmContainer::maxlev, nodesize(), prob(), startpos, table, tb_offset, tbltype, and word().

Here is the call graph for this function:

template<typename T >

T lmtable::prob	(	node	nd,
		LMT_TYPE	ndt,
		T	value
	)			`[inline]`

Definition at line 440 of file lmtable.h.

References INTERNAL, LEAF, LMTCODESIZE, putmem(), QINTERNAL, QLEAF, and T.

Here is the call graph for this function:

float lmtable::prob	(	node	nd,
		LMT_TYPE	ndt
	)			`[inline]`

Definition at line 415 of file lmtable.h.

References getmem(), INTERNAL, LEAF, LMTCODESIZE, QINTERNAL, and QLEAF.

Referenced by add(), addwithoffset(), get(), lmmacro::lprob(), ngcnt(), printTable(), pscale(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::pscale	(	int	lev,
		table_entry_pos_t	ipos,
		table_entry_pos_t	epos,
		double	s
	)

Definition at line 2713 of file lmtable.cpp.

References nodesize(), NOPROB, prob(), table, and tbltype.

Referenced by wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

void lmtable::putmem	(	char *	ptr,
		T	value,
		int	offs
	)			`[inline]`

Definition at line 361 of file lmtable.h.

References NULL, and T.

void lmtable::putmem	(	char *	ptr,
		int	value,
		int	offs,
		int	size
	)			`[inline]`

Definition at line 346 of file lmtable.h.

References NULL.

Referenced by bound(), bow(), prob(), search(), and word().

Here is the caller graph for this function:

int lmtable::reload ( std::set< string > words )

Definition at line 375 of file lmtable.cpp.

References dict, dictionary::encode(), dictionary::incflag(), and NULL.

Here is the call graph for this function:

void lmtable::remove_all_levels ( const char * filename )

Definition at line 1683 of file lmtable.cpp.

References maxlevel(), and remove_single_level().

Here is the call graph for this function:

void lmtable::remove_single_level	(	int	level,
		const char *	filename
	)

Definition at line 1691 of file lmtable.cpp.

References removefile().

Referenced by remove_all_levels().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::reset_caches ( ) [virtual]

Reimplemented from lmContainer.

Definition at line 259 of file lmtable.cpp.

References reset_lmtcaches(), and reset_prob_and_state_cache().

Here is the call graph for this function:

void lmtable::reset_lmtcaches ( )

Definition at line 251 of file lmtable.cpp.

References cursize, lmtcache, MAX, max_cache_lev, and maxsize.

Referenced by reset_caches().

Here is the caller graph for this function:

void lmtable::reset_mmap ( ) [virtual]

Reimplemented from lmContainer.

Definition at line 2529 of file lmtable.cpp.

References cursize, diskid, memmap, MMap(), Munmap(), nodesize(), table, tableOffs, and tbltype.

Here is the call graph for this function:

void lmtable::reset_prob_and_state_cache ( )

Definition at line 243 of file lmtable.cpp.

References MAX, and prob_and_state_cache.

Referenced by reset_caches(), and reset_probcache().

Here is the caller graph for this function:

void lmtable::reset_probcache ( ) [inline]

Definition at line 209 of file lmtable.h.

References reset_prob_and_state_cache().

Here is the call graph for this function:

void lmtable::reset_statecache ( ) [inline]

Definition at line 212 of file lmtable.h.

void lmtable::resize_level	(	int	level,
		const char *	outfilename,
		int	mmap
	)

Definition at line 1756 of file lmtable.cpp.

References getCurrentSize(), lmContainer::maxlev, resize_level_mmap(), and resize_level_nommap().

Referenced by mdiadaptlm::saveBIN_per_level(), and mdiadaptlm::saveBIN_per_word().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::resize_level_mmap	(	int	level,
		const char *	filename
	)

Definition at line 1768 of file lmtable.cpp.

References cursize, MMap(), Munmap(), nodesize(), NULL, table, tableGaps, and tbltype.

Referenced by resize_level().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::resize_level_nommap ( int level )

Definition at line 1788 of file lmtable.cpp.

References cursize, nodesize(), table, tbltype, and VERBOSE.

Referenced by resize_level().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::savebin ( const char * filename ) [virtual]

Reimplemented from lmContainer.

Definition at line 1449 of file lmtable.cpp.

References Bcenters, cursize, getDict(), isInverted, isPruned, lmContainer::maxlev, nodesize(), dictionary::save(), table, tbltype, and VERBOSE.

Here is the call graph for this function:

void lmtable::savebin_dict ( std::fstream & out )

Definition at line 1493 of file lmtable.cpp.

References getDict(), and dictionary::save().

Here is the call graph for this function:

void lmtable::savebin_level	(	int	level,
		const char *	filename,
		int	mmap
	)

Definition at line 1563 of file lmtable.cpp.

References savebin_level_mmap(), and savebin_level_nommap().

Referenced by mdiadaptlm::saveBIN_per_level(), and mdiadaptlm::saveBIN_per_word().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::savebin_level_mmap	(	int	level,
		const char *	filename
	)

Definition at line 1628 of file lmtable.cpp.

References VERBOSE.

Referenced by savebin_level().

Here is the caller graph for this function:

void lmtable::savebin_level_nommap	(	int	level,
		const char *	filename
	)

Definition at line 1572 of file lmtable.cpp.

References cursize, lmContainer::maxlev, nodesize(), removefile(), lmContainer::requiredMaxlev, table, tbltype, and VERBOSE.

Referenced by savebin_level().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::savetxt ( const char * filename ) [virtual]

Reimplemented from lmContainer.

Definition at line 1393 of file lmtable.cpp.

References Bcenters, cursize, dumplm(), getDict(), isPruned, MAX_NGRAM, lmContainer::maxlev, ngcnt(), and ngram::size.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void * lmtable::search	(	int	lev,
		table_entry_pos_t	offs,
		table_entry_pos_t	n,
		int	sz,
		int *	w,
		LMT_ACTION	action,
		char **	found = `(char **)NULL`
	)

Definition at line 1150 of file lmtable.cpp.

References error(), LMT_FIND, LMTCODESIZE, mybsearch(), NULL, putmem(), table, and totbsearch.

Referenced by add(), addwithoffset(), and get().

Here is the call graph for this function:

Here is the caller graph for this function:

virtual void lmtable::setDict ( dictionary * d ) [inline, virtual]

Definition at line 600 of file lmtable.h.

References delete_dict, and dict.

Referenced by mdiadaptlm::saveBIN_per_level(), and mdiadaptlm::saveBIN_per_word().

Here is the caller graph for this function:

double lmtable::setlogOOVpenalty ( double oovp ) [inline, virtual]

Reimplemented from lmContainer.

Definition at line 252 of file lmtable.h.

References logOOVpenalty.

double lmtable::setlogOOVpenalty ( int dub ) [inline, virtual]

Reimplemented from lmContainer.

Definition at line 246 of file lmtable.h.

References dict, dictionary_upperbound, logOOVpenalty, and dictionary::size().

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::setOrderQuery ( bool v ) [inline]

Definition at line 614 of file lmtable.h.

References orderQuery.

Referenced by lmtable().

Here is the caller graph for this function:

void lmtable::stat ( int lev = 0 ) [virtual]

Reimplemented from lmContainer.

Definition at line 2500 of file lmtable.cpp.

References cursize, getDict(), nodesize(), dictionary::stat(), tbltype, totbsearch, and totget.

Here is the call graph for this function:

int lmtable::succrange	(	node	ndp,
		int	level,
		table_entry_pos_t *	isucc = `NULL`,
		table_entry_pos_t *	esucc = `NULL`
	)

Definition at line 2481 of file lmtable.cpp.

References bound(), nodesize(), table, and tbltype.

Referenced by lprob(), ngcnt(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::succscan	(	ngram &	h,
		ngram &	ng,
		LMT_ACTION	action,
		int	lev
	)

Definition at line 2101 of file lmtable.cpp.

References bound(), ngram::lev, ngram::link, LMT_CONT, LMT_INIT, ngram::midx, nodesize(), ngram::size, ngram::succ, ngram::succlink, table, tbltype, ngram::trans(), word(), and ngram::wordp().

Here is the call graph for this function:

void lmtable::update_offset	(	int	level,
		table_entry_pos_t	value
	)			`[inline]`

Definition at line 299 of file lmtable.h.

References tb_offset.

Referenced by mdiadaptlm::saveBIN_per_word().

Here is the caller graph for this function:

void lmtable::used_caches ( ) [virtual]

Reimplemented from lmContainer.

Definition at line 214 of file lmtable.cpp.

References used_lmtcaches(), and used_prob_and_state_cache().

Here is the call graph for this function:

void lmtable::used_lmtcaches ( )

Definition at line 204 of file lmtable.cpp.

References lmtcache, and max_cache_lev.

Referenced by used_caches().

Here is the caller graph for this function:

void lmtable::used_prob_and_state_cache ( )

Definition at line 196 of file lmtable.cpp.

References prob_and_state_cache.

Referenced by used_caches().

Here is the caller graph for this function:

table_entry_pos_t lmtable::wdprune	(	float *	thr,
		int	aflag,
		ngram	ng,
		int	ilev,
		int	elev,
		table_entry_pos_t	ipos,
		table_entry_pos_t	epos,
		double	lk = `0`,
		double	bo = `0`,
		double *	ts = `0`,
		double *	tbs = `0`
	)

Definition at line 2621 of file lmtable.cpp.

References BOS_, bow(), cursize, dictionary::getcode(), getDict(), k, lprob(), nodesize(), NOPROB, prob(), pscale(), ngram::pushc(), ngram::size, succrange(), table, tbltype, wdprune(), word(), and ngram::wordp().

Here is the call graph for this function:

table_entry_pos_t lmtable::wdprune	(	float *	thr,
		int	aflag = `0`
	)

Definition at line 2601 of file lmtable.cpp.

References cursize, getDict(), isPruned, and ngram::size.

Referenced by main(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::word	(	node	nd,
		int	value = `-1`
	)			`[inline]`

Definition at line 389 of file lmtable.h.

References getmem(), LMTCODESIZE, and putmem().

Referenced by add(), addwithoffset(), codediff(), cpsublm(), ngcnt(), printTable(), succscan(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

int lmtable::backoff_state [protected]

Definition at line 135 of file lmtable.h.

float* lmtable::Bcenters[LMTMAXLEV+1] [protected]

Definition at line 131 of file lmtable.h.

Referenced by cpsublm(), lprob(), lprobx(), savebin(), savetxt(), and ~lmtable().

table_entry_pos_t lmtable::cursize[LMTMAXLEV+1] [protected]

Definition at line 102 of file lmtable.h.

Referenced by add(), addwithoffset(), appendbin_level_nommap(), check_lmtcaches_levels(), checkbounds(), cpsublm(), delete_level_mmap(), delete_level_nommap(), get(), getCurrentSize(), lmtable(), ngcnt(), print_table_stat(), reset_lmtcaches(), reset_mmap(), resize_level_mmap(), resize_level_nommap(), savebin(), savebin_level_nommap(), savetxt(), stat(), wdprune(), and ~lmtable().

bool lmtable::delete_dict [protected]

Definition at line 156 of file lmtable.h.

Referenced by lmtable(), setDict(), and ~lmtable().

dictionary* lmtable::dict

Definition at line 165 of file lmtable.h.

Referenced by clprob(), cpsublm(), filter(), getDict(), is_OOV(), lmtable(), lmclass::loadMapElement(), lprob(), lprobx(), main(), printTable(), reload(), setDict(), setlogOOVpenalty(), and ~lmtable().

float lmtable::dictionary_load_factor [protected]

Definition at line 143 of file lmtable.h.

Referenced by lmtable().

int lmtable::dictionary_upperbound [protected]

Definition at line 134 of file lmtable.h.

Referenced by setlogOOVpenalty().

int lmtable::diskid [protected]

Definition at line 147 of file lmtable.h.

Referenced by reset_mmap().

char lmtable::info[100] [protected]

Definition at line 111 of file lmtable.h.

Referenced by lmtable().

bool lmtable::isInverted [protected]

Definition at line 124 of file lmtable.h.

Referenced by is_inverted(), lmtable(), lprob(), maxsuffptr(), and savebin().

bool lmtable::isItable [protected]

Definition at line 121 of file lmtable.h.

bool lmtable::isPruned [protected]

Definition at line 127 of file lmtable.h.

Referenced by lmtable(), savebin(), savetxt(), and wdprune().

bool lmtable::isQtable [protected]

Definition at line 118 of file lmtable.h.

Referenced by isQuantized(), and ~lmtable().

NGRAMCACHE_t* lmtable::lmtcache[LMTMAXLEV+1] [protected]

Definition at line 141 of file lmtable.h.

Referenced by are_lmtcaches_active(), check_lmtcaches_levels(), delete_lmtcaches(), get(), init_lmtcaches(), lmtable(), reset_lmtcaches(), and used_lmtcaches().

double lmtable::logOOVpenalty [protected]

Definition at line 133 of file lmtable.h.

Referenced by getlogOOVpenalty(), lmtable(), lprob(), and setlogOOVpenalty().

int lmtable::max_cache_lev [protected]

Definition at line 138 of file lmtable.h.

Referenced by are_lmtcaches_active(), check_lmtcaches_levels(), delete_lmtcaches(), init_lmtcaches(), lmtable(), reset_lmtcaches(), and used_lmtcaches().

table_entry_pos_t lmtable::maxsize[LMTMAXLEV+1] [protected]

Definition at line 109 of file lmtable.h.

Referenced by lmtable(), and reset_lmtcaches().

int lmtable::memmap [protected]

Definition at line 146 of file lmtable.h.

Referenced by lmtable(), reset_mmap(), and ~lmtable().

float lmtable::ngramcache_load_factor [protected]

Definition at line 142 of file lmtable.h.

Referenced by GetDictioanryLoadFactor(), GetNgramcacheLoadFactor(), init_lmtcaches(), init_prob_and_state_cache(), and lmtable().

int lmtable::NumCenters[LMTMAXLEV+1] [protected]

Definition at line 129 of file lmtable.h.

Referenced by cpsublm(), and lmtable().

bool lmtable::orderQuery [protected]

Definition at line 153 of file lmtable.h.

Referenced by isOrderQuery(), and setOrderQuery().

float* lmtable::Pcenters[LMTMAXLEV+1] [protected]

Definition at line 130 of file lmtable.h.

Referenced by cpsublm(), and ~lmtable().

NGRAMCACHE_t* lmtable::prob_and_state_cache [protected]

Definition at line 140 of file lmtable.h.

Referenced by are_prob_and_state_cache_active(), check_prob_and_state_cache_levels(), clprob(), cmaxsuffptr(), delete_prob_and_state_cache(), init_prob_and_state_cache(), lmtable(), reset_prob_and_state_cache(), and used_prob_and_state_cache().

table_entry_pos_t* lmtable::startpos[LMTMAXLEV+1] [protected]

Definition at line 110 of file lmtable.h.

Referenced by add(), addwithoffset(), checkbounds(), expand_level_mmap(), expand_level_nommap(), and printTable().

char* lmtable::table[LMTMAXLEV+1] [protected]

Definition at line 100 of file lmtable.h.

Referenced by add(), addwithoffset(), appendbin_level_nommap(), checkbounds(), cpsublm(), delete_level_mmap(), delete_level_nommap(), expand_level_mmap(), expand_level_nommap(), get(), lmtable(), ngcnt(), print_table_stat(), printTable(), pscale(), reset_mmap(), resize_level_mmap(), resize_level_nommap(), savebin(), savebin_level_nommap(), search(), succrange(), succscan(), wdprune(), and ~lmtable().

off_t lmtable::tableGaps[LMTMAXLEV+1] [protected]

Definition at line 149 of file lmtable.h.

Referenced by delete_level_mmap(), expand_level_mmap(), lmtable(), print_table_stat(), resize_level_mmap(), and ~lmtable().

off_t lmtable::tableOffs[LMTMAXLEV+1] [protected]

Definition at line 148 of file lmtable.h.

Referenced by reset_mmap().

table_entry_pos_t lmtable::tb_offset[LMTMAXLEV+1] [protected]

Definition at line 107 of file lmtable.h.

Referenced by boundwithoffset(), lmtable(), print_table_stat(), printTable(), and update_offset().

LMT_TYPE lmtable::tbltype[LMTMAXLEV+1] [protected]

Definition at line 101 of file lmtable.h.

Referenced by add(), addwithoffset(), appendbin_level_nommap(), checkbounds(), configure(), cpsublm(), delete_level_mmap(), expand_level_mmap(), expand_level_nommap(), get(), lmtable(), lprob(), ngcnt(), printTable(), pscale(), reset_mmap(), resize_level_mmap(), resize_level_nommap(), savebin(), savebin_level_nommap(), stat(), succrange(), succscan(), wdprune(), and ~lmtable().

int lmtable::totbsearch[LMTMAXLEV+1] [protected]

Definition at line 115 of file lmtable.h.

Referenced by lmtable(), search(), and stat().

int lmtable::totget[LMTMAXLEV+1] [protected]

Definition at line 114 of file lmtable.h.

Referenced by get(), lmtable(), and stat().

The documentation for this class was generated from the following files:

/disk4/html/www/moses/doxygen/irstlm/trunk/src/lmtable.h
/disk4/html/www/moses/doxygen/irstlm/trunk/src/lmtable.cpp


Public Member Functions
	lmtable (float nlf=0.0, float dlfi=0.0)
virtual	~lmtable ()
table_entry_pos_t	wdprune (float *thr, int aflag=0)
table_entry_pos_t	wdprune (float thr, int aflag, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos, double lk=0, double bo=0, double ts=0, double *tbs=0)
double	lprobx (ngram ong, double lkp=0, double bop=0, int *bol=0)
table_entry_pos_t	ngcnt (table_entry_pos_t *cnt)
table_entry_pos_t	ngcnt (table_entry_pos_t *cnt, ngram ng, int l, table_entry_pos_t ipos, table_entry_pos_t epos)
int	pscale (int lev, table_entry_pos_t ipos, table_entry_pos_t epos, double s)
void	init_prob_and_state_cache ()
void	init_probcache ()
void	init_statecache ()
void	init_lmtcaches (int uptolev)
void	init_caches (int uptolev)
void	used_prob_and_state_cache ()
void	used_lmtcaches ()
void	used_caches ()
void	delete_prob_and_state_cache ()
void	delete_probcache ()
void	delete_statecache ()
void	delete_lmtcaches ()
void	delete_caches ()
void	check_prob_and_state_cache_levels ()
void	check_probcache_levels ()
void	check_statecache_levels ()
void	check_lmtcaches_levels ()
void	check_caches_levels ()
void	reset_prob_and_state_cache ()
void	reset_probcache ()
void	reset_statecache ()
void	reset_lmtcaches ()
void	reset_caches ()
bool	are_prob_and_state_cache_active ()
bool	is_probcache_active ()
bool	is_statecache_active ()
bool	are_lmtcaches_active ()
bool	are_caches_active ()
void	reset_mmap ()
bool	is_inverted (const bool flag)
bool	is_inverted ()
void	configure (int n, bool quantized)
double	getlogOOVpenalty () const
double	setlogOOVpenalty (int dub)
double	setlogOOVpenalty (double oovp)
virtual int	maxlevel () const
bool	isQuantized () const
void	savetxt (const char *filename)
void	savebin (const char *filename)
void	appendbin_level (int level, fstream &out, int mmap)
void	appendbin_level_nommap (int level, fstream &out)
void	appendbin_level_mmap (int level, fstream &out)
void	savebin_level (int level, const char *filename, int mmap)
void	savebin_level_nommap (int level, const char *filename)
void	savebin_level_mmap (int level, const char *filename)
void	savebin_dict (std::fstream &out)
void	compact_all_levels (const char *filename)
void	compact_single_level (int level, const char *filename)
void	concatenate_all_levels (const char fromfilename, const char tofilename)
void	concatenate_single_level (int level, const char fromfilename, const char tofilename)
void	remove_all_levels (const char *filename)
void	remove_single_level (int level, const char *filename)
void	print_table_stat ()
void	print_table_stat (int level)
void	dumplm (std::fstream &out, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos)
void	delete_level (int level, const char *outfilename, int mmap)
void	delete_level_nommap (int level)
void	delete_level_mmap (int level, const char *filename)
void	resize_level (int level, const char *outfilename, int mmap)
void	resize_level_nommap (int level)
void	resize_level_mmap (int level, const char *filename)
void	update_offset (int level, table_entry_pos_t value)
void	load (const std::string filename, int mmap=0)
void	load (std::istream &inp, const char filename=NULL, const char outfilename=NULL, int mmap=0, OUTFILE_TYPE outtype=NONE)
void	load_centers (std::istream &inp, int l)
void	expand_level (int level, table_entry_pos_t size, const char *outfilename, int mmap)
void	expand_level_nommap (int level, table_entry_pos_t size)
void	expand_level_mmap (int level, table_entry_pos_t size, const char *outfilename)
void	cpsublm (lmtable sublmt, dictionary subdict, bool keepunigr=true)
int	reload (std::set< string > words)
void	filter (const char *)
virtual double	lprob (ngram ng, double bow=NULL, int bol=NULL, char *maxsuffptr=NULL, unsigned int statesize=NULL, bool extendible=NULL, double lastbow=NULL)
virtual double	clprob (ngram ng, double bow=NULL, int bol=NULL, char *maxsuffptr=NULL, unsigned int statesize=NULL, bool *extendible=NULL)
virtual double	clprob (int ng, int ngsize, double bow=NULL, int bol=NULL, char maxsuffptr=NULL, unsigned int statesize=NULL, bool *extendible=NULL)
void *	search (int lev, table_entry_pos_t offs, table_entry_pos_t n, int sz, int w, LMT_ACTION action, char found=(char *) NULL)
int	mybsearch (char ar, table_entry_pos_t n, int size, char key, table_entry_pos_t *idx)
int	add (ngram &ng, float prob, float bow)
int	addwithoffset (ngram &ng, float prob, float bow)
void	checkbounds (int level)
int	get (ngram &ng)
int	get (ngram &ng, int n, int lev)
int	succscan (ngram &h, ngram &ng, LMT_ACTION action, int lev)
virtual const char *	maxsuffptr (ngram ong, unsigned int *size=NULL)
virtual const char *	cmaxsuffptr (ngram ong, unsigned int *size=NULL)
void	putmem (char *ptr, int value, int offs, int size)
void	getmem (char ptr, int value, int offs, int size)
template<typename T >
void	putmem (char *ptr, T value, int offs)
template<typename T >
void	getmem (char ptr, T value, int offs)
int	nodesize (LMT_TYPE ndt)
int	word (node nd, int value=-1)
int	codecmp (node a, node b)
int	codediff (node a, node b)
float	prob (node nd, LMT_TYPE ndt)
template<typename T >
T	prob (node nd, LMT_TYPE ndt, T value)
float	bow (node nd, LMT_TYPE ndt)
template<typename T >
T	bow (node nd, LMT_TYPE ndt, T value)
table_entry_pos_t	boundwithoffset (node nd, LMT_TYPE ndt, int level)
table_entry_pos_t	boundwithoffset (node nd, LMT_TYPE ndt, table_entry_pos_t value, int level)
table_entry_pos_t	bound (node nd, LMT_TYPE ndt)
table_entry_pos_t	bound (node nd, LMT_TYPE ndt, table_entry_pos_t value)
int	succrange (node ndp, int level, table_entry_pos_t isucc=NULL, table_entry_pos_t esucc=NULL)
void	stat (int lev=0)
void	printTable (int level)
virtual void	setDict (dictionary *d)
virtual dictionary *	getDict () const
table_entry_pos_t	getCurrentSize (int l) const
void	setOrderQuery (bool v)
bool	isOrderQuery () const
float	GetNgramcacheLoadFactor ()
float	GetDictioanryLoadFactor ()
virtual void	dictionary_incflag (const bool flag)
virtual bool	filter (const string sfilter, lmtable *sublmt, const string skeepunigrams)
virtual bool	is_OOV (int code)
Public Attributes
dictionary *	dict
Protected Attributes
char *	table [LMTMAXLEV+1]
LMT_TYPE	tbltype [LMTMAXLEV+1]
table_entry_pos_t	cursize [LMTMAXLEV+1]
table_entry_pos_t	tb_offset [LMTMAXLEV+1]
table_entry_pos_t	maxsize [LMTMAXLEV+1]
table_entry_pos_t *	startpos [LMTMAXLEV+1]
char	info [100]
int	totget [LMTMAXLEV+1]
int	totbsearch [LMTMAXLEV+1]
bool	isQtable
bool	isItable
bool	isInverted
bool	isPruned
int	NumCenters [LMTMAXLEV+1]
float *	Pcenters [LMTMAXLEV+1]
float *	Bcenters [LMTMAXLEV+1]
double	logOOVpenalty
int	dictionary_upperbound
int	backoff_state
int	max_cache_lev
NGRAMCACHE_t *	prob_and_state_cache
NGRAMCACHE_t *	lmtcache [LMTMAXLEV+1]
float	ngramcache_load_factor
float	dictionary_load_factor
int	memmap
int	diskid
off_t	tableOffs [LMTMAXLEV+1]
off_t	tableGaps [LMTMAXLEV+1]
bool	orderQuery
bool	delete_dict