lm Namespace Reference

Namespaces
namespace	base
namespace	builder
namespace	ngram
namespace	np
namespace	phrase
namespace	vocab
Classes
class	Comparator
class	SuffixOrder
class	ContextOrder
class	PrefixOrder
class	ModelBuffer
class	NGramHeader
class	NGram
class	ProxyStream
class	NGramStream
class	NGramStreams
class	VocabReconstitute
class	PrintARPA
class	Renumber
class	SpecialVocab
class	EnumerateVocab
class	ARPAInputException
class	ARPAOutput
class	CountOutput
class	CountBatch
class	MultipleOutput
class	MultipleARPAOutput
class	DispatchInput
class	DispatchARPAInput
struct	ARPAFormat
struct	CountFormat
class	InputBuffer
class	BinaryOutputBuffer
class	MultipleOutputBuffer
class	ThreadBatch
class	FilterWorker
class	OutputWorker
class	Controller
class	BinaryFilter
class	ContextFilter
class	ConfigException
class	LoadException
class	FormatLoadException
class	VocabLoadException
class	SpecialWordMissingException
class	PositiveProbWarn
struct	FullScoreReturn
struct	Prob
struct	ProbBackoff
struct	RestWeights
Typedefs
typedef unsigned int	WordIndex
Enumerations
enum	WarningAction { THROW_UP, COMPLAIN, SILENT }
Functions
template<class Callback , class Compare >
void	JointOrder (const util::stream::ChainPositions &positions, Callback &callback)
boost::program_options::typed_value < std::string > *	SizeOption (std::size_t &to, const char *default_value)
template<class Stream >
void	WriteCounts (Stream &out, const std::vector< uint64_t > &number)
size_t	SizeNeededForCounts (const std::vector< uint64_t > &number)
bool	IsEntirelyWhiteSpace (const StringPiece &line)
template<class Output >
void	ReadNGrams (util::FilePiece &in, unsigned int length, uint64_t number, Output &out)
template<class Output >
void	ReadARPA (util::FilePiece &in_lm, Output &out)
template<class Output >
void	ReadCount (util::FilePiece &in_file, Output &out)
void	ReadARPACounts (util::FilePiece &in, std::vector< uint64_t > &number)
void	ReadNGramHeader (util::FilePiece &in, unsigned int length)
void	ReadBackoff (util::FilePiece &in, Prob &)
void	ReadBackoff (util::FilePiece &in, float &backoff)
void	ReadEnd (util::FilePiece &in)
void	ReadBackoff (util::FilePiece &in, ProbBackoff &weights)
void	ReadBackoff (util::FilePiece &in, RestWeights &weights)
template<class Voc , class Weights >
void	Read1Gram (util::FilePiece &f, Voc &vocab, Weights *unigrams, PositiveProbWarn &warn)
template<class Voc , class Weights >
void	Read1Grams (util::FilePiece &f, std::size_t count, Voc &vocab, Weights *unigrams, PositiveProbWarn &warn)
template<class Voc , class Weights , class Iterator >
void	ReadNGram (util::FilePiece &f, const unsigned char n, const Voc &vocab, Iterator indices_out, Weights &weights, PositiveProbWarn &warn)
Variables
const bool	kARPASpaces [256] = {0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}
const WordIndex	kMaxWordIndex = UINT_MAX
const WordIndex	kUNK = 0

---

Typedef Documentation

typedef unsigned int lm::WordIndex

Definition at line 8 of file word_index.hh.

---

Enumeration Type Documentation

enum lm::WarningAction

Enumerator:

THROW_UP
COMPLAIN
SILENT

Definition at line 14 of file lm_exception.hh.

---

Function Documentation

bool lm::IsEntirelyWhiteSpace

(

const StringPiece &

line

)

Definition at line 41 of file arpa_io.cc.

References StringPiece::data(), and StringPiece::size().

Referenced by ReadARPACounts(), ReadEnd(), and ReadNGramHeader().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Callback , class Compare >

void lm::JointOrder	(	const util::stream::ChainPositions &	positions,
		Callback &	callback
	)			[inline]

Definition at line 16 of file joint_order.hh.

References begin, NULL, util::FixedArray< T >::push_back(), util::FixedArray< T >::size(), and UTIL_THROW_IF.

Here is the call graph for this function:

template<class Voc , class Weights >

void lm::Read1Gram	(	util::FilePiece &	f,
		Voc &	vocab,
		Weights *	unigrams,
		PositiveProbWarn &	warn
	)			[inline]

Definition at line 44 of file read_arpa.hh.

References util::FilePiece::get(), kARPASpaces, util::FilePiece::Offset(), prob(), ReadBackoff(), util::FilePiece::ReadDelimited(), util::FilePiece::ReadFloat(), UTIL_THROW_IF, and lm::PositiveProbWarn::Warn().

Referenced by Read1Grams().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Voc , class Weights >

void lm::Read1Grams	(	util::FilePiece &	f,
		std::size_t	count,
		Voc &	vocab,
		Weights *	unigrams,
		PositiveProbWarn &	warn
	)			[inline]

Definition at line 62 of file read_arpa.hh.

References Read1Gram(), and ReadNGramHeader().

Referenced by lm::ngram::detail::HashedSearch< Value >::InitializeFromARPA().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Output >

void lm::ReadARPA	(	util::FilePiece &	in_lm,
		Output &	out
	)			[inline]

Definition at line 86 of file arpa_io.hh.

References ReadARPACounts(), ReadEnd(), ReadNGrams(), and SizeNeededForCounts().

Referenced by lm::ARPAFormat::Copy(), and lm::ARPAFormat::RunFilter().

Here is the call graph for this function:

Here is the caller graph for this function:

void lm::ReadARPACounts	(	util::FilePiece &	in,
		std::vector< uint64_t > &	number
	)

Definition at line 47 of file read_arpa.cc.

References StringPiece::data(), util::FilePiece::FileName(), IsEntirelyWhiteSpace(), ReadCount(), util::FilePiece::ReadLine(), StringPiece::size(), starts_with(), UTIL_THROW, and UTIL_THROW_IF.

Referenced by lm::ngram::LowerRestBuild< Model >::LowerRestBuild(), ReadARPA(), and lm::ngram::ShowSizes().

Here is the call graph for this function:

Here is the caller graph for this function:

void lm::ReadBackoff	(	util::FilePiece &	in,
		RestWeights &	weights
	)			[inline]

Definition at line 23 of file read_arpa.hh.

References lm::RestWeights::backoff, and ReadBackoff().

Here is the call graph for this function:

void lm::ReadBackoff	(	util::FilePiece &	in,
		ProbBackoff &	weights
	)			[inline]

Definition at line 20 of file read_arpa.hh.

References lm::ProbBackoff::backoff, and ReadBackoff().

Here is the call graph for this function:

void lm::ReadBackoff	(	util::FilePiece &	in,
		float &	backoff
	)

Definition at line 104 of file read_arpa.cc.

References util::FilePiece::get(), lm::ngram::kExtensionBackoff, lm::ngram::kNoExtensionBackoff, util::FilePiece::ReadFloat(), UTIL_THROW, and UTIL_THROW_IF.

Here is the call graph for this function:

void lm::ReadBackoff	(	util::FilePiece &	in,
		Prob &
	)

Definition at line 88 of file read_arpa.cc.

References util::FilePiece::get(), util::FilePiece::ReadFloat(), and UTIL_THROW.

Referenced by Read1Gram(), ReadBackoff(), and ReadNGram().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Output >

void lm::ReadCount	(	util::FilePiece &	in_file,
		Output &	out
	)			[inline]

Definition at line 73 of file count_io.hh.

References util::FilePiece::ReadLine().

Referenced by lm::CountFormat::Copy(), ReadARPACounts(), and lm::CountFormat::RunFilter().

Here is the call graph for this function:

Here is the caller graph for this function:

void lm::ReadEnd

(

util::FilePiece &

in

)

Definition at line 133 of file read_arpa.cc.

References IsEntirelyWhiteSpace(), util::FilePiece::ReadLine(), and UTIL_THROW.

Referenced by ReadARPA().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Voc , class Weights , class Iterator >

void lm::ReadNGram	(	util::FilePiece &	f,
		const unsigned char	n,
		const Voc &	vocab,
		Iterator	indices_out,
		Weights &	weights,
		PositiveProbWarn &	warn
	)			[inline]

Definition at line 71 of file read_arpa.hh.

References index, kARPASpaces, util::FilePiece::Offset(), ReadBackoff(), util::FilePiece::ReadDelimited(), util::FilePiece::ReadFloat(), UTIL_THROW_IF, and lm::PositiveProbWarn::Warn().

Referenced by lm::ngram::LowerRestBuild< Model >::LowerRestBuild().

Here is the call graph for this function:

Here is the caller graph for this function:

void lm::ReadNGramHeader	(	util::FilePiece &	in,
		unsigned int	length
	)

Definition at line 80 of file read_arpa.cc.

References IsEntirelyWhiteSpace(), util::FilePiece::ReadLine(), and UTIL_THROW.

Referenced by lm::ngram::LowerRestBuild< Model >::LowerRestBuild(), Read1Grams(), and ReadNGrams().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Output >

void lm::ReadNGrams	(	util::FilePiece &	in,
		unsigned int	length,
		uint64_t	number,
		Output &	out
	)			[inline]

Definition at line 72 of file arpa_io.hh.

References util::FilePiece::ReadLine(), and ReadNGramHeader().

Referenced by ReadARPA().

Here is the call graph for this function:

Here is the caller graph for this function:

size_t lm::SizeNeededForCounts

(

const std::vector< uint64_t > &

number

)

Definition at line 35 of file arpa_io.cc.

References util::StringStream::str(), and WriteCounts().

Referenced by ReadARPA().

Here is the call graph for this function:

Here is the caller graph for this function:

boost::program_options::typed_value< std::string > * lm::SizeOption	(	std::size_t &	to,
		const char *	default_value
	)

Definition at line 20 of file size_option.cc.

Referenced by main().

Here is the caller graph for this function:

template<class Stream >

void lm::WriteCounts	(	Stream &	out,
		const std::vector< uint64_t > &	number
	)			[inline]

Definition at line 27 of file arpa_io.cc.

Referenced by lm::ARPAOutput::Finish(), and SizeNeededForCounts().

Here is the caller graph for this function:

---

Variable Documentation

const bool lm::kARPASpaces = {0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}

Definition at line 23 of file read_arpa.cc.

Referenced by Read1Gram(), and ReadNGram().

const WordIndex lm::kMaxWordIndex = UINT_MAX

Definition at line 9 of file word_index.hh.

const WordIndex lm::kUNK = 0

Definition at line 10 of file word_index.hh.

Referenced by lm::SpecialVocab::IsSpecial(), lm::builder::CorpusCount::Run(), lm::builder::AdjustCounts::Run(), and lm::SpecialVocab::UNK().