lm::ngram Namespace Reference


Namespaces

namespace  detail
namespace  trie

Classes

struct  FixedWidthParameters
struct  Parameters
class  BinaryFormat
struct  Config
class  RuleScore
class  QueryPrinter
struct  ExtendReturn
class  DontQuantize
class  SeparatelyQuantize
class  State
struct  Left
struct  ChartState
class  GenericProbingProxy
class  GenericTrieUnigramProxy
struct  BackoffValue
struct  RestValue
class  NoRestBuild
class  MaxRestBuild
class  LowerRestBuild
class  ImmediateWriteWordsWrapper
class  WriteWordsWrapper
class  SortedVocabulary
struct  ProbingVocabularyEntry
class  ProbingVocabulary
class  WriteUniqueWords
class  NoOpUniqueWords
class  GrowableVocab

Typedefs

typedef
::lm::ngram::ProbingVocabulary 		Vocabulary
typedef ProbingModel Model
typedef State Right

Enumerations

enum  ModelType {
  PROBING = 0, REST_PROBING = 1, TRIE = 2, QUANT_TRIE = 3,
  ARRAY_TRIE = 4, QUANT_ARRAY_TRIE = 5
}

Functions

bool IsBinaryFormat (int fd)
void ReadHeader (int fd, Parameters &out)
void MatchCheck (ModelType model_type, unsigned int search_version, const Parameters &params)
bool RecognizeBinary (const char *file, ModelType &recognized)
void SetExtension (float &backoff)
bool HasExtension (const float &backoff)
base::Model * LoadVirtual (const char *file_name, const Config &config, ModelType model_type)
 LM_NAME_MODEL (ProbingModel, detail::GenericModel< detail::HashedSearch< BackoffValue > LM_COMMA() ProbingVocabulary >)
 LM_NAME_MODEL (RestProbingModel, detail::GenericModel< detail::HashedSearch< RestValue > LM_COMMA() ProbingVocabulary >)
 LM_NAME_MODEL (TrieModel, detail::GenericModel< trie::TrieSearch< DontQuantize LM_COMMA() trie::DontBhiksha > LM_COMMA() SortedVocabulary >)
 LM_NAME_MODEL (ArrayTrieModel, detail::GenericModel< trie::TrieSearch< DontQuantize LM_COMMA() trie::ArrayBhiksha > LM_COMMA() SortedVocabulary >)
 LM_NAME_MODEL (QuantTrieModel, detail::GenericModel< trie::TrieSearch< SeparatelyQuantize LM_COMMA() trie::DontBhiksha > LM_COMMA() SortedVocabulary >)
 LM_NAME_MODEL (QuantArrayTrieModel, detail::GenericModel< trie::TrieSearch< SeparatelyQuantize LM_COMMA() trie::ArrayBhiksha > LM_COMMA() SortedVocabulary >)
std::ostream & operator<< (std::ostream &o, const State &state)
template<class Model , class Printer >
void Query (const Model &model, bool sentence_context, Printer &printer)
template<class Model >
void Query (const char *file, const Config &config, bool sentence_context, QueryPrinter &printer)
template<class Model >
ExtendReturn ExtendLoop (const Model &model, unsigned char seen, const WordIndex *add_rbegin, const WordIndex *add_rend, const float *backoff_start, const uint64_t *pointers, const uint64_t *pointers_end, uint64_t *&pointers_write, float *backoff_write)
template<class Model >
float RevealBefore (const Model &model, const Right &reveal, const unsigned char seen, bool reveal_full, Left &left, Right &right)
template<class Model >
float RevealAfter (const Model &model, Left &left, Right &right, const Left &reveal, unsigned char seen)
template<class Model >
float Subsume (const Model &model, Left &first_left, const Right &first_right, const Left &second_left, Right &second_right, const unsigned int between_length)
void ShowSizes (const std::vector< uint64_t > &counts, const lm::ngram::Config &config)
void ShowSizes (const std::vector< uint64_t > &counts)
void ShowSizes (const char *file, const lm::ngram::Config &config)
uint64_t hash_value (const State &state, uint64_t seed=0)
uint64_t hash_value (const Left &left)
uint64_t hash_value (const ChartState &state)
void MissingUnknown (const Config &config) throw (SpecialWordMissingException)
void MissingSentenceMarker (const Config &config, const char *str) throw (SpecialWordMissingException)
template<class Vocab >
void CheckSpecials (const Config &config, const Vocab &vocab) throw (SpecialWordMissingException)

Variables

const char * kModelNames [6] = {"probing hash tables", "probing hash tables with rest costs", "trie", "trie with quantization", "trie with array-compressed pointers", "trie with quantization and array-compressed pointers"}
const std::size_t kInvalidSize = static_cast<std::size_t>(-1)
const float kNoExtensionBackoff = -0.0
const float kExtensionBackoff = 0.0
const uint64_t kNoExtensionQuant = 0
const uint64_t kExtensionQuant = 1
const ModelType HASH_PROBING = PROBING
const ModelType TRIE_SORTED = TRIE
const ModelType QUANT_TRIE_SORTED = QUANT_TRIE
const ModelType ARRAY_TRIE_SORTED = ARRAY_TRIE
const ModelType QUANT_ARRAY_TRIE_SORTED = QUANT_ARRAY_TRIE

Typedef Documentation

typedef ProbingModel lm::ngram::Model

Definition at line 145 of file model.hh.

typedef State lm::ngram::Right

Definition at line 50 of file state.hh.

typedef ::lm::ngram::ProbingVocabulary lm::ngram::Vocabulary

Definition at line 144 of file model.hh.

Enumeration Type Documentation

enum lm::ngram::ModelType

Enumerator:
PROBING 
REST_PROBING 
TRIE 
QUANT_TRIE 
ARRAY_TRIE 
QUANT_ARRAY_TRIE 

Definition at line 9 of file model_type.hh.

Function Documentation

template<class Vocab >
void lm::ngram::CheckSpecials ( const Config &  config,
const Vocab &  vocab 
) throw (SpecialWordMissingException) [inline]

Definition at line 213 of file vocab.hh.

References MissingSentenceMarker(), and MissingUnknown().

Referenced by lm::ngram::detail::HashedSearch< Value >::InitializeFromARPA().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Model >
ExtendReturn lm::ngram::ExtendLoop ( const Model &  model,
unsigned char  seen,
const WordIndex *  add_rbegin,
const WordIndex *  add_rend,
const float *  backoff_start,
const uint64_t *  pointers,
const uint64_t *  pointers_end,
uint64_t *&  pointers_write,
float *  backoff_write 
) [inline]

Definition at line 19 of file partial.hh.

References lm::ngram::ExtendReturn::adjust, lm::ngram::ExtendReturn::make_full, lm::ngram::ExtendReturn::next_use, and swap().

Referenced by RevealAfter(), RevealBefore(), and Subsume().

Here is the call graph for this function:

Here is the caller graph for this function:

bool lm::ngram::HasExtension ( const float &  backoff  )  [inline]

Definition at line 32 of file blank.hh.

References kNoExtensionBackoff.

Referenced by lm::ngram::detail::GenericModel< Search, VocabularyT >::GetState().

Here is the caller graph for this function:

uint64_t lm::ngram::hash_value ( const ChartState &  state  )  [inline]

Definition at line 117 of file state.hh.

References hash_value(), lm::ngram::ChartState::left, and lm::ngram::ChartState::right.

Here is the call graph for this function:

uint64_t lm::ngram::hash_value ( const Left &  left  )  [inline]

Definition at line 86 of file state.hh.

References lm::ngram::Left::full, lm::ngram::Left::length, util::MurmurHashNative(), and lm::ngram::Left::pointers.

Here is the call graph for this function:

uint64_t lm::ngram::hash_value ( const State &  state,
uint64_t  seed = 0 
) [inline]

Definition at line 52 of file state.hh.

References lm::ngram::State::length, util::MurmurHashNative(), and lm::ngram::State::words.

Referenced by hash_value().

Here is the call graph for this function:

Here is the caller graph for this function:

bool lm::ngram::IsBinaryFormat ( int  fd  ) 

Definition at line 85 of file binary_format.cc.

References util::scoped_memory::get(), util::kBadSize, util::LAZY, util::MapRead(), util::SizeFile(), UTIL_THROW, and UTIL_THROW_IF.

Referenced by lm::ngram::detail::GenericModel< Search, VocabularyT >::GenericModel(), and RecognizeBinary().

Here is the call graph for this function:

Here is the caller graph for this function:

lm::ngram::LM_NAME_MODEL ( QuantArrayTrieModel  ,
detail::GenericModel< trie::TrieSearch< SeparatelyQuantize LM_COMMA() trie::ArrayBhiksha > LM_COMMA() SortedVocabulary >   
)

lm::ngram::LM_NAME_MODEL ( QuantTrieModel  ,
detail::GenericModel< trie::TrieSearch< SeparatelyQuantize LM_COMMA() trie::DontBhiksha > LM_COMMA() SortedVocabulary >   
)

lm::ngram::LM_NAME_MODEL ( ArrayTrieModel  ,
detail::GenericModel< trie::TrieSearch< DontQuantize LM_COMMA() trie::ArrayBhiksha > LM_COMMA() SortedVocabulary >   
)

lm::ngram::LM_NAME_MODEL ( TrieModel  ,
detail::GenericModel< trie::TrieSearch< DontQuantize LM_COMMA() trie::DontBhiksha > LM_COMMA() SortedVocabulary >   
)

lm::ngram::LM_NAME_MODEL ( RestProbingModel  ,
detail::GenericModel< detail::HashedSearch< RestValue > LM_COMMA() ProbingVocabulary >   
)

lm::ngram::LM_NAME_MODEL ( ProbingModel  ,
detail::GenericModel< detail::HashedSearch< BackoffValue > LM_COMMA() ProbingVocabulary >   
)

base::Model * lm::ngram::LoadVirtual ( const char *  file_name,
const Config &  config,
ModelType  model_type 
)

Definition at line 328 of file model.cc.

References ARRAY_TRIE, PROBING, QUANT_ARRAY_TRIE, QUANT_TRIE, RecognizeBinary(), REST_PROBING, TRIE, and UTIL_THROW.

Here is the call graph for this function:

void lm::ngram::MatchCheck ( ModelType  model_type,
unsigned int  search_version,
const Parameters &  params 
)

Definition at line 127 of file binary_format.cc.

References lm::ngram::Parameters::fixed, kModelNames, lm::ngram::FixedWidthParameters::model_type, lm::ngram::FixedWidthParameters::search_version, UTIL_THROW, and UTIL_THROW_IF.

Referenced by lm::ngram::BinaryFormat::InitializeBinary().

Here is the caller graph for this function:

void lm::ngram::MissingSentenceMarker ( const Config &  config,
const char *  str 
) throw (SpecialWordMissingException)

Definition at line 316 of file vocab.cc.

References lm::COMPLAIN, lm::SILENT, str, lm::THROW_UP, and UTIL_THROW.

Referenced by CheckSpecials().

Here is the caller graph for this function:

void lm::ngram::MissingUnknown ( const Config &  config  )  throw (SpecialWordMissingException)

Definition at line 304 of file vocab.cc.

References lm::COMPLAIN, lm::SILENT, lm::THROW_UP, and UTIL_THROW.

Referenced by CheckSpecials().

Here is the caller graph for this function:

std::ostream& lm::ngram::operator<< ( std::ostream &  o,
const State &  state 
)

Definition at line 16 of file model_test.cc.

References lm::ngram::State::length, and lm::ngram::State::words.

template<class Model >
void lm::ngram::Query ( const char *  file,
const Config &  config,
bool  sentence_context,
QueryPrinter &  printer 
) [inline]

Definition at line 103 of file ngram_query.hh.

template<class Model , class Printer >
void lm::ngram::Query ( const Model &  model,
bool  sentence_context,
Printer &  printer 
) [inline]

Definition at line 52 of file ngram_query.hh.

References util::FilePiece::get(), lm::FullScoreReturn::prob, util::FilePiece::ReadWordSameLine(), state, and UTIL_THROW_IF.

Here is the call graph for this function:

void lm::ngram::ReadHeader ( int  fd,
Parameters &  out 
)

Definition at line 117 of file binary_format.cc.

References lm::ngram::Parameters::counts, lm::ngram::Parameters::fixed, lm::ngram::FixedWidthParameters::order, lm::ngram::FixedWidthParameters::probing_multiplier, util::ReadOrThrow(), util::SeekOrThrow(), and UTIL_THROW.

Referenced by lm::ngram::BinaryFormat::InitializeBinary(), and RecognizeBinary().

Here is the call graph for this function:

Here is the caller graph for this function:

bool lm::ngram::RecognizeBinary ( const char *  file,
ModelType &  recognized 
)

Definition at line 290 of file binary_format.cc.

References lm::ngram::Parameters::fixed, util::scoped_fd::get(), IsBinaryFormat(), lm::ngram::FixedWidthParameters::model_type, util::OpenReadOrThrow(), and ReadHeader().

Referenced by Moses::ConstructBackwardLM(), Moses::ConstructDsgLM(), Moses::ConstructKenLM(), Moses::ConstructOSMLM(), Moses::ConstructReloadingLM(), LoadVirtual(), and main().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Model >
float lm::ngram::RevealAfter ( const Model &  model,
Left &  left,
Right &  right,
const Left &  reveal,
unsigned char  seen 
) [inline]

Definition at line 111 of file partial.hh.

References lm::ngram::ExtendReturn::adjust, lm::ngram::State::backoff, ExtendLoop(), lm::ngram::Left::full, lm::ngram::State::length, lm::ngram::Left::length, lm::ngram::ExtendReturn::make_full, lm::ngram::ExtendReturn::next_use, NULL, lm::ngram::Left::pointers, and lm::ngram::State::words.

Here is the call graph for this function:

template<class Model >
float lm::ngram::RevealBefore ( const Model &  model,
const Right &  reveal,
const unsigned char  seen,
bool  reveal_full,
Left &  left,
Right &  right 
) [inline]

Definition at line 83 of file partial.hh.

References lm::ngram::ExtendReturn::adjust, lm::ngram::State::backoff, ExtendLoop(), lm::ngram::Left::full, lm::ngram::Left::length, lm::ngram::State::length, lm::ngram::ExtendReturn::make_full, lm::ngram::ExtendReturn::next_use, NULL, lm::ngram::Left::pointers, and lm::ngram::State::words.

Here is the call graph for this function:

void lm::ngram::SetExtension ( float &  backoff  )  [inline]

Definition at line 27 of file blank.hh.

References kExtensionBackoff, and kNoExtensionBackoff.

void lm::ngram::ShowSizes ( const char *  file,
const lm::ngram::Config &  config 
)

Definition at line 56 of file sizes.cc.

References lm::ReadARPACounts(), and ShowSizes().

Here is the call graph for this function:

void lm::ngram::ShowSizes ( const std::vector< uint64_t > &  counts  ) 

Definition at line 51 of file sizes.cc.

References ShowSizes().

Here is the call graph for this function:

void lm::ngram::ShowSizes ( const std::vector< uint64_t > &  counts,
const lm::ngram::Config &  config 
)

Definition at line 11 of file sizes.cc.

References lm::ngram::Config::backoff_bits, min_element(), lm::ngram::Config::pointer_bhiksha_bits, lm::ngram::Config::prob_bits, and lm::ngram::Config::probing_multiplier.

Referenced by main(), and ShowSizes().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Model >
float lm::ngram::Subsume ( const Model &  model,
Left &  first_left,
const Right &  first_right,
const Left &  second_left,
Right &  second_right,
const unsigned int  between_length 
) [inline]

Definition at line 135 of file partial.hh.

References lm::ngram::ExtendReturn::adjust, lm::ngram::State::backoff, ExtendLoop(), lm::ngram::Left::full, lm::ngram::Left::length, lm::ngram::State::length, lm::ngram::ExtendReturn::make_full, lm::ngram::ExtendReturn::next_use, NULL, lm::ngram::Left::pointers, and lm::ngram::State::words.

Here is the call graph for this function:

Variable Documentation

const ModelType lm::ngram::ARRAY_TRIE_SORTED = ARRAY_TRIE

Definition at line 15 of file model_type.hh.

const ModelType lm::ngram::HASH_PROBING = PROBING

Definition at line 12 of file model_type.hh.

const float lm::ngram::kExtensionBackoff = 0.0

Definition at line 23 of file blank.hh.

Referenced by lm::ReadBackoff(), SetExtension(), and lm::ngram::SeparatelyQuantize::Train().

const uint64_t lm::ngram::kExtensionQuant = 1

Definition at line 25 of file blank.hh.

const std::size_t lm::ngram::kInvalidSize = static_cast<std::size_t>(-1)

Definition at line 136 of file binary_format.cc.

Referenced by lm::ngram::BinaryFormat::GrowForSearch(), lm::ngram::BinaryFormat::LoadBinary(), lm::ngram::BinaryFormat::ReadForConfig(), and lm::ngram::BinaryFormat::WriteVocabWords().

const char * lm::ngram::kModelNames = {"probing hash tables", "probing hash tables with rest costs", "trie", "trie with quantization", "trie with array-compressed pointers", "trie with quantization and array-compressed pointers"}

Definition at line 18 of file binary_format.cc.

Referenced by MatchCheck().

const float lm::ngram::kNoExtensionBackoff = -0.0

Definition at line 22 of file blank.hh.

Referenced by HasExtension(), lm::ReadBackoff(), SetExtension(), and lm::ngram::SeparatelyQuantize::Train().

const uint64_t lm::ngram::kNoExtensionQuant = 0

Definition at line 24 of file blank.hh.

const ModelType lm::ngram::QUANT_ARRAY_TRIE_SORTED = QUANT_ARRAY_TRIE

Definition at line 16 of file model_type.hh.

const ModelType lm::ngram::QUANT_TRIE_SORTED = QUANT_TRIE

Definition at line 14 of file model_type.hh.

const ModelType lm::ngram::TRIE_SORTED = TRIE

Definition at line 13 of file model_type.hh.

Generated on Thu Jul 6 00:32:43 2017 for Moses by  doxygen 1.5.9