#include "lm/search_trie.hh"
#include "lm/bhiksha.hh"
#include "lm/binary_format.hh"
#include "lm/blank.hh"
#include "lm/lm_exception.hh"
#include "lm/max_order.hh"
#include "lm/quantize.hh"
#include "lm/trie.hh"
#include "lm/trie_sort.hh"
#include "lm/vocab.hh"
#include "lm/weights.hh"
#include "lm/word_index.hh"
#include "util/ersatz_progress.hh"
#include "util/mmap.hh"
#include "util/proxy_iterator.hh"
#include "util/scoped.hh"
#include "util/sized_iterator.hh"
#include <algorithm>
#include <cstring>
#include <cstdio>
#include <cstdlib>
#include <queue>
#include <limits>
#include <numeric>
#include <vector>
Go to the source code of this file.
Namespaces | |
namespace | lm |
namespace | lm::ngram |
namespace | lm::ngram::trie |
Functions | |
template<class Quant , class Bhiksha > | |
void | lm::ngram::trie::BuildTrie (SortedFiles &files, std::vector< uint64_t > &counts, const Config &config, TrieSearch< Quant, Bhiksha > &out, Quant &quant, SortedVocabulary &vocab, BinaryFormat &backing) |
uint8_t * allocated_ |
Definition at line 173 of file search_trie.cc.
unsigned char array |
Definition at line 54 of file search_trie.cc.
Definition at line 171 of file search_trie.cc.
float basis_[KENLM_MAX_ORDER] |
Definition at line 372 of file search_trie.cc.
Definition at line 369 of file search_trie.cc.
unsigned char been_length_ |
Definition at line 370 of file search_trie.cc.
Definition at line 322 of file search_trie.cc.
Referenced by MosesTuning::M2::M2::AddStats(), as_utf8_end(), as_wide_impl(), Moses::dsgHypothesis::calculateDsgProb(), Moses::LanguageModelKen< Model >::EvaluateWhenApplied(), Moses::LanguageModelIRST::EvaluateWhenApplied(), Moses::LanguageModelDALM::EvaluateWhenApplied(), Moses::WordTranslationFeature::EvaluateWithSourceContext(), Moses::SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(), Moses::PhrasePairFeature::EvaluateWithSourceContext(), Moses::BleuScoreFeature::GetClippedNgramMatchesAndCounts(), invertAlignment(), lm::JointOrder(), main(), Moses::VWFeatureSourceWindow::operator()(), Moses::VWFeatureSourceSenseWindow::operator()(), Moses::VWFeatureSourcePhraseInternal::operator()(), Moses::VWFeatureSourceIndicator::operator()(), Moses::TargetPreferencesFeatureState::operator==(), Moses::TargetConstituentAdjacencyFeatureState::operator==(), outputPhrasePair(), MosesTraining::ParseXmlTagAttributes(), Moses::TargetPreferencesPhraseProperty::ProcessValue(), Moses::SourceLabelsPhraseProperty::ProcessValue(), MosesTraining::ReadAlignment(), util::stream::LineInput::Run(), and MosesTuning::Tokenize().
BitPacked& bigram_pack_ |
Definition at line 314 of file search_trie.cc.
RecordReader* contexts_ |
Definition at line 309 of file search_trie.cc.
std::vector<uint64_t> counts_ |
Definition at line 261 of file search_trie.cc.
uint8_t* current_ |
Definition at line 173 of file search_trie.cc.
Doing& doing_ |
Definition at line 374 of file search_trie.cc.
Definition at line 322 of file search_trie.cc.
Referenced by lmtable::add(), Moses::FactorCollection::AddFactor(), MosesTuning::ReferenceSet::AddLine(), Moses::Syntax::S2T::PChart::AddVertex(), lmtable::addwithoffset(), Alignments::Alignments(), as_utf8_end(), as_wide_impl(), Moses::calcNgramExpectations(), Moses::dsgHypothesis::calculateDsgProb(), TERCPPNS_TERCpp::terCalc::calculerPermutations(), lmtable::checkbounds(), lmtable::cpsublm(), TERCPPNS_TERCpp::terCalc::createConcordMots(), Moses::Syntax::S2T::Manager< Parser >::Decode(), utf_decoder< Traits, opt_swap >::decode_utf16_block(), utf_decoder< Traits, opt_swap >::decode_utf32_block(), Moses::DistortionScoreProducer::EmptyHypothesisState(), Moses::Syntax::S2T::Scope3Parser< Callback >::EnumerateHyperedges(), Moses::Syntax::S2T::RecursiveCYKPlusParser< Callback >::EnumerateHyperedges(), Moses::LanguageModelKen< Model >::EvaluateWhenApplied(), Moses::LanguageModelIRST::EvaluateWhenApplied(), Moses::LanguageModelDALM::EvaluateWhenApplied(), Moses::Syntax::S2T::PatternApplicationTrie::Extend(), MosesTuning::StreamingHypPackEnumerator::finished(), Moses::Scope3Parser::GetChartRuleCollection(), Moses::FactorCollection::GetFactor(), Moses::Bitmap::GetID(), Moses::Bitmap::GetIDPlus(), Moses::IncrementIterators(), invertAlignment(), Moses::PhraseDecoder::Load(), Moses::BlockHashIndex::Load(), load_stream_data_seek(), Moses::BlockHashIndex::LoadIndex(), main(), util::MurmurHash64A(), search::NBestList::NBestList(), Moses::VWFeatureSourceWindow::operator()(), Moses::VWFeatureSourceSenseWindow::operator()(), Moses::VWFeatureSourcePhraseInternal::operator()(), Moses::VWFeatureSourceIndicator::operator()(), lm::ngram::trie::EntryCompare::operator()(), outputPhrasePair(), Moses::Hypothesis::PrintHypothesis(), Moses::ProcessAndStripSGML(), MosesTraining::ReadAlignment(), util::stream::LineInput::Run(), SAMT(), MosesTuning::StatisticsBasedScorer::score(), MosesTuning::SentenceLevelScorer::score(), MosesTraining::unescape(), writeLeftHandSideLabelCounts(), and Moses::ChartManager::~ChartManager().
std::size_t entry_size_ |
Definition at line 175 of file search_trie.cc.
uint64_t index |
Definition at line 55 of file search_trie.cc.
Referenced by MosesTraining::PropertiesConsolidator::ActivatePartsOfSpeechProcessing(), MosesTraining::PropertiesConsolidator::ActivateSourceLabelsProcessing(), MosesTraining::PropertiesConsolidator::ActivateTargetSyntacticPreferencesProcessing(), Moses::StackLatticeBuilder::Build(), Moses::calcNgramExpectations(), Moses::BackwardLanguageModel< Model >::CalcScore(), Moses::PrefixTreeMap::ConvertPhrase(), Moses::BackwardLanguageModel< Model >::Evaluate(), Moses::LanguageModelImplementation::EvaluateWhenApplied(), Moses::ApplicableRuleTrie::Extend(), double_conversion::PowersOfTenCache::GetCachedPowerForBinaryExponentRange(), double_conversion::PowersOfTenCache::GetCachedPowerForDecimalExponent(), ngramtable::getfreq(), Moses::Edge::GetPhraseSuffix(), Moses::LexicalReorderingTableCompact::GetScore(), GetScore(), sapt::imTSA< TOKEN >::imTSA(), Moses::SoftSourceSyntacticConstraintsFeature::LoadSourceLabelSet(), Moses::SoftSourceSyntacticConstraintsFeature::LoadTargetSourceLeftHandSideJointCountFile(), MosesTraining::Syntax::GHKM::Node::LowestCommonAncestor(), MosesTraining::Syntax::GHKM::ExtractGHKM::Main(), main(), numberize(), sapt::snt_adder< L2R_Token< SimpleWordId > >::operator()(), MosesTraining::Syntax::Tree< T >::LeafIter< T >::operator++(), MosesTraining::Syntax::Tree< T >::PreOrderIter< T >::operator++(), processPhrasePairs(), lm::ReadNGram(), sapt::imTSA< TOKEN >::save_as_mm_tsa(), ngramtable::setfreq(), sapt::mmTtrack< TKN >::sntEnd(), sapt::mmTtrack< TKN >::sntStart(), sapt::write_mm_2d_table(), and sapt::write_tokenindex_to_disk().
float* it_[KENLM_MAX_ORDER-1] |
Definition at line 227 of file search_trie.cc.
BitPackedLongest& longest_ |
Definition at line 313 of file search_trie.cc.
BackoffMessages messages_[KENLM_MAX_ORDER-1] |
Definition at line 225 of file search_trie.cc.
BitPackedMiddle<Bhiksha>* const middle_ |
Definition at line 312 of file search_trie.cc.
const unsigned char order_ |
Definition at line 315 of file search_trie.cc.
Referenced by lm::base::ModelFacade< Model, State, Vocabulary >::Init().
const Quant& quant_ |
Definition at line 310 of file search_trie.cc.
SRISucks& sri_ |
Definition at line 265 of file search_trie.cc.
const unsigned char total_order_ |
Definition at line 367 of file search_trie.cc.
UnigramValue* const unigrams_ |
Definition at line 263 of file search_trie.cc.
std::vector<float> values_[KENLM_MAX_ORDER-1] |
Definition at line 224 of file search_trie.cc.