#include "lm/builder/adjust_counts.hh"
#include "lm/common/ngram_stream.hh"
#include "lm/builder/payload.hh"
#include "util/stream/timer.hh"
#include <algorithm>
#include <iostream>
#include <limits>
Go to the source code of this file.
Namespaces | |
namespace | lm |
namespace | lm::builder |
Definition at line 214 of file adjust_counts.cc.
uint8_t* copy_from_ |
Definition at line 211 of file adjust_counts.cc.
uint64_t count |
Definition at line 96 of file adjust_counts.cc.
Referenced by MosesTuning::ReferenceSet::AddLine(), xpath_node_set_raw::append(), Moses::Bitmap::Bitmap(), Moses::TranslationOptionCollection::CalcEstimatedScore(), Moses::calcNgramExpectations(), Moses::PhraseDictionaryTree::Create(), create_xml(), MosesTraining::Syntax::PCFG::RuleCollection::CreatePcfg(), Moses::Syntax::T2S::Manager< RuleMatcher >::Decode(), Moses::Syntax::S2T::Manager< Parser >::Decode(), Moses::Syntax::F2S::Manager< RuleMatcher >::Decode(), xpath_ast_node::eval_string_concat(), interplm::gencorrcounts(), Moses::NonTermContextProperty::ProbStore::GetProb(), Moses::LanguageModelRemote::GetValue(), Moses::PhraseDictionaryFuzzyMatch::InitializeForInput(), Moses::RuleTableLoaderStandard::Load(), Moses::Syntax::T2S::RuleTrieLoader::Load(), Moses::Syntax::S2T::RuleTrieLoader::Load(), Moses::Syntax::F2S::HyperTreeLoader::Load(), loadOrientationPriors(), Moses::SoftSourceSyntacticConstraintsFeature::LoadTargetSourceLeftHandSideJointCountFile(), MosesTraining::Syntax::ScoreStsg::ScoreStsg::Main(), main(), Moses::MakePair(), outputPhrasePair(), Moses::Manager::OutputSearchGraphAsHypergraph(), Moses::TargetConstituentBoundariesRightAdjacentPhraseProperty::ProcessValue(), Moses::TargetConstituentBoundariesLeftPhraseProperty::ProcessValue(), Moses::SpanLengthPhraseProperty::ProcessValue(), Moses::NonTermContextProperty::ProcessValue(), Moses::FVector::pruneSparseFeatures(), Moses::FVector::pruneZeroWeightFeatures(), lm::builder::CorpusCount::Run(), Moses::Edge::storeNgramHistory(), MosesTuning::Viterbi(), and writeUnknownWordLabel().
uint64_t count_pruned |
Definition at line 97 of file adjust_counts.cc.
std::vector<uint64_t>& counts_ |
Definition at line 103 of file adjust_counts.cc.
std::vector<uint64_t>& counts_pruned_ |
Definition at line 104 of file adjust_counts.cc.
NGram<BuildingPayload> current_ |
Definition at line 208 of file adjust_counts.cc.
Referenced by util::FileStream::AdvanceTo(), util::FileStream::Ensure(), lm::phrase::detail::Vertex::LowerBound(), and lm::phrase::detail::Arc::LowerBound().
std::vector<Discount>& discounts_ |
Definition at line 105 of file adjust_counts.cc.
OrderStat& full_ |
Definition at line 101 of file adjust_counts.cc.
uint64_t n[5] |
Definition at line 95 of file adjust_counts.cc.
Referenced by _generator< T >::_push(), pugi::xml_node::append_child(), MosesTraining::Syntax::GHKM::ComposedRule::AttemptComposition(), BOOST_AUTO_TEST_CASE(), Moses::VarSpanNode::CalculateRanges(), dictionary::cleanfreq(), Moses::ConsistentPhrases::ConsistentPhrases(), Moses::CreateDiff(), Moses::Syntax::S2T::PatternApplicationTrie::DetermineEndRange(), Moses::Syntax::S2T::PatternApplicationTrie::DetermineStartRange(), Moses::TargetNgramFeature::EvaluateWhenApplied(), sapt::findExternalRoot(), sapt::findInternalRoot(), fisher_exact(), sapt::imTSA< TOKEN >::imTSA(), pugi::xml_node::insert_child_after(), pugi::xml_node::insert_child_before(), sapt::isCoherent(), MosesTuning::M2::M2::LevenshteinMatrix(), main(), sapt::R2L_Token< T >::next(), sapt::L2R_Token< T >::next(), ngramtable::ngramtable(), node_output(), Moses::Manager::OutputBest(), pugi::xml_node::prepend_child(), MosesTraining::ProcessAndStripXMLTags(), lm::ngram::detail::HashedSearch< Value >::Size(), dictionary::size(), SPrintParams(), string_value(), Moses::SyntacticLanguageModelFiles< MH, MO >::SyntacticLanguageModelFiles(), interplm::test_ngt(), interplm::test_txt(), sapt::topNode(), and MosesTraining::unescape().
std::vector<OrderStat> orders_ |
Definition at line 100 of file adjust_counts.cc.
uint64_t prune_threshold_ |
Definition at line 212 of file adjust_counts.cc.
const std::vector<bool>& prune_words_ |
Definition at line 213 of file adjust_counts.cc.