/disk4/html/www/moses/doxygen/mosesdecoder/phrase-extract/consolidate-main.cpp File Reference
#include <cstdlib>#include <vector>#include <string>#include "util/exception.hh"#include "moses/Util.h"#include "InputFileStream.h"#include "OutputFileStream.h"#include "PropertiesConsolidator.h"Go to the source code of this file.
Functions | |
| void | processFiles (const std::string &, const std::string &, const std::string &, const std::string &, const std::string &, const std::string &, const std::string &) |
| void | loadCountOfCounts (const std::string &) |
| void | breakdownCoreAndSparse (const std::string &combined, std::string &core, std::string &sparse) |
| bool | getLine (Moses::InputFileStream &file, std::vector< std::string > &item) |
| float | maybeLogProb (float a) |
| bool | isNonTerminal (const std::string &word) |
| int | main (int argc, char *argv[]) |
Variables | |
| bool | countsProperty = false |
| bool | goodTuringFlag = false |
| bool | hierarchicalFlag = false |
| bool | kneserNeyFlag = false |
| bool | logProbFlag = false |
| bool | lowCountFlag = false |
| bool | onlyDirectFlag = false |
| bool | partsOfSpeechFlag = false |
| bool | phraseCountFlag = false |
| bool | sourceLabelsFlag = false |
| bool | targetSyntacticPreferencesFlag = false |
| bool | sparseCountBinFeatureFlag = false |
| std::vector< int > | countBin |
| float | minScore0 = 0 |
| float | minScore2 = 0 |
| std::vector< float > | countOfCounts |
| std::vector< float > | goodTuringDiscount |
| float | kneserNey_D1 |
| float | kneserNey_D2 |
| float | kneserNey_D3 |
| float | totalCount = -1 |
Function Documentation
| void breakdownCoreAndSparse | ( | const std::string & | combined, | |
| std::string & | core, | |||
| std::string & | sparse | |||
| ) |
Definition at line 458 of file consolidate-main.cpp.
References Moses::Tokenize().
Referenced by processFiles().
| bool getLine | ( | Moses::InputFileStream & | file, | |
| std::vector< std::string > & | item | |||
| ) |
Definition at line 477 of file consolidate-main.cpp.
References Moses::TokenizeMultiCharSeparator().
| bool isNonTerminal | ( | const std::string & | word | ) | [inline] |
Definition at line 65 of file consolidate-main.cpp.
Referenced by Moses::FactorCollection::AddFactor(), calcCrossedNonTerm(), Moses::PhraseDictionaryOnDisk::ConvertFromMoses(), Moses::Phrase::CreateFromString(), NumNonTerminal(), outputPhrasePair(), processFiles(), and Moses::Word::Word().
| void loadCountOfCounts | ( | const std::string & | fileNameCountOfCounts | ) |
Definition at line 196 of file consolidate-main.cpp.
References Moses::InputFileStream::Close(), countOfCounts, goodTuringDiscount, goodTuringFlag, kneserNey_D1, kneserNey_D2, kneserNey_D3, totalCount, and UTIL_THROW_IF2.
Referenced by processFiles().
| int main | ( | int | argc, | |
| char * | argv[] | |||
| ) |
Definition at line 71 of file consolidate-main.cpp.
References countBin, countsProperty, goodTuringFlag, hierarchicalFlag, kneserNeyFlag, logProbFlag, lowCountFlag, minScore0, minScore2, onlyDirectFlag, partsOfSpeechFlag, phraseCountFlag, processFiles(), sourceLabelsFlag, sparseCountBinFeatureFlag, targetSyntacticPreferencesFlag, UTIL_THROW2, and UTIL_THROW_IF2.
| float maybeLogProb | ( | float | a | ) | [inline] |
Definition at line 59 of file consolidate-main.cpp.
References logProbFlag.
Referenced by main(), outputPhrasePair(), and processFiles().
| void processFiles | ( | const std::string & | fileNameDirect, | |
| const std::string & | fileNameIndirect, | |||
| const std::string & | fileNameConsolidated, | |||
| const std::string & | fileNameCountOfCounts, | |||
| const std::string & | fileNameSourceLabelSet, | |||
| const std::string & | fileNamePartsOfSpeechVocabulary, | |||
| const std::string & | fileNameTargetSyntacticPreferencesLabelSet | |||
| ) |
Definition at line 236 of file consolidate-main.cpp.
References MosesTraining::PropertiesConsolidator::ActivatePartsOfSpeechProcessing(), MosesTraining::PropertiesConsolidator::ActivateSourceLabelsProcessing(), MosesTraining::PropertiesConsolidator::ActivateTargetSyntacticPreferencesProcessing(), breakdownCoreAndSparse(), Moses::InputFileStream::Close(), countBin, countsProperty, getLine(), MosesTraining::PropertiesConsolidator::GetPOSPropertyValueFromPropertiesString(), goodTuringDiscount, goodTuringFlag, isNonTerminal(), kneserNey_D1, kneserNey_D2, kneserNey_D3, kneserNeyFlag, loadCountOfCounts(), lowCountFlag, maybeLogProb(), minScore0, minScore2, onlyDirectFlag, partsOfSpeechFlag, phraseCountFlag, MosesTraining::PropertiesConsolidator::ProcessPropertiesString(), sourceLabelsFlag, sparseCountBinFeatureFlag, targetSyntacticPreferencesFlag, Moses::Tokenize(), totalCount, and UTIL_THROW_IF2.
Referenced by main().
Variable Documentation
| std::vector< int > countBin |
| std::vector< float > countOfCounts |
Definition at line 48 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), main(), outputPhrasePair(), and writeCountOfCounts().
| bool countsProperty = false |
| std::vector< float > goodTuringDiscount |
Definition at line 49 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), and processFiles().
| bool goodTuringFlag = false |
Definition at line 32 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), main(), outputPhrasePair(), and processFiles().
| bool hierarchicalFlag = false |
Definition at line 33 of file consolidate-main.cpp.
Referenced by main(), outputPhrasePair(), and processLine().
| float kneserNey_D1 |
Definition at line 50 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), and processFiles().
| float kneserNey_D2 |
Definition at line 50 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), and processFiles().
| float kneserNey_D3 |
Definition at line 50 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), and processFiles().
| bool kneserNeyFlag = false |
Definition at line 34 of file consolidate-main.cpp.
Referenced by main(), outputPhrasePair(), and processFiles().
| bool logProbFlag = false |
Definition at line 35 of file consolidate-main.cpp.
Referenced by main(), maybeLogProb(), and processFiles().
| bool lowCountFlag = false |
| float minScore0 = 0 |
| float minScore2 = 0 |
| bool onlyDirectFlag = false |
| bool partsOfSpeechFlag = false |
Definition at line 38 of file consolidate-main.cpp.
Referenced by main(), outputPhrasePair(), and processFiles().
| bool phraseCountFlag = false |
| bool sourceLabelsFlag = false |
| bool sparseCountBinFeatureFlag = false |
| bool targetSyntacticPreferencesFlag = false |
Definition at line 41 of file consolidate-main.cpp.
Referenced by main(), outputPhrasePair(), and processFiles().
| float totalCount = -1 |
Definition at line 50 of file consolidate-main.cpp.
Referenced by sapt::Ttrack< TKN >::count_tokens(), Moses::TargetPreferencesFeature::EvaluateWhenApplied(), Moses::SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(), loadCountOfCounts(), processFiles(), processPhrasePairs(), MosesTraining::PropertiesConsolidator::ProcessSourceLabelsPropertyValue(), MosesTraining::PropertiesConsolidator::ProcessTargetSyntacticPreferencesPropertyValue(), and Moses::ConfusionNet::ReadFormat0().
