#include <Word.h>
Public Member Functions | |
Word (const Word ©) | |
Word (bool isNonTerminal=false) | |
~Word () | |
const Factor *& | operator[] (FactorType index) |
returns Factor pointer for particular FactorType | |
const Factor *const & | operator[] (FactorType index) const |
const Factor * | GetFactor (FactorType factorType) const |
Deprecated. should use operator[]. | |
void | SetFactor (FactorType factorType, const Factor *factor) |
bool | IsNonTerminal () const |
void | SetIsNonTerminal (bool val) |
bool | IsOOV () const |
void | SetIsOOV (bool val) |
bool | IsEpsilon () const |
void | Merge (const Word &sourceWord) |
std::string | GetString (const std::vector< FactorType > factorType, bool endWithBlank) const |
StringPiece | GetString (FactorType factorType) const |
TO_STRING () | |
bool | operator== (const Word &compare) const |
bool | operator!= (const Word &compare) const |
void | CreateFromString (FactorDirection direction, const std::vector< FactorType > &factorOrder, const StringPiece &str, bool isNonTerminal, bool strict=true) |
void | CreateUnknownWord (const Word &sourceWord) |
void | OnlyTheseFactors (const FactorMask &factors) |
size_t | hash () const |
Static Public Member Functions | |
static int | Compare (const Word &targetWord, const Word &sourceWord) |
Protected Types | |
typedef const Factor * | FactorArray [MAX_NUM_FACTORS] |
Protected Attributes | |
FactorArray | m_factorArray |
bool | m_isNonTerminal |
bool | m_isOOV |
Friends | |
std::ostream & | operator<< (std::ostream &, const Word &) |
Definition at line 44 of file Word.h.
typedef const Factor* Moses::Word::FactorArray[MAX_NUM_FACTORS] [protected] |
Moses::Word::Word | ( | const Word & | copy | ) | [inline] |
Moses::Word::Word | ( | bool | isNonTerminal = false |
) | [inline, explicit] |
empty word
Definition at line 65 of file Word.h.
References isNonTerminal(), m_factorArray, m_isNonTerminal, and m_isOOV.
transitive comparison of 2 word objects. Used by operator<. Only compare the co-joined factors, ie. where factor exists for both words. Should make it non-static
Definition at line 49 of file Word.cpp.
References IsNonTerminal(), and NULL.
void Word::CreateFromString | ( | FactorDirection | direction, | |
const std::vector< FactorType > & | factorOrder, | |||
const StringPiece & | str, | |||
bool | isNonTerminal, | |||
bool | strict = true | |||
) |
Definition at line 131 of file Word.cpp.
References Moses::FactorCollection::AddFactor(), Moses::StaticData::GetFactorDelimiter(), Moses::StaticData::Instance(), Moses::FactorCollection::Instance(), k, m_factorArray, m_isNonTerminal, and UTIL_THROW_IF.
Referenced by Moses::InternalTree::AddSubTree(), Moses::Phrase::CreateFromString(), Moses::ExamplePT::CreateTargetPhrase(), Moses::PhraseDecoder::DecodeCollection(), SimplePhrase::init(), Moses::WordLattice::InitializeFromPCNDataType(), Moses::InternalTree::InternalTree(), Moses::SoftMatchingFeature::Load(), Moses::ChartParserUnknown::Process(), Moses::RDLM::RDLM(), Moses::ForestInput::Read(), Moses::ConfusionNet::ReadFormat0(), Moses::ConfusionNet::ReadFormat1(), and Moses::TreeStructureFeature::TreeStructureFeature().
void Word::CreateUnknownWord | ( | const Word & | sourceWord | ) |
Definition at line 172 of file Word.cpp.
References Moses::FactorCollection::AddFactor(), Moses::Factor::GetString(), Moses::FactorCollection::Instance(), IsNonTerminal(), m_isNonTerminal, m_isOOV, Moses::max_fax(), NULL, Moses::Output, SetFactor(), and UNKNOWN_FACTOR.
Referenced by Moses::ChartParserUnknown::Process().
const Factor* Moses::Word::GetFactor | ( | FactorType | factorType | ) | const [inline] |
Deprecated. should use operator[].
Definition at line 83 of file Word.h.
References m_factorArray.
Referenced by Moses::LanguageModelKen< Model >::CalcScore(), Moses::LanguageModelDALM::CalcScore(), Moses::BackwardLanguageModel< Model >::CalcScore(), Moses::TargetWordInsertionFeature::ComputeFeatures(), Moses::SourceWordDeletionFeature::ComputeFeatures(), Moses::EditOps::ComputeFeatures(), Moses::OxLMMapper::convert(), Moses::PhraseDictionaryOnDisk::ConvertFromMoses(), Moses::OxLMParallelMapper::convertSource(), Moses::ExamplePT::CreateTargetPhrase(), Moses::PhrasePairFeature::EvaluateInIsolation(), Moses::OpSequenceModel::EvaluateInIsolation(), Moses::DesegModel::EvaluateInIsolation(), Moses::LanguageModelKen< Model >::EvaluateWhenApplied(), Moses::LanguageModelDALM::EvaluateWhenApplied(), Moses::TargetBigramFeature::EvaluateWhenApplied(), Moses::OpSequenceModel::EvaluateWhenApplied(), Moses::DesegModel::EvaluateWhenApplied(), Moses::DistortionScoreProducer::EvaluateWhenApplied(), Moses::WordTranslationFeature::EvaluateWithSourceContext(), Moses::PhrasePairFeature::EvaluateWithSourceContext(), Moses::LanguageModelIRST::GetLmID(), Moses::ChartKBestExtractor::GetOutputPhrase(), Moses::ChartHypothesis::GetOutputPhrase(), Moses::VWFeatureContext::GetWord(), Moses::operator<<(), operator==(), Moses::ChartParserUnknown::Process(), and Moses::LanguageModelKen< lm::ngram::ProbingModel >::TranslateID().
StringPiece Word::GetString | ( | FactorType | factorType | ) | const |
std::string Moses::Word::GetString | ( | const std::vector< FactorType > | factorType, | |
bool | endWithBlank | |||
) | const |
get string representation of list of factors. Used by PDTimp so supposed to be invariant to changes in format of debuggin output, therefore, doesn't use streaming output or ToString() from any class so not dependant on these debugging functions.
Referenced by Moses::CorrectionPattern::ComputeFeatures(), Moses::PhraseDictionaryOnDisk::ConvertToMoses(), Moses::PhraseDecoder::DecodeCollection(), Moses::TargetNgramFeature::EvaluateWhenApplied(), Moses::TargetBigramFeature::EvaluateWhenApplied(), Moses::GlobalLexicalModelUnlimited::EvaluateWhenApplied(), Moses::PDTAimp::Factors2String(), Moses::fillIdSeq(), Moses::GetContext(), Moses::RDLM::GetHead(), Moses::SoftMatchingFeature::GetOrSetFeatureName(), Moses::VWFeatureContext::GetSourceWord(), Moses::InternalTree::GetString(), Moses::VWFeatureTarget::GetWord(), Moses::VWFeatureSource::GetWord(), Moses::VWFeatureContext::GetWord(), Moses::RDLM::Load(), Moses::ChartParserUnknown::Process(), Moses::RDLM::Score(), Moses::PhraseOrientationFeature::SparseWordL2RScore(), Moses::PhraseOrientationFeature::SparseWordR2LScore(), and Moses::InternalTree::Unbinarize().
size_t Moses::Word::hash | ( | ) | const [inline] |
Definition at line 143 of file Word.h.
References m_factorArray, m_isNonTerminal, and util::MurmurHashNative().
Referenced by Moses::hash_value().
bool Word::IsEpsilon | ( | ) | const |
Definition at line 202 of file Word.cpp.
References StringPiece::compare(), EPSILON, Moses::Factor::GetString(), and m_factorArray.
Referenced by Moses::TranslationOptionCollectionLattice::Extend(), and Moses::TranslationOptionCollectionLattice::TranslationOptionCollectionLattice().
bool Moses::Word::IsNonTerminal | ( | ) | const [inline] |
Definition at line 90 of file Word.h.
References m_isNonTerminal.
Referenced by Moses::TreeInput::AddChartLabel(), Moses::Syntax::S2T::PChart::AddVertex(), Moses::LanguageModelKen< Model >::CalcScore(), Moses::LanguageModelImplementation::CalcScore(), Moses::LanguageModelDALM::CalcScore(), Moses::BackwardLanguageModel< Model >::CalcScore(), Compare(), Moses::TargetWordInsertionFeature::ComputeFeatures(), Moses::SourceWordDeletionFeature::ComputeFeatures(), Moses::EditOps::ComputeFeatures(), Moses::PhraseDictionaryOnDisk::ConvertFromMoses(), CreateUnknownWord(), Moses::BackwardLanguageModel< Model >::Evaluate(), Moses::UnalignedWordCountFeature::EvaluateInIsolation(), Moses::RuleScope::EvaluateInIsolation(), Moses::RulePairUnlexicalizedSource::EvaluateInIsolation(), Moses::CountNonTerms::EvaluateInIsolation(), Moses::RDLM::EvaluateWhenApplied(), Moses::LanguageModelKen< Model >::EvaluateWhenApplied(), Moses::LanguageModelImplementation::EvaluateWhenApplied(), Moses::LanguageModelDALM::EvaluateWhenApplied(), Moses::TreeStructureFeature::EvaluateWhenApplied(), Moses::TargetPreferencesFeature::EvaluateWhenApplied(), Moses::TargetNgramFeature::EvaluateWhenApplied(), Moses::SoftMatchingFeature::EvaluateWhenApplied(), Moses::BleuScoreFeature::EvaluateWhenApplied(), Moses::WordTranslationFeature::EvaluateWithSourceContext(), Moses::SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(), Moses::NieceTerminal::EvaluateWithSourceContext(), Moses::Model1Feature::EvaluateWithSourceContext(), Moses::MaxSpanFreeNonTermSource::EvaluateWithSourceContext(), Moses::PhraseDictionaryNodeMemory::GetChild(), Moses::Syntax::T2S::RuleTrie::Node::GetChild(), Moses::Syntax::S2T::RuleTrieCYKPlus::Node::GetChild(), Moses::Syntax::T2S::RuleTrie::Node::GetNonTerminalChild(), Moses::Syntax::S2T::RuleTrieCYKPlus::Node::GetNonTerminalChild(), Moses::Phrase::GetNumTerminals(), Moses::Syntax::GetOneBestTargetYield(), Moses::PhraseDictionaryNodeMemory::GetOrCreateChild(), Moses::PhraseDictionaryFuzzyMatch::GetOrCreateNode(), Moses::PhraseDictionaryMemory::GetOrCreateNode(), Moses::UTrieNode::GetOrCreateNonTerminalChild(), Moses::Syntax::T2S::RuleTrie::Node::GetOrCreateNonTerminalChild(), Moses::Syntax::S2T::RuleTrieScope3::Node::GetOrCreateNonTerminalChild(), Moses::Syntax::S2T::RuleTrieCYKPlus::Node::GetOrCreateNonTerminalChild(), Moses::Syntax::T2S::RuleTrie::Node::GetOrCreateTargetPhraseCollection(), Moses::UTrieNode::GetOrCreateTerminalChild(), Moses::Syntax::S2T::RuleTrieScope3::Node::GetOrCreateTerminalChild(), Moses::Syntax::KBestExtractor::GetOutputPhrase(), Moses::ChartKBestExtractor::GetOutputPhrase(), Moses::ChartHypothesis::GetOutputPhrase(), Moses::ChartKBestExtractor::GetOutputScoreBreakdown(), Moses::Syntax::KBestExtractor::GetOutputTree(), Moses::ChartKBestExtractor::GetOutputTree(), Moses::InternalTree::IsLeafNT(), Moses::DottedRule::IsNonTerminal(), Moses::InternalTree::IsTerminal(), Moses::operator<<(), operator==(), Moses::Syntax::S2T::PHyperedgeToSHyperedgeBundle(), Moses::PhraseOrientationFeature::SparseWordL2RScore(), and Moses::PhraseOrientationFeature::SparseWordR2LScore().
bool Moses::Word::IsOOV | ( | ) | const [inline] |
Definition at line 97 of file Word.h.
References m_isOOV.
Referenced by Moses::PhraseOrientationFeature::EvaluateInIsolation(), Moses::OpSequenceModel::EvaluateInIsolation(), Moses::TargetPreferencesFeature::EvaluateWhenApplied(), Moses::TargetConstituentAdjacencyFeature::EvaluateWhenApplied(), Moses::PhraseOrientationFeature::EvaluateWhenApplied(), Moses::OpSequenceModel::EvaluateWhenApplied(), Moses::SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(), Moses::Phrase::Find(), Moses::Manager::OutputSurface(), and Moses::BaseManager::OutputSurface().
void Word::Merge | ( | const Word & | sourceWord | ) |
add the factors from sourceWord into this representation, NULL elements in sourceWord will be skipped
Definition at line 86 of file Word.cpp.
References m_factorArray, and NULL.
void Word::OnlyTheseFactors | ( | const FactorMask & | factors | ) |
Definition at line 193 of file Word.cpp.
References NULL, and SetFactor().
Referenced by Moses::PhraseDictionaryOnDisk::GetTargetPhraseCollectionBatch(), and Moses::PhraseDictionaryMemory::GetTargetPhraseCollectionBatch().
bool Moses::Word::operator!= | ( | const Word & | compare | ) | const [inline] |
bool Word::operator== | ( | const Word & | compare | ) | const |
Definition at line 69 of file Word.cpp.
References GetFactor(), and IsNonTerminal().
const Factor* const& Moses::Word::operator[] | ( | FactorType | index | ) | const [inline] |
const Factor*& Moses::Word::operator[] | ( | FactorType | index | ) | [inline] |
returns Factor pointer for particular FactorType
Definition at line 74 of file Word.h.
References m_factorArray.
void Moses::Word::SetFactor | ( | FactorType | factorType, | |
const Factor * | factor | |||
) | [inline] |
Definition at line 86 of file Word.h.
References m_factorArray.
Referenced by Moses::BilingualLM::BilingualLM(), Moses::BilingualLM_NPLM::BilingualLM_NPLM(), Moses::PhraseDictionaryOnDisk::ConvertToMoses(), CreateUnknownWord(), Moses::Phrase::GetSubString(), Moses::GlobalLexicalModel::GlobalLexicalModel(), Moses::Phrase::InitStartEndWord(), Moses::GenerationDictionary::Load(), Moses::StaticData::LoadNonTerminals(), Moses::SyntaxOptions::LoadNonTerminals(), Moses::BleuScoreFeature::LoadReferences(), Moses::MaxSpanFreeNonTermSource::MaxSpanFreeNonTermSource(), OnlyTheseFactors(), Moses::ProcessAndStripXMLTags(), Moses::TargetBigramFeature::TargetBigramFeature(), Moses::TargetNgramFeature::TargetNgramFeature(), and Moses::VW::VW().
void Moses::Word::SetIsNonTerminal | ( | bool | val | ) | [inline] |
Definition at line 93 of file Word.h.
References m_isNonTerminal.
Referenced by Moses::StaticData::LoadNonTerminals(), and Moses::SyntaxOptions::LoadNonTerminals().
void Moses::Word::SetIsOOV | ( | bool | val | ) | [inline] |
Definition at line 100 of file Word.h.
References m_isOOV.
Referenced by Moses::ChartParserUnknown::Process(), and Moses::TranslationOptionCollection::ProcessOneUnknownWord().
Moses::Word::TO_STRING | ( | ) |
std::ostream& operator<< | ( | std::ostream & | , | |
const Word & | ||||
) | [friend] |
FactorArray Moses::Word::m_factorArray [protected] |
set of factors
Definition at line 52 of file Word.h.
Referenced by CreateFromString(), GetFactor(), GetString(), hash(), IsEpsilon(), Merge(), operator[](), SetFactor(), and Word().
bool Moses::Word::m_isNonTerminal [protected] |
Definition at line 53 of file Word.h.
Referenced by CreateFromString(), CreateUnknownWord(), hash(), IsNonTerminal(), SetIsNonTerminal(), and Word().
bool Moses::Word::m_isOOV [protected] |
Definition at line 54 of file Word.h.
Referenced by CreateUnknownWord(), IsOOV(), SetIsOOV(), and Word().