/disk4/html/www/moses/doxygen/mosesdecoder/moses/Util.h File Reference

#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
#include <cmath>
#include <cassert>
#include <limits>
#include <map>
#include <cstdlib>
#include <cstring>
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "TypeDef.h"

Go to the source code of this file.


Classes

class  Moses::UnorderedComparer< T >

Namespaces

namespace  Moses
 $Id$

Defines

#define TRACE_ERR(str)   do {} while (false)
#define VERBOSE(level, str)   { IFVERBOSE(level) { TRACE_ERR(str); } }
#define IFVERBOSE(level)   if (Moses::StaticData::Instance().GetVerboseLevel() >= level)
#define XVERBOSE(level, str)   VERBOSE(level, "[" << HERE << "] " << str)
#define HERE   __FILE__ << ":" << __LINE__
#define FEATUREVERBOSE(level, str)   FEATUREVERBOSE2(level, "[" << GetScoreProducerDescription() << "] " << str)
#define FEATUREVERBOSE2(level, str)   { IFFEATUREVERBOSE(level) { TRACE_ERR(str); } }
#define IFFEATUREVERBOSE(level)   if ((m_verbosity == std::numeric_limits<std::size_t>::max() && StaticData::Instance().GetVerboseLevel() >= level) || (m_verbosity != std::numeric_limits<std::size_t>::max() && m_verbosity >= level))
#define NTH_ELEMENT3(begin, middle, end)   std::nth_element(begin, middle, end)
#define NTH_ELEMENT4(begin, middle, end, orderer)   std::nth_element(begin, middle, end, orderer)
#define TO_STRING()   std::string ToString() const;
#define TO_STRING_BODY(CLASS)
 definition of ToString() function to go in .cpp file. Can be used for any class that can be piped to a stream

Functions

const std::string Moses::ToLower (const std::string &str)
std::string Moses::Trim (const std::string &str, const std::string dropChars=" \t\n\r")
 delete white spaces at beginning and end of string
template<typename T >
std::string Moses::SPrint (const T &input)
 get string representation of any object/variable, as long as it can pipe to a stream
template<typename T >
T Moses::Scan (const std::string &input)
 convert string to variable of type T. Used to reading floats, int etc from files
template<>
std::string Moses::Scan< std::string > (const std::string &input)
 just return input
template<>
WordAlignmentSort Moses::Scan< WordAlignmentSort > (const std::string &input)
template<>
InputTypeEnum Moses::Scan< InputTypeEnum > (const std::string &input)
template<>
SearchAlgorithm Moses::Scan< SearchAlgorithm > (const std::string &input)
template<>
S2TParsingAlgorithm Moses::Scan< S2TParsingAlgorithm > (const std::string &input)
template<>
SourceLabelOverlap Moses::Scan< SourceLabelOverlap > (const std::string &input)
template<>
XmlInputType Moses::Scan< XmlInputType > (const std::string &input)
template<>
bool Moses::Scan< bool > (const std::string &input)
 Specialisation to understand yes/no y/n true/false 0/1.
template<typename T >
std::vector< TMoses::Scan (const std::vector< std::string > &input)
 convert vectors of string to vectors of type T variables
template<typename T >
void Moses::Scan (std::vector< T > &output, const std::vector< std::string > &input)
 speeded up version of above
std::string Moses::Replace (const std::string &str, const std::string &todelStr, const std::string &toaddStr)
std::vector< std::string > Moses::Tokenize (const std::string &input, const std::string &delimiters=" \t")
 tokenise input string to vector of type T
void Moses::Tokenize (std::vector< std::string > &output, const std::string &str, const std::string &delimiters=" \t")
template<typename T >
void Moses::Tokenize (std::vector< T > &output, const std::string &input, const std::string &delimiters=" \t")
std::vector< std::string > Moses::TokenizeMultiCharSeparator (const std::string &str, const std::string &separator)
void Moses::TokenizeMultiCharSeparator (std::vector< std::string > &output, const std::string &str, const std::string &separator)
std::vector< std::string > Moses::TokenizeFirstOnly (const std::string &str, const std::string &delimiters=" \t")
template<typename T >
std::string Moses::Join (const std::string &delimiter, const std::vector< T > &items)
template<typename It >
std::string Moses::Join (const std::string &delim, It begin, It end)
float Moses::TransformScore (float prob)
 transform prob to natural log score
float Moses::UntransformScore (float score)
 transform natural log score to prob. Not currently used
float Moses::TransformLMScore (float irstScore)
 irst number are in log 10, transform to natural log
float Moses::UntransformLMScore (float logNScore)
float Moses::FloorScore (float logScore)
 make sure score doesn't fall below LOWEST_SCORE
float Moses::CalcTranslationScore (const std::vector< float > &probVector, const std::vector< float > &weightT)
template<class COLL >
void Moses::RemoveAllInColl (COLL &coll)
 delete and remove every element of a collection object such as set, list etc
template<class COLL >
void Moses::RemoveAllInMap (COLL &coll)
 delete and remove every element of map
string Moses::GetTempFolder ()
 x-platform reference to temp folder
std::string Moses::GetMD5Hash (const std::string &filePath)
 MD5 hash of a file.
template<typename T >
void Moses::ShrinkToFit (T &v)
 save memory by getting rid of spare, unused elements in a collection
bool Moses::FileExists (const std::string &filePath)
std::vector< std::map
< std::string, std::string > > 
Moses::ProcessAndStripDLT (std::string &line)
std::map< std::string,
std::string > 
Moses::ProcessAndStripSGML (std::string &line)
std::string Moses::PassthroughSGML (std::string &line, const std::string tagName, const std::string &lbrackStr, const std::string &rbrackStr)
std::string Moses::GetFirstString (const std::string &str, int &first_pos, const std::string &delimiters=" \t")
template<class T >
T Moses::log_sum (T log_a, T log_b)
bool Moses::Equals (float a, float b)
void Moses::FixPrecision (std::ostream &stream, size_t size=3)
void Moses::PrintFeatureWeight (const FeatureFunction *ff)
void Moses::ShowWeights ()


Define Documentation

#define FEATUREVERBOSE ( level,
str   )     FEATUREVERBOSE2(level, "[" << GetScoreProducerDescription() << "] " << str)

#define FEATUREVERBOSE2 ( level,
str   )     { IFFEATUREVERBOSE(level) { TRACE_ERR(str); } }

#define HERE   __FILE__ << ":" << __LINE__

#define IFFEATUREVERBOSE ( level   )     if ((m_verbosity == std::numeric_limits<std::size_t>::max() && StaticData::Instance().GetVerboseLevel() >= level) || (m_verbosity != std::numeric_limits<std::size_t>::max() && m_verbosity >= level))

#define IFVERBOSE ( level   )     if (Moses::StaticData::Instance().GetVerboseLevel() >= level)

Definition at line 75 of file Util.h.

Referenced by Moses::ChartManager::AddXmlChartOptions(), batch_run(), Moses::Manager::CalcDecoderStatistics(), Moses::TranslationOptionCollection::CalcEstimatedScore(), Moses::calcNgramExpectations(), Moses::DistortionScoreProducer::CalculateDistortionScore(), Moses::PhraseDictionaryDynamicCacheBased::ClearSource(), Moses::SearchNormal::Decode(), Moses::SearchCubePruning::Decode(), Moses::Manager::Decode(), Moses::ChartManager::Decode(), decoder_main(), Moses::BitmapContainer::Enqueue(), Moses::LanguageModelImplementation::EvaluateWhenApplied(), Moses::PhraseDictionaryDynamicCacheBased::Execute(), Moses::SearchNormal::ExpandHypothesis(), Moses::getLatticeMBRNBest(), Moses::WordLattice::InitializeFromPCNDataType(), Moses::PhraseDictionaryDynamicCacheBased::Insert(), Moses::RuleTableLoaderStandard::Load(), Moses::Syntax::T2S::RuleTrieLoader::Load(), Moses::Syntax::S2T::RuleTrieLoader::Load(), Moses::Syntax::F2S::HyperTreeLoader::Load(), Moses::PhraseDictionaryDynamicCacheBased::Load_Single_File(), main(), Moses::Manager::OutputBest(), Moses::ChartManager::OutputNBest(), Moses::BitmapContainer::ProcessBestHypothesis(), Moses::DecodeStepTranslation::ProcessInitialTranslation(), Moses::DecodeStepTranslation::ProcessInitialTransLEGACY(), Moses::SearchNormal::ProcessOneStack(), Moses::pruneLatticeFB(), Moses::HypothesisStackNormal::PruneToSize(), Moses::HypothesisStackCubePruning::PruneToSize(), Moses::ChartHypothesisCollection::PruneToSize(), Moses::TranslationTask::Run(), and Moses::SyntacticLanguageModelState< MY, MX, YS, B >::SyntacticLanguageModelState().

#define NTH_ELEMENT3 ( begin,
middle,
end   )     std::nth_element(begin, middle, end)

Definition at line 88 of file Util.h.

Referenced by main(), and Moses::PDTAimp::PruneTargetCandidates().

#define NTH_ELEMENT4 ( begin,
middle,
end,
orderer   )     std::nth_element(begin, middle, end, orderer)

 
#define TO_STRING (  )     std::string ToString() const;

declaration of ToString() function to go in header for each class. This function, as well as the operator<< fn for each class, is for debugging purposes only. The output format is likely to change from time-to-time as classes are updated so shouldn't be relied upon for any decoding algorithm

Definition at line 421 of file Util.h.

#define TO_STRING_BODY ( CLASS   ) 

Value:

std::string CLASS::ToString() const     \
        {                                                                                                                       \
                std::stringstream out;                  \
                out << *this;                                                           \
                return out.str();                                               \
        }                                                                                                                       \
definition of ToString() function to go in .cpp file. Can be used for any class that can be piped to a stream

Definition at line 424 of file Util.h.

#define TRACE_ERR ( str   )     do {} while (false)

Outputting debugging/verbose information to stderr. Use TRACE_ENABLE flag to redirect tracing output into oblivion so that you can output your own ad-hoc debugging info. However, if you use stderr directly, please delete calls to it once you finished debugging so that it won't clutter up. Also use TRACE_ENABLE to turn off output of any debugging info when compiling for a gui front-end so that running gui won't generate output on command line

Definition at line 59 of file Util.h.

Referenced by batch_run(), Moses::PDTAimp::CacheSource(), Moses::Manager::CalcDecoderStatistics(), Moses::TranslationOptionCollection::CalcEstimatedScore(), Moses::PrefixTreeF< T, D >::changeData(), MosesTuning::Timer::check(), Moses::LanguageModelIRST::CleanUpAfterSentenceProcessing(), Moses::WordLattice::ComputeDistortionDistance(), Moses::PhraseDictionaryTree::Create(), Moses::PrefixTreeF< T, D >::create(), Moses::LexicalReorderingTableTree::Create(), Moses::Manager::Decode(), decoder_main(), Moses::FeatureFactory::DefaultSetup(), Moses::PhraseDistanceFeature::EvaluateWithSourceContext(), Moses::ConfusionNet::GetStringRep(), Moses::PhraseDictionaryFuzzyMatch::InitializeForInput(), Moses::WordLattice::InitializeFromPCNDataType(), Moses::RuleTableLoaderStandard::Load(), Moses::Syntax::T2S::RuleTrieLoader::Load(), Moses::Syntax::S2T::RuleTrieLoader::Load(), Moses::Parameter::LoadParam(), main(), Moses::SearchNormal::OutputHypoStack(), Moses::SearchCubePruning::OutputHypoStack(), Moses::SearchNormal::OutputHypoStackSize(), Moses::SearchCubePruning::OutputHypoStackSize(), Moses::BaseManager::OutputSearchGraphAsHypergraph(), Moses::Manager::OutputSearchGraphSLF(), Moses::ParseXmlTagAttribute(), Moses::PassthroughSGML(), Moses::Hypothesis::PrintHypothesis(), Moses::PhraseDictionaryTree::PrintTargetCandidates(), Moses::ProcessAndStripDLT(), Moses::ProcessAndStripSGML(), Moses::ProcessAndStripXMLTags(), Moses::BitmapContainer::ProcessBestHypothesis(), Moses::DecodeStepTranslation::ProcessInitialTranslation(), Moses::DecodeStepTranslation::ProcessInitialTransLEGACY(), Moses::HypothesisStackNormal::PruneToSize(), Moses::HypothesisStackCubePruning::PruneToSize(), Moses::ChartHypothesisCollection::PruneToSize(), Moses::PrefixTreeMap::Read(), Moses::ConfusionNet::ReadFormat0(), MosesTuning::Timer::restart(), Moses::AllOptions::sanity_check(), Moses::Parameter::SetBooleanSwitch(), MosesTuning::Timer::start(), Moses::TokenizeXml(), Moses::LanguageModelIRST::~LanguageModelIRST(), and Moses::PDTAimp::~PDTAimp().

#define VERBOSE ( level,
str   )     { IFVERBOSE(level) { TRACE_ERR(str); } }

verbose macros

Definition at line 69 of file Util.h.

Referenced by Moses::HypothesisStackNormal::Add(), Moses::HypothesisStackCubePruning::Add(), Moses::ChartHypothesisCollection::Add(), Moses::ChartHypothesisCollection::AddHypothesis(), Moses::HypothesisStackNormal::AddPrune(), Moses::HypothesisStackCubePruning::AddPrune(), lmtable::appendbin_level_nommap(), Moses::BackwardsEdge::BackwardsEdge(), batch_run(), Moses::calcNgramExpectations(), Moses::Timer::check(), Moses::ReorderingConstraint::Check(), Timer::check(), Moses::LexicalReorderingTableCompact::CheckAndLoad(), lmtable::checkbounds(), lmclass::checkMap(), Moses::StaticData::CheckWeights(), Moses::PhraseDictionaryDynamicCacheBased::ClearEntries(), Moses::DynamicCacheBasedLanguageModel::ClearEntries(), Moses::PhraseDictionaryDynamicCacheBased::ClearSource(), lmtable::clprob(), lmmacro::clprob(), lmmacro::cmaxsuffptr(), lmtable::compact_single_level(), Moses::WordLattice::ComputeDistortionDistance(), lmtable::concatenate_single_level(), lmtable::configure(), Moses::PDTAimp::Create(), Moses::TranslationOptionCollectionLattice::CreateTranslationOptions(), Moses::TranslationOptionCollection::CreateTranslationOptions(), Moses::PhraseDictionaryDynamicCacheBased::Decay(), Moses::SearchCubePruning::Decode(), Moses::Manager::Decode(), Moses::ChartManager::Decode(), Moses::DecodeFeature::DecodeFeature(), Moses::DecodeStep::DecodeStep(), Moses::FeatureFactory::DefaultSetup(), Moses::doConsensusDecoding(), Moses::DynamicCacheBasedLanguageModel::DynamicCacheBasedLanguageModel(), Moses::OpSequenceModel::EmptyHypothesisState(), Moses::DesegModel::EmptyHypothesisState(), Moses::VW::EvaluateTranslationOptionListWithSourceContext(), Moses::VW::EvaluateWhenApplied(), Moses::LexicalReordering::EvaluateWhenApplied(), Moses::PhraseDictionaryDynamicCacheBased::Execute(), Moses::DynamicCacheBasedLanguageModel::Execute(), Moses::PhraseDictionaryDynamicCacheBased::Execute_Single_Command(), lmtable::expand_level_nommap(), Moses::ReorderingConstraint::FinalizeWalls(), lmContainer::getLanguageModelType(), Moses::getLatticeMBRNBest(), Moses::InMemoryPerSentenceOnDemandLM::GetPerThreadLM(), Moses::InputOptions::init(), Moses::StaticData::initialize_features(), Moses::PhraseDictionaryMemoryPerSentenceOnDemand::InitializeForInput(), Moses::StaticData::InitializeForInput(), Moses::ReloadingLanguageModel< Model >::InitializeForInput(), Moses::PhraseDictionaryDynamicCacheBased::Insert(), Moses::DynamicCacheBasedLanguageModel::Insert(), Moses::IOWrapper::IOWrapper(), Moses::LanguageModelIRST::LanguageModelIRST(), Moses::LexicalReordering::LexicalReordering(), Moses::PhraseDictionaryDynamicCacheBased::Load(), Moses::LanguageModelIRST::Load(), Moses::DynamicCacheBasedLanguageModel::Load(), lmtable::load(), lmmacro::load(), lmInterpolation::load(), lmclass::load(), Moses::PhraseDictionaryDynamicCacheBased::Load_Multiple_Files(), Moses::PhraseDictionaryDynamicCacheBased::Load_Single_File(), Moses::StaticData::LoadAlternateWeightSettings(), Moses::StaticData::LoadDecodeGraphsNew(), Moses::StaticData::LoadFeatureFunctions(), lmclass::loadMapElement(), Moses::LanguageModelKen< Model >::LoadModel(), lmtable::lprob(), lmmacro::lprob(), lmclass::lprob(), lmmacro::map(), lmclass::mapping(), lmmacro::maxsuffptr(), lmmacro::Micro2MacroMapping(), Moses::Model1Feature::Model1Feature(), Moses::Search::out_of_time(), Moses::Syntax::Manager::OutputBest(), Moses::Manager::OutputBest(), Moses::ChartManager::OutputNBest(), Moses::Manager::OutputSearchGraph(), Moses::Manager::OutputSearchGraphAsHypergraph(), Moses::StaticData::OverrideFeatures(), Moses::PhraseOrientationFeature::PhraseOrientationFeature(), Moses::PhrasePairFeature::PhrasePairFeature(), Moses::PhraseDictionaryDynamicCacheBased::Print(), Moses::ProcessAndStripXMLTags(), Moses::DecodeStepTranslation::ProcessInitialTranslation(), Moses::DecodeStepTranslation::ProcessInitialTransLEGACY(), Moses::SearchNormal::ProcessOneStack(), Moses::TranslationOptionCollection::Prune(), Moses::pruneLatticeFB(), Moses::HypothesisStackNormal::PruneToSize(), Moses::HypothesisStackCubePruning::PruneToSize(), Moses::ChartHypothesisCollection::PruneToSize(), Moses::PhraseDictionaryTree::Read(), Moses::PDTimp::Read(), Moses::ConfusionNet::ReadF(), Moses::ConfusionNet::ReadFormat0(), Moses::ConfusionNet::ReadFormat1(), Moses::PhraseDictionary::ReduceCache(), Moses::ScoreComponentCollection::RegisterScoreProducer(), Moses::ReloadingLanguageModel< Model >::ReloadingLanguageModel(), lmtable::resize_level_nommap(), Moses::RulePairUnlexicalizedSource::RulePairUnlexicalizedSource(), Moses::TranslationTask::Run(), MosesServer::Server::run(), mdiadaptlm::saveARPA_per_level(), mdiadaptlm::saveARPA_per_word(), lmtable::savebin(), lmtable::savebin_level_mmap(), lmtable::savebin_level_nommap(), mdiadaptlm::saveBIN_per_level(), mdiadaptlm::saveBIN_per_word(), Moses::SearchNormal::SearchNormal(), Moses::StaticData::SetExecPath(), Moses::PhraseDictionaryDynamicCacheBased::SetMaxAge(), Moses::DynamicCacheBasedLanguageModel::SetMaxAge(), Moses::PhraseDictionaryDynamicCacheBased::SetParameter(), Moses::DynamicCacheBasedLanguageModel::SetParameter(), Moses::PhraseDictionaryDynamicCacheBased::SetPreComputedScores(), Moses::DynamicCacheBasedLanguageModel::SetQueryType(), Moses::PhraseDictionaryDynamicCacheBased::SetScoreType(), Moses::DynamicCacheBasedLanguageModel::SetScoreType(), Moses::ReorderingConstraint::SetWall(), Moses::ReorderingConstraint::SetZone(), Moses::SoftSourceSyntacticConstraintsFeature::SoftSourceSyntacticConstraintsFeature(), Moses::SourceWordDeletionFeature::SourceWordDeletionFeature(), Moses::Timer::start(), Timer::start(), storage::stat(), strstack::stat(), mempool::stat(), Moses::Timer::stop(), Moses::SyntacticLanguageModelState< MY, MX, YS, B >::SyntacticLanguageModelState(), Moses::TargetConstituentAdjacencyFeature::TargetConstituentAdjacencyFeature(), Moses::TargetPreferencesFeature::TargetPreferencesFeature(), Moses::TargetWordInsertionFeature::TargetWordInsertionFeature(), Moses::UnalignedWordCountFeature::UnalignedWordCountFeature(), Moses::PhraseDictionaryDynamicCacheBased::Update(), Moses::VW::VW(), Moses::VWState::VWState(), Moses::WordTranslationFeature::WordTranslationFeature(), Moses::ChartManager::~ChartManager(), Moses::SyntacticLanguageModelFiles< MH, MO >::~SyntacticLanguageModelFiles(), and Moses::SyntacticLanguageModelState< MY, MX, YS, B >::~SyntacticLanguageModelState().

#define XVERBOSE ( level,
str   )     VERBOSE(level, "[" << HERE << "] " << str)


Generated on Thu Jul 6 00:31:35 2017 for Moses by  doxygen 1.5.9