Namespaces | |
namespace | CheckOffT< 8 > |
namespace | detail |
namespace | internal |
Implementation detail. For the random module's internal use only. | |
namespace | stream |
Classes | |
union | FloatEnc |
struct | BitsMask |
struct | BitAddress |
class | ErsatzProgress |
class | Exception |
class | ErrnoException |
class | FileOpenException |
class | OverflowException |
class | FakeOStream |
class | scoped_fd |
struct | scoped_FILE_closer |
class | FDException |
class | EndOfFileException |
class | ParseNumberException |
class | FilePiece |
class | FileStream |
class | FixedArray |
struct | ToStringBuf< double > |
struct | ToStringBuf< float > |
struct | ToStringBuf< bool > |
struct | ToStringBuf< uint16_t > |
struct | ToStringBuf< int16_t > |
struct | ToStringBuf< uint32_t > |
struct | ToStringBuf< int32_t > |
struct | ToStringBuf< uint64_t > |
struct | ToStringBuf< int64_t > |
struct | ToStringBuf< const void * > |
class | PairedIterator |
class | scoped_mmap |
class | scoped_memory |
class | Rolling |
class | PCQueue |
class | Pool |
class | ProbingSizeException |
struct | IdentityHash |
class | DivMod |
class | Power2Mod |
class | ProbingHashTable |
class | AutoProbing |
class | ProxyIterator |
class | ReadBase |
class | CompressedException |
class | GZException |
class | BZException |
class | XZException |
class | ReadCompressed |
class | MallocException |
class | scoped_base |
class | scoped |
class | scoped< void, Closer > |
struct | scoped_c_forward |
class | scoped_c |
class | scoped_malloc |
struct | scoped_delete_array_forward |
class | scoped_array |
struct | scoped_delete_forward |
class | scoped_ptr |
class | SizedInnerIterator |
class | SizedProxy |
class | SizedCompare |
class | IdentityAccessor |
struct | Pivot64 |
struct | Pivot32 |
struct | PivotSelect< 8 > |
struct | PivotSelect< 4 > |
struct | PivotSelect< 2 > |
class | StringStream |
class | temp_dir |
class | temp_file |
class | Worker |
class | ThreadPool |
class | OutOfTokens |
class | SingleCharacter |
class | MultiCharacter |
class | AnyCharacter |
class | BoolCharacter |
class | AnyCharacterLast |
class | TokenIter |
Typedefs | |
typedef scoped< std::FILE, scoped_FILE_closer > | scoped_FILE |
typedef boost::interprocess::interprocess_semaphore | Semaphore |
typedef ProxyIterator< SizedProxy > | SizedIterator |
Enumerations | |
enum | { kToStringMaxBytes = 20 } |
enum | LoadMethod { LAZY, POPULATE_OR_LAZY, POPULATE_OR_READ, READ, PARALLEL_READ } |
Functions | |
uint8_t | RequiredBits (uint64_t max_value) |
void | BitPackingSanity () |
uint8_t | BitPackShift (uint8_t bit, uint8_t) |
uint64_t | ReadOff (const void *base, uint64_t bit_off) |
uint64_t | ReadInt57 (const void *base, uint64_t bit_off, uint8_t length, uint64_t mask) |
void | WriteInt57 (void *base, uint64_t bit_off, uint8_t length, uint64_t value) |
uint32_t | ReadInt25 (const void *base, uint64_t bit_off, uint8_t length, uint32_t mask) |
void | WriteInt25 (void *base, uint64_t bit_off, uint8_t length, uint32_t value) |
float | ReadFloat32 (const void *base, uint64_t bit_off) |
void | WriteFloat32 (void *base, uint64_t bit_off, float value) |
void | SetSign (float &to) |
void | UnsetSign (float &to) |
float | ReadNonPositiveFloat31 (const void *base, uint64_t bit_off) |
void | WriteNonPositiveFloat31 (void *base, uint64_t bit_off, float value) |
template<class Except , class Data > | |
Except::template ExceptionTag < Except & >::Identity | operator<< (Except &e, const Data &data) |
template<unsigned len> | |
std::size_t | CheckOverflowInternal (uint64_t value) |
template<> | |
std::size_t | CheckOverflowInternal< 8 > (uint64_t value) |
std::size_t | CheckOverflow (uint64_t value) |
bool | InputFileIsStdin (StringPiece path) |
bool | OutputFileIsStdout (StringPiece path) |
int | OpenReadOrThrow (const char *name) |
int | CreateOrThrow (const char *name) |
uint64_t | SizeFile (int fd) |
uint64_t | SizeOrThrow (int fd) |
void | ResizeOrThrow (int fd, uint64_t to) |
std::size_t | PartialRead (int fd, void *to, std::size_t amount) |
void | ReadOrThrow (int fd, void *to_void, std::size_t amount) |
std::size_t | ReadOrEOF (int fd, void *to_void, std::size_t amount) |
void | WriteOrThrow (int fd, const void *data_void, std::size_t size) |
void | WriteOrThrow (FILE *to, const void *data, std::size_t size) |
void | ErsatzPRead (int fd, void *to_void, std::size_t size, uint64_t off) |
void | ErsatzPWrite (int fd, const void *from_void, std::size_t size, uint64_t off) |
void | FSyncOrThrow (int fd) |
void | SeekOrThrow (int fd, uint64_t off) |
void | AdvanceOrThrow (int fd, int64_t off) |
void | SeekEnd (int fd) |
std::FILE * | FDOpenOrThrow (scoped_fd &file) |
std::FILE * | FDOpenReadOrThrow (scoped_fd &file) |
int | mkstemp_and_unlink (char *tmpl) |
void | NormalizeTempPrefix (std::string &base) |
int | MakeTemp (const StringPiece &base) |
std::FILE * | FMakeTemp (const StringPiece &base) |
int | DupOrThrow (int fd) |
std::string | NameFromFD (int fd) |
bool | InputPathIsStdin (StringPiece path) |
bool | OutputPathIsStdout (StringPiece path) |
char * | ToString (double value, char *to) |
char * | ToString (float value, char *to) |
char * | ToString (uint32_t value, char *buffer) |
char * | ToString (uint64_t value, char *buffer) |
char * | ToString (int32_t value, char *to) |
char * | ToString (int64_t value, char *to) |
char * | ToString (int16_t value, char *to) |
char * | ToString (uint16_t value, char *to) |
char * | ToString (const void *v, char *to) |
char * | ToString (bool value, char *to) |
template<class KeyIter , class ValueIter , class Less > | |
void | JointSort (const KeyIter &key_begin, const KeyIter &key_end, const ValueIter &value_begin, const Less &less) |
template<class KeyIter , class ValueIter > | |
void | JointSort (const KeyIter &key_begin, const KeyIter &key_end, const ValueIter &value_begin) |
std::size_t | SizePage () |
void * | MapOrThrow (std::size_t size, bool for_write, int flags, bool prefault, int fd, uint64_t offset) |
void | SyncOrThrow (void *start, size_t length) |
void | UnmapOrThrow (void *start, size_t length) |
void | HugeMalloc (std::size_t size, bool zeroed, scoped_memory &to) |
void | HugeRealloc (std::size_t to, bool zero_new, scoped_memory &mem) |
void | MapRead (LoadMethod method, int fd, uint64_t offset, std::size_t size, scoped_memory &out) |
void * | MapZeroedWrite (int fd, std::size_t size) |
void * | MapZeroedWrite (const char *name, std::size_t size, scoped_fd &file) |
template<class Iterator , class Less > | |
boost::optional< typename std::iterator_traits< Iterator > ::value_type > | FirstIntersection (std::vector< boost::iterator_range< Iterator > > &sets, const Less less) |
template<class Iterator > | |
boost::optional< typename std::iterator_traits< Iterator > ::value_type > | FirstIntersection (std::vector< boost::iterator_range< Iterator > > &sets) |
template<class Iterator , class Output , class Less > | |
void | AllIntersection (std::vector< boost::iterator_range< Iterator > > &sets, Output &out, const Less less) |
template<class Iterator , class Output > | |
void | AllIntersection (std::vector< boost::iterator_range< Iterator > > &sets, Output &out) |
uint64_t | MurmurHash64A (const void *key, std::size_t len, uint64_t seed) |
uint64_t | MurmurHash64B (const void *key, std::size_t len, uint64_t seed) |
uint64_t | MurmurHashNative (const void *key, std::size_t len, uint64_t seed) |
void | ParallelRead (int fd, void *to, std::size_t amount, uint64_t offset) |
void | WaitSemaphore (Semaphore &on) |
template<class Proxy > | |
ProxyIterator< Proxy > | operator+ (std::ptrdiff_t amount, const ProxyIterator< Proxy > &it) |
void | rand_init (unsigned int seed) |
void | rand_init () |
template<typename T > | |
T | rand () |
template<typename T > | |
T | rand_excl (T bottom, T top) |
template<typename T > | |
T | rand_excl (T top) |
template<typename T > | |
T | rand_incl (T bottom, T top) |
template<typename T > | |
T | rand_incl (T top) |
template<typename T > | |
T | wide_rand () |
template<typename T > | |
T | wide_rand_excl (T top) |
template<typename T > | |
T | wide_rand_excl (T bottom, T top) |
template<typename T > | |
T | wide_rand_incl (T top) |
template<typename T > | |
T | wide_rand_incl (T bottom, T top) |
void * | MallocOrThrow (std::size_t requested) |
void * | CallocOrThrow (std::size_t requested) |
void | AdviseHugePages (const void *addr, std::size_t size) |
SizedIterator | SizedIt (void *ptr, std::size_t size) |
template<class Iterator , class Accessor > | |
bool | BinaryFind (const Accessor &accessor, Iterator begin, Iterator end, const typename Accessor::Key key, Iterator &out) |
template<class Iterator , class Accessor , class Pivot > | |
bool | BoundedSortedUniformFind (const Accessor &accessor, Iterator before_it, typename Accessor::Key before_v, Iterator after_it, typename Accessor::Key after_v, const typename Accessor::Key key, Iterator &out) |
template<class Iterator , class Accessor , class Pivot > | |
bool | SortedUniformFind (const Accessor &accessor, Iterator begin, Iterator end, const typename Accessor::Key key, Iterator &out) |
std::string | temp_location () |
Obtain a directory for temporary files, e.g. /tmp. | |
void | posix_tmp_template (char *buf) |
std::vector< std::string > | tokenize (const char input[]) |
std::vector< std::string > | tokenize (const std::string &input) |
double | WallTime () |
double | CPUTime () |
uint64_t | RSSMax () |
void | PrintUsage (std::ostream &out) |
uint64_t | GuessPhysicalMemory () |
uint64_t | ParseSize (const std::string &arg) |
Variables | |
const uint32_t | kSignBit = 0x80000000 |
const char | kProgressBanner [] = "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n" |
const uint64_t | kBadSize = (uint64_t)-1 |
const bool | kSpaces [256] = {0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} |
const int | kFileFlags |
typedef scoped<std::FILE, scoped_FILE_closer> util::scoped_FILE |
typedef boost::interprocess::interprocess_semaphore util::Semaphore |
Definition at line 57 of file pcqueue.hh.
typedef ProxyIterator<SizedProxy> util::SizedIterator |
Definition at line 91 of file sized_iterator.hh.
anonymous enum |
enum util::LoadMethod |
void util::AdviseHugePages | ( | const void * | addr, | |
std::size_t | size | |||
) |
Definition at line 37 of file scoped.cc.
Referenced by lm::ngram::BinaryFormat::GrowForSearch(), and lm::ngram::BinaryFormat::SetupJustVocab().
void util::AllIntersection | ( | std::vector< boost::iterator_range< Iterator > > & | sets, | |
Output & | out | |||
) | [inline] |
Definition at line 74 of file multi_intersection.hh.
References AllIntersection().
void util::AllIntersection | ( | std::vector< boost::iterator_range< Iterator > > & | sets, | |
Output & | out, | |||
const Less | less | |||
) | [inline] |
Definition at line 63 of file multi_intersection.hh.
References util::detail::FirstIntersectionSorted(), and sort().
Referenced by lm::vocab::Multiple::AddNGram(), and AllIntersection().
bool util::BinaryFind | ( | const Accessor & | accessor, | |
Iterator | begin, | |||
Iterator | end, | |||
const typename Accessor::Key | key, | |||
Iterator & | out | |||
) | [inline] |
Definition at line 39 of file sorted_uniform.hh.
void util::BitPackingSanity | ( | ) |
Definition at line 24 of file bit_packing.cc.
References util::FloatEnc::i, ReadInt57(), UTIL_THROW, and WriteInt57().
Referenced by lm::ngram::trie::BitPacked::BaseInit().
uint8_t util::BitPackShift | ( | uint8_t | bit, | |
uint8_t | ||||
) | [inline] |
Definition at line 37 of file bit_packing.hh.
Referenced by ReadFloat32(), ReadInt25(), ReadInt57(), ReadNonPositiveFloat31(), WriteInt25(), and WriteInt57().
bool util::BoundedSortedUniformFind | ( | const Accessor & | accessor, | |
Iterator | before_it, | |||
typename Accessor::Key | before_v, | |||
Iterator | after_it, | |||
typename Accessor::Key | after_v, | |||
const typename Accessor::Key | key, | |||
Iterator & | out | |||
) | [inline] |
Definition at line 64 of file sorted_uniform.hh.
Referenced by lm::ngram::SortedVocabulary::Index().
std::size_t util::CheckOverflow | ( | uint64_t | value | ) | [inline] |
Definition at line 150 of file exception.hh.
Referenced by lm::ngram::BinaryFormat::LoadBinary(), and lm::ngram::BinaryFormat::SetupJustVocab().
std::size_t util::CheckOverflowInternal | ( | uint64_t | value | ) | [inline] |
std::size_t util::CheckOverflowInternal< 8 > | ( | uint64_t | value | ) | [inline] |
double util::CPUTime | ( | ) |
Definition at line 138 of file usage.cc.
Referenced by MosesTuning::Timer::get_elapsed_cpu_time_microseconds(), and MosesTuning::Timer::ToString().
int util::CreateOrThrow | ( | const char * | name | ) |
Definition at line 81 of file file.cc.
References UTIL_THROW_IF.
Referenced by lm::builder::Print< V >::DumpSeparateFiles(), main(), MapZeroedWrite(), lm::ngram::BinaryFormat::SetupJustVocab(), and lm::ModelBuffer::Sink().
int util::DupOrThrow | ( | int | fd | ) |
void util::ErsatzPRead | ( | int | fd, | |
void * | to_void, | |||
std::size_t | size, | |||
uint64_t | off | |||
) |
Definition at line 231 of file file.cc.
References NameFromFD(), UTIL_THROW_ARG, and UTIL_THROW_IF.
Referenced by ParallelRead(), lm::ngram::BinaryFormat::ReadForConfig(), and util::stream::PRead::Run().
void util::ErsatzPWrite | ( | int | fd, | |
const void * | from_void, | |||
std::size_t | size, | |||
uint64_t | off | |||
) |
Definition at line 266 of file file.cc.
References NameFromFD(), UTIL_THROW_ARG, and UTIL_THROW_IF.
Referenced by util::stream::PWriteAndRecycle::Run().
std::FILE * util::FDOpenOrThrow | ( | scoped_fd & | file | ) |
Definition at line 351 of file file.cc.
References util::scoped_fd::get(), util::scoped_fd::release(), and UTIL_THROW_IF_ARG.
Referenced by lm::ngram::trie::BuildTrie(), and FMakeTemp().
std::FILE * util::FDOpenReadOrThrow | ( | scoped_fd & | file | ) |
Definition at line 358 of file file.cc.
References util::scoped_fd::get(), util::scoped_fd::release(), and UTIL_THROW_IF_ARG.
boost::optional<typename std::iterator_traits<Iterator>::value_type> util::FirstIntersection | ( | std::vector< boost::iterator_range< Iterator > > & | sets | ) | [inline] |
Definition at line 59 of file multi_intersection.hh.
References FirstIntersection().
boost::optional<typename std::iterator_traits<Iterator>::value_type> util::FirstIntersection | ( | std::vector< boost::iterator_range< Iterator > > & | sets, | |
const Less | less | |||
) | [inline] |
Definition at line 52 of file multi_intersection.hh.
References util::detail::FirstIntersectionSorted(), and sort().
Referenced by FirstIntersection(), and lm::vocab::Union::PassNGram().
std::FILE * util::FMakeTemp | ( | const StringPiece & | base | ) |
Definition at line 513 of file file.cc.
References FDOpenOrThrow(), and MakeTemp().
Referenced by Moses::LexicalReorderingTableCreator::LexicalReorderingTableCreator(), and Moses::PhraseTableCreator::PhraseTableCreator().
void util::FSyncOrThrow | ( | int | fd | ) |
Definition at line 302 of file file.cc.
References UTIL_THROW_IF_ARG.
Referenced by lm::ngram::BinaryFormat::FinishFile().
uint64_t util::GuessPhysicalMemory | ( | ) |
void util::HugeMalloc | ( | std::size_t | size, | |
bool | zeroed, | |||
scoped_memory & | to | |||
) |
Definition at line 222 of file mmap.cc.
References util::scoped_memory::get(), util::scoped_memory::MALLOC_ALLOCATED, util::scoped_memory::reset(), and UTIL_THROW_IF.
Referenced by util::FilePiece::FilePiece(), lm::ngram::BinaryFormat::GrowForSearch(), HugeRealloc(), MapRead(), util::scoped_memory::scoped_memory(), and lm::ngram::BinaryFormat::SetupJustVocab().
void util::HugeRealloc | ( | std::size_t | to, | |
bool | zero_new, | |||
scoped_memory & | mem | |||
) |
Definition at line 243 of file mmap.cc.
References util::scoped_memory::get(), util::scoped_base< T, Closer >::get(), HugeMalloc(), util::scoped_memory::MALLOC_ALLOCATED, util::scoped_memory::MMAP_ALLOCATED, util::scoped_memory::MMAP_ROUND_UP_ALLOCATED, util::scoped_memory::NONE_ALLOCATED, util::scoped_base< T, Closer >::release(), util::scoped_memory::reset(), util::scoped_memory::size(), SizePage(), util::scoped_memory::source(), util::scoped_memory::steal(), UTIL_THROW, and UTIL_THROW_IF.
bool util::InputFileIsStdin | ( | StringPiece | path | ) |
bool util::InputPathIsStdin | ( | StringPiece | path | ) |
Does the given input file path denote standard input?
Returns true if, and only if, path is either "-" or "/dev/stdin".
Opening standard input as a file may need some special treatment for portability. There's a convention that a dash ("-") in place of an input file path denotes standard input, but opening "/dev/stdin" may need to be special as well.
void util::JointSort | ( | const KeyIter & | key_begin, | |
const KeyIter & | key_end, | |||
const ValueIter & | value_begin | |||
) | [inline] |
Definition at line 140 of file joint_sort.hh.
References JointSort().
void util::JointSort | ( | const KeyIter & | key_begin, | |
const KeyIter & | key_end, | |||
const ValueIter & | value_begin, | |||
const Less & | less | |||
) | [inline] |
Definition at line 133 of file joint_sort.hh.
References sort().
Referenced by JointSort().
int util::MakeTemp | ( | const StringPiece & | base | ) |
Definition at line 504 of file file.cc.
References StringPiece::data(), mkstemp_and_unlink(), StringPiece::size(), and UTIL_THROW_IF.
Referenced by FMakeTemp(), util::stream::Sort< Compare, Combine >::Merge(), and lm::ModelBuffer::Sink().
void * util::MallocOrThrow | ( | std::size_t | requested | ) |
Definition at line 25 of file scoped.cc.
Referenced by util::stream::MergingReader< Compare, Combine >::Run(), and util::stream::Chain::Start().
void * util::MapOrThrow | ( | std::size_t | size, | |
bool | for_write, | |||
int | flags, | |||
bool | prefault, | |||
int | fd, | |||
uint64_t | offset | |||
) |
Definition at line 104 of file mmap.cc.
References NULL, and UTIL_THROW_IF.
Referenced by Moses::MmapAllocator< T >::allocate(), util::Rolling::ExtractNonRolling(), MapRead(), and MapZeroedWrite().
void util::MapRead | ( | LoadMethod | method, | |
int | fd, | |||
uint64_t | offset, | |||
std::size_t | size, | |||
scoped_memory & | out | |||
) |
Definition at line 300 of file mmap.cc.
References util::scoped_memory::get(), HugeMalloc(), kFileFlags, LAZY, MapOrThrow(), util::scoped_memory::MMAP_ALLOCATED, PARALLEL_READ, ParallelRead(), POPULATE_OR_LAZY, POPULATE_OR_READ, READ, ReadOrThrow(), util::scoped_memory::reset(), and SeekOrThrow().
Referenced by lm::ngram::trie::BuildTrie(), lm::ngram::SortedVocabulary::ComputeRenumbering(), lm::ngram::IsBinaryFormat(), lm::ngram::BinaryFormat::LoadBinary(), and lm::VocabReconstitute::VocabReconstitute().
void * util::MapZeroedWrite | ( | const char * | name, | |
std::size_t | size, | |||
scoped_fd & | file | |||
) |
Definition at line 332 of file mmap.cc.
References CreateOrThrow(), util::scoped_fd::get(), MapZeroedWrite(), and util::scoped_fd::reset().
void * util::MapZeroedWrite | ( | int | fd, | |
std::size_t | size | |||
) |
Definition at line 326 of file mmap.cc.
References kFileFlags, MapOrThrow(), and ResizeOrThrow().
Referenced by MapZeroedWrite(), and lm::ngram::BinaryFormat::SetupJustVocab().
int util::mkstemp_and_unlink | ( | char * | tmpl | ) |
Definition at line 478 of file file.cc.
References UTIL_THROW_IF.
Referenced by MakeTemp().
uint64_t util::MurmurHash64A | ( | const void * | key, | |
std::size_t | len, | |||
uint64_t | seed | |||
) |
Definition at line 26 of file murmur_hash.cc.
Referenced by lm::ngram::detail::HashForVocab().
uint64_t util::MurmurHash64B | ( | const void * | key, | |
std::size_t | len, | |||
uint64_t | seed | |||
) |
uint64_t util::MurmurHashNative | ( | const void * | key, | |
std::size_t | len, | |||
uint64_t | seed | |||
) |
Definition at line 171 of file murmur_hash.cc.
Referenced by lm::phrase::Substrings::AddPhrase(), Moses::GlobalLexicalModelUnlimited::EvaluateWhenApplied(), lm::ngram::GrowableVocab< NewWordAction >::FindOrInsert(), Moses::Word::hash(), Moses::DALMChartState::hash(), MosesTuning::hash_value(), lm::ngram::hash_value(), lm::phrase::detail::MakeHashes(), Moses::Murmur::operator()(), Moses::HashSparseReorderingFeatureKey::operator()(), MosesTuning::NgramHash::operator()(), and lm::phrase::ReadMultiple().
std::string util::NameFromFD | ( | int | fd | ) |
Definition at line 559 of file file.cc.
Referenced by ErsatzPRead(), ErsatzPWrite(), ReadOrThrow(), and util::stream::LineInput::Run().
void util::NormalizeTempPrefix | ( | std::string & | base | ) |
int util::OpenReadOrThrow | ( | const char * | name | ) |
Definition at line 71 of file file.cc.
References UTIL_THROW_IF.
Referenced by lm::ngram::detail::GenericModel< Search, VocabularyT >::GenericModel(), main(), lm::ModelBuffer::ModelBuffer(), lm::np::Model::Recognize(), and lm::ngram::RecognizeBinary().
ProxyIterator<Proxy> util::operator+ | ( | std::ptrdiff_t | amount, | |
const ProxyIterator< Proxy > & | it | |||
) | [inline] |
Definition at line 95 of file proxy_iterator.hh.
Except::template ExceptionTag< Except & >::Identity util::operator<< | ( | Except & | e, | |
const Data & | data | |||
) | [inline] |
Definition at line 48 of file exception.hh.
bool util::OutputFileIsStdout | ( | StringPiece | path | ) |
bool util::OutputPathIsStdout | ( | StringPiece | path | ) |
Does the given output file path denote standard output?
Returns true if, and only if, path is either "-" or "/dev/stdout".
Opening standard output as a file may need some special treatment for portability. There's a convention that a dash ("-") in place of an output file path denotes standard output, but opening "/dev/stdout" may need to be special as well.
void util::ParallelRead | ( | int | fd, | |
void * | to, | |||
std::size_t | amount, | |||
uint64_t | offset | |||
) |
Definition at line 64 of file parallel_read.cc.
References ErsatzPRead().
Referenced by MapRead().
uint64_t util::ParseSize | ( | const std::string & | arg | ) |
std::size_t util::PartialRead | ( | int | fd, | |
void * | to, | |||
std::size_t | amount | |||
) |
Definition at line 156 of file file.cc.
References NULL, read(), UTIL_THROW, and UTIL_THROW_IF_ARG.
Referenced by ReadOrEOF(), and ReadOrThrow().
void util::posix_tmp_template | ( | char * | buf | ) |
POSIX helper: create template for temporary filename.
Writes the template into buf, which must have room for at least PATH_MAX bytes. The function fails if the template is too long.
Definition at line 64 of file tempfile.hh.
References temp_location().
Referenced by util::temp_dir::temp_dir(), and util::temp_file::temp_file().
void util::PrintUsage | ( | std::ostream & | out | ) |
Definition at line 160 of file usage.cc.
References WallTime().
Referenced by batch_run(), and main().
T util::rand | ( | ) | [inline] |
Return a pseudorandom number between 0 and RAND_MAX inclusive.
Initialize (seed) the randomizer before starting to call this.
Definition at line 174 of file random.hh.
References util::internal::rand_int(), and T.
Referenced by main(), rand01(), util::internal::rand_int(), MosesTuning::randomStats(), and doc::savernd().
Return a pseudorandom number in the half-open interval [bottom, top).
Generates a value between "bottom" (inclusive) and "top" (exclusive), assuming that (top - bottom) <= RAND_MAX.
Definition at line 185 of file random.hh.
Referenced by Moses::Credit::Credit(), main(), sapt::TSA_tree_iterator< TKN >::randomSample(), and Moses::randomSample().
Return a pseudorandom number in the open interval [bottom, top].
Generates a value between "bottom" and "top" inclusive, assuming that (top - bottom) < RAND_MAX.
Definition at line 196 of file random.hh.
Referenced by Moses::Manager::CalcLatticeSamples(), and MosesTuning::Point::Randomize().
void util::rand_init | ( | ) |
Initialize randomizer based on current time.
Call this to make the randomizer return hard-to-predict numbers. It won't produce high-grade randomness, but enough to make the program act differently on different runs.
The seed will be based on the current time in seconds. So calling it twice within the same second will just reset the randomizer to where it was before. Don't do that.
Definition at line 29 of file random.cc.
References NULL, and rand_init().
void util::rand_init | ( | unsigned | int | ) |
Thread-safe, cross-platform random number generator.
This is not for proper security-grade randomness, but should be "good enough" for producing arbitrary values of various numeric types.
Before starting, call rand_init() to seed the randomizer. There is no need to do this more than once; in fact doing it more often is likely to make the randomizer less effective. Once that is done, call the rand(), rand_excl(), and rand_incl() functions as needed to generate pseudo-random numbers.
Probability distribution is roughly uniform, but for integral types is skewed slightly towards lower numbers depending on how close "top" comes to RAND_MAX.
For floating-point types, resolution is limited; there will actually be only RAND_MAX different possible values. Initialize randomizer with a fixed seed.
After this, unless the randomizer gets seeded again, consecutive calls to the random functions will return a sequence of pseudo-random numbers determined by the seed. Every time the randomizer is seeded with this same seed, it will again start returning the same sequence of numbers.
Definition at line 22 of file random.cc.
Referenced by batch_run(), main(), and rand_init().
float util::ReadFloat32 | ( | const void * | base, | |
uint64_t | bit_off | |||
) | [inline] |
Definition at line 109 of file bit_packing.hh.
References BitPackShift(), util::FloatEnc::f, util::FloatEnc::i, and ReadOff().
Referenced by lm::ngram::DontQuantize::MiddlePointer::Backoff().
uint32_t util::ReadInt25 | ( | const void * | base, | |
uint64_t | bit_off, | |||
uint8_t | length, | |||
uint32_t | mask | |||
) | [inline] |
Definition at line 83 of file bit_packing.hh.
References BitPackShift().
Referenced by lm::ngram::SeparatelyQuantize::MiddlePointer::Backoff(), lm::ngram::SeparatelyQuantize::LongestPointer::Prob(), and lm::ngram::SeparatelyQuantize::MiddlePointer::Prob().
uint64_t util::ReadInt57 | ( | const void * | base, | |
uint64_t | bit_off, | |||
uint8_t | length, | |||
uint64_t | mask | |||
) | [inline] |
Definition at line 63 of file bit_packing.hh.
References BitPackShift(), and ReadOff().
Referenced by BitPackingSanity(), lm::ngram::trie::ArrayBhiksha::ReadNext(), and lm::ngram::trie::DontBhiksha::ReadNext().
float util::ReadNonPositiveFloat31 | ( | const void * | base, | |
uint64_t | bit_off | |||
) | [inline] |
Definition at line 136 of file bit_packing.hh.
References BitPackShift(), util::FloatEnc::f, util::FloatEnc::i, kSignBit, and ReadOff().
Referenced by lm::ngram::DontQuantize::LongestPointer::Prob(), and lm::ngram::DontQuantize::MiddlePointer::Prob().
uint64_t util::ReadOff | ( | const void * | base, | |
uint64_t | bit_off | |||
) | [inline] |
Definition at line 48 of file bit_packing.hh.
Referenced by ReadFloat32(), ReadInt57(), and ReadNonPositiveFloat31().
std::size_t util::ReadOrEOF | ( | int | fd, | |
void * | to_void, | |||
std::size_t | amount | |||
) |
Definition at line 190 of file file.cc.
References PartialRead().
Referenced by util::stream::Read::Run().
void util::ReadOrThrow | ( | int | fd, | |
void * | to_void, | |||
std::size_t | amount | |||
) |
Definition at line 180 of file file.cc.
References NameFromFD(), PartialRead(), and UTIL_THROW_IF.
Referenced by util::stream::Offsets::FinishedAppending(), MapRead(), util::stream::Offsets::NextSize(), lm::ngram::ReadHeader(), and lm::np::Model::Recognize().
uint8_t util::RequiredBits | ( | uint64_t | max_value | ) |
Definition at line 17 of file bit_packing.cc.
Referenced by lm::ngram::trie::BitPacked::BaseInit(), lm::ngram::trie::BitPacked::BaseSize(), util::BitsMask::FromMax(), lm::ngram::trie::DontBhiksha::InlineBits(), and lm::ngram::trie::ArrayBhiksha::InlineBits().
void util::ResizeOrThrow | ( | int | fd, | |
uint64_t | to | |||
) |
Definition at line 121 of file file.cc.
References UTIL_THROW_IF_ARG.
Referenced by lm::ngram::BinaryFormat::GrowForSearch(), MapZeroedWrite(), util::stream::Sort< Compare, Combine >::Merge(), util::stream::Offsets::Reset(), util::stream::PWriteAndRecycle::Run(), and lm::ngram::BinaryFormat::SetupJustVocab().
void util::SeekOrThrow | ( | int | fd, | |
uint64_t | off | |||
) |
Definition at line 339 of file file.cc.
Referenced by util::stream::Offsets::FinishedAppending(), lm::ngram::BinaryFormat::FinishFile(), MapRead(), util::stream::Sort< Compare, Combine >::Merge(), lm::ngram::ReadHeader(), util::stream::Offsets::Reset(), util::FileStream::seekp(), util::stream::FileBuffer::Sink(), util::stream::Sort< Compare, Combine >::StealCompleted(), lm::ngram::WriteWordsWrapper::Write(), and lm::ngram::BinaryFormat::WriteVocabWords().
void util::SetSign | ( | float & | to | ) | [inline] |
Definition at line 122 of file bit_packing.hh.
References util::FloatEnc::f, util::FloatEnc::i, and kSignBit.
Referenced by lm::ngram::MaxRestBuild::SetRest().
SizedIterator util::SizedIt | ( | void * | ptr, | |
std::size_t | size | |||
) | [inline] |
Definition at line 93 of file sized_iterator.hh.
Referenced by util::stream::BlockSorter< Compare >::Run().
uint64_t util::SizeFile | ( | int | fd | ) |
Definition at line 91 of file file.cc.
References kBadSize.
Referenced by lm::ngram::IsBinaryFormat(), lm::ngram::BinaryFormat::LoadBinary(), and SizeOrThrow().
uint64_t util::SizeOrThrow | ( | int | fd | ) |
Definition at line 115 of file file.cc.
References kBadSize, SizeFile(), and UTIL_THROW_IF_ARG.
Referenced by lm::ngram::SortedVocabulary::ComputeRenumbering(), util::stream::Sort< Compare, Combine >::Merge(), util::stream::PRead::Run(), util::stream::Sort< Compare, Combine >::Size(), util::stream::FileBuffer::Size(), and lm::VocabReconstitute::VocabReconstitute().
std::size_t util::SizePage | ( | ) |
Definition at line 30 of file mmap.cc.
Referenced by util::Rolling::ExtractNonRolling(), HugeRealloc(), and util::scoped_memory::reset().
bool util::SortedUniformFind | ( | const Accessor & | accessor, | |
Iterator | begin, | |||
Iterator | end, | |||
const typename Accessor::Key | key, | |||
Iterator & | out | |||
) | [inline] |
Definition at line 86 of file sorted_uniform.hh.
void util::SyncOrThrow | ( | void * | start, | |
size_t | length | |||
) |
Definition at line 133 of file mmap.cc.
References UTIL_THROW_IF.
Referenced by lm::ngram::BinaryFormat::FinishFile(), and util::scoped_mmap::~scoped_mmap().
std::string util::temp_location | ( | ) |
Obtain a directory for temporary files, e.g. /tmp.
Definition at line 26 of file tempfile.hh.
Referenced by posix_tmp_template().
std::vector<std::string> util::tokenize | ( | const std::string & | input | ) | [inline] |
Split input string into a series of tokens.
Like tokenize(const char[]), but takes a std::string.
Definition at line 44 of file tokenize.hh.
References tokenize().
std::vector<std::string> util::tokenize | ( | const char | input[] | ) | [inline] |
Split input text into a series of tokens.
Splits on spaces and tabs, no other whitespace characters, and is not locale-sensitive.
The spaces themselves are not included. A sequence of consecutive space/tab characters count as one.
Definition at line 18 of file tokenize.hh.
Referenced by MosesTraining::SentenceAlignment::create(), MosesTraining::LexicalTable::load(), MosesTraining::Domain::load(), main(), MosesTraining::Syntax::XmlTreeParser::Parse(), processFiles(), MosesTraining::SentenceAlignmentWithSyntax::processSourceSentence(), MosesTraining::SentenceAlignment::processSourceSentence(), MosesTraining::SentenceAlignmentWithSyntax::processTargetSentence(), MosesTraining::SentenceAlignment::processTargetSentence(), reverseAlignment(), and tokenize().
char* util::ToString | ( | bool | value, | |
char * | to | |||
) | [inline] |
Definition at line 23 of file integer_to_string.hh.
char * util::ToString | ( | const void * | v, | |
char * | to | |||
) |
Definition at line 646 of file integer_to_string.cc.
char * util::ToString | ( | uint16_t | value, | |
char * | to | |||
) |
Definition at line 637 of file integer_to_string.cc.
References ToString().
char * util::ToString | ( | int16_t | value, | |
char * | to | |||
) |
Definition at line 634 of file integer_to_string.cc.
References ToString().
char * util::ToString | ( | int64_t | value, | |
char * | to | |||
) |
Definition at line 624 of file integer_to_string.cc.
References ToString().
char * util::ToString | ( | int32_t | value, | |
char * | to | |||
) |
Definition at line 615 of file integer_to_string.cc.
References ToString().
char * util::ToString | ( | uint64_t | value, | |
char * | buffer | |||
) |
Definition at line 455 of file integer_to_string.cc.
char * util::ToString | ( | uint32_t | value, | |
char * | buffer | |||
) |
Definition at line 382 of file integer_to_string.cc.
char * util::ToString | ( | float | value, | |
char * | to | |||
) |
Definition at line 17 of file float_to_string.cc.
References util::ToStringBuf< float >::kBytes, and double_conversion::StringBuilder::position().
char * util::ToString | ( | double | value, | |
char * | to | |||
) |
Definition at line 11 of file float_to_string.cc.
References util::ToStringBuf< double >::kBytes, and double_conversion::StringBuilder::position().
Referenced by Moses::Manager::CalcDecoderStatistics(), main(), Moses::PhraseDictionaryDynamicCacheBased::Print(), and ToString().
void util::UnmapOrThrow | ( | void * | start, | |
size_t | length | |||
) |
Definition at line 141 of file mmap.cc.
References UTIL_THROW_IF.
Referenced by Moses::MmapAllocator< T >::deallocate(), Moses::MmapAllocator< T >::~MmapAllocator(), and util::scoped_mmap::~scoped_mmap().
void util::UnsetSign | ( | float & | to | ) | [inline] |
Definition at line 129 of file bit_packing.hh.
References util::FloatEnc::f, util::FloatEnc::i, and kSignBit.
Referenced by lm::ngram::LowerRestBuild< Model >::MarkExtends(), lm::ngram::MaxRestBuild::MarkExtends(), and lm::ngram::NoRestBuild::MarkExtends().
void util::WaitSemaphore | ( | Semaphore & | on | ) | [inline] |
Definition at line 59 of file pcqueue.hh.
Referenced by util::PCQueue< Block >::Consume(), and util::PCQueue< Block >::Produce().
double util::WallTime | ( | ) |
Definition at line 134 of file usage.cc.
Referenced by Moses::Timer::get_elapsed_time(), sapt::imTSA< TOKEN >::imTSA(), PrintUsage(), Moses::Timer::start(), and Moses::Timer::stop().
T util::wide_rand | ( | ) | [inline] |
Return a pseudorandom number which may be larger than RAND_MAX.
The requested type must be integral, and its size must be an even multiple of the size of an int. The return value will combine one or more random ints into a single value, which could get quite large.
The result is nonnegative. Because the constituent ints are also nonnegative, the most significant bit in each of the ints will be zero, so for a wider type, there will be "gaps" in the range of possible outputs.
Definition at line 201 of file random.hh.
References T.
Return a pseudorandom number in [bottom, top), not limited to RAND_MAX.
Works like wide_rand(), but if the requested type is wider than an int, it accommodates larger value ranges than an int can represent.
Definition at line 212 of file random.hh.
References wide_rand_excl().
Return a pseudorandom number in [0, top), not limited to RAND_MAX.
Works like wide_rand(), but if the requested type is wider than an int, it accommodates larger top values than an int can represent.
Definition at line 206 of file random.hh.
Referenced by wide_rand_excl().
Return a pseudorandom number in [bottom, top], not limited to RAND_MAX.
Works like wide_rand(), but if the requested type is wider than an int, it accommodates larger top values than an int can represent.
Definition at line 223 of file random.hh.
References wide_rand_incl().
Return a pseudorandom number in [0, top], not limited to RAND_MAX.
Works like wide_rand(), but if the requested type is wider than an int, it accommodates larger top values than an int can represent.
Definition at line 217 of file random.hh.
Referenced by wide_rand_incl().
void util::WriteFloat32 | ( | void * | base, | |
uint64_t | bit_off, | |||
float | value | |||
) | [inline] |
Definition at line 114 of file bit_packing.hh.
References util::FloatEnc::f, util::FloatEnc::i, and WriteInt57().
Referenced by lm::ngram::DontQuantize::MiddlePointer::Write().
void util::WriteInt25 | ( | void * | base, | |
uint64_t | bit_off, | |||
uint8_t | length, | |||
uint32_t | value | |||
) | [inline] |
Definition at line 94 of file bit_packing.hh.
References BitPackShift().
Referenced by lm::ngram::SeparatelyQuantize::LongestPointer::Write().
void util::WriteInt57 | ( | void * | base, | |
uint64_t | bit_off, | |||
uint8_t | length, | |||
uint64_t | value | |||
) | [inline] |
Definition at line 69 of file bit_packing.hh.
References BitPackShift().
Referenced by BitPackingSanity(), lm::ngram::trie::BitPackedLongest::Insert(), lm::ngram::trie::BitPackedMiddle< Bhiksha >::Insert(), lm::ngram::SeparatelyQuantize::MiddlePointer::Write(), WriteFloat32(), lm::ngram::trie::ArrayBhiksha::WriteNext(), lm::ngram::trie::DontBhiksha::WriteNext(), and WriteNonPositiveFloat31().
void util::WriteNonPositiveFloat31 | ( | void * | base, | |
uint64_t | bit_off, | |||
float | value | |||
) | [inline] |
Definition at line 143 of file bit_packing.hh.
References util::FloatEnc::f, util::FloatEnc::i, kSignBit, and WriteInt57().
Referenced by lm::ngram::DontQuantize::LongestPointer::Write(), and lm::ngram::DontQuantize::MiddlePointer::Write().
void util::WriteOrThrow | ( | FILE * | to, | |
const void * | data, | |||
std::size_t | size | |||
) |
void util::WriteOrThrow | ( | int | fd, | |
const void * | data_void, | |||
std::size_t | size | |||
) |
Definition at line 202 of file file.cc.
References UTIL_THROW_IF_ARG, and write().
Referenced by util::stream::Offsets::Append(), util::stream::Offsets::FinishedAppending(), lm::ngram::BinaryFormat::FinishFile(), util::FileStream::flush(), lm::ngram::trie::RecordReader::Overwrite(), util::stream::WriteAndRecycle::Run(), util::stream::Write::Run(), util::FileStream::write(), lm::ngram::WriteWordsWrapper::Write(), and lm::ngram::BinaryFormat::WriteVocabWords().
const uint64_t util::kBadSize = (uint64_t)-1 |
Definition at line 104 of file file.hh.
Referenced by lm::ngram::IsBinaryFormat(), lm::ngram::BinaryFormat::LoadBinary(), SizeFile(), and SizeOrThrow().
const int util::kFileFlags |
Initial value:
MAP_SHARED
Definition at line 94 of file mmap.cc.
Referenced by util::Rolling::ExtractNonRolling(), MapRead(), and MapZeroedWrite().
const char util::kProgressBanner = "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n" |
Definition at line 12 of file ersatz_progress.cc.
Referenced by util::ErsatzProgress::ErsatzProgress(), and util::stream::MultiProgress::SetTarget().
const uint32_t util::kSignBit = 0x80000000 |
Definition at line 120 of file bit_packing.hh.
Referenced by lm::ngram::GenericProbingProxy< Weights >::IndependentLeft(), lm::ngram::GenericProbingProxy< Weights >::Prob(), ReadNonPositiveFloat31(), SetSign(), UnsetSign(), and WriteNonPositiveFloat31().
const bool util::kSpaces = {0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} |
Definition at line 33 of file file_piece.cc.