util Namespace Reference


Namespaces

namespace  CheckOffT< 8 >
namespace  detail
namespace  internal
 Implementation detail. For the random module's internal use only.
namespace  stream

Classes

union  FloatEnc
struct  BitsMask
struct  BitAddress
class  ErsatzProgress
class  Exception
class  ErrnoException
class  FileOpenException
class  OverflowException
class  FakeOStream
class  scoped_fd
struct  scoped_FILE_closer
class  FDException
class  EndOfFileException
class  ParseNumberException
class  FilePiece
class  FileStream
class  FixedArray
struct  ToStringBuf< double >
struct  ToStringBuf< float >
struct  ToStringBuf< bool >
struct  ToStringBuf< uint16_t >
struct  ToStringBuf< int16_t >
struct  ToStringBuf< uint32_t >
struct  ToStringBuf< int32_t >
struct  ToStringBuf< uint64_t >
struct  ToStringBuf< int64_t >
struct  ToStringBuf< const void * >
class  PairedIterator
class  scoped_mmap
class  scoped_memory
class  Rolling
class  PCQueue
class  Pool
class  ProbingSizeException
struct  IdentityHash
class  DivMod
class  Power2Mod
class  ProbingHashTable
class  AutoProbing
class  ProxyIterator
class  ReadBase
class  CompressedException
class  GZException
class  BZException
class  XZException
class  ReadCompressed
class  MallocException
class  scoped_base
class  scoped
class  scoped< void, Closer >
struct  scoped_c_forward
class  scoped_c
class  scoped_malloc
struct  scoped_delete_array_forward
class  scoped_array
struct  scoped_delete_forward
class  scoped_ptr
class  SizedInnerIterator
class  SizedProxy
class  SizedCompare
class  IdentityAccessor
struct  Pivot64
struct  Pivot32
struct  PivotSelect< 8 >
struct  PivotSelect< 4 >
struct  PivotSelect< 2 >
class  StringStream
class  temp_dir
class  temp_file
class  Worker
class  ThreadPool
class  OutOfTokens
class  SingleCharacter
class  MultiCharacter
class  AnyCharacter
class  BoolCharacter
class  AnyCharacterLast
class  TokenIter

Typedefs

typedef scoped< std::FILE,
scoped_FILE_closer
scoped_FILE
typedef
boost::interprocess::interprocess_semaphore 
Semaphore
typedef ProxyIterator< SizedProxySizedIterator

Enumerations

enum  { kToStringMaxBytes = 20 }
enum  LoadMethod {
  LAZY, POPULATE_OR_LAZY, POPULATE_OR_READ, READ,
  PARALLEL_READ
}

Functions

uint8_t RequiredBits (uint64_t max_value)
void BitPackingSanity ()
uint8_t BitPackShift (uint8_t bit, uint8_t)
uint64_t ReadOff (const void *base, uint64_t bit_off)
uint64_t ReadInt57 (const void *base, uint64_t bit_off, uint8_t length, uint64_t mask)
void WriteInt57 (void *base, uint64_t bit_off, uint8_t length, uint64_t value)
uint32_t ReadInt25 (const void *base, uint64_t bit_off, uint8_t length, uint32_t mask)
void WriteInt25 (void *base, uint64_t bit_off, uint8_t length, uint32_t value)
float ReadFloat32 (const void *base, uint64_t bit_off)
void WriteFloat32 (void *base, uint64_t bit_off, float value)
void SetSign (float &to)
void UnsetSign (float &to)
float ReadNonPositiveFloat31 (const void *base, uint64_t bit_off)
void WriteNonPositiveFloat31 (void *base, uint64_t bit_off, float value)
template<class Except , class Data >
Except::template ExceptionTag
< Except & >::Identity 
operator<< (Except &e, const Data &data)
template<unsigned len>
std::size_t CheckOverflowInternal (uint64_t value)
template<>
std::size_t CheckOverflowInternal< 8 > (uint64_t value)
std::size_t CheckOverflow (uint64_t value)
bool InputFileIsStdin (StringPiece path)
bool OutputFileIsStdout (StringPiece path)
int OpenReadOrThrow (const char *name)
int CreateOrThrow (const char *name)
uint64_t SizeFile (int fd)
uint64_t SizeOrThrow (int fd)
void ResizeOrThrow (int fd, uint64_t to)
std::size_t PartialRead (int fd, void *to, std::size_t amount)
void ReadOrThrow (int fd, void *to_void, std::size_t amount)
std::size_t ReadOrEOF (int fd, void *to_void, std::size_t amount)
void WriteOrThrow (int fd, const void *data_void, std::size_t size)
void WriteOrThrow (FILE *to, const void *data, std::size_t size)
void ErsatzPRead (int fd, void *to_void, std::size_t size, uint64_t off)
void ErsatzPWrite (int fd, const void *from_void, std::size_t size, uint64_t off)
void FSyncOrThrow (int fd)
void SeekOrThrow (int fd, uint64_t off)
void AdvanceOrThrow (int fd, int64_t off)
void SeekEnd (int fd)
std::FILE * FDOpenOrThrow (scoped_fd &file)
std::FILE * FDOpenReadOrThrow (scoped_fd &file)
int mkstemp_and_unlink (char *tmpl)
void NormalizeTempPrefix (std::string &base)
int MakeTemp (const StringPiece &base)
std::FILE * FMakeTemp (const StringPiece &base)
int DupOrThrow (int fd)
std::string NameFromFD (int fd)
bool InputPathIsStdin (StringPiece path)
bool OutputPathIsStdout (StringPiece path)
char * ToString (double value, char *to)
char * ToString (float value, char *to)
char * ToString (uint32_t value, char *buffer)
char * ToString (uint64_t value, char *buffer)
char * ToString (int32_t value, char *to)
char * ToString (int64_t value, char *to)
char * ToString (int16_t value, char *to)
char * ToString (uint16_t value, char *to)
char * ToString (const void *v, char *to)
char * ToString (bool value, char *to)
template<class KeyIter , class ValueIter , class Less >
void JointSort (const KeyIter &key_begin, const KeyIter &key_end, const ValueIter &value_begin, const Less &less)
template<class KeyIter , class ValueIter >
void JointSort (const KeyIter &key_begin, const KeyIter &key_end, const ValueIter &value_begin)
std::size_t SizePage ()
void * MapOrThrow (std::size_t size, bool for_write, int flags, bool prefault, int fd, uint64_t offset)
void SyncOrThrow (void *start, size_t length)
void UnmapOrThrow (void *start, size_t length)
void HugeMalloc (std::size_t size, bool zeroed, scoped_memory &to)
void HugeRealloc (std::size_t to, bool zero_new, scoped_memory &mem)
void MapRead (LoadMethod method, int fd, uint64_t offset, std::size_t size, scoped_memory &out)
void * MapZeroedWrite (int fd, std::size_t size)
void * MapZeroedWrite (const char *name, std::size_t size, scoped_fd &file)
template<class Iterator , class Less >
boost::optional< typename
std::iterator_traits< Iterator >
::value_type > 
FirstIntersection (std::vector< boost::iterator_range< Iterator > > &sets, const Less less)
template<class Iterator >
boost::optional< typename
std::iterator_traits< Iterator >
::value_type > 
FirstIntersection (std::vector< boost::iterator_range< Iterator > > &sets)
template<class Iterator , class Output , class Less >
void AllIntersection (std::vector< boost::iterator_range< Iterator > > &sets, Output &out, const Less less)
template<class Iterator , class Output >
void AllIntersection (std::vector< boost::iterator_range< Iterator > > &sets, Output &out)
uint64_t MurmurHash64A (const void *key, std::size_t len, uint64_t seed)
uint64_t MurmurHash64B (const void *key, std::size_t len, uint64_t seed)
uint64_t MurmurHashNative (const void *key, std::size_t len, uint64_t seed)
void ParallelRead (int fd, void *to, std::size_t amount, uint64_t offset)
void WaitSemaphore (Semaphore &on)
template<class Proxy >
ProxyIterator< Proxy > operator+ (std::ptrdiff_t amount, const ProxyIterator< Proxy > &it)
void rand_init (unsigned int seed)
void rand_init ()
template<typename T >
T rand ()
template<typename T >
T rand_excl (T bottom, T top)
template<typename T >
T rand_excl (T top)
template<typename T >
T rand_incl (T bottom, T top)
template<typename T >
T rand_incl (T top)
template<typename T >
T wide_rand ()
template<typename T >
T wide_rand_excl (T top)
template<typename T >
T wide_rand_excl (T bottom, T top)
template<typename T >
T wide_rand_incl (T top)
template<typename T >
T wide_rand_incl (T bottom, T top)
void * MallocOrThrow (std::size_t requested)
void * CallocOrThrow (std::size_t requested)
void AdviseHugePages (const void *addr, std::size_t size)
SizedIterator SizedIt (void *ptr, std::size_t size)
template<class Iterator , class Accessor >
bool BinaryFind (const Accessor &accessor, Iterator begin, Iterator end, const typename Accessor::Key key, Iterator &out)
template<class Iterator , class Accessor , class Pivot >
bool BoundedSortedUniformFind (const Accessor &accessor, Iterator before_it, typename Accessor::Key before_v, Iterator after_it, typename Accessor::Key after_v, const typename Accessor::Key key, Iterator &out)
template<class Iterator , class Accessor , class Pivot >
bool SortedUniformFind (const Accessor &accessor, Iterator begin, Iterator end, const typename Accessor::Key key, Iterator &out)
std::string temp_location ()
 Obtain a directory for temporary files, e.g. /tmp.
void posix_tmp_template (char *buf)
std::vector< std::string > tokenize (const char input[])
std::vector< std::string > tokenize (const std::string &input)
double WallTime ()
double CPUTime ()
uint64_t RSSMax ()
void PrintUsage (std::ostream &out)
uint64_t GuessPhysicalMemory ()
uint64_t ParseSize (const std::string &arg)

Variables

const uint32_t kSignBit = 0x80000000
const char kProgressBanner [] = "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n"
const uint64_t kBadSize = (uint64_t)-1
const bool kSpaces [256] = {0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}
const int kFileFlags

Detailed Description

For loading from the feature data file.


Typedef Documentation

Definition at line 48 of file file.hh.

typedef boost::interprocess::interprocess_semaphore util::Semaphore

Definition at line 57 of file pcqueue.hh.

Definition at line 91 of file sized_iterator.hh.


Enumeration Type Documentation

anonymous enum

Enumerator:
kToStringMaxBytes 

Definition at line 62 of file integer_to_string.hh.

Enumerator:
LAZY 
POPULATE_OR_LAZY 
POPULATE_OR_READ 
READ 
PARALLEL_READ 

Definition at line 131 of file mmap.hh.


Function Documentation

void util::AdvanceOrThrow ( int  fd,
int64_t  off 
)

Definition at line 343 of file file.cc.

void util::AdviseHugePages ( const void *  addr,
std::size_t  size 
)

Definition at line 37 of file scoped.cc.

Referenced by lm::ngram::BinaryFormat::GrowForSearch(), and lm::ngram::BinaryFormat::SetupJustVocab().

Here is the caller graph for this function:

template<class Iterator , class Output >
void util::AllIntersection ( std::vector< boost::iterator_range< Iterator > > &  sets,
Output &  out 
) [inline]

Definition at line 74 of file multi_intersection.hh.

References AllIntersection().

Here is the call graph for this function:

template<class Iterator , class Output , class Less >
void util::AllIntersection ( std::vector< boost::iterator_range< Iterator > > &  sets,
Output &  out,
const Less  less 
) [inline]

Definition at line 63 of file multi_intersection.hh.

References util::detail::FirstIntersectionSorted(), and sort().

Referenced by lm::vocab::Multiple::AddNGram(), and AllIntersection().

Here is the call graph for this function:

Here is the caller graph for this function:

template<class Iterator , class Accessor >
bool util::BinaryFind ( const Accessor &  accessor,
Iterator  begin,
Iterator  end,
const typename Accessor::Key  key,
Iterator &  out 
) [inline]

Definition at line 39 of file sorted_uniform.hh.

void util::BitPackingSanity (  ) 

Definition at line 24 of file bit_packing.cc.

References util::FloatEnc::i, ReadInt57(), UTIL_THROW, and WriteInt57().

Referenced by lm::ngram::trie::BitPacked::BaseInit().

Here is the call graph for this function:

Here is the caller graph for this function:

uint8_t util::BitPackShift ( uint8_t  bit,
uint8_t   
) [inline]

Definition at line 37 of file bit_packing.hh.

Referenced by ReadFloat32(), ReadInt25(), ReadInt57(), ReadNonPositiveFloat31(), WriteInt25(), and WriteInt57().

Here is the caller graph for this function:

template<class Iterator , class Accessor , class Pivot >
bool util::BoundedSortedUniformFind ( const Accessor &  accessor,
Iterator  before_it,
typename Accessor::Key  before_v,
Iterator  after_it,
typename Accessor::Key  after_v,
const typename Accessor::Key  key,
Iterator &  out 
) [inline]

Definition at line 64 of file sorted_uniform.hh.

Referenced by lm::ngram::SortedVocabulary::Index().

Here is the caller graph for this function:

void * util::CallocOrThrow ( std::size_t  requested  ) 

Definition at line 29 of file scoped.cc.

std::size_t util::CheckOverflow ( uint64_t  value  )  [inline]

Definition at line 150 of file exception.hh.

Referenced by lm::ngram::BinaryFormat::LoadBinary(), and lm::ngram::BinaryFormat::SetupJustVocab().

Here is the caller graph for this function:

template<unsigned len>
std::size_t util::CheckOverflowInternal ( uint64_t  value  )  [inline]

Definition at line 141 of file exception.hh.

References UTIL_THROW_IF.

template<>
std::size_t util::CheckOverflowInternal< 8 > ( uint64_t  value  )  [inline]

double util::CPUTime (  ) 

Definition at line 138 of file usage.cc.

Referenced by MosesTuning::Timer::get_elapsed_cpu_time_microseconds(), and MosesTuning::Timer::ToString().

Here is the caller graph for this function:

int util::CreateOrThrow ( const char *  name  ) 

Definition at line 81 of file file.cc.

References UTIL_THROW_IF.

Referenced by lm::builder::Print< V >::DumpSeparateFiles(), main(), MapZeroedWrite(), lm::ngram::BinaryFormat::SetupJustVocab(), and lm::ModelBuffer::Sink().

Here is the caller graph for this function:

int util::DupOrThrow ( int  fd  ) 

Definition at line 518 of file file.cc.

References UTIL_THROW_IF_ARG.

void util::ErsatzPRead ( int  fd,
void *  to_void,
std::size_t  size,
uint64_t  off 
)

Definition at line 231 of file file.cc.

References NameFromFD(), UTIL_THROW_ARG, and UTIL_THROW_IF.

Referenced by ParallelRead(), lm::ngram::BinaryFormat::ReadForConfig(), and util::stream::PRead::Run().

Here is the call graph for this function:

Here is the caller graph for this function:

void util::ErsatzPWrite ( int  fd,
const void *  from_void,
std::size_t  size,
uint64_t  off 
)

Definition at line 266 of file file.cc.

References NameFromFD(), UTIL_THROW_ARG, and UTIL_THROW_IF.

Referenced by util::stream::PWriteAndRecycle::Run().

Here is the call graph for this function:

Here is the caller graph for this function:

std::FILE * util::FDOpenOrThrow ( scoped_fd &  file  ) 

Definition at line 351 of file file.cc.

References util::scoped_fd::get(), util::scoped_fd::release(), and UTIL_THROW_IF_ARG.

Referenced by lm::ngram::trie::BuildTrie(), and FMakeTemp().

Here is the call graph for this function:

Here is the caller graph for this function:

std::FILE * util::FDOpenReadOrThrow ( scoped_fd &  file  ) 

Definition at line 358 of file file.cc.

References util::scoped_fd::get(), util::scoped_fd::release(), and UTIL_THROW_IF_ARG.

Here is the call graph for this function:

template<class Iterator >
boost::optional<typename std::iterator_traits<Iterator>::value_type> util::FirstIntersection ( std::vector< boost::iterator_range< Iterator > > &  sets  )  [inline]

Definition at line 59 of file multi_intersection.hh.

References FirstIntersection().

Here is the call graph for this function:

template<class Iterator , class Less >
boost::optional<typename std::iterator_traits<Iterator>::value_type> util::FirstIntersection ( std::vector< boost::iterator_range< Iterator > > &  sets,
const Less  less 
) [inline]

Definition at line 52 of file multi_intersection.hh.

References util::detail::FirstIntersectionSorted(), and sort().

Referenced by FirstIntersection(), and lm::vocab::Union::PassNGram().

Here is the call graph for this function:

Here is the caller graph for this function:

std::FILE * util::FMakeTemp ( const StringPiece base  ) 

Definition at line 513 of file file.cc.

References FDOpenOrThrow(), and MakeTemp().

Referenced by Moses::LexicalReorderingTableCreator::LexicalReorderingTableCreator(), and Moses::PhraseTableCreator::PhraseTableCreator().

Here is the call graph for this function:

Here is the caller graph for this function:

void util::FSyncOrThrow ( int  fd  ) 

Definition at line 302 of file file.cc.

References UTIL_THROW_IF_ARG.

Referenced by lm::ngram::BinaryFormat::FinishFile().

Here is the caller graph for this function:

uint64_t util::GuessPhysicalMemory (  ) 

Definition at line 210 of file usage.cc.

References NULL, and TEXT.

Referenced by main().

Here is the caller graph for this function:

void util::HugeMalloc ( std::size_t  size,
bool  zeroed,
scoped_memory &  to 
)

void util::HugeRealloc ( std::size_t  to,
bool  zero_new,
scoped_memory &  mem 
)

bool util::InputFileIsStdin ( StringPiece  path  ) 

Definition at line 63 of file file.cc.

bool util::InputPathIsStdin ( StringPiece  path  ) 

Does the given input file path denote standard input?

Returns true if, and only if, path is either "-" or "/dev/stdin".

Opening standard input as a file may need some special treatment for portability. There's a convention that a dash ("-") in place of an input file path denotes standard input, but opening "/dev/stdin" may need to be special as well.

template<class KeyIter , class ValueIter >
void util::JointSort ( const KeyIter &  key_begin,
const KeyIter &  key_end,
const ValueIter &  value_begin 
) [inline]

Definition at line 140 of file joint_sort.hh.

References JointSort().

Here is the call graph for this function:

template<class KeyIter , class ValueIter , class Less >
void util::JointSort ( const KeyIter &  key_begin,
const KeyIter &  key_end,
const ValueIter &  value_begin,
const Less &  less 
) [inline]

Definition at line 133 of file joint_sort.hh.

References sort().

Referenced by JointSort().

Here is the call graph for this function:

Here is the caller graph for this function:

int util::MakeTemp ( const StringPiece base  ) 

Definition at line 504 of file file.cc.

References StringPiece::data(), mkstemp_and_unlink(), StringPiece::size(), and UTIL_THROW_IF.

Referenced by FMakeTemp(), util::stream::Sort< Compare, Combine >::Merge(), and lm::ModelBuffer::Sink().

Here is the call graph for this function:

Here is the caller graph for this function:

void * util::MallocOrThrow ( std::size_t  requested  ) 

Definition at line 25 of file scoped.cc.

Referenced by util::stream::MergingReader< Compare, Combine >::Run(), and util::stream::Chain::Start().

Here is the caller graph for this function:

void * util::MapOrThrow ( std::size_t  size,
bool  for_write,
int  flags,
bool  prefault,
int  fd,
uint64_t  offset 
)

Definition at line 104 of file mmap.cc.

References NULL, and UTIL_THROW_IF.

Referenced by Moses::MmapAllocator< T >::allocate(), util::Rolling::ExtractNonRolling(), MapRead(), and MapZeroedWrite().

Here is the caller graph for this function:

void util::MapRead ( LoadMethod  method,
int  fd,
uint64_t  offset,
std::size_t  size,
scoped_memory &  out 
)

void * util::MapZeroedWrite ( const char *  name,
std::size_t  size,
scoped_fd &  file 
)

Definition at line 332 of file mmap.cc.

References CreateOrThrow(), util::scoped_fd::get(), MapZeroedWrite(), and util::scoped_fd::reset().

Here is the call graph for this function:

void * util::MapZeroedWrite ( int  fd,
std::size_t  size 
)

Definition at line 326 of file mmap.cc.

References kFileFlags, MapOrThrow(), and ResizeOrThrow().

Referenced by MapZeroedWrite(), and lm::ngram::BinaryFormat::SetupJustVocab().

Here is the call graph for this function:

Here is the caller graph for this function:

int util::mkstemp_and_unlink ( char *  tmpl  ) 

Definition at line 478 of file file.cc.

References UTIL_THROW_IF.

Referenced by MakeTemp().

Here is the caller graph for this function:

uint64_t util::MurmurHash64A ( const void *  key,
std::size_t  len,
uint64_t  seed 
)

Definition at line 26 of file murmur_hash.cc.

References end, k, and m.

Referenced by lm::ngram::detail::HashForVocab().

Here is the caller graph for this function:

uint64_t util::MurmurHash64B ( const void *  key,
std::size_t  len,
uint64_t  seed 
)

Definition at line 84 of file murmur_hash.cc.

References m.

uint64_t util::MurmurHashNative ( const void *  key,
std::size_t  len,
uint64_t  seed 
)

std::string util::NameFromFD ( int  fd  ) 

Definition at line 559 of file file.cc.

Referenced by ErsatzPRead(), ErsatzPWrite(), ReadOrThrow(), and util::stream::LineInput::Run().

Here is the caller graph for this function:

void util::NormalizeTempPrefix ( std::string &  base  ) 

Definition at line 489 of file file.cc.

Referenced by main().

Here is the caller graph for this function:

int util::OpenReadOrThrow ( const char *  name  ) 

template<class Proxy >
ProxyIterator<Proxy> util::operator+ ( std::ptrdiff_t  amount,
const ProxyIterator< Proxy > &  it 
) [inline]

Definition at line 95 of file proxy_iterator.hh.

template<class Except , class Data >
Except::template ExceptionTag< Except & >::Identity util::operator<< ( Except &  e,
const Data &  data 
) [inline]

Definition at line 48 of file exception.hh.

bool util::OutputFileIsStdout ( StringPiece  path  ) 

Definition at line 67 of file file.cc.

bool util::OutputPathIsStdout ( StringPiece  path  ) 

Does the given output file path denote standard output?

Returns true if, and only if, path is either "-" or "/dev/stdout".

Opening standard output as a file may need some special treatment for portability. There's a convention that a dash ("-") in place of an output file path denotes standard output, but opening "/dev/stdout" may need to be special as well.

void util::ParallelRead ( int  fd,
void *  to,
std::size_t  amount,
uint64_t  offset 
)

Definition at line 64 of file parallel_read.cc.

References ErsatzPRead().

Referenced by MapRead().

Here is the call graph for this function:

Here is the caller graph for this function:

uint64_t util::ParseSize ( const std::string &  arg  ) 

Definition at line 301 of file usage.cc.

Referenced by main().

Here is the caller graph for this function:

std::size_t util::PartialRead ( int  fd,
void *  to,
std::size_t  amount 
)

Definition at line 156 of file file.cc.

References NULL, read(), UTIL_THROW, and UTIL_THROW_IF_ARG.

Referenced by ReadOrEOF(), and ReadOrThrow().

Here is the call graph for this function:

Here is the caller graph for this function:

void util::posix_tmp_template ( char *  buf  ) 

POSIX helper: create template for temporary filename.

Writes the template into buf, which must have room for at least PATH_MAX bytes. The function fails if the template is too long.

Definition at line 64 of file tempfile.hh.

References temp_location().

Referenced by util::temp_dir::temp_dir(), and util::temp_file::temp_file().

Here is the call graph for this function:

Here is the caller graph for this function:

void util::PrintUsage ( std::ostream &  out  ) 

Definition at line 160 of file usage.cc.

References WallTime().

Referenced by batch_run(), and main().

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >
T util::rand (  )  [inline]

Return a pseudorandom number between 0 and RAND_MAX inclusive.

Initialize (seed) the randomizer before starting to call this.

Definition at line 174 of file random.hh.

References util::internal::rand_int(), and T.

Referenced by main(), rand01(), util::internal::rand_int(), MosesTuning::randomStats(), and doc::savernd().

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >
T util::rand_excl ( T  top  )  [inline]

Return a pseudorandom number in the half-open interval [0, top).

Generates a value between 0 (inclusive) and "top" (exclusive), assuming that bottom <= RAND_MAX.

Definition at line 179 of file random.hh.

template<typename T >
T util::rand_excl ( T  bottom,
T  top 
) [inline]

Return a pseudorandom number in the half-open interval [bottom, top).

Generates a value between "bottom" (inclusive) and "top" (exclusive), assuming that (top - bottom) <= RAND_MAX.

Definition at line 185 of file random.hh.

Referenced by Moses::Credit::Credit(), main(), sapt::TSA_tree_iterator< TKN >::randomSample(), and Moses::randomSample().

Here is the caller graph for this function:

template<typename T >
T util::rand_incl ( T  top  )  [inline]

Return a pseudorandom number in the open interval [0, top].

Generates a value between 0 and "top" inclusive, assuming that bottom < RAND_MAX.

Definition at line 190 of file random.hh.

template<typename T >
T util::rand_incl ( T  bottom,
T  top 
) [inline]

Return a pseudorandom number in the open interval [bottom, top].

Generates a value between "bottom" and "top" inclusive, assuming that (top - bottom) < RAND_MAX.

Definition at line 196 of file random.hh.

Referenced by Moses::Manager::CalcLatticeSamples(), and MosesTuning::Point::Randomize().

Here is the caller graph for this function:

void util::rand_init (  ) 

Initialize randomizer based on current time.

Call this to make the randomizer return hard-to-predict numbers. It won't produce high-grade randomness, but enough to make the program act differently on different runs.

The seed will be based on the current time in seconds. So calling it twice within the same second will just reset the randomizer to where it was before. Don't do that.

Definition at line 29 of file random.cc.

References NULL, and rand_init().

Here is the call graph for this function:

void util::rand_init ( unsigned  int  ) 

Thread-safe, cross-platform random number generator.

This is not for proper security-grade randomness, but should be "good enough" for producing arbitrary values of various numeric types.

Before starting, call rand_init() to seed the randomizer. There is no need to do this more than once; in fact doing it more often is likely to make the randomizer less effective. Once that is done, call the rand(), rand_excl(), and rand_incl() functions as needed to generate pseudo-random numbers.

Probability distribution is roughly uniform, but for integral types is skewed slightly towards lower numbers depending on how close "top" comes to RAND_MAX.

For floating-point types, resolution is limited; there will actually be only RAND_MAX different possible values. Initialize randomizer with a fixed seed.

After this, unless the randomizer gets seeded again, consecutive calls to the random functions will return a sequence of pseudo-random numbers determined by the seed. Every time the randomizer is seeded with this same seed, it will again start returning the same sequence of numbers.

Definition at line 22 of file random.cc.

Referenced by batch_run(), main(), and rand_init().

Here is the caller graph for this function:

float util::ReadFloat32 ( const void *  base,
uint64_t  bit_off 
) [inline]

Definition at line 109 of file bit_packing.hh.

References BitPackShift(), util::FloatEnc::f, util::FloatEnc::i, and ReadOff().

Referenced by lm::ngram::DontQuantize::MiddlePointer::Backoff().

Here is the call graph for this function:

Here is the caller graph for this function:

uint32_t util::ReadInt25 ( const void *  base,
uint64_t  bit_off,
uint8_t  length,
uint32_t  mask 
) [inline]

Definition at line 83 of file bit_packing.hh.

References BitPackShift().

Referenced by lm::ngram::SeparatelyQuantize::MiddlePointer::Backoff(), lm::ngram::SeparatelyQuantize::LongestPointer::Prob(), and lm::ngram::SeparatelyQuantize::MiddlePointer::Prob().

Here is the call graph for this function:

Here is the caller graph for this function:

uint64_t util::ReadInt57 ( const void *  base,
uint64_t  bit_off,
uint8_t  length,
uint64_t  mask 
) [inline]

Definition at line 63 of file bit_packing.hh.

References BitPackShift(), and ReadOff().

Referenced by BitPackingSanity(), lm::ngram::trie::ArrayBhiksha::ReadNext(), and lm::ngram::trie::DontBhiksha::ReadNext().

Here is the call graph for this function:

Here is the caller graph for this function:

float util::ReadNonPositiveFloat31 ( const void *  base,
uint64_t  bit_off 
) [inline]

Definition at line 136 of file bit_packing.hh.

References BitPackShift(), util::FloatEnc::f, util::FloatEnc::i, kSignBit, and ReadOff().

Referenced by lm::ngram::DontQuantize::LongestPointer::Prob(), and lm::ngram::DontQuantize::MiddlePointer::Prob().

Here is the call graph for this function:

Here is the caller graph for this function:

uint64_t util::ReadOff ( const void *  base,
uint64_t  bit_off 
) [inline]

Definition at line 48 of file bit_packing.hh.

Referenced by ReadFloat32(), ReadInt57(), and ReadNonPositiveFloat31().

Here is the caller graph for this function:

std::size_t util::ReadOrEOF ( int  fd,
void *  to_void,
std::size_t  amount 
)

Definition at line 190 of file file.cc.

References PartialRead().

Referenced by util::stream::Read::Run().

Here is the call graph for this function:

Here is the caller graph for this function:

void util::ReadOrThrow ( int  fd,
void *  to_void,
std::size_t  amount 
)

Definition at line 180 of file file.cc.

References NameFromFD(), PartialRead(), and UTIL_THROW_IF.

Referenced by util::stream::Offsets::FinishedAppending(), MapRead(), util::stream::Offsets::NextSize(), lm::ngram::ReadHeader(), and lm::np::Model::Recognize().

Here is the call graph for this function:

Here is the caller graph for this function:

uint8_t util::RequiredBits ( uint64_t  max_value  ) 

void util::ResizeOrThrow ( int  fd,
uint64_t  to 
)

uint64_t util::RSSMax (  ) 

Definition at line 149 of file usage.cc.

void util::SeekEnd ( int  fd  ) 

Definition at line 347 of file file.cc.

void util::SeekOrThrow ( int  fd,
uint64_t  off 
)

void util::SetSign ( float &  to  )  [inline]

Definition at line 122 of file bit_packing.hh.

References util::FloatEnc::f, util::FloatEnc::i, and kSignBit.

Referenced by lm::ngram::MaxRestBuild::SetRest().

Here is the caller graph for this function:

SizedIterator util::SizedIt ( void *  ptr,
std::size_t  size 
) [inline]

Definition at line 93 of file sized_iterator.hh.

Referenced by util::stream::BlockSorter< Compare >::Run().

Here is the caller graph for this function:

uint64_t util::SizeFile ( int  fd  ) 

Definition at line 91 of file file.cc.

References kBadSize.

Referenced by lm::ngram::IsBinaryFormat(), lm::ngram::BinaryFormat::LoadBinary(), and SizeOrThrow().

Here is the caller graph for this function:

uint64_t util::SizeOrThrow ( int  fd  ) 

std::size_t util::SizePage (  ) 

Definition at line 30 of file mmap.cc.

Referenced by util::Rolling::ExtractNonRolling(), HugeRealloc(), and util::scoped_memory::reset().

Here is the caller graph for this function:

template<class Iterator , class Accessor , class Pivot >
bool util::SortedUniformFind ( const Accessor &  accessor,
Iterator  begin,
Iterator  end,
const typename Accessor::Key  key,
Iterator &  out 
) [inline]

Definition at line 86 of file sorted_uniform.hh.

void util::SyncOrThrow ( void *  start,
size_t  length 
)

Definition at line 133 of file mmap.cc.

References UTIL_THROW_IF.

Referenced by lm::ngram::BinaryFormat::FinishFile(), and util::scoped_mmap::~scoped_mmap().

Here is the caller graph for this function:

std::string util::temp_location (  ) 

Obtain a directory for temporary files, e.g. /tmp.

Definition at line 26 of file tempfile.hh.

Referenced by posix_tmp_template().

Here is the caller graph for this function:

std::vector<std::string> util::tokenize ( const std::string &  input  )  [inline]

Split input string into a series of tokens.

Like tokenize(const char[]), but takes a std::string.

Definition at line 44 of file tokenize.hh.

References tokenize().

Here is the call graph for this function:

std::vector<std::string> util::tokenize ( const char  input[]  )  [inline]

Split input text into a series of tokens.

Splits on spaces and tabs, no other whitespace characters, and is not locale-sensitive.

The spaces themselves are not included. A sequence of consecutive space/tab characters count as one.

Definition at line 18 of file tokenize.hh.

Referenced by MosesTraining::SentenceAlignment::create(), MosesTraining::LexicalTable::load(), MosesTraining::Domain::load(), main(), MosesTraining::Syntax::XmlTreeParser::Parse(), processFiles(), MosesTraining::SentenceAlignmentWithSyntax::processSourceSentence(), MosesTraining::SentenceAlignment::processSourceSentence(), MosesTraining::SentenceAlignmentWithSyntax::processTargetSentence(), MosesTraining::SentenceAlignment::processTargetSentence(), reverseAlignment(), and tokenize().

Here is the caller graph for this function:

char* util::ToString ( bool  value,
char *  to 
) [inline]

Definition at line 23 of file integer_to_string.hh.

char * util::ToString ( const void *  v,
char *  to 
)

Definition at line 646 of file integer_to_string.cc.

char * util::ToString ( uint16_t  value,
char *  to 
)

Definition at line 637 of file integer_to_string.cc.

References ToString().

Here is the call graph for this function:

char * util::ToString ( int16_t  value,
char *  to 
)

Definition at line 634 of file integer_to_string.cc.

References ToString().

Here is the call graph for this function:

char * util::ToString ( int64_t  value,
char *  to 
)

Definition at line 624 of file integer_to_string.cc.

References ToString().

Here is the call graph for this function:

char * util::ToString ( int32_t  value,
char *  to 
)

Definition at line 615 of file integer_to_string.cc.

References ToString().

Here is the call graph for this function:

char * util::ToString ( uint64_t  value,
char *  buffer 
)

Definition at line 455 of file integer_to_string.cc.

char * util::ToString ( uint32_t  value,
char *  buffer 
)

Definition at line 382 of file integer_to_string.cc.

char * util::ToString ( float  value,
char *  to 
)

Definition at line 17 of file float_to_string.cc.

References util::ToStringBuf< float >::kBytes, and double_conversion::StringBuilder::position().

Here is the call graph for this function:

char * util::ToString ( double  value,
char *  to 
)

Definition at line 11 of file float_to_string.cc.

References util::ToStringBuf< double >::kBytes, and double_conversion::StringBuilder::position().

Referenced by Moses::Manager::CalcDecoderStatistics(), main(), Moses::PhraseDictionaryDynamicCacheBased::Print(), and ToString().

Here is the call graph for this function:

Here is the caller graph for this function:

void util::UnmapOrThrow ( void *  start,
size_t  length 
)

Definition at line 141 of file mmap.cc.

References UTIL_THROW_IF.

Referenced by Moses::MmapAllocator< T >::deallocate(), Moses::MmapAllocator< T >::~MmapAllocator(), and util::scoped_mmap::~scoped_mmap().

Here is the caller graph for this function:

void util::UnsetSign ( float &  to  )  [inline]

void util::WaitSemaphore ( Semaphore &  on  )  [inline]

Definition at line 59 of file pcqueue.hh.

Referenced by util::PCQueue< Block >::Consume(), and util::PCQueue< Block >::Produce().

Here is the caller graph for this function:

double util::WallTime (  ) 

Definition at line 134 of file usage.cc.

Referenced by Moses::Timer::get_elapsed_time(), sapt::imTSA< TOKEN >::imTSA(), PrintUsage(), Moses::Timer::start(), and Moses::Timer::stop().

Here is the caller graph for this function:

template<typename T >
T util::wide_rand (  )  [inline]

Return a pseudorandom number which may be larger than RAND_MAX.

The requested type must be integral, and its size must be an even multiple of the size of an int. The return value will combine one or more random ints into a single value, which could get quite large.

The result is nonnegative. Because the constituent ints are also nonnegative, the most significant bit in each of the ints will be zero, so for a wider type, there will be "gaps" in the range of possible outputs.

Definition at line 201 of file random.hh.

References T.

template<typename T >
T util::wide_rand_excl ( T  bottom,
T  top 
) [inline]

Return a pseudorandom number in [bottom, top), not limited to RAND_MAX.

Works like wide_rand(), but if the requested type is wider than an int, it accommodates larger value ranges than an int can represent.

Definition at line 212 of file random.hh.

References wide_rand_excl().

Here is the call graph for this function:

template<typename T >
T util::wide_rand_excl ( T  top  )  [inline]

Return a pseudorandom number in [0, top), not limited to RAND_MAX.

Works like wide_rand(), but if the requested type is wider than an int, it accommodates larger top values than an int can represent.

Definition at line 206 of file random.hh.

Referenced by wide_rand_excl().

Here is the caller graph for this function:

template<typename T >
T util::wide_rand_incl ( T  bottom,
T  top 
) [inline]

Return a pseudorandom number in [bottom, top], not limited to RAND_MAX.

Works like wide_rand(), but if the requested type is wider than an int, it accommodates larger top values than an int can represent.

Definition at line 223 of file random.hh.

References wide_rand_incl().

Here is the call graph for this function:

template<typename T >
T util::wide_rand_incl ( T  top  )  [inline]

Return a pseudorandom number in [0, top], not limited to RAND_MAX.

Works like wide_rand(), but if the requested type is wider than an int, it accommodates larger top values than an int can represent.

Definition at line 217 of file random.hh.

Referenced by wide_rand_incl().

Here is the caller graph for this function:

void util::WriteFloat32 ( void *  base,
uint64_t  bit_off,
float  value 
) [inline]

Definition at line 114 of file bit_packing.hh.

References util::FloatEnc::f, util::FloatEnc::i, and WriteInt57().

Referenced by lm::ngram::DontQuantize::MiddlePointer::Write().

Here is the call graph for this function:

Here is the caller graph for this function:

void util::WriteInt25 ( void *  base,
uint64_t  bit_off,
uint8_t  length,
uint32_t  value 
) [inline]

Definition at line 94 of file bit_packing.hh.

References BitPackShift().

Referenced by lm::ngram::SeparatelyQuantize::LongestPointer::Write().

Here is the call graph for this function:

Here is the caller graph for this function:

void util::WriteInt57 ( void *  base,
uint64_t  bit_off,
uint8_t  length,
uint64_t  value 
) [inline]

void util::WriteNonPositiveFloat31 ( void *  base,
uint64_t  bit_off,
float  value 
) [inline]

Definition at line 143 of file bit_packing.hh.

References util::FloatEnc::f, util::FloatEnc::i, kSignBit, and WriteInt57().

Referenced by lm::ngram::DontQuantize::LongestPointer::Write(), and lm::ngram::DontQuantize::MiddlePointer::Write().

Here is the call graph for this function:

Here is the caller graph for this function:

void util::WriteOrThrow ( FILE *  to,
const void *  data,
std::size_t  size 
)

Definition at line 226 of file file.cc.

References UTIL_THROW_IF.

void util::WriteOrThrow ( int  fd,
const void *  data_void,
std::size_t  size 
)


Variable Documentation

const uint64_t util::kBadSize = (uint64_t)-1

const int util::kFileFlags

Initial value:






  MAP_SHARED

Definition at line 94 of file mmap.cc.

Referenced by util::Rolling::ExtractNonRolling(), MapRead(), and MapZeroedWrite().

const char util::kProgressBanner = "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n"

const uint32_t util::kSignBit = 0x80000000

const bool util::kSpaces = {0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}

Definition at line 33 of file file_piece.cc.


Generated on Thu Jul 6 00:34:48 2017 for Moses by  doxygen 1.5.9