00001 #ifndef LM_FILTER_WRAPPER_H
00002 #define LM_FILTER_WRAPPER_H
00003
00004 #include "util/string_piece.hh"
00005
00006 #include <algorithm>
00007 #include <string>
00008 #include <vector>
00009
00010 namespace lm {
00011
00012
00013
00014 template <class Binary> class BinaryFilter {
00015 public:
00016
00017 explicit BinaryFilter(Binary binary) : binary_(binary) {}
00018
00019 template <class Iterator, class Output> void AddNGram(const Iterator &begin, const Iterator &end, const StringPiece &line, Output &output) {
00020 if (binary_.PassNGram(begin, end))
00021 output.AddNGram(line);
00022 }
00023
00024 template <class Output> void AddNGram(const StringPiece &ngram, const StringPiece &line, Output &output) {
00025 AddNGram(util::TokenIter<util::SingleCharacter, true>(ngram, ' '), util::TokenIter<util::SingleCharacter, true>::end(), line, output);
00026 }
00027
00028 void Flush() const {}
00029
00030 private:
00031 Binary binary_;
00032 };
00033
00034
00035 template <class FilterT> class ContextFilter {
00036 public:
00037 typedef FilterT Filter;
00038
00039 explicit ContextFilter(Filter &backend) : backend_(backend) {}
00040
00041 template <class Output> void AddNGram(const StringPiece &ngram, const StringPiece &line, Output &output) {
00042
00043 const char *last_space;
00044 for (last_space = ngram.data() + ngram.size() - 1; last_space > ngram.data() && *last_space != ' '; --last_space) {}
00045 backend_.AddNGram(StringPiece(ngram.data(), last_space - ngram.data()), line, output);
00046 }
00047
00048 void Flush() const {}
00049
00050 private:
00051 Filter backend_;
00052 };
00053
00054 }
00055
00056 #endif // LM_FILTER_WRAPPER_H