00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023 #include "LexicalReorderingTableCompact.h"
00024 #include "moses/parameters/OOVHandlingOptions.h"
00025
00026 namespace Moses
00027 {
00028 bool LexicalReorderingTableCompact::s_inMemoryByDefault = false;
00029
00030 LexicalReorderingTableCompact::
00031 LexicalReorderingTableCompact(const std::string& filePath,
00032 const std::vector<FactorType>& f_factors,
00033 const std::vector<FactorType>& e_factors,
00034 const std::vector<FactorType>& c_factors)
00035 : LexicalReorderingTable(f_factors, e_factors, c_factors)
00036 , m_inMemory(s_inMemoryByDefault)
00037 , m_numScoreComponent(6)
00038 , m_multipleScoreTrees(true)
00039 , m_hash(10, 16)
00040 , m_scoreTrees(1)
00041 {
00042 Load(filePath);
00043 }
00044
00045 LexicalReorderingTableCompact::
00046 LexicalReorderingTableCompact(const std::vector<FactorType>& f_factors,
00047 const std::vector<FactorType>& e_factors,
00048 const std::vector<FactorType>& c_factors)
00049 : LexicalReorderingTable(f_factors, e_factors, c_factors)
00050 , m_inMemory(s_inMemoryByDefault)
00051 , m_numScoreComponent(6)
00052 , m_multipleScoreTrees(true)
00053 , m_hash(10, 16)
00054 , m_scoreTrees(1)
00055 { }
00056
00057 LexicalReorderingTableCompact::
00058 ~LexicalReorderingTableCompact()
00059 {
00060 for(size_t i = 0; i < m_scoreTrees.size(); i++)
00061 delete m_scoreTrees[i];
00062 }
00063
00064 std::vector<float>
00065 LexicalReorderingTableCompact::
00066 GetScore(const Phrase& f, const Phrase& e, const Phrase& c)
00067 {
00068 std::string key;
00069 Scores scores;
00070
00071 if(0 == c.GetSize())
00072 key = MakeKey(f, e, c);
00073 else
00074 for(size_t i = 0; i <= c.GetSize(); ++i) {
00075 Phrase sub_c(c.GetSubString(Range(i,c.GetSize()-1)));
00076 key = MakeKey(f,e,sub_c);
00077 }
00078
00079 size_t index = m_hash[key];
00080 if(m_hash.GetSize() != index) {
00081 std::string scoresString;
00082 if(m_inMemory)
00083 scoresString = m_scoresMemory[index].str();
00084 else
00085 scoresString = m_scoresMapped[index].str();
00086
00087 BitWrapper<> bitStream(scoresString);
00088 for(size_t i = 0; i < m_numScoreComponent; i++)
00089 scores.push_back(m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream));
00090
00091 return scores;
00092 }
00093
00094 return Scores();
00095 }
00096
00097 std::string
00098 LexicalReorderingTableCompact::
00099 MakeKey(const Phrase& f,
00100 const Phrase& e,
00101 const Phrase& c) const
00102 {
00103 return MakeKey(Trim(f.GetStringRep(m_FactorsF)),
00104 Trim(e.GetStringRep(m_FactorsE)),
00105 Trim(c.GetStringRep(m_FactorsC)));
00106 }
00107
00108 std::string
00109 LexicalReorderingTableCompact::
00110 MakeKey(const std::string& f,
00111 const std::string& e,
00112 const std::string& c) const
00113 {
00114 std::string key;
00115 if(!f.empty()) key += f;
00116 if(!m_FactorsE.empty()) {
00117 if(!key.empty()) key += " ||| ";
00118 key += e;
00119 }
00120 if(!m_FactorsC.empty()) {
00121 if(!key.empty()) key += " ||| ";
00122 key += c;
00123 }
00124 key += " ||| ";
00125 return key;
00126 }
00127
00128 LexicalReorderingTable*
00129 LexicalReorderingTableCompact::
00130 CheckAndLoad
00131 (const std::string& filePath,
00132 const std::vector<FactorType>& f_factors,
00133 const std::vector<FactorType>& e_factors,
00134 const std::vector<FactorType>& c_factors)
00135 {
00136 #ifdef HAVE_CMPH
00137 std::string minlexr = ".minlexr";
00138
00139 if(FileExists(filePath + minlexr)) {
00140
00141 VERBOSE(2,"Using compact lexical reordering table" << std::endl);
00142 return new LexicalReorderingTableCompact(filePath + minlexr, f_factors, e_factors, c_factors);
00143 }
00144
00145 if(filePath.substr(filePath.length() - minlexr.length(), minlexr.length()) == minlexr
00146 && FileExists(filePath)) {
00147
00148 VERBOSE(2,"Using compact lexical reordering table" << std::endl);
00149 return new LexicalReorderingTableCompact(filePath, f_factors, e_factors, c_factors);
00150 }
00151 #endif
00152 return 0;
00153 }
00154
00155 void
00156 LexicalReorderingTableCompact::
00157 Load(std::string filePath)
00158 {
00159 std::FILE* pFile = std::fopen(filePath.c_str(), "r");
00160 UTIL_THROW_IF2(pFile == NULL, "File " << filePath << " could not be opened");
00161
00162
00163 m_hash.Load(pFile);
00164
00165
00166
00167 size_t read = 0;
00168 read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, pFile);
00169 read += std::fread(&m_multipleScoreTrees,
00170 sizeof(m_multipleScoreTrees), 1, pFile);
00171
00172 if(m_multipleScoreTrees) {
00173 m_scoreTrees.resize(m_numScoreComponent);
00174 for(size_t i = 0; i < m_numScoreComponent; i++)
00175 m_scoreTrees[i] = new CanonicalHuffman<float>(pFile);
00176 } else {
00177 m_scoreTrees.resize(1);
00178 m_scoreTrees[0] = new CanonicalHuffman<float>(pFile);
00179 }
00180
00181 if(m_inMemory)
00182 m_scoresMemory.load(pFile, false);
00183 else
00184 m_scoresMapped.load(pFile, true);
00185 }
00186
00187 void
00188 LexicalReorderingTableCompact::
00189 SetStaticDefaultParameters(Parameter const& param)
00190 {
00191 param.SetParameter(s_inMemoryByDefault, "minlexr-memory", false);
00192 }
00193
00194
00195 }