00001 #include "TranslationRequest.h"
00002 #include "PackScores.h"
00003 #include "moses/ContextScope.h"
00004 #include <boost/foreach.hpp>
00005 #include "moses/Util.h"
00006 #include "moses/Hypothesis.h"
00007
00008 namespace MosesServer
00009 {
00010 using namespace std;
00011 using Moses::Hypothesis;
00012 using Moses::StaticData;
00013 using Moses::Range;
00014 using Moses::ChartHypothesis;
00015 using Moses::Phrase;
00016 using Moses::Manager;
00017 using Moses::SearchGraphNode;
00018 using Moses::TrellisPathList;
00019 using Moses::TranslationOptionCollection;
00020 using Moses::TranslationOptionList;
00021 using Moses::TranslationOption;
00022 using Moses::TargetPhrase;
00023 using Moses::FValue;
00024 using Moses::PhraseDictionaryMultiModel;
00025 using Moses::FindPhraseDictionary;
00026 using Moses::Sentence;
00027 using Moses::TokenizeMultiCharSeparator;
00028 using Moses::FeatureFunction;
00029 using Moses::Scan;
00030
00031 boost::shared_ptr<TranslationRequest>
00032 TranslationRequest::
00033 create(Translator* translator, xmlrpc_c::paramList const& paramList,
00034 boost::condition_variable& cond, boost::mutex& mut)
00035 {
00036 boost::shared_ptr<TranslationRequest> ret;
00037 ret.reset(new TranslationRequest(paramList, cond, mut));
00038 ret->m_self = ret;
00039 ret->m_translator = translator;
00040 return ret;
00041 }
00042
00043 void
00044 SetContextWeights(Moses::ContextScope& s, xmlrpc_c::value const& w)
00045 {
00046 SPTR<std::map<std::string,float> > M(new std::map<std::string, float>);
00047 typedef std::map<std::string,xmlrpc_c::value> tmap;
00048 tmap const tmp = static_cast<tmap>(xmlrpc_c::value_struct(w));
00049 for(tmap::const_iterator m = tmp.begin(); m != tmp.end(); ++m)
00050 (*M)[m->first] = xmlrpc_c::value_double(m->second);
00051 s.SetContextWeights(M);
00052 }
00053
00054 void
00055 TranslationRequest::
00056 Run()
00057 {
00058 typedef std::map<std::string,xmlrpc_c::value> param_t;
00059 param_t const& params = m_paramList.getStruct(0);
00060 parse_request(params);
00061
00062
00063
00064
00065 param_t::const_iterator si = params.find("context-weights");
00066 if (si != params.end()) SetContextWeights(*m_scope, si->second);
00067
00068 Moses::StaticData const& SD = Moses::StaticData::Instance();
00069
00070 if (is_syntax(m_options->search.algo))
00071 run_chart_decoder();
00072 else
00073 run_phrase_decoder();
00074
00075 {
00076 boost::lock_guard<boost::mutex> lock(m_mutex);
00077 m_done = true;
00078 }
00079 m_cond.notify_one();
00080
00081 }
00082
00084 void
00085 TranslationRequest::
00086 add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
00087 {
00088 if (!m_withAlignInfo) return;
00089
00090 Range const& trg = h.GetCurrTargetWordsRange();
00091 Range const& src = h.GetCurrSourceWordsRange();
00092
00093 std::map<std::string, xmlrpc_c::value> pAlnInfo;
00094 pAlnInfo["tgt-start"] = xmlrpc_c::value_int(trg.GetStartPos());
00095 pAlnInfo["tgt-end"] = xmlrpc_c::value_int(trg.GetEndPos());
00096 pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos());
00097 pAlnInfo["src-end"] = xmlrpc_c::value_int(src.GetEndPos());
00098 aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo));
00099 }
00100
00101 void
00102 TranslationRequest::
00103 outputChartHypo(ostream& out, const ChartHypothesis* hypo)
00104 {
00105 Phrase outPhrase(20);
00106 hypo->GetOutputPhrase(outPhrase);
00107
00108
00109 assert(outPhrase.GetSize() >= 2);
00110 outPhrase.RemoveWord(0);
00111 outPhrase.RemoveWord(outPhrase.GetSize() - 1);
00112 for (size_t pos = 0 ; pos < outPhrase.GetSize() ; pos++)
00113 out << *outPhrase.GetFactor(pos, 0) << " ";
00114 }
00115
00116 bool
00117 TranslationRequest::
00118 compareSearchGraphNode(const Moses::SearchGraphNode& a,
00119 const Moses::SearchGraphNode& b)
00120 {
00121 return a.hypo->GetId() < b.hypo->GetId();
00122 }
00123
00124 void
00125 TranslationRequest::
00126 insertGraphInfo(Manager& manager, map<string, xmlrpc_c::value>& retData)
00127 {
00128 using xmlrpc_c::value_int;
00129 using xmlrpc_c::value_double;
00130 using xmlrpc_c::value_struct;
00131 using xmlrpc_c::value_string;
00132 vector<xmlrpc_c::value> searchGraphXml;
00133 vector<SearchGraphNode> searchGraph;
00134 manager.GetSearchGraph(searchGraph);
00135 std::sort(searchGraph.begin(), searchGraph.end());
00136 BOOST_FOREACH(Moses::SearchGraphNode const& n, searchGraph) {
00137 map<string, xmlrpc_c::value> x;
00138 x["forward"] = value_double(n.forward);
00139 x["fscore"] = value_double(n.fscore);
00140 const Hypothesis* hypo = n.hypo;
00141 x["hyp"] = value_int(hypo->GetId());
00142 x["stack"] = value_int(hypo->GetWordsBitmap().GetNumWordsCovered());
00143 if (hypo->GetId() != 0) {
00144 const Hypothesis *prevHypo = hypo->GetPrevHypo();
00145 x["back"] = value_int(prevHypo->GetId());
00146 x["score"] = value_double(hypo->GetScore());
00147 x["transition"] = value_double(hypo->GetScore() - prevHypo->GetScore());
00148 if (n.recombinationHypo)
00149 x["recombined"] = value_int(n.recombinationHypo->GetId());
00150 x["cover-start"] = value_int(hypo->GetCurrSourceWordsRange().GetStartPos());
00151 x["cover-end"] = value_int(hypo->GetCurrSourceWordsRange().GetEndPos());
00152 x["out"] = value_string(hypo->GetCurrTargetPhrase().GetStringRep(options()->output.factor_order));
00153 }
00154 searchGraphXml.push_back(value_struct(x));
00155 }
00156 retData["sg"] = xmlrpc_c::value_array(searchGraphXml);
00157 }
00158
00159 void
00160 TranslationRequest::
00161 outputNBest(const Manager& manager, map<string, xmlrpc_c::value>& retData)
00162 {
00163 TrellisPathList nBestList;
00164 vector<xmlrpc_c::value> nBestXml;
00165
00166 Moses::NBestOptions const& nbo = m_options->nbest;
00167 manager.CalcNBest(nbo.nbest_size, nBestList, nbo.only_distinct);
00168 manager.OutputNBest(cout, nBestList);
00169
00170 BOOST_FOREACH(Moses::TrellisPath const* path, nBestList) {
00171 vector<const Hypothesis *> const& E = path->GetEdges();
00172 if (!E.size()) continue;
00173 std::map<std::string, xmlrpc_c::value> nBestXmlItem;
00174 pack_hypothesis(manager, E, "hyp", nBestXmlItem);
00175 if (m_withScoreBreakdown) {
00176
00177 ostringstream buf;
00178 bool with_labels = nbo.include_feature_labels;
00179 path->GetScoreBreakdown()->OutputAllFeatureScores(buf, with_labels);
00180 nBestXmlItem["fvals"] = xmlrpc_c::value_string(buf.str());
00181 nBestXmlItem["scores"] = PackScores(*path->GetScoreBreakdown());
00182 }
00183
00184
00185 nBestXmlItem["totalScore"] = xmlrpc_c::value_double(path->GetFutureScore());
00186 nBestXml.push_back(xmlrpc_c::value_struct(nBestXmlItem));
00187 }
00188 retData["nbest"] = xmlrpc_c::value_array(nBestXml);
00189 }
00190
00191 void
00192 TranslationRequest::
00193 insertTranslationOptions(Moses::Manager& manager,
00194 std::map<std::string, xmlrpc_c::value>& retData)
00195 {
00196 std::vector<Moses::FactorType> const& ofactor_order = options()->output.factor_order;
00197
00198 const TranslationOptionCollection* toptsColl = manager.getSntTranslationOptions();
00199 vector<xmlrpc_c::value> toptsXml;
00200 size_t const stop = toptsColl->GetSource().GetSize();
00201 TranslationOptionList const* tol;
00202 for (size_t s = 0 ; s < stop ; ++s) {
00203 for (size_t e=s;(tol=toptsColl->GetTranslationOptionList(s,e))!=NULL;++e) {
00204 BOOST_FOREACH(TranslationOption const* topt, *tol) {
00205 std::map<std::string, xmlrpc_c::value> toptXml;
00206 TargetPhrase const& tp = topt->GetTargetPhrase();
00207 std::string tphrase = tp.GetStringRep(ofactor_order);
00208 toptXml["phrase"] = xmlrpc_c::value_string(tphrase);
00209 toptXml["fscore"] = xmlrpc_c::value_double(topt->GetFutureScore());
00210 toptXml["start"] = xmlrpc_c::value_int(s);
00211 toptXml["end"] = xmlrpc_c::value_int(e);
00212 vector<xmlrpc_c::value> scoresXml;
00213 const std::valarray<FValue> &scores
00214 = topt->GetScoreBreakdown().getCoreFeatures();
00215 for (size_t j = 0; j < scores.size(); ++j)
00216 scoresXml.push_back(xmlrpc_c::value_double(scores[j]));
00217 toptXml["scores"] = xmlrpc_c::value_array(scoresXml);
00218 ostringstream buf;
00219 topt->GetScoreBreakdown().OutputAllFeatureScores(buf, true);
00220 toptXml["labelledScores"] = PackScores(topt->GetScoreBreakdown());
00221 toptsXml.push_back(xmlrpc_c::value_struct(toptXml));
00222 }
00223 }
00224 }
00225 retData["topt"] = xmlrpc_c::value_array(toptsXml);
00226 }
00227
00228 TranslationRequest::
00229 TranslationRequest(xmlrpc_c::paramList const& paramList,
00230 boost::condition_variable& cond, boost::mutex& mut)
00231 : m_cond(cond), m_mutex(mut), m_done(false), m_paramList(paramList)
00232 , m_session_id(0)
00233 {
00234
00235 }
00236
00237 bool
00238 check(std::map<std::string, xmlrpc_c::value> const& param,
00239 std::string const key)
00240 {
00241 std::map<std::string, xmlrpc_c::value>::const_iterator m = param.find(key);
00242 if(m == param.end()) return false;
00243
00244 if (m->second.type() == xmlrpc_c::value::TYPE_BOOLEAN)
00245 return xmlrpc_c::value_boolean(m->second);
00246
00247 std::string val = string(xmlrpc_c::value_string(m->second));
00248 if(val == "true" || val == "True" || val == "TRUE" || val == "1") return true;
00249 return false;
00250 }
00251
00252 void
00253 TranslationRequest::
00254 parse_request(std::map<std::string, xmlrpc_c::value> const& params)
00255 {
00256
00257 m_paramList.verifyEnd(1);
00258
00259 typedef std::map<std::string, xmlrpc_c::value> params_t;
00260 params_t::const_iterator si;
00261
00262 si = params.find("session-id");
00263 if (si != params.end())
00264 {
00265 m_session_id = xmlrpc_c::value_int(si->second);
00266 Session const& S = m_translator->get_session(m_session_id);
00267 m_scope = S.scope;
00268 m_session_id = S.id;
00269 }
00270 else
00271 {
00272 m_session_id = 0;
00273 m_scope.reset(new Moses::ContextScope);
00274 }
00275
00276 boost::shared_ptr<Moses::AllOptions> opts(new Moses::AllOptions(*StaticData::Instance().options()));
00277 opts->update(params);
00278
00279 m_withGraphInfo = check(params, "sg");
00280 if (m_withGraphInfo || opts->nbest.nbest_size > 0) {
00281 opts->output.SearchGraph = "true";
00282 opts->nbest.enabled = true;
00283 }
00284
00285 m_options = opts;
00286
00287
00288 si = params.find("text");
00289 if (si == params.end())
00290 throw xmlrpc_c::fault("Missing source text", xmlrpc_c::fault::CODE_PARSE);
00291 m_source_string = xmlrpc_c::value_string(si->second);
00292 XVERBOSE(1,"Input: " << m_source_string << endl);
00293
00294 m_withTopts = check(params, "topt");
00295 m_withScoreBreakdown = check(params, "add-score-breakdown");
00296 si = params.find("lambda");
00297 if (si != params.end())
00298 {
00299
00300 xmlrpc_c::value_array muMoArray = xmlrpc_c::value_array(si->second);
00301 vector<xmlrpc_c::value> muMoValVec(muMoArray.vectorValueValue());
00302 vector<float> w(muMoValVec.size());
00303 for (size_t i = 0; i < muMoValVec.size(); ++i)
00304 w[i] = xmlrpc_c::value_double(muMoValVec[i]);
00305 if (w.size() && (si = params.find("model_name")) != params.end())
00306 {
00307 string const model_name = xmlrpc_c::value_string(si->second);
00308 PhraseDictionaryMultiModel* pdmm
00309 = (PhraseDictionaryMultiModel*) FindPhraseDictionary(model_name);
00310 pdmm->SetTemporaryMultiModelWeightsVector(w);
00311 }
00312 }
00313
00314 si = params.find("context");
00315 if (si != params.end())
00316 {
00317 string context = xmlrpc_c::value_string(si->second);
00318 VERBOSE(1,"CONTEXT " << context);
00319 m_context.reset(new std::vector<std::string>(1,context));
00320 }
00321
00322 si = params.find("context-scope");
00323 if (si != params.end())
00324 {
00325
00326 string context = xmlrpc_c::value_string(si->second);
00327
00328 string groupSeparator("Moses::ContextScope::GroupSeparator");
00329 string recordSeparator("Moses::ContextScope::RecordSeparator");
00330
00331
00332
00333
00334
00335
00336
00337
00338
00339
00340
00341
00342
00343 boost::shared_ptr<Moses::ContextScope> contextScope = GetScope();
00344
00345 BOOST_FOREACH(string group, TokenizeMultiCharSeparator(context, groupSeparator)) {
00346
00347 vector<string> record = TokenizeMultiCharSeparator(group, recordSeparator);
00348
00349
00350 FeatureFunction& ff = Moses::FeatureFunction::FindFeatureFunction(record[0]);
00351 void const* key = static_cast<void const*>(&ff);
00352
00353
00354 boost::shared_ptr<string> value = contextScope->get<string>(key,true);
00355 value->replace(value->begin(), value->end(), record[1]);
00356
00357 }
00358 }
00359
00360
00361 m_withAlignInfo = options()->output.ReportSegmentation || check(params, "align");
00362
00363
00364 m_withWordAlignInfo = options()->output.PrintAlignmentInfo || check(params, "word-align");
00365
00366 si = params.find("weights");
00367 if (si != params.end())
00368 {
00369
00370 boost::unordered_map<string, FeatureFunction*> map;
00371 {
00372 const vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
00373 BOOST_FOREACH(FeatureFunction* const& ff, ffs) {
00374 map[ff->GetScoreProducerDescription()] = ff;
00375 }
00376 }
00377
00378 string allValues = xmlrpc_c::value_string(si->second);
00379
00380 BOOST_FOREACH(string values, TokenizeMultiCharSeparator(allValues, "\t")) {
00381
00382 vector<string> record = TokenizeMultiCharSeparator(values, "=");
00383
00384 if (record.size() == 2) {
00385 string featureName = record[0];
00386 string featureWeights = record[1];
00387
00388 boost::unordered_map<string, FeatureFunction*>::iterator ffi = map.find(featureName);
00389
00390 if (ffi != map.end()) {
00391 FeatureFunction* ff = ffi->second;
00392
00393 size_t prevNumWeights = ff->GetNumScoreComponents();
00394
00395 vector<float> ffWeights;
00396 BOOST_FOREACH(string weight, TokenizeMultiCharSeparator(featureWeights, " ")) {
00397 ffWeights.push_back(Scan<float>(weight));
00398 }
00399
00400 if (ffWeights.size() == ff->GetNumScoreComponents()) {
00401
00402
00403 Moses::StaticData::InstanceNonConst().SetWeights(ff, ffWeights);
00404 VERBOSE(1, "WARNING: THIS IS NOT THREAD-SAFE!\tUpdating weights for " << featureName << " to " << featureWeights << "\n");
00405
00406 } else {
00407 TRACE_ERR("ERROR: Unable to update weights for " << featureName << " because " << ff->GetNumScoreComponents() << " weights are required but only " << ffWeights.size() << " were provided\n");
00408 }
00409
00410 } else {
00411 TRACE_ERR("ERROR: No FeatureFunction with name " << featureName << ", no weight update\n");
00412 }
00413
00414 } else {
00415 TRACE_ERR("WARNING: XML-RPC weights update was improperly formatted:\t" << values << "\n");
00416 }
00417
00418 }
00419
00420 }
00421
00422
00423
00424
00425
00426
00427
00428
00429
00430
00431 if (is_syntax(m_options->search.algo)) {
00432 m_source.reset(new Sentence(m_options,0,m_source_string));
00433 } else {
00434 m_source.reset(new Sentence(m_options,0,m_source_string));
00435 }
00436 }
00437
00438
00439 void
00440 TranslationRequest::
00441 run_chart_decoder()
00442 {
00443 Moses::ChartManager manager(this->self());
00444 manager.Decode();
00445
00446 const Moses::ChartHypothesis *hypo = manager.GetBestHypothesis();
00447 ostringstream out;
00448 if (hypo) outputChartHypo(out,hypo);
00449
00450 m_target_string = out.str();
00451 m_retData["text"] = xmlrpc_c::value_string(m_target_string);
00452
00453 if (m_withGraphInfo) {
00454 std::ostringstream sgstream;
00455 manager.OutputSearchGraphMoses(sgstream);
00456 m_retData["sg"] = xmlrpc_c::value_string(sgstream.str());
00457 }
00458 }
00459
00460 void
00461 TranslationRequest::
00462 pack_hypothesis(const Moses::Manager& manager,
00463 vector<Hypothesis const* > const& edges, string const& key,
00464 map<string, xmlrpc_c::value> & dest) const
00465 {
00466
00467 ostringstream target;
00468 BOOST_REVERSE_FOREACH(Hypothesis const* e, edges) {
00469 manager.OutputSurface(target, *e);
00470 }
00471 XVERBOSE(1, "BEST TRANSLATION: " << *(manager.GetBestHypothesis())
00472 << std::endl);
00473 dest[key] = xmlrpc_c::value_string(target.str());
00474
00475 if (m_withAlignInfo) {
00476
00477
00478
00479 vector<xmlrpc_c::value> p_aln;
00480 BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
00481 add_phrase_aln_info(*e, p_aln);
00482 dest["align"] = xmlrpc_c::value_array(p_aln);
00483 }
00484
00485 if (m_withWordAlignInfo) {
00486
00487
00488 vector<xmlrpc_c::value> w_aln;
00489 BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
00490 e->OutputLocalWordAlignment(w_aln);
00491 dest["word-align"] = xmlrpc_c::value_array(w_aln);
00492 }
00493 }
00494
00495 void
00496 TranslationRequest::
00497 pack_hypothesis(const Moses::Manager& manager, Hypothesis const* h, string const& key,
00498 map<string, xmlrpc_c::value>& dest) const
00499 {
00500 using namespace std;
00501 vector<Hypothesis const*> edges;
00502 for (; h; h = h->GetPrevHypo())
00503 edges.push_back(h);
00504 pack_hypothesis(manager, edges, key, dest);
00505 }
00506
00507
00508 void
00509 TranslationRequest::
00510 run_phrase_decoder()
00511 {
00512 Manager manager(this->self());
00513 manager.Decode();
00514 pack_hypothesis(manager, manager.GetBestHypothesis(), "text", m_retData);
00515 if (m_session_id)
00516 m_retData["session-id"] = xmlrpc_c::value_int(m_session_id);
00517
00518 if (m_withGraphInfo) insertGraphInfo(manager,m_retData);
00519 if (m_withTopts) insertTranslationOptions(manager,m_retData);
00520 if (m_options->nbest.nbest_size) outputNBest(manager, m_retData);
00521
00522 }
00523 }