00001 #include <fstream>
00002 #include "OpSequenceModel.h"
00003 #include "osmHyp.h"
00004 #include "moses/Util.h"
00005 #include "util/exception.hh"
00006
00007 using namespace std;
00008 using namespace lm::ngram;
00009
00010 namespace Moses
00011 {
00012
00013 OpSequenceModel::OpSequenceModel(const std::string &line)
00014 :StatefulFeatureFunction(5, line )
00015 {
00016 sFactor = 0;
00017 tFactor = 0;
00018 numFeatures = 5;
00019 ReadParameters();
00020 load_method = util::READ;
00021 }
00022
00023 OpSequenceModel::~OpSequenceModel()
00024 {
00025 delete OSM;
00026 }
00027
00028 void OpSequenceModel :: readLanguageModel(const char *lmFile)
00029 {
00030 string unkOp = "_TRANS_SLF_";
00031 OSM = ConstructOSMLM(m_lmPath.c_str(), load_method);
00032
00033 State startState = OSM->NullContextState();
00034 State endState;
00035 unkOpProb = OSM->Score(startState,unkOp,endState);
00036 }
00037
00038
00039 void OpSequenceModel::Load(AllOptions::ptr const& opts)
00040 {
00041 m_options = opts;
00042 readLanguageModel(m_lmPath.c_str());
00043 }
00044
00045
00046
00047 void OpSequenceModel:: EvaluateInIsolation(const Phrase &source
00048 , const TargetPhrase &targetPhrase
00049 , ScoreComponentCollection &scoreBreakdown
00050 , ScoreComponentCollection &estimatedScores) const
00051 {
00052
00053 osmHypothesis obj;
00054 obj.setState(OSM->NullContextState());
00055 Bitmap myBitmap(source.GetSize());
00056 vector <string> mySourcePhrase;
00057 vector <string> myTargetPhrase;
00058 vector<float> scores;
00059 vector <int> alignments;
00060 int startIndex = 0;
00061 int endIndex = source.GetSize();
00062
00063 const AlignmentInfo &align = targetPhrase.GetAlignTerm();
00064 AlignmentInfo::const_iterator iter;
00065
00066 for (iter = align.begin(); iter != align.end(); ++iter) {
00067 alignments.push_back(iter->first);
00068 alignments.push_back(iter->second);
00069 }
00070
00071 for (size_t i = 0; i < targetPhrase.GetSize(); i++) {
00072 if (targetPhrase.GetWord(i).IsOOV() && sFactor == 0 && tFactor == 0)
00073 myTargetPhrase.push_back("_TRANS_SLF_");
00074 else
00075 myTargetPhrase.push_back(targetPhrase.GetWord(i).GetFactor(tFactor)->GetString().as_string());
00076 }
00077
00078 for (size_t i = 0; i < source.GetSize(); i++) {
00079 mySourcePhrase.push_back(source.GetWord(i).GetFactor(sFactor)->GetString().as_string());
00080 }
00081
00082 obj.setPhrases(mySourcePhrase , myTargetPhrase);
00083 obj.constructCepts(alignments,startIndex,endIndex-1,targetPhrase.GetSize());
00084 obj.computeOSMFeature(startIndex,myBitmap);
00085 obj.calculateOSMProb(*OSM);
00086 obj.populateScores(scores,numFeatures);
00087 estimatedScores.PlusEquals(this, scores);
00088
00089 }
00090
00091
00092 FFState* OpSequenceModel::EvaluateWhenApplied(
00093 const Hypothesis& cur_hypo,
00094 const FFState* prev_state,
00095 ScoreComponentCollection* accumulator) const
00096 {
00097 const TargetPhrase &target = cur_hypo.GetCurrTargetPhrase();
00098 const Bitmap &bitmap = cur_hypo.GetWordsBitmap();
00099 Bitmap myBitmap(bitmap);
00100 const Manager &manager = cur_hypo.GetManager();
00101 const InputType &source = manager.GetSource();
00102
00103 osmHypothesis obj;
00104 vector <string> mySourcePhrase;
00105 vector <string> myTargetPhrase;
00106 vector<float> scores;
00107
00108
00109
00110
00111
00112
00113
00114
00115
00116
00117
00118
00119
00120
00121
00122
00123
00124
00125 const Range & sourceRange = cur_hypo.GetCurrSourceWordsRange();
00126 int startIndex = sourceRange.GetStartPos();
00127 int endIndex = sourceRange.GetEndPos();
00128 const AlignmentInfo &align = cur_hypo.GetCurrTargetPhrase().GetAlignTerm();
00129
00130
00131 vector <int> alignments;
00132
00133
00134
00135 AlignmentInfo::const_iterator iter;
00136
00137 for (iter = align.begin(); iter != align.end(); ++iter) {
00138
00139 alignments.push_back(iter->first);
00140 alignments.push_back(iter->second);
00141 }
00142
00143
00144
00145
00146
00147
00148 for (int i = startIndex; i <= endIndex; i++) {
00149 myBitmap.SetValue(i,0);
00150 mySourcePhrase.push_back(source.GetWord(i).GetFactor(sFactor)->GetString().as_string());
00151
00152 }
00153
00154 for (size_t i = 0; i < target.GetSize(); i++) {
00155
00156 if (target.GetWord(i).IsOOV() && sFactor == 0 && tFactor == 0)
00157 myTargetPhrase.push_back("_TRANS_SLF_");
00158 else
00159 myTargetPhrase.push_back(target.GetWord(i).GetFactor(tFactor)->GetString().as_string());
00160
00161 }
00162
00163
00164
00165
00166 obj.setState(prev_state);
00167 obj.constructCepts(alignments,startIndex,endIndex,target.GetSize());
00168 obj.setPhrases(mySourcePhrase , myTargetPhrase);
00169 obj.computeOSMFeature(startIndex,myBitmap);
00170 obj.calculateOSMProb(*OSM);
00171 obj.populateScores(scores,numFeatures);
00172
00173
00174
00175
00176
00177
00178
00179
00180
00181
00182
00183
00184
00185
00186
00187
00188 accumulator->PlusEquals(this, scores);
00189
00190 return obj.saveState();
00191
00192
00193
00194
00195
00196
00197 }
00198
00199 FFState* OpSequenceModel::EvaluateWhenApplied(
00200 const ChartHypothesis& ,
00201 int ,
00202 ScoreComponentCollection* accumulator) const
00203 {
00204 UTIL_THROW2("Chart decoding not support by OpSequenceModel");
00205
00206 }
00207
00208 const FFState* OpSequenceModel::EmptyHypothesisState(const InputType &input) const
00209 {
00210 VERBOSE(3,"OpSequenceModel::EmptyHypothesisState()" << endl);
00211
00212 State startState = OSM->BeginSentenceState();
00213
00214 return new osmState(startState);
00215 }
00216
00217 std::string OpSequenceModel::GetScoreProducerWeightShortName(unsigned idx) const
00218 {
00219 return "osm";
00220 }
00221
00222 std::vector<float> OpSequenceModel::GetFutureScores(const Phrase &source, const Phrase &target) const
00223 {
00224 ParallelPhrase pp(source, target);
00225 std::map<ParallelPhrase, Scores>::const_iterator iter;
00226 iter = m_futureCost.find(pp);
00227
00228 if (iter == m_futureCost.end()) {
00229 vector<float> scores(numFeatures, 0);
00230 scores[0] = unkOpProb;
00231 return scores;
00232 } else {
00233 const vector<float> &scores = iter->second;
00234 return scores;
00235 }
00236 }
00237
00238 void OpSequenceModel::SetParameter(const std::string& key, const std::string& value)
00239 {
00240
00241 if (key == "path") {
00242 m_lmPath = value;
00243 } else if (key == "support-features") {
00244 if(value == "no")
00245 numFeatures = 1;
00246 else
00247 numFeatures = 5;
00248 } else if (key == "input-factor") {
00249 sFactor = Scan<int>(value);
00250 } else if (key == "output-factor") {
00251 tFactor = Scan<int>(value);
00252 } else if (key == "load") {
00253 if (value == "lazy") {
00254 load_method = util::LAZY;
00255 } else if (value == "populate_or_lazy") {
00256 load_method = util::POPULATE_OR_LAZY;
00257 } else if (value == "populate_or_read" || value == "populate") {
00258 load_method = util::POPULATE_OR_READ;
00259 } else if (value == "read") {
00260 load_method = util::READ;
00261 } else if (value == "parallel_read") {
00262 load_method = util::PARALLEL_READ;
00263 } else {
00264 UTIL_THROW2("Unknown KenLM load method " << value);
00265 }
00266 } else {
00267 StatefulFeatureFunction::SetParameter(key, value);
00268 }
00269 }
00270
00271 bool OpSequenceModel::IsUseable(const FactorMask &mask) const
00272 {
00273 bool ret = mask[0];
00274 return ret;
00275 }
00276
00277 }