00001 #include <vector>
00002 #include "BilingualLM.h"
00003 #include "moses/ScoreComponentCollection.h"
00004
00005 using namespace std;
00006
00007 namespace Moses
00008 {
00009
00011 BilingualLM::BilingualLM(const std::string &line)
00012 : StatefulFeatureFunction(1, line),
00013 word_factortype(0)
00014 {
00015 FactorCollection& factorFactory = FactorCollection::Instance();
00016 BOS_factor = factorFactory.AddFactor(BOS_);
00017 BOS_word.SetFactor(0, BOS_factor);
00018 EOS_factor = factorFactory.AddFactor(EOS_);
00019 EOS_word.SetFactor(0, EOS_factor);
00020
00021 }
00022
00023 void BilingualLM::Load(AllOptions::ptr const& opts)
00024 {
00025 m_options = opts;
00026 ReadParameters();
00027 loadModel();
00028 }
00029
00030
00031
00032 void BilingualLM::requestPrevTargetNgrams(
00033 const Hypothesis &cur_hypo, int amount, std::vector<int> &words) const
00034 {
00035 const Hypothesis * prev_hyp = cur_hypo.GetPrevHypo();
00036 int found = 0;
00037
00038 while (prev_hyp && found != amount) {
00039 const TargetPhrase& currTargetPhrase = prev_hyp->GetCurrTargetPhrase();
00040 for (int i = currTargetPhrase.GetSize() - 1; i> -1; i--) {
00041 if (found != amount) {
00042 const Word& word = currTargetPhrase.GetWord(i);
00043 words[found] = getNeuralLMId(word, false);
00044 found++;
00045 } else {
00046 return;
00047 }
00048 }
00049
00050 prev_hyp = prev_hyp->GetPrevHypo();
00051 }
00052
00053 int neuralLM_wordID = getNeuralLMId(BOS_word, false);
00054 for (int i = found; i < amount; i++) {
00055 words[i] = neuralLM_wordID;
00056 }
00057 }
00058
00059
00060
00061 void BilingualLM::getTargetWords(
00062 const Hypothesis &cur_hypo,
00063 const TargetPhrase &targetPhrase,
00064 int current_word_index,
00065 std::vector<int> &words) const
00066 {
00067
00068 int additional_needed = current_word_index - target_ngrams;
00069 if (additional_needed < 0) {
00070 additional_needed = -additional_needed;
00071 std::vector<int> prev_words(additional_needed);
00072 requestPrevTargetNgrams(cur_hypo, additional_needed, prev_words);
00073 for (int i = additional_needed - 1; i >= 0; i--) {
00074 words.push_back(prev_words[i]);
00075 }
00076 }
00077
00078 if (words.size() > 0) {
00079
00080
00081 for (int i = 0; i <= current_word_index; i++) {
00082 const Word& word = targetPhrase.GetWord(i);
00083 words.push_back(getNeuralLMId(word, false));
00084 }
00085 } else {
00086
00087 for (int i = current_word_index - target_ngrams; i <= current_word_index; i++) {
00088 const Word& word = targetPhrase.GetWord(i);
00089 words.push_back(getNeuralLMId(word, false));
00090 }
00091 }
00092 }
00093
00094
00095
00096 size_t BilingualLM::selectMiddleAlignment(
00097 const set<size_t>& alignment_links) const
00098 {
00099
00100 set<size_t>::iterator it = alignment_links.begin();
00101 for (size_t i = 0; i < (alignment_links.size() - 1) / 2; ++i) {
00102 ++it;
00103 }
00104
00105 return *it;
00106 }
00107
00108 void BilingualLM::getSourceWords(
00109 const TargetPhrase &targetPhrase,
00110 int targetWordIdx,
00111 const Sentence &source_sent,
00112 const Range &sourceWordRange,
00113 std::vector<int> &words) const
00114 {
00115
00116
00117
00118 const AlignmentInfo& alignments = targetPhrase.GetAlignTerm();
00119
00120
00121
00122 std::set<size_t> last_word_al;
00123 for (int j = 0; j < targetPhrase.GetSize(); j++) {
00124
00125 if ((targetWordIdx + j) < targetPhrase.GetSize()) {
00126 last_word_al = alignments.GetAlignmentsForTarget(targetWordIdx + j);
00127 if (!last_word_al.empty()) {
00128 break;
00129 }
00130 }
00131
00132
00133 if ((targetWordIdx - j) >= 0) {
00134 last_word_al = alignments.GetAlignmentsForTarget(targetWordIdx - j);
00135 if (!last_word_al.empty()) {
00136 break;
00137 }
00138 }
00139 }
00140
00141
00142
00143
00144
00145
00146
00147 UTIL_THROW_IF2(last_word_al.size() == 0,
00148 "A target phrase with no alignments detected! " << targetPhrase << "Check if there is something wrong with your phrase table.");
00149 size_t source_center_index = selectMiddleAlignment(last_word_al);
00150
00151 size_t phrase_start_pos = sourceWordRange.GetStartPos();
00152
00153 size_t source_word_mid_idx = phrase_start_pos + source_center_index;
00154
00155 appendSourceWordsToVector(source_sent, words, source_word_mid_idx);
00156 }
00157
00158 size_t BilingualLM::getState(const Hypothesis& cur_hypo) const
00159 {
00160 const TargetPhrase &targetPhrase = cur_hypo.GetCurrTargetPhrase();
00161 size_t hashCode = 0;
00162
00163
00164 int additional_needed = targetPhrase.GetSize() - target_ngrams;
00165 if (additional_needed < 0) {
00166 additional_needed = -additional_needed;
00167 std::vector<int> prev_words(additional_needed);
00168 requestPrevTargetNgrams(cur_hypo, additional_needed, prev_words);
00169 for (int i = additional_needed - 1; i >= 0; i--) {
00170 boost::hash_combine(hashCode, prev_words[i]);
00171 }
00172
00173
00174 for (int i = 0; i < targetPhrase.GetSize(); i++) {
00175 const Word& word = targetPhrase.GetWord(i);
00176 int neuralLM_wordID = getNeuralLMId(word, false);
00177 boost::hash_combine(hashCode, neuralLM_wordID);
00178 }
00179 } else {
00180
00181 for (int i = targetPhrase.GetSize() - target_ngrams; i < targetPhrase.GetSize(); i++) {
00182 const Word& word = targetPhrase.GetWord(i);
00183 int neuralLM_wordID = getNeuralLMId(word, false);
00184
00185 boost::hash_combine(hashCode, neuralLM_wordID);
00186 }
00187 }
00188
00189 return hashCode;
00190 }
00191
00192 FFState* BilingualLM::EvaluateWhenApplied(
00193 const Hypothesis& cur_hypo,
00194 const FFState* prev_state,
00195 ScoreComponentCollection* accumulator) const
00196 {
00197 Manager& manager = cur_hypo.GetManager();
00198 const Sentence& source_sent = static_cast<const Sentence&>(manager.GetSource());
00199
00200
00201 std::vector<int> source_words;
00202 source_words.reserve(source_ngrams);
00203 std::vector<int> target_words;
00204 target_words.reserve(target_ngrams);
00205
00206 float value = 0;
00207 const TargetPhrase& currTargetPhrase = cur_hypo.GetCurrTargetPhrase();
00208 const Range& sourceWordRange = cur_hypo.GetCurrSourceWordsRange();
00209
00210
00211 for (int i = 0; i < currTargetPhrase.GetSize(); i++) {
00212 getSourceWords(
00213 currTargetPhrase, i, source_sent, sourceWordRange, source_words);
00214 getTargetWords(cur_hypo, currTargetPhrase, i, target_words);
00215 value += Score(source_words, target_words);
00216
00217
00218 source_words.clear();
00219 target_words.clear();
00220 }
00221
00222 size_t new_state = getState(cur_hypo);
00223 accumulator->PlusEquals(this, value);
00224
00225 return new BilingualLMState(new_state);
00226 }
00227
00228 void BilingualLM::getAllTargetIdsChart(const ChartHypothesis& cur_hypo, size_t featureID, std::vector<int>& wordIds) const
00229 {
00230 const TargetPhrase targetPhrase = cur_hypo.GetCurrTargetPhrase();
00231
00232 for (int i = 0; i < targetPhrase.GetSize(); i++) {
00233 if (targetPhrase.GetWord(i).IsNonTerminal()) {
00234 const ChartHypothesis * prev_hypo = cur_hypo.GetPrevHypo(targetPhrase.GetAlignNonTerm().GetNonTermIndexMap()[i]);
00235 const BilingualLMState * prev_state = static_cast<const BilingualLMState *>(prev_hypo->GetFFState(featureID));
00236 const std::vector<int> prevWordIDs = prev_state->GetWordIdsVector();
00237 for (std::vector<int>::const_iterator it = prevWordIDs.begin(); it!= prevWordIDs.end(); it++) {
00238 wordIds.push_back(*it);
00239 }
00240 } else {
00241 wordIds.push_back(getNeuralLMId(targetPhrase.GetWord(i), false));
00242 }
00243 }
00244 }
00245
00246 void BilingualLM::getAllAlignments(const ChartHypothesis& cur_hypo, size_t featureID, std::vector<int>& word_alignments) const
00247 {
00248 const TargetPhrase targetPhrase = cur_hypo.GetCurrTargetPhrase();
00249 int source_word_mid_idx;
00250
00251
00252 const AlignmentInfo& alignments = targetPhrase.GetAlignTerm();
00253
00254
00255 std::vector<int> absolute_source_position (cur_hypo.GetCurrSourceRange().GetNumWordsCovered(), 0);
00256
00257 absolute_source_position[0] = cur_hypo.GetCurrSourceRange().GetStartPos();
00258
00259 for (int i = 0; i < targetPhrase.GetSize(); i++) {
00260 if (targetPhrase.GetWord(i).IsNonTerminal()) {
00261 const ChartHypothesis * prev_hypo = cur_hypo.GetPrevHypo(targetPhrase.GetAlignNonTerm().GetNonTermIndexMap()[i]);
00262 absolute_source_position[targetPhrase.GetAlignNonTerm().GetNonTermIndexMap2()[i]] = prev_hypo->GetCurrSourceRange().GetEndPos();
00263 }
00264 }
00265
00266
00267 for (int i = 0; i != absolute_source_position.size(); i++) {
00268 if (i && absolute_source_position[i] == 0) {
00269 absolute_source_position[i] = absolute_source_position[i-1] + 1;
00270 }
00271 }
00272
00273 for (int i = 0; i < targetPhrase.GetSize(); i++) {
00274
00275
00276 if (targetPhrase.GetWord(i).IsNonTerminal()) {
00277
00278 const ChartHypothesis * prev_hypo = cur_hypo.GetPrevHypo(targetPhrase.GetAlignNonTerm().GetNonTermIndexMap()[i]);
00279 const BilingualLMState * prev_state = static_cast<const BilingualLMState *>(prev_hypo->GetFFState(featureID));
00280 const std::vector<int> prevWordAls = prev_state->GetWordAlignmentVector();
00281 for (std::vector<int>::const_iterator it = prevWordAls.begin(); it!= prevWordAls.end(); it++) {
00282 word_alignments.push_back(*it);
00283 }
00284 } else {
00285 bool resolvedIndexis = false;
00286 std::set<size_t> word_al = alignments.GetAlignmentsForTarget(i);
00287 if (word_al.empty()) {
00288 for (int j = 1; j < targetPhrase.GetSize(); j++) {
00289
00290
00291 if ((i+j) < targetPhrase.GetSize()) {
00292
00293 if (targetPhrase.GetWord(i + j).IsNonTerminal()) {
00294 const ChartHypothesis * prev_hypo = cur_hypo.GetPrevHypo(targetPhrase.GetAlignNonTerm().GetNonTermIndexMap()[i+j]);
00295 const BilingualLMState * prev_state = static_cast<const BilingualLMState *>(prev_hypo->GetFFState(featureID));
00296 source_word_mid_idx = prev_state->GetWordAlignmentVector().front();
00297 resolvedIndexis = true;
00298 break;
00299 }
00300 word_al = alignments.GetAlignmentsForTarget(i + j);
00301 if (!word_al.empty()) {
00302 break;
00303 }
00304 }
00305
00306 if ((i - j) >= 0) {
00307
00308 if (targetPhrase.GetWord(i - j).IsNonTerminal()) {
00309 const ChartHypothesis * prev_hypo = cur_hypo.GetPrevHypo(targetPhrase.GetAlignNonTerm().GetNonTermIndexMap()[i-j]);
00310 const BilingualLMState * prev_state = static_cast<const BilingualLMState *>(prev_hypo->GetFFState(featureID));
00311 source_word_mid_idx = prev_state->GetWordAlignmentVector().back();
00312 resolvedIndexis = true;
00313 break;
00314 }
00315
00316 word_al = alignments.GetAlignmentsForTarget(i - j);
00317 if (!word_al.empty()) {
00318 break;
00319 }
00320 }
00321 }
00322 }
00323
00324 if (!resolvedIndexis) {
00325
00326
00327 UTIL_THROW_IF2(word_al.size() == 0,
00328 "A target phrase with no alignments detected! " << targetPhrase << "Check if there is something wrong with your phrase table.");
00329 size_t source_center_index = selectMiddleAlignment(word_al);
00330
00331 source_word_mid_idx = absolute_source_position[source_center_index];
00332 }
00333 word_alignments.push_back(source_word_mid_idx);
00334 }
00335 }
00336
00337 }
00338
00339 size_t BilingualLM::getStateChart(std::vector<int>& neuralLMids) const
00340 {
00341 size_t hashCode = 0;
00342 for (int i = neuralLMids.size() - target_ngrams; i < neuralLMids.size(); i++) {
00343 int neuralLM_wordID;
00344 if (i < 0) {
00345 neuralLM_wordID = getNeuralLMId(BOS_word, false);
00346 } else {
00347 neuralLM_wordID = neuralLMids[i];
00348 }
00349 boost::hash_combine(hashCode, neuralLM_wordID);
00350 }
00351 return hashCode;
00352 }
00353
00354 void BilingualLM::getTargetWordsChart(
00355 std::vector<int>& neuralLMids,
00356 int current_word_index,
00357 std::vector<int>& words,
00358 bool sentence_begin) const
00359 {
00360
00361 for (int i = current_word_index - target_ngrams; i <= current_word_index; i++) {
00362 if (i < 0) {
00363 if (sentence_begin) {
00364 words.push_back(getNeuralLMId(BOS_word, false));
00365 } else {
00366 words.push_back(getNeuralLMId(getNullWord(), false));
00367 }
00368 } else {
00369 words.push_back(neuralLMids[i]);
00370 }
00371 }
00372 }
00373
00374 void BilingualLM::appendSourceWordsToVector(const Sentence &source_sent, std::vector<int> &words, int source_word_mid_idx) const
00375 {
00376
00377
00378
00379 int begin_idx;
00380 int end_idx;
00381
00382 if (source_ngrams % 2 == 0) {
00383 begin_idx = source_word_mid_idx - source_ngrams / 2 + 1;
00384 end_idx = source_word_mid_idx + source_ngrams / 2;
00385 } else {
00386 begin_idx = source_word_mid_idx - (source_ngrams - 1) / 2;
00387 end_idx = source_word_mid_idx + (source_ngrams - 1) / 2;
00388 }
00389
00390
00391 for (int j = begin_idx; j <= end_idx; j++) {
00392 int neuralLM_wordID;
00393 if (j < 0) {
00394 neuralLM_wordID = getNeuralLMId(BOS_word, true);
00395 } else if (j >= source_sent.GetSize()) {
00396 neuralLM_wordID = getNeuralLMId(EOS_word, true);
00397 } else {
00398 const Word& word = source_sent.GetWord(j);
00399 neuralLM_wordID = getNeuralLMId(word, true);
00400 }
00401 words.push_back(neuralLM_wordID);
00402 }
00403 }
00404
00405 FFState* BilingualLM::EvaluateWhenApplied(
00406 const ChartHypothesis& cur_hypo,
00407 int featureID,
00408 ScoreComponentCollection* accumulator) const
00409 {
00410
00411 std::vector<int> source_words;
00412 source_words.reserve(source_ngrams);
00413 std::vector<int> target_words;
00414 target_words.reserve(target_ngrams+1);
00415
00416 float value = 0;
00417 const TargetPhrase& currTargetPhrase = cur_hypo.GetCurrTargetPhrase();
00418
00419 std::vector<int> neuralLMids;
00420 std::vector<int> alignments;
00421
00422 int future_size = currTargetPhrase.GetNumTerminals();
00423 for (int i =0; i<currTargetPhrase.GetNumNonTerminals(); i++) {
00424 const ChartHypothesis * prev_hypo = cur_hypo.GetPrevHypo(i);
00425 future_size += prev_hypo->GetCurrTargetPhrase().GetSize();
00426 }
00427 neuralLMids.reserve(future_size);
00428 alignments.reserve(future_size);
00429
00430 getAllTargetIdsChart(cur_hypo, featureID, neuralLMids);
00431 getAllAlignments(cur_hypo, featureID, alignments);
00432
00433 bool sentence_begin = false;
00434 if (neuralLMids[0] == getNeuralLMId(BOS_word, false)) {
00435 sentence_begin = true;
00436 }
00437
00438
00439 const ChartManager& manager = cur_hypo.GetManager();
00440 const Sentence& source_sent = static_cast<const Sentence&>(manager.GetSource());
00441
00442 for (int i = 0; i < neuralLMids.size(); i++) {
00443
00444
00445 appendSourceWordsToVector(source_sent, source_words, alignments[i]);
00446 getTargetWordsChart(neuralLMids, i, target_words, sentence_begin);
00447
00448 value += Score(source_words, target_words);
00449
00450
00451 source_words.clear();
00452 target_words.clear();
00453
00454 }
00455 size_t new_state = getStateChart(neuralLMids);
00456
00457
00458 for (std::vector<const ChartHypothesis*>::const_iterator iter = cur_hypo.GetPrevHypos().begin(); iter != cur_hypo.GetPrevHypos().end(); ++iter) {
00459 const ChartHypothesis &prevHypo = **iter;
00460 value -= (prevHypo.GetScoreBreakdown().GetScoreForProducer(this));
00461 }
00462
00463 accumulator->PlusEquals(this, value);
00464
00465 return new BilingualLMState(new_state, alignments, neuralLMids);
00466 }
00467
00468 void BilingualLM::SetParameter(const std::string& key, const std::string& value)
00469 {
00470 if (key == "path") {
00471 m_filePath = value;
00472 } else {
00473 StatefulFeatureFunction::SetParameter(key, value);
00474 }
00475 }
00476
00477 }
00478