00001 // $Id$ 00002 00003 /*********************************************************************** 00004 Moses - factored phrase-based language decoder 00005 Copyright (C) 2006 University of Edinburgh 00006 00007 This library is free software; you can redistribute it and/or 00008 modify it under the terms of the GNU Lesser General Public 00009 License as published by the Free Software Foundation; either 00010 version 2.1 of the License, or (at your option) any later version. 00011 00012 This library is distributed in the hope that it will be useful, 00013 but WITHOUT ANY WARRANTY; without even the implied warranty of 00014 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00015 Lesser General Public License for more details. 00016 00017 You should have received a copy of the GNU Lesser General Public 00018 License along with this library; if not, write to the Free Software 00019 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 00020 ***********************************************************************/ 00021 00022 #ifndef moses_LanguageModelBackward_h 00023 #define moses_LanguageModelBackward_h 00024 00025 #include <string> 00026 00027 #include "moses/LM/Ken.h" 00028 #include "moses/LM/BackwardLMState.h" 00029 00030 #include "lm/state.hh" 00031 00032 namespace Moses 00033 { 00034 00036 LanguageModel *ConstructBackwardLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy); 00037 00038 class FFState; 00039 // template<typename M> class BackwardLanguageModelTest; 00040 class BackwardLanguageModelTest; 00041 00042 /* 00043 * An implementation of single factor backward LM using Kenneth's code. 00044 */ 00045 template <class Model> class BackwardLanguageModel : public LanguageModelKen<Model> 00046 { 00047 public: 00048 BackwardLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy); 00049 00050 virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const; 00051 00052 virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const; 00053 00054 virtual FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const; 00055 00056 FFState *Evaluate(const Phrase &phrase, const FFState *ps, float &returnedScore) const; 00057 00058 private: 00059 00060 // These lines are required to make the parent class's protected members visible to this class 00061 using LanguageModelKen<Model>::m_ngram; 00062 using LanguageModelKen<Model>::m_beginSentenceFactor; 00063 using LanguageModelKen<Model>::m_factorType; 00064 using LanguageModelKen<Model>::TranslateID; 00065 00066 // friend class Moses::BackwardLanguageModelTest<Model>; 00067 friend class Moses::BackwardLanguageModelTest; 00068 /* 00069 lm::ngram::ChartState* GetState(FFState *ffState) { 00070 return NULL; 00071 } 00072 */ 00073 /* 00074 double Score(FFState *ffState) { 00075 BackwardLMState *lmState = static_cast< BackwardLMState* >(ffState); 00076 lm::ngram::ChartState &state = lmState->state; 00077 lm::ngram::RuleScore<Model> ruleScore(*m_ngram, lmState); 00078 return ruleScore.Finish(); 00079 } 00080 */ 00081 }; 00082 00083 } // namespace Moses 00084 00085 #endif 00086 00087 // To create a sample backward language model using SRILM: 00088 // 00089 // (ngram-count and reverse-text are SRILM programs) 00090 // 00091 // head -n 49 ./contrib/synlm/hhmm/LICENSE | tail -n 45 | tr '\n' ' ' | ./scripts/ems/support/split-sentences.perl | ./scripts/tokenizer/lowercase.perl | ./scripts/tokenizer/tokenizer.perl | reverse-text | ngram-count -order 3 -text - -lm - > lm/backward.arpa