#include <IRST.h>
Public Member Functions | |
LanguageModelIRST (const std::string &line) | |
~LanguageModelIRST () | |
void | SetParameter (const std::string &key, const std::string &value) |
bool | IsUseable (const FactorMask &mask) const |
void | Load (AllOptions::ptr const &opts) |
override to load model files | |
const FFState * | EmptyHypothesisState (const InputType &) const |
return the state associated with the empty hypothesis for a given sentence | |
virtual LMResult | GetValue (const std::vector< const Word * > &contextFactor, State *finalState=NULL) const |
virtual void | CalcScore (const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const |
virtual FFState * | EvaluateWhenApplied (const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const |
This interface should be implemented. Notes: When evaluating the value of this feature function, you should avoid calling hypo.GetPrevHypo(). If you need something from the "previous" hypothesis, you should store it in an FFState object which will be passed in as prev_state. If you don't do this, you will get in trouble. | |
void | InitializeForInput (ttasksptr const &ttask) |
Called before search and collecting of translation options. | |
void | CleanUpAfterSentenceProcessing (const InputType &source) |
void | set_dictionary_upperbound (int dub) |
Protected Member Functions | |
void | CreateFactors (FactorCollection &factorCollection) |
int | GetLmID (const Word &word) const |
int | GetLmID (const std::string &str) const |
int | GetLmID (const Factor *factor) const |
Protected Attributes | |
std::vector< int > | m_lmIdLookup |
lmContainer * | m_lmtb |
int | m_unknownId |
int | m_empty |
int | m_lmtb_sentenceStart |
int | m_lmtb_sentenceEnd |
int | m_lmtb_dub |
int | m_lmtb_size |
dictionary * | d |
std::string | m_mapFilePath |
Definition at line 58 of file IRST.h.
Moses::LanguageModelIRST::LanguageModelIRST | ( | const std::string & | line | ) |
Definition at line 63 of file IRST.cpp.
References Moses::FeatureFunction::GetScoreProducerDescription(), Moses::StaticData::Instance(), Moses::LanguageModelSingleFactor::m_factorType, Moses::LanguageModelImplementation::m_filePath, m_lmtb_dub, m_lmtb_size, Moses::FeatureFunction::ReadParameters(), Moses::SPrint(), Moses::StaticData::ThreadCount(), and VERBOSE.
Moses::LanguageModelIRST::~LanguageModelIRST | ( | ) |
Definition at line 81 of file IRST.cpp.
References m_lmtb, lmContainer::reset_mmap(), and TRACE_ERR.
void Moses::LanguageModelIRST::CalcScore | ( | const Phrase & | phrase, | |
float & | fullScore, | |||
float & | ngramScore, | |||
size_t & | oovCount | |||
) | const [virtual] |
Reimplemented from Moses::LanguageModelImplementation.
Definition at line 241 of file IRST.cpp.
References lmContainer::clprob(), GetLmID(), Moses::Phrase::GetSize(), Moses::Phrase::GetWord(), m_lmtb, m_lmtb_sentenceStart, m_lmtb_size, m_unknownId, NULL, and Moses::TransformLMScore().
void Moses::LanguageModelIRST::CleanUpAfterSentenceProcessing | ( | const InputType & | source | ) | [virtual] |
Reimplemented from Moses::FeatureFunction.
Definition at line 414 of file IRST.cpp.
References Moses::StaticData::GetLMCacheCleanupThreshold(), Moses::StaticData::Instance(), Moses::LMCacheCleanup(), m_lmtb, lmContainer::reset_caches(), and TRACE_ERR.
void Moses::LanguageModelIRST::CreateFactors | ( | FactorCollection & | factorCollection | ) | [protected] |
Definition at line 126 of file IRST.cpp.
References Moses::FactorCollection::AddFactor(), BOS_, dict_entry::code, d, EOS_, fill(), Moses::Factor::GetId(), GetLmID(), lmIdMap, m_empty, Moses::LanguageModelSingleFactor::m_factorType, m_lmIdLookup, m_lmtb_sentenceEnd, m_lmtb_sentenceStart, Moses::LanguageModelSingleFactor::m_sentenceEnd, Moses::LanguageModelImplementation::m_sentenceEndWord, Moses::LanguageModelSingleFactor::m_sentenceStart, Moses::LanguageModelImplementation::m_sentenceStartWord, dictionary_iter::next(), NULL, Moses::Output, and dict_entry::word.
Referenced by Load().
const FFState * Moses::LanguageModelIRST::EmptyHypothesisState | ( | const InputType & | input | ) | const [virtual] |
return the state associated with the empty hypothesis for a given sentence
Reimplemented from Moses::LanguageModelImplementation.
FFState * Moses::LanguageModelIRST::EvaluateWhenApplied | ( | const Hypothesis & | cur_hypo, | |
const FFState * | prev_state, | |||
ScoreComponentCollection * | accumulator | |||
) | const [virtual] |
This interface should be implemented. Notes: When evaluating the value of this feature function, you should avoid calling hypo.GetPrevHypo(). If you need something from the "previous" hypothesis, you should store it in an FFState object which will be passed in as prev_state. If you don't do this, you will get in trouble.
Reimplemented from Moses::LanguageModelImplementation.
Definition at line 282 of file IRST.cpp.
References begin, lmContainer::clprob(), end, Moses::Hypothesis::GetCurrTargetLength(), Moses::Hypothesis::GetCurrTargetWordsRange(), Moses::Range::GetEndPos(), GetLmID(), Moses::Range::GetStartPos(), Moses::Hypothesis::GetWord(), Moses::Hypothesis::IsSourceCompleted(), m_lmtb, m_lmtb_sentenceEnd, m_lmtb_sentenceStart, m_lmtb_size, NULL, Moses::ScoreComponentCollection::PlusEquals(), and Moses::TransformLMScore().
int Moses::LanguageModelIRST::GetLmID | ( | const Factor * | factor | ) | const [protected] |
poiche' non c'e' distinzione tra i factorIDs delle parole sorgenti e delle parole target in Moses, puo' accadere che una parola target di cui non sia stato ancora calcolato il suo codice target abbia comunque un factorID noto (e quindi minore di m_lmIdLookup.size()) E' necessario dunque identificare questi casi di indeterminatezza del codice target. Attualmente, questo controllo e' stato implementato impostando a m_empty tutti i termini che non hanno ancora
OLD PROBLEM - SOLVED
IL PPROBLEMA ERA QUI m_lmIdLookup.push_back(code); PERCHE' USANDO PUSH_BACK IN REALTA' INSEREVIVAMO L'ELEMENTO NUOVO IN POSIZIONE (factorID-1) invece che in posizione factrID dove dopo andiamo a leggerlo (vedi caso C Cosi' funziona .... ho un dubbio su cosa c'e' nelle prime posizioni di m_lmIdLookup quindi e scopro che rimane vuota una entry ogni due perche' factorID cresce di due in due (perche' codifica sia source che target) "vuota" la posizione (factorID-1) non da problemi di correttezza, ma solo di "spreco" di memoria potremmo sostituirerendere m_lmIdLookup una std:map invece che un std::vector, ma si perde in efficienza nell'accesso perche' non e' piu' possibile quello random dei vettori a te la scelta!!!!
Definition at line 178 of file IRST.cpp.
References StringPiece::as_string(), d, dictionary::encode(), Moses::Factor::GetId(), Moses::Factor::GetString(), dictionary::incflag(), m_empty, m_lmIdLookup, and m_unknownId.
int Moses::LanguageModelIRST::GetLmID | ( | const std::string & | str | ) | const [protected] |
Definition at line 168 of file IRST.cpp.
References d, and dictionary::encode().
int Moses::LanguageModelIRST::GetLmID | ( | const Word & | word | ) | const [protected] |
Definition at line 173 of file IRST.cpp.
References Moses::Word::GetFactor(), and Moses::LanguageModelSingleFactor::m_factorType.
Referenced by CalcScore(), CreateFactors(), and EvaluateWhenApplied().
virtual LMResult Moses::LanguageModelIRST::GetValue | ( | const std::vector< const Word * > & | contextFactor, | |
State * | finalState = NULL | |||
) | const [virtual] |
Implements Moses::LanguageModelSingleFactor.
void Moses::LanguageModelIRST::InitializeForInput | ( | ttasksptr const & | ttask | ) | [virtual] |
Called before search and collecting of translation options.
Reimplemented from Moses::FeatureFunction.
Definition at line 406 of file IRST.cpp.
References m_lmtb.
bool Moses::LanguageModelIRST::IsUseable | ( | const FactorMask & | mask | ) | const [virtual] |
Reimplemented from Moses::LanguageModelSingleFactor.
Definition at line 93 of file IRST.cpp.
References Moses::LanguageModelSingleFactor::m_factorType.
void Moses::LanguageModelIRST::Load | ( | AllOptions::ptr const & | opts | ) | [virtual] |
override to load model files
Reimplemented from Moses::FeatureFunction.
Definition at line 99 of file IRST.cpp.
References CreateFactors(), lmContainer::CreateLanguageModel(), d, lmContainer::getDict(), Moses::FeatureFunction::GetScoreProducerDescription(), dictionary::incflag(), lmContainer::init_caches(), Moses::FactorCollection::Instance(), lmContainer::load(), m_empty, Moses::LanguageModelImplementation::m_filePath, m_lmtb, m_lmtb_dub, m_lmtb_size, Moses::LanguageModelImplementation::m_nGramOrder, m_unknownId, lmContainer::maxlevel(), dictionary::oovcode(), lmContainer::setlogOOVpenalty(), lmContainer::setMaxLoadedLevel(), and VERBOSE.
void Moses::LanguageModelIRST::set_dictionary_upperbound | ( | int | dub | ) | [inline] |
void Moses::LanguageModelIRST::SetParameter | ( | const std::string & | key, | |
const std::string & | value | |||
) | [virtual] |
Reimplemented from Moses::LanguageModelSingleFactor.
Definition at line 428 of file IRST.cpp.
References m_lmtb_dub, and Moses::LanguageModelImplementation::m_nGramOrder.
dictionary* Moses::LanguageModelIRST::d [protected] |
int Moses::LanguageModelIRST::m_empty [protected] |
std::vector<int> Moses::LanguageModelIRST::m_lmIdLookup [mutable, protected] |
lmContainer* Moses::LanguageModelIRST::m_lmtb [protected] |
Definition at line 62 of file IRST.h.
Referenced by CalcScore(), CleanUpAfterSentenceProcessing(), EvaluateWhenApplied(), InitializeForInput(), Load(), and ~LanguageModelIRST().
int Moses::LanguageModelIRST::m_lmtb_dub [protected] |
Definition at line 68 of file IRST.h.
Referenced by LanguageModelIRST(), Load(), and SetParameter().
int Moses::LanguageModelIRST::m_lmtb_sentenceEnd [protected] |
int Moses::LanguageModelIRST::m_lmtb_sentenceStart [protected] |
Definition at line 66 of file IRST.h.
Referenced by CalcScore(), CreateFactors(), and EvaluateWhenApplied().
int Moses::LanguageModelIRST::m_lmtb_size [protected] |
Definition at line 69 of file IRST.h.
Referenced by CalcScore(), EvaluateWhenApplied(), LanguageModelIRST(), Load(), and set_dictionary_upperbound().
std::string Moses::LanguageModelIRST::m_mapFilePath [protected] |
int Moses::LanguageModelIRST::m_unknownId [protected] |