#include <vocab.h>
Public Types | |
typedef std::map< Word, wordID_t > | Word2Id |
typedef std::map< wordID_t, Word > | Id2Word |
Public Member Functions | |
Vocab (bool sntMarkers=true) | |
Vocab (const std::string &vocab_path, const FactorDirection &direction, const FactorList &factors, bool closed=true) | |
Vocab (FileHandler *fin, const FactorDirection &direction, const FactorList &factors, bool closed=true) | |
Vocab (FileHandler *fin) | |
~Vocab () | |
wordID_t | GetWordID (const std::string &word, const FactorDirection &direction, const FactorList &factors, bool isNonTerminal) |
wordID_t | GetWordID (const Word &word) |
wordID_t | GetWordID (const std::string &word) |
Word & | GetWord (wordID_t id) |
wordID_t | GetkOOVWordID () |
wordID_t | GetBOSWordID () |
const Word & | GetkOOVWord () |
const Word & | GetkBOSWord () |
const Word & | GetkEOSWord () |
bool | InVocab (wordID_t id) |
bool | InVocab (const Word &word) |
uint32_t | Size () |
void | MakeClosed () |
void | MakeOpen () |
bool | IsClosed () |
bool | Save (const std::string &vocab_path) |
bool | Save (FileHandler *fout) |
bool | Load (const std::string &vocab_path, const FactorDirection &direction, const FactorList &factors, bool closed=true) |
bool | Load (FileHandler *fin, const FactorDirection &direction, const FactorList &factors, bool closed=true) |
bool | Load (FileHandler *fin) |
void | PrintVocab () |
Word2Id::const_iterator | VocabStart () |
Word2Id::const_iterator | VocabEnd () |
Protected Member Functions | |
const Word | InitSpecialWord (const std::string &type) |
void | InitSpecialWords () |
Protected Attributes | |
bool | m_closed |
const wordID_t | m_kOOVWordID |
const wordID_t | m_kBOSWordID |
Word | m_kBOSWord |
Word | m_kEOSWord |
Word | m_kOOVWord |
Word2Id | m_words2ids |
Id2Word | m_ids2words |
Definition at line 16 of file vocab.h.
typedef std::map<wordID_t, Word> Moses::Vocab::Id2Word |
typedef std::map<Word, wordID_t> Moses::Vocab::Word2Id |
Moses::Vocab::Vocab | ( | bool | sntMarkers = true |
) | [inline] |
Definition at line 23 of file vocab.h.
References GetWordID(), InitSpecialWords(), m_kBOSWord, and m_kEOSWord.
Moses::Vocab::Vocab | ( | const std::string & | vocab_path, | |
const FactorDirection & | direction, | |||
const FactorList & | factors, | |||
bool | closed = true | |||
) | [inline] |
Definition at line 36 of file vocab.h.
References InitSpecialWords(), Load(), and UTIL_THROW_IF2.
Moses::Vocab::Vocab | ( | FileHandler * | fin, | |
const FactorDirection & | direction, | |||
const FactorList & | factors, | |||
bool | closed = true | |||
) | [inline] |
Definition at line 44 of file vocab.h.
References InitSpecialWords(), Load(), and UTIL_THROW_IF2.
Moses::Vocab::Vocab | ( | FileHandler * | fin | ) | [inline] |
wordID_t Moses::Vocab::GetBOSWordID | ( | ) | [inline] |
Definition at line 67 of file vocab.h.
References m_kBOSWordID.
Referenced by OnlineRLM< T >::insert().
const Word& Moses::Vocab::GetkBOSWord | ( | ) | [inline] |
const Word& Moses::Vocab::GetkEOSWord | ( | ) | [inline] |
const Word& Moses::Vocab::GetkOOVWord | ( | ) | [inline] |
Definition at line 70 of file vocab.h.
References m_kOOVWord.
Referenced by Moses::BilingualDynSuffixArray::GetMosesFactorIDs(), and Load().
wordID_t Moses::Vocab::GetkOOVWordID | ( | ) | [inline] |
Definition at line 64 of file vocab.h.
References m_kOOVWordID.
Referenced by OnlineRLM< T >::getProb().
Definition at line 65 of file vocab.cpp.
References m_ids2words, and m_kOOVWord.
Referenced by Moses::BilingualDynSuffixArray::GetMosesFactorIDs().
wordID_t Moses::Vocab::GetWordID | ( | const std::string & | word | ) |
Definition at line 29 of file vocab.cpp.
References Moses::Word::CreateFromString(), GetWordID(), and Moses::Input.
Definition at line 48 of file vocab.cpp.
References m_closed, m_ids2words, m_kOOVWordID, and m_words2ids.
wordID_t Moses::Vocab::GetWordID | ( | const std::string & | word, | |
const FactorDirection & | direction, | |||
const FactorList & | factors, | |||
bool | isNonTerminal | |||
) |
Definition at line 39 of file vocab.cpp.
References Moses::Word::CreateFromString().
Referenced by GetWordID(), OnlineRLM< T >::insert(), OnlineRLM< T >::remove(), OnlineRLM< T >::sbsqQuery(), OnlineRLM< T >::update(), and Vocab().
const Word Moses::Vocab::InitSpecialWord | ( | const std::string & | type | ) | [protected] |
Definition at line 15 of file vocab.cpp.
References Moses::Word::CreateFromString(), and Moses::Input.
Referenced by InitSpecialWords().
void Moses::Vocab::InitSpecialWords | ( | ) | [protected] |
Definition at line 8 of file vocab.cpp.
References BOS_, EOS_, InitSpecialWord(), m_kBOSWord, m_kEOSWord, m_kOOVWord, and UNKNOWN_FACTOR.
Referenced by Vocab().
bool Moses::Vocab::InVocab | ( | const Word & | word | ) |
bool Moses::Vocab::InVocab | ( | wordID_t | id | ) |
bool Moses::Vocab::IsClosed | ( | ) | [inline] |
bool Moses::Vocab::Load | ( | FileHandler * | fin | ) |
Definition at line 106 of file vocab.cpp.
References Moses::Input, and Load().
bool Moses::Vocab::Load | ( | FileHandler * | fin, | |
const FactorDirection & | direction, | |||
const FactorList & | factors, | |||
bool | closed = true | |||
) |
Definition at line 112 of file vocab.cpp.
References Moses::Word::CreateFromString(), GetkOOVWord(), m_closed, m_ids2words, m_words2ids, and UTIL_THROW_IF2.
bool Moses::Vocab::Load | ( | const std::string & | vocab_path, | |
const FactorDirection & | direction, | |||
const FactorList & | factors, | |||
bool | closed = true | |||
) |
void Moses::Vocab::MakeClosed | ( | ) | [inline] |
void Moses::Vocab::MakeOpen | ( | ) | [inline] |
Definition at line 88 of file vocab.h.
References m_closed.
Referenced by OnlineRLM< T >::update().
void Moses::Vocab::PrintVocab | ( | ) |
bool Moses::Vocab::Save | ( | FileHandler * | fout | ) |
bool Moses::Vocab::Save | ( | const std::string & | vocab_path | ) |
Definition at line 81 of file vocab.cpp.
Referenced by OnlineRLM< T >::save().
uint32_t Moses::Vocab::Size | ( | ) | [inline] |
Definition at line 82 of file vocab.h.
References m_words2ids.
Referenced by OnlineRLM< T >::getProb().
Word2Id::const_iterator Moses::Vocab::VocabEnd | ( | ) | [inline] |
Word2Id::const_iterator Moses::Vocab::VocabStart | ( | ) | [inline] |
bool Moses::Vocab::m_closed [protected] |
Definition at line 110 of file vocab.h.
Referenced by GetWordID(), IsClosed(), Load(), MakeClosed(), and MakeOpen().
Id2Word Moses::Vocab::m_ids2words [protected] |
Definition at line 122 of file vocab.h.
Referenced by GetWord(), GetWordID(), InVocab(), Load(), PrintVocab(), and Save().
Word Moses::Vocab::m_kBOSWord [protected] |
Definition at line 114 of file vocab.h.
Referenced by GetkBOSWord(), InitSpecialWords(), and Vocab().
const wordID_t Moses::Vocab::m_kBOSWordID [protected] |
Word Moses::Vocab::m_kEOSWord [protected] |
Definition at line 115 of file vocab.h.
Referenced by GetkEOSWord(), InitSpecialWords(), and Vocab().
Word Moses::Vocab::m_kOOVWord [protected] |
Definition at line 116 of file vocab.h.
Referenced by GetkOOVWord(), GetWord(), and InitSpecialWords().
const wordID_t Moses::Vocab::m_kOOVWordID [protected] |
Word2Id Moses::Vocab::m_words2ids [protected] |
Definition at line 121 of file vocab.h.
Referenced by GetWordID(), InVocab(), Load(), PrintVocab(), Size(), VocabEnd(), and VocabStart().