Namespaces | |
namespace | detail |
namespace | trie |
Classes | |
struct | FixedWidthParameters |
struct | Parameters |
class | BinaryFormat |
struct | Config |
class | RuleScore |
class | QueryPrinter |
struct | ExtendReturn |
class | DontQuantize |
class | SeparatelyQuantize |
class | State |
struct | Left |
struct | ChartState |
class | GenericProbingProxy |
class | GenericTrieUnigramProxy |
struct | BackoffValue |
struct | RestValue |
class | NoRestBuild |
class | MaxRestBuild |
class | LowerRestBuild |
class | ImmediateWriteWordsWrapper |
class | WriteWordsWrapper |
class | SortedVocabulary |
struct | ProbingVocabularyEntry |
class | ProbingVocabulary |
class | WriteUniqueWords |
class | NoOpUniqueWords |
class | GrowableVocab |
Typedefs | |
typedef ::lm::ngram::ProbingVocabulary | Vocabulary |
typedef ProbingModel | Model |
typedef State | Right |
Enumerations | |
enum | ModelType { PROBING = 0, REST_PROBING = 1, TRIE = 2, QUANT_TRIE = 3, ARRAY_TRIE = 4, QUANT_ARRAY_TRIE = 5 } |
Functions | |
bool | IsBinaryFormat (int fd) |
void | ReadHeader (int fd, Parameters &out) |
void | MatchCheck (ModelType model_type, unsigned int search_version, const Parameters ¶ms) |
bool | RecognizeBinary (const char *file, ModelType &recognized) |
void | SetExtension (float &backoff) |
bool | HasExtension (const float &backoff) |
base::Model * | LoadVirtual (const char *file_name, const Config &config, ModelType model_type) |
LM_NAME_MODEL (ProbingModel, detail::GenericModel< detail::HashedSearch< BackoffValue > LM_COMMA() ProbingVocabulary >) | |
LM_NAME_MODEL (RestProbingModel, detail::GenericModel< detail::HashedSearch< RestValue > LM_COMMA() ProbingVocabulary >) | |
LM_NAME_MODEL (TrieModel, detail::GenericModel< trie::TrieSearch< DontQuantize LM_COMMA() trie::DontBhiksha > LM_COMMA() SortedVocabulary >) | |
LM_NAME_MODEL (ArrayTrieModel, detail::GenericModel< trie::TrieSearch< DontQuantize LM_COMMA() trie::ArrayBhiksha > LM_COMMA() SortedVocabulary >) | |
LM_NAME_MODEL (QuantTrieModel, detail::GenericModel< trie::TrieSearch< SeparatelyQuantize LM_COMMA() trie::DontBhiksha > LM_COMMA() SortedVocabulary >) | |
LM_NAME_MODEL (QuantArrayTrieModel, detail::GenericModel< trie::TrieSearch< SeparatelyQuantize LM_COMMA() trie::ArrayBhiksha > LM_COMMA() SortedVocabulary >) | |
std::ostream & | operator<< (std::ostream &o, const State &state) |
template<class Model , class Printer > | |
void | Query (const Model &model, bool sentence_context, Printer &printer) |
template<class Model > | |
void | Query (const char *file, const Config &config, bool sentence_context, QueryPrinter &printer) |
template<class Model > | |
ExtendReturn | ExtendLoop (const Model &model, unsigned char seen, const WordIndex *add_rbegin, const WordIndex *add_rend, const float *backoff_start, const uint64_t *pointers, const uint64_t *pointers_end, uint64_t *&pointers_write, float *backoff_write) |
template<class Model > | |
float | RevealBefore (const Model &model, const Right &reveal, const unsigned char seen, bool reveal_full, Left &left, Right &right) |
template<class Model > | |
float | RevealAfter (const Model &model, Left &left, Right &right, const Left &reveal, unsigned char seen) |
template<class Model > | |
float | Subsume (const Model &model, Left &first_left, const Right &first_right, const Left &second_left, Right &second_right, const unsigned int between_length) |
void | ShowSizes (const std::vector< uint64_t > &counts, const lm::ngram::Config &config) |
void | ShowSizes (const std::vector< uint64_t > &counts) |
void | ShowSizes (const char *file, const lm::ngram::Config &config) |
uint64_t | hash_value (const State &state, uint64_t seed=0) |
uint64_t | hash_value (const Left &left) |
uint64_t | hash_value (const ChartState &state) |
void | MissingUnknown (const Config &config) throw (SpecialWordMissingException) |
void | MissingSentenceMarker (const Config &config, const char *str) throw (SpecialWordMissingException) |
template<class Vocab > | |
void | CheckSpecials (const Config &config, const Vocab &vocab) throw (SpecialWordMissingException) |
Variables | |
const char * | kModelNames [6] = {"probing hash tables", "probing hash tables with rest costs", "trie", "trie with quantization", "trie with array-compressed pointers", "trie with quantization and array-compressed pointers"} |
const std::size_t | kInvalidSize = static_cast<std::size_t>(-1) |
const float | kNoExtensionBackoff = -0.0 |
const float | kExtensionBackoff = 0.0 |
const uint64_t | kNoExtensionQuant = 0 |
const uint64_t | kExtensionQuant = 1 |
const ModelType | HASH_PROBING = PROBING |
const ModelType | TRIE_SORTED = TRIE |
const ModelType | QUANT_TRIE_SORTED = QUANT_TRIE |
const ModelType | ARRAY_TRIE_SORTED = ARRAY_TRIE |
const ModelType | QUANT_ARRAY_TRIE_SORTED = QUANT_ARRAY_TRIE |
typedef ProbingModel lm::ngram::Model |
typedef State lm::ngram::Right |
enum lm::ngram::ModelType |
Definition at line 9 of file model_type.hh.
void lm::ngram::CheckSpecials | ( | const Config & | config, | |
const Vocab & | vocab | |||
) | throw (SpecialWordMissingException) [inline] |
Definition at line 213 of file vocab.hh.
References MissingSentenceMarker(), and MissingUnknown().
Referenced by lm::ngram::detail::HashedSearch< Value >::InitializeFromARPA().
ExtendReturn lm::ngram::ExtendLoop | ( | const Model & | model, | |
unsigned char | seen, | |||
const WordIndex * | add_rbegin, | |||
const WordIndex * | add_rend, | |||
const float * | backoff_start, | |||
const uint64_t * | pointers, | |||
const uint64_t * | pointers_end, | |||
uint64_t *& | pointers_write, | |||
float * | backoff_write | |||
) | [inline] |
Definition at line 19 of file partial.hh.
References lm::ngram::ExtendReturn::adjust, lm::ngram::ExtendReturn::make_full, lm::ngram::ExtendReturn::next_use, and swap().
Referenced by RevealAfter(), RevealBefore(), and Subsume().
bool lm::ngram::HasExtension | ( | const float & | backoff | ) | [inline] |
Definition at line 32 of file blank.hh.
References kNoExtensionBackoff.
Referenced by lm::ngram::detail::GenericModel< Search, VocabularyT >::GetState().
uint64_t lm::ngram::hash_value | ( | const ChartState & | state | ) | [inline] |
Definition at line 117 of file state.hh.
References hash_value(), lm::ngram::ChartState::left, and lm::ngram::ChartState::right.
uint64_t lm::ngram::hash_value | ( | const Left & | left | ) | [inline] |
Definition at line 86 of file state.hh.
References lm::ngram::Left::full, lm::ngram::Left::length, util::MurmurHashNative(), and lm::ngram::Left::pointers.
uint64_t lm::ngram::hash_value | ( | const State & | state, | |
uint64_t | seed = 0 | |||
) | [inline] |
Definition at line 52 of file state.hh.
References lm::ngram::State::length, util::MurmurHashNative(), and lm::ngram::State::words.
Referenced by hash_value().
bool lm::ngram::IsBinaryFormat | ( | int | fd | ) |
Definition at line 85 of file binary_format.cc.
References util::scoped_memory::get(), util::kBadSize, util::LAZY, util::MapRead(), util::SizeFile(), UTIL_THROW, and UTIL_THROW_IF.
Referenced by lm::ngram::detail::GenericModel< Search, VocabularyT >::GenericModel(), and RecognizeBinary().
lm::ngram::LM_NAME_MODEL | ( | QuantArrayTrieModel | , | |
detail::GenericModel< trie::TrieSearch< SeparatelyQuantize LM_COMMA() trie::ArrayBhiksha > LM_COMMA() SortedVocabulary > | ||||
) |
lm::ngram::LM_NAME_MODEL | ( | QuantTrieModel | , | |
detail::GenericModel< trie::TrieSearch< SeparatelyQuantize LM_COMMA() trie::DontBhiksha > LM_COMMA() SortedVocabulary > | ||||
) |
lm::ngram::LM_NAME_MODEL | ( | ArrayTrieModel | , | |
detail::GenericModel< trie::TrieSearch< DontQuantize LM_COMMA() trie::ArrayBhiksha > LM_COMMA() SortedVocabulary > | ||||
) |
lm::ngram::LM_NAME_MODEL | ( | TrieModel | , | |
detail::GenericModel< trie::TrieSearch< DontQuantize LM_COMMA() trie::DontBhiksha > LM_COMMA() SortedVocabulary > | ||||
) |
lm::ngram::LM_NAME_MODEL | ( | RestProbingModel | , | |
detail::GenericModel< detail::HashedSearch< RestValue > LM_COMMA() ProbingVocabulary > | ||||
) |
lm::ngram::LM_NAME_MODEL | ( | ProbingModel | , | |
detail::GenericModel< detail::HashedSearch< BackoffValue > LM_COMMA() ProbingVocabulary > | ||||
) |
base::Model * lm::ngram::LoadVirtual | ( | const char * | file_name, | |
const Config & | config, | |||
ModelType | model_type | |||
) |
Definition at line 328 of file model.cc.
References ARRAY_TRIE, PROBING, QUANT_ARRAY_TRIE, QUANT_TRIE, RecognizeBinary(), REST_PROBING, TRIE, and UTIL_THROW.
void lm::ngram::MatchCheck | ( | ModelType | model_type, | |
unsigned int | search_version, | |||
const Parameters & | params | |||
) |
Definition at line 127 of file binary_format.cc.
References lm::ngram::Parameters::fixed, kModelNames, lm::ngram::FixedWidthParameters::model_type, lm::ngram::FixedWidthParameters::search_version, UTIL_THROW, and UTIL_THROW_IF.
Referenced by lm::ngram::BinaryFormat::InitializeBinary().
void lm::ngram::MissingSentenceMarker | ( | const Config & | config, | |
const char * | str | |||
) | throw (SpecialWordMissingException) |
Definition at line 316 of file vocab.cc.
References lm::COMPLAIN, lm::SILENT, str, lm::THROW_UP, and UTIL_THROW.
Referenced by CheckSpecials().
void lm::ngram::MissingUnknown | ( | const Config & | config | ) | throw (SpecialWordMissingException) |
Definition at line 304 of file vocab.cc.
References lm::COMPLAIN, lm::SILENT, lm::THROW_UP, and UTIL_THROW.
Referenced by CheckSpecials().
std::ostream& lm::ngram::operator<< | ( | std::ostream & | o, | |
const State & | state | |||
) |
Definition at line 16 of file model_test.cc.
References lm::ngram::State::length, and lm::ngram::State::words.
void lm::ngram::Query | ( | const char * | file, | |
const Config & | config, | |||
bool | sentence_context, | |||
QueryPrinter & | printer | |||
) | [inline] |
Definition at line 103 of file ngram_query.hh.
void lm::ngram::Query | ( | const Model & | model, | |
bool | sentence_context, | |||
Printer & | printer | |||
) | [inline] |
Definition at line 52 of file ngram_query.hh.
References util::FilePiece::get(), lm::FullScoreReturn::prob, util::FilePiece::ReadWordSameLine(), state, and UTIL_THROW_IF.
void lm::ngram::ReadHeader | ( | int | fd, | |
Parameters & | out | |||
) |
Definition at line 117 of file binary_format.cc.
References lm::ngram::Parameters::counts, lm::ngram::Parameters::fixed, lm::ngram::FixedWidthParameters::order, lm::ngram::FixedWidthParameters::probing_multiplier, util::ReadOrThrow(), util::SeekOrThrow(), and UTIL_THROW.
Referenced by lm::ngram::BinaryFormat::InitializeBinary(), and RecognizeBinary().
bool lm::ngram::RecognizeBinary | ( | const char * | file, | |
ModelType & | recognized | |||
) |
Definition at line 290 of file binary_format.cc.
References lm::ngram::Parameters::fixed, util::scoped_fd::get(), IsBinaryFormat(), lm::ngram::FixedWidthParameters::model_type, util::OpenReadOrThrow(), and ReadHeader().
Referenced by Moses::ConstructBackwardLM(), Moses::ConstructDsgLM(), Moses::ConstructKenLM(), Moses::ConstructOSMLM(), Moses::ConstructReloadingLM(), LoadVirtual(), and main().
float lm::ngram::RevealAfter | ( | const Model & | model, | |
Left & | left, | |||
Right & | right, | |||
const Left & | reveal, | |||
unsigned char | seen | |||
) | [inline] |
Definition at line 111 of file partial.hh.
References lm::ngram::ExtendReturn::adjust, lm::ngram::State::backoff, ExtendLoop(), lm::ngram::Left::full, lm::ngram::State::length, lm::ngram::Left::length, lm::ngram::ExtendReturn::make_full, lm::ngram::ExtendReturn::next_use, NULL, lm::ngram::Left::pointers, and lm::ngram::State::words.
float lm::ngram::RevealBefore | ( | const Model & | model, | |
const Right & | reveal, | |||
const unsigned char | seen, | |||
bool | reveal_full, | |||
Left & | left, | |||
Right & | right | |||
) | [inline] |
Definition at line 83 of file partial.hh.
References lm::ngram::ExtendReturn::adjust, lm::ngram::State::backoff, ExtendLoop(), lm::ngram::Left::full, lm::ngram::Left::length, lm::ngram::State::length, lm::ngram::ExtendReturn::make_full, lm::ngram::ExtendReturn::next_use, NULL, lm::ngram::Left::pointers, and lm::ngram::State::words.
void lm::ngram::SetExtension | ( | float & | backoff | ) | [inline] |
void lm::ngram::ShowSizes | ( | const char * | file, | |
const lm::ngram::Config & | config | |||
) |
Definition at line 56 of file sizes.cc.
References lm::ReadARPACounts(), and ShowSizes().
void lm::ngram::ShowSizes | ( | const std::vector< uint64_t > & | counts | ) |
Definition at line 51 of file sizes.cc.
References ShowSizes().
void lm::ngram::ShowSizes | ( | const std::vector< uint64_t > & | counts, | |
const lm::ngram::Config & | config | |||
) |
Definition at line 11 of file sizes.cc.
References lm::ngram::Config::backoff_bits, min_element(), lm::ngram::Config::pointer_bhiksha_bits, lm::ngram::Config::prob_bits, and lm::ngram::Config::probing_multiplier.
Referenced by main(), and ShowSizes().
float lm::ngram::Subsume | ( | const Model & | model, | |
Left & | first_left, | |||
const Right & | first_right, | |||
const Left & | second_left, | |||
Right & | second_right, | |||
const unsigned int | between_length | |||
) | [inline] |
Definition at line 135 of file partial.hh.
References lm::ngram::ExtendReturn::adjust, lm::ngram::State::backoff, ExtendLoop(), lm::ngram::Left::full, lm::ngram::Left::length, lm::ngram::State::length, lm::ngram::ExtendReturn::make_full, lm::ngram::ExtendReturn::next_use, NULL, lm::ngram::Left::pointers, and lm::ngram::State::words.
const ModelType lm::ngram::ARRAY_TRIE_SORTED = ARRAY_TRIE |
Definition at line 15 of file model_type.hh.
const ModelType lm::ngram::HASH_PROBING = PROBING |
Definition at line 12 of file model_type.hh.
const float lm::ngram::kExtensionBackoff = 0.0 |
Definition at line 23 of file blank.hh.
Referenced by lm::ReadBackoff(), SetExtension(), and lm::ngram::SeparatelyQuantize::Train().
const uint64_t lm::ngram::kExtensionQuant = 1 |
const std::size_t lm::ngram::kInvalidSize = static_cast<std::size_t>(-1) |
Definition at line 136 of file binary_format.cc.
Referenced by lm::ngram::BinaryFormat::GrowForSearch(), lm::ngram::BinaryFormat::LoadBinary(), lm::ngram::BinaryFormat::ReadForConfig(), and lm::ngram::BinaryFormat::WriteVocabWords().
const char * lm::ngram::kModelNames = {"probing hash tables", "probing hash tables with rest costs", "trie", "trie with quantization", "trie with array-compressed pointers", "trie with quantization and array-compressed pointers"} |
const float lm::ngram::kNoExtensionBackoff = -0.0 |
Definition at line 22 of file blank.hh.
Referenced by HasExtension(), lm::ReadBackoff(), SetExtension(), and lm::ngram::SeparatelyQuantize::Train().
const uint64_t lm::ngram::kNoExtensionQuant = 0 |
const ModelType lm::ngram::QUANT_ARRAY_TRIE_SORTED = QUANT_ARRAY_TRIE |
Definition at line 16 of file model_type.hh.
const ModelType lm::ngram::QUANT_TRIE_SORTED = QUANT_TRIE |
Definition at line 14 of file model_type.hh.
const ModelType lm::ngram::TRIE_SORTED = TRIE |
Definition at line 13 of file model_type.hh.