#include <vocab.hh>
Public Member Functions | |
ProbingVocabulary () | |
WordIndex | Index (const StringPiece &str) const |
WordIndex | Bound () const |
void | SetupMemory (void *start, std::size_t allocated) |
void | SetupMemory (void *start, std::size_t allocated, std::size_t, const Config &) |
void | Relocate (void *new_start) |
void | ConfigureEnumerate (EnumerateVocab *to, std::size_t max_entries) |
WordIndex | Insert (const StringPiece &str) |
template<class Weights > | |
void | FinishedLoading (Weights *) |
std::size_t | UnkCountChangePadding () const |
bool | SawUnk () const |
void | LoadedBinary (bool have_words, int fd, EnumerateVocab *to, uint64_t offset) |
Static Public Member Functions | |
static uint64_t | Size (uint64_t entries, float probing_multiplier) |
static uint64_t | Size (uint64_t entries, const Config &config) |
Definition at line 156 of file vocab.hh.
WordIndex lm::ngram::ProbingVocabulary::Bound | ( | ) | const [inline] |
void lm::ngram::ProbingVocabulary::ConfigureEnumerate | ( | EnumerateVocab * | to, | |
std::size_t | max_entries | |||
) |
Definition at line 270 of file vocab.cc.
References lm::EnumerateVocab::Add().
void lm::ngram::ProbingVocabulary::FinishedLoading | ( | Weights * | ) | [inline] |
WordIndex lm::ngram::ProbingVocabulary::Index | ( | const StringPiece & | str | ) | const [inline, virtual] |
Implements lm::base::Vocabulary.
Definition at line 160 of file vocab.hh.
References util::ProbingHashTable< EntryT, HashT, EqualT, ModT >::Find(), and lm::ngram::detail::HashForVocab().
Referenced by LoadedBinary().
WordIndex lm::ngram::ProbingVocabulary::Insert | ( | const StringPiece & | str | ) |
Definition at line 277 of file vocab.cc.
References lm::EnumerateVocab::Add(), lm::ngram::detail::HashForVocab(), util::ProbingHashTable< EntryT, HashT, EqualT, ModT >::Insert(), and lm::ngram::ProbingVocabularyEntry::Make().
void lm::ngram::ProbingVocabulary::LoadedBinary | ( | bool | have_words, | |
int | fd, | |||
EnumerateVocab * | to, | |||
uint64_t | offset | |||
) |
Definition at line 297 of file vocab.cc.
References lm::ngram::detail::ProbingVocabularyHeader::bound, Index(), lm::base::Vocabulary::SetSpecial(), UTIL_THROW_IF, and lm::ngram::detail::ProbingVocabularyHeader::version.
void lm::ngram::ProbingVocabulary::Relocate | ( | void * | new_start | ) |
Definition at line 265 of file vocab.cc.
References ALIGN8, and util::ProbingHashTable< EntryT, HashT, EqualT, ModT >::Relocate().
Referenced by lm::ngram::detail::HashedSearch< Value >::InitializeFromARPA().
bool lm::ngram::ProbingVocabulary::SawUnk | ( | ) | const [inline] |
void lm::ngram::ProbingVocabulary::SetupMemory | ( | void * | start, | |
std::size_t | allocated, | |||
std::size_t | , | |||
const Config & | ||||
) | [inline] |
Definition at line 174 of file vocab.hh.
References SetupMemory().
void lm::ngram::ProbingVocabulary::SetupMemory | ( | void * | start, | |
std::size_t | allocated | |||
) |
Definition at line 258 of file vocab.cc.
References ALIGN8.
Referenced by SetupMemory().
uint64_t lm::ngram::ProbingVocabulary::Size | ( | uint64_t | entries, | |
const Config & | config | |||
) | [static] |
Definition at line 254 of file vocab.cc.
References lm::ngram::Config::probing_multiplier, and Size().
uint64_t lm::ngram::ProbingVocabulary::Size | ( | uint64_t | entries, | |
float | probing_multiplier | |||
) | [static] |
Definition at line 250 of file vocab.cc.
References ALIGN8, and util::ProbingHashTable< ProbingVocabularyEntry, util::IdentityHash >::Size().
Referenced by Size().
std::size_t lm::ngram::ProbingVocabulary::UnkCountChangePadding | ( | ) | const [inline] |
Definition at line 188 of file vocab.hh.
Referenced by lm::ngram::detail::HashedSearch< Value >::InitializeFromARPA().