#include <search_trie.hh>
Public Types | |
typedef NodeRange | Node |
typedef ::lm::ngram::trie::UnigramPointer | UnigramPointer |
typedef Quant::MiddlePointer | MiddlePointer |
typedef Quant::LongestPointer | LongestPointer |
Public Member Functions | |
TrieSearch () | |
~TrieSearch () | |
uint8_t * | SetupMemory (uint8_t *start, const std::vector< uint64_t > &counts, const Config &config) |
void | InitializeFromARPA (const char *file, util::FilePiece &f, std::vector< uint64_t > &counts, const Config &config, SortedVocabulary &vocab, BinaryFormat &backing) |
unsigned char | Order () const |
ProbBackoff & | UnknownUnigram () |
UnigramPointer | LookupUnigram (WordIndex word, Node &next, bool &independent_left, uint64_t &extend_left) const |
MiddlePointer | Unpack (uint64_t extend_pointer, unsigned char extend_length, Node &node) const |
MiddlePointer | LookupMiddle (unsigned char order_minus_2, WordIndex word, Node &node, bool &independent_left, uint64_t &extend_left) const |
LongestPointer | LookupLongest (WordIndex word, const Node &node) const |
bool | FastMakeNode (const WordIndex *begin, const WordIndex *end, Node &node) const |
Static Public Member Functions | |
static void | UpdateConfigFromBinary (const BinaryFormat &file, const std::vector< uint64_t > &counts, uint64_t offset, Config &config) |
static uint64_t | Size (const std::vector< uint64_t > &counts, const Config &config) |
Static Public Attributes | |
static const bool | kDifferentRest = false |
static const ModelType | kModelType = static_cast<ModelType>(TRIE_SORTED + Quant::kModelTypeAdd + Bhiksha::kModelTypeAdd) |
static const unsigned int | kVersion = 1 |
Friends | |
void | BuildTrie (SortedFiles &files, std::vector< uint64_t > &counts, const Config &config, TrieSearch< Quant, Bhiksha > &out, Quant &quant, SortedVocabulary &vocab, BinaryFormat &backing) |
Definition at line 27 of file search_trie.hh.
typedef Quant::LongestPointer lm::ngram::trie::TrieSearch< Quant, Bhiksha >::LongestPointer |
Definition at line 33 of file search_trie.hh.
typedef Quant::MiddlePointer lm::ngram::trie::TrieSearch< Quant, Bhiksha >::MiddlePointer |
Definition at line 32 of file search_trie.hh.
typedef NodeRange lm::ngram::trie::TrieSearch< Quant, Bhiksha >::Node |
Definition at line 29 of file search_trie.hh.
typedef ::lm::ngram::trie::UnigramPointer lm::ngram::trie::TrieSearch< Quant, Bhiksha >::UnigramPointer |
Definition at line 31 of file search_trie.hh.
lm::ngram::trie::TrieSearch< Quant, Bhiksha >::TrieSearch | ( | ) | [inline] |
Definition at line 56 of file search_trie.hh.
lm::ngram::trie::TrieSearch< Quant, Bhiksha >::~TrieSearch | ( | ) | [inline] |
Definition at line 58 of file search_trie.hh.
bool lm::ngram::trie::TrieSearch< Quant, Bhiksha >::FastMakeNode | ( | const WordIndex * | begin, | |
const WordIndex * | end, | |||
Node & | node | |||
) | const [inline] |
Definition at line 91 of file search_trie.hh.
References lm::ngram::trie::TrieSearch< Quant, Bhiksha >::LookupMiddle(), and lm::ngram::trie::TrieSearch< Quant, Bhiksha >::LookupUnigram().
void lm::ngram::trie::TrieSearch< Quant, Bhiksha >::InitializeFromARPA | ( | const char * | file, | |
util::FilePiece & | f, | |||
std::vector< uint64_t > & | counts, | |||
const Config & | config, | |||
SortedVocabulary & | vocab, | |||
BinaryFormat & | backing | |||
) | [inline] |
Definition at line 578 of file search_trie.cc.
References lm::ngram::Config::building_memory, lm::ngram::trie::TrieSearch< Quant, Bhiksha >::BuildTrie, lm::ngram::Config::temporary_directory_prefix, and lm::ngram::Config::write_mmap.
LongestPointer lm::ngram::trie::TrieSearch< Quant, Bhiksha >::LookupLongest | ( | WordIndex | word, | |
const Node & | node | |||
) | const [inline] |
Definition at line 87 of file search_trie.hh.
References lm::ngram::trie::BitPackedLongest::Find().
MiddlePointer lm::ngram::trie::TrieSearch< Quant, Bhiksha >::LookupMiddle | ( | unsigned char | order_minus_2, | |
WordIndex | word, | |||
Node & | node, | |||
bool & | independent_left, | |||
uint64_t & | extend_left | |||
) | const [inline] |
Definition at line 81 of file search_trie.hh.
References util::BitAddress::base, lm::ngram::trie::NodeRange::begin, lm::ngram::trie::NodeRange::end, and NULL.
Referenced by lm::ngram::trie::TrieSearch< Quant, Bhiksha >::FastMakeNode().
UnigramPointer lm::ngram::trie::TrieSearch< Quant, Bhiksha >::LookupUnigram | ( | WordIndex | word, | |
Node & | next, | |||
bool & | independent_left, | |||
uint64_t & | extend_left | |||
) | const [inline] |
Definition at line 70 of file search_trie.hh.
References lm::ngram::trie::NodeRange::begin, lm::ngram::trie::NodeRange::end, and lm::ngram::trie::Unigram::Find().
Referenced by lm::ngram::trie::TrieSearch< Quant, Bhiksha >::FastMakeNode().
unsigned char lm::ngram::trie::TrieSearch< Quant, Bhiksha >::Order | ( | ) | const [inline] |
Definition at line 64 of file search_trie.hh.
uint8_t * lm::ngram::trie::TrieSearch< Quant, Bhiksha >::SetupMemory | ( | uint8_t * | start, | |
const std::vector< uint64_t > & | counts, | |||
const Config & | config | |||
) | [inline] |
Definition at line 549 of file search_trie.cc.
References lm::ngram::trie::BitPackedLongest::Init(), lm::ngram::trie::Unigram::Init(), lm::ngram::trie::BitPackedLongest::Size(), lm::ngram::trie::BitPackedMiddle< Bhiksha >::Size(), lm::ngram::trie::Unigram::Size(), and lm::ngram::trie::TrieSearch< Quant, Bhiksha >::Size().
Referenced by lm::ngram::trie::BuildTrie().
static uint64_t lm::ngram::trie::TrieSearch< Quant, Bhiksha >::Size | ( | const std::vector< uint64_t > & | counts, | |
const Config & | config | |||
) | [inline, static] |
Definition at line 48 of file search_trie.hh.
References lm::ngram::trie::BitPackedLongest::Size(), lm::ngram::trie::BitPackedMiddle< Bhiksha >::Size(), and lm::ngram::trie::Unigram::Size().
Referenced by lm::ngram::trie::TrieSearch< Quant, Bhiksha >::SetupMemory().
ProbBackoff& lm::ngram::trie::TrieSearch< Quant, Bhiksha >::UnknownUnigram | ( | ) | [inline] |
Definition at line 68 of file search_trie.hh.
References lm::ngram::trie::Unigram::Unknown().
MiddlePointer lm::ngram::trie::TrieSearch< Quant, Bhiksha >::Unpack | ( | uint64_t | extend_pointer, | |
unsigned char | extend_length, | |||
Node & | node | |||
) | const [inline] |
Definition at line 77 of file search_trie.hh.
static void lm::ngram::trie::TrieSearch< Quant, Bhiksha >::UpdateConfigFromBinary | ( | const BinaryFormat & | file, | |
const std::vector< uint64_t > & | counts, | |||
uint64_t | offset, | |||
Config & | config | |||
) | [inline, static] |
Definition at line 41 of file search_trie.hh.
References lm::ngram::trie::Unigram::Size().
void BuildTrie | ( | SortedFiles & | files, | |
std::vector< uint64_t > & | counts, | |||
const Config & | config, | |||
TrieSearch< Quant, Bhiksha > & | out, | |||
Quant & | quant, | |||
SortedVocabulary & | vocab, | |||
BinaryFormat & | backing | |||
) | [friend] |
Definition at line 462 of file search_trie.cc.
Referenced by lm::ngram::trie::TrieSearch< Quant, Bhiksha >::InitializeFromARPA().
const bool lm::ngram::trie::TrieSearch< Quant, Bhiksha >::kDifferentRest = false [static] |
Definition at line 35 of file search_trie.hh.
const ModelType lm::ngram::trie::TrieSearch< Quant, Bhiksha >::kModelType = static_cast<ModelType>(TRIE_SORTED + Quant::kModelTypeAdd + Bhiksha::kModelTypeAdd) [static] |
Definition at line 37 of file search_trie.hh.
const unsigned int lm::ngram::trie::TrieSearch< Quant, Bhiksha >::kVersion = 1 [static] |
Definition at line 39 of file search_trie.hh.