#include <binary_format.hh>
Public Member Functions | |
BinaryFormat (const Config &config) | |
void | InitializeBinary (int fd, ModelType model_type, unsigned int search_version, Parameters ¶ms) |
void | ReadForConfig (void *to, std::size_t amount, uint64_t offset_excluding_header) const |
void * | LoadBinary (std::size_t size) |
uint64_t | VocabStringReadingOffset () const |
void * | SetupJustVocab (std::size_t memory_size, uint8_t order) |
void * | GrowForSearch (std::size_t memory_size, std::size_t vocab_pad, void *&vocab_base) |
void | WriteVocabWords (const std::string &buffer, void *&vocab_base, void *&search_base) |
void | FinishFile (const Config &config, ModelType model_type, unsigned int search_version, const std::vector< uint64_t > &counts) |
Definition at line 47 of file binary_format.hh.
lm::ngram::BinaryFormat::BinaryFormat | ( | const Config & | config | ) | [explicit] |
Definition at line 138 of file binary_format.cc.
void lm::ngram::BinaryFormat::FinishFile | ( | const Config & | config, | |
ModelType | model_type, | |||
unsigned int | search_version, | |||
const std::vector< uint64_t > & | counts | |||
) |
Definition at line 245 of file binary_format.cc.
References lm::ngram::Parameters::counts, lm::ngram::Parameters::fixed, util::FSyncOrThrow(), util::scoped_fd::get(), util::scoped_memory::get(), lm::ngram::FixedWidthParameters::has_vocabulary, lm::ngram::Config::include_vocab, lm::ngram::FixedWidthParameters::model_type, lm::ngram::FixedWidthParameters::order, lm::ngram::Config::probing_multiplier, lm::ngram::FixedWidthParameters::probing_multiplier, lm::ngram::FixedWidthParameters::search_version, util::SeekOrThrow(), util::scoped_memory::size(), util::SyncOrThrow(), lm::ngram::Config::WRITE_AFTER, lm::ngram::Config::WRITE_MMAP, and util::WriteOrThrow().
void * lm::ngram::BinaryFormat::GrowForSearch | ( | std::size_t | memory_size, | |
std::size_t | vocab_pad, | |||
void *& | vocab_base | |||
) |
Definition at line 196 of file binary_format.cc.
References util::AdviseHugePages(), util::scoped_fd::get(), util::scoped_memory::get(), util::HugeMalloc(), lm::ngram::kInvalidSize, util::scoped_memory::reset(), util::ResizeOrThrow(), lm::ngram::Config::WRITE_AFTER, and lm::ngram::Config::WRITE_MMAP.
Referenced by lm::ngram::trie::BuildTrie(), and lm::ngram::detail::HashedSearch< Value >::InitializeFromARPA().
void lm::ngram::BinaryFormat::InitializeBinary | ( | int | fd, | |
ModelType | model_type, | |||
unsigned int | search_version, | |||
Parameters & | params | |||
) |
Definition at line 142 of file binary_format.cc.
References lm::ngram::Parameters::counts, lm::ngram::MatchCheck(), NULL, lm::ngram::ReadHeader(), and util::scoped_fd::reset().
Referenced by lm::ngram::detail::GenericModel< Search, VocabularyT >::GenericModel().
void * lm::ngram::BinaryFormat::LoadBinary | ( | std::size_t | size | ) |
Definition at line 155 of file binary_format.cc.
References util::CheckOverflow(), util::scoped_memory::get(), util::scoped_fd::get(), util::kBadSize, lm::ngram::kInvalidSize, util::MapRead(), util::SizeFile(), and UTIL_THROW_IF.
Referenced by lm::ngram::detail::GenericModel< Search, VocabularyT >::GenericModel().
void lm::ngram::BinaryFormat::ReadForConfig | ( | void * | to, | |
std::size_t | amount, | |||
uint64_t | offset_excluding_header | |||
) | const |
Definition at line 150 of file binary_format.cc.
References util::ErsatzPRead(), util::scoped_fd::get(), and lm::ngram::kInvalidSize.
Referenced by lm::ngram::SeparatelyQuantize::UpdateConfigFromBinary(), and lm::ngram::trie::ArrayBhiksha::UpdateConfigFromBinary().
void * lm::ngram::BinaryFormat::SetupJustVocab | ( | std::size_t | memory_size, | |
uint8_t | order | |||
) |
Definition at line 168 of file binary_format.cc.
References util::AdviseHugePages(), util::CheckOverflow(), util::CreateOrThrow(), util::scoped_fd::get(), util::scoped_memory::get(), util::HugeMalloc(), util::MapZeroedWrite(), util::scoped_memory::MMAP_ALLOCATED, NULL, util::scoped_memory::reset(), util::scoped_fd::reset(), util::ResizeOrThrow(), lm::ngram::Config::WRITE_AFTER, and lm::ngram::Config::WRITE_MMAP.
uint64_t lm::ngram::BinaryFormat::VocabStringReadingOffset | ( | ) | const [inline] |
Definition at line 59 of file binary_format.hh.
Referenced by lm::ngram::detail::GenericModel< Search, VocabularyT >::GenericModel(), and WriteVocabWords().
void lm::ngram::BinaryFormat::WriteVocabWords | ( | const std::string & | buffer, | |
void *& | vocab_base, | |||
void *& | search_base | |||
) |
Definition at line 223 of file binary_format.cc.
References util::scoped_fd::get(), util::scoped_memory::get(), lm::ngram::kInvalidSize, util::scoped_memory::reset(), util::SeekOrThrow(), VocabStringReadingOffset(), lm::ngram::Config::WRITE_MMAP, and util::WriteOrThrow().