Namespaces | |
namespace | base |
namespace | builder |
namespace | ngram |
namespace | np |
namespace | phrase |
namespace | vocab |
Classes | |
class | Comparator |
class | SuffixOrder |
class | ContextOrder |
class | PrefixOrder |
class | ModelBuffer |
class | NGramHeader |
class | NGram |
class | ProxyStream |
class | NGramStream |
class | NGramStreams |
class | VocabReconstitute |
class | PrintARPA |
class | Renumber |
class | SpecialVocab |
class | EnumerateVocab |
class | ARPAInputException |
class | ARPAOutput |
class | CountOutput |
class | CountBatch |
class | MultipleOutput |
class | MultipleARPAOutput |
class | DispatchInput |
class | DispatchARPAInput |
struct | ARPAFormat |
struct | CountFormat |
class | InputBuffer |
class | BinaryOutputBuffer |
class | MultipleOutputBuffer |
class | ThreadBatch |
class | FilterWorker |
class | OutputWorker |
class | Controller |
class | BinaryFilter |
class | ContextFilter |
class | ConfigException |
class | LoadException |
class | FormatLoadException |
class | VocabLoadException |
class | SpecialWordMissingException |
class | PositiveProbWarn |
struct | FullScoreReturn |
struct | Prob |
struct | ProbBackoff |
struct | RestWeights |
Typedefs | |
typedef unsigned int | WordIndex |
Enumerations | |
enum | WarningAction { THROW_UP, COMPLAIN, SILENT } |
Functions | |
template<class Callback , class Compare > | |
void | JointOrder (const util::stream::ChainPositions &positions, Callback &callback) |
boost::program_options::typed_value < std::string > * | SizeOption (std::size_t &to, const char *default_value) |
template<class Stream > | |
void | WriteCounts (Stream &out, const std::vector< uint64_t > &number) |
size_t | SizeNeededForCounts (const std::vector< uint64_t > &number) |
bool | IsEntirelyWhiteSpace (const StringPiece &line) |
template<class Output > | |
void | ReadNGrams (util::FilePiece &in, unsigned int length, uint64_t number, Output &out) |
template<class Output > | |
void | ReadARPA (util::FilePiece &in_lm, Output &out) |
template<class Output > | |
void | ReadCount (util::FilePiece &in_file, Output &out) |
void | ReadARPACounts (util::FilePiece &in, std::vector< uint64_t > &number) |
void | ReadNGramHeader (util::FilePiece &in, unsigned int length) |
void | ReadBackoff (util::FilePiece &in, Prob &) |
void | ReadBackoff (util::FilePiece &in, float &backoff) |
void | ReadEnd (util::FilePiece &in) |
void | ReadBackoff (util::FilePiece &in, ProbBackoff &weights) |
void | ReadBackoff (util::FilePiece &in, RestWeights &weights) |
template<class Voc , class Weights > | |
void | Read1Gram (util::FilePiece &f, Voc &vocab, Weights *unigrams, PositiveProbWarn &warn) |
template<class Voc , class Weights > | |
void | Read1Grams (util::FilePiece &f, std::size_t count, Voc &vocab, Weights *unigrams, PositiveProbWarn &warn) |
template<class Voc , class Weights , class Iterator > | |
void | ReadNGram (util::FilePiece &f, const unsigned char n, const Voc &vocab, Iterator indices_out, Weights &weights, PositiveProbWarn &warn) |
Variables | |
const bool | kARPASpaces [256] = {0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} |
const WordIndex | kMaxWordIndex = UINT_MAX |
const WordIndex | kUNK = 0 |
typedef unsigned int lm::WordIndex |
Definition at line 8 of file word_index.hh.
enum lm::WarningAction |
bool lm::IsEntirelyWhiteSpace | ( | const StringPiece & | line | ) |
Definition at line 41 of file arpa_io.cc.
References StringPiece::data(), and StringPiece::size().
Referenced by ReadARPACounts(), ReadEnd(), and ReadNGramHeader().
void lm::JointOrder | ( | const util::stream::ChainPositions & | positions, | |
Callback & | callback | |||
) | [inline] |
Definition at line 16 of file joint_order.hh.
References begin, NULL, util::FixedArray< T >::push_back(), util::FixedArray< T >::size(), and UTIL_THROW_IF.
void lm::Read1Gram | ( | util::FilePiece & | f, | |
Voc & | vocab, | |||
Weights * | unigrams, | |||
PositiveProbWarn & | warn | |||
) | [inline] |
Definition at line 44 of file read_arpa.hh.
References util::FilePiece::get(), kARPASpaces, util::FilePiece::Offset(), prob(), ReadBackoff(), util::FilePiece::ReadDelimited(), util::FilePiece::ReadFloat(), UTIL_THROW_IF, and lm::PositiveProbWarn::Warn().
Referenced by Read1Grams().
void lm::Read1Grams | ( | util::FilePiece & | f, | |
std::size_t | count, | |||
Voc & | vocab, | |||
Weights * | unigrams, | |||
PositiveProbWarn & | warn | |||
) | [inline] |
Definition at line 62 of file read_arpa.hh.
References Read1Gram(), and ReadNGramHeader().
Referenced by lm::ngram::detail::HashedSearch< Value >::InitializeFromARPA().
void lm::ReadARPA | ( | util::FilePiece & | in_lm, | |
Output & | out | |||
) | [inline] |
Definition at line 86 of file arpa_io.hh.
References ReadARPACounts(), ReadEnd(), ReadNGrams(), and SizeNeededForCounts().
Referenced by lm::ARPAFormat::Copy(), and lm::ARPAFormat::RunFilter().
void lm::ReadARPACounts | ( | util::FilePiece & | in, | |
std::vector< uint64_t > & | number | |||
) |
Definition at line 47 of file read_arpa.cc.
References StringPiece::data(), util::FilePiece::FileName(), IsEntirelyWhiteSpace(), ReadCount(), util::FilePiece::ReadLine(), StringPiece::size(), starts_with(), UTIL_THROW, and UTIL_THROW_IF.
Referenced by lm::ngram::LowerRestBuild< Model >::LowerRestBuild(), ReadARPA(), and lm::ngram::ShowSizes().
void lm::ReadBackoff | ( | util::FilePiece & | in, | |
RestWeights & | weights | |||
) | [inline] |
Definition at line 23 of file read_arpa.hh.
References lm::RestWeights::backoff, and ReadBackoff().
void lm::ReadBackoff | ( | util::FilePiece & | in, | |
ProbBackoff & | weights | |||
) | [inline] |
Definition at line 20 of file read_arpa.hh.
References lm::ProbBackoff::backoff, and ReadBackoff().
void lm::ReadBackoff | ( | util::FilePiece & | in, | |
float & | backoff | |||
) |
Definition at line 104 of file read_arpa.cc.
References util::FilePiece::get(), lm::ngram::kExtensionBackoff, lm::ngram::kNoExtensionBackoff, util::FilePiece::ReadFloat(), UTIL_THROW, and UTIL_THROW_IF.
void lm::ReadBackoff | ( | util::FilePiece & | in, | |
Prob & | ||||
) |
Definition at line 88 of file read_arpa.cc.
References util::FilePiece::get(), util::FilePiece::ReadFloat(), and UTIL_THROW.
Referenced by Read1Gram(), ReadBackoff(), and ReadNGram().
void lm::ReadCount | ( | util::FilePiece & | in_file, | |
Output & | out | |||
) | [inline] |
Definition at line 73 of file count_io.hh.
References util::FilePiece::ReadLine().
Referenced by lm::CountFormat::Copy(), ReadARPACounts(), and lm::CountFormat::RunFilter().
void lm::ReadEnd | ( | util::FilePiece & | in | ) |
Definition at line 133 of file read_arpa.cc.
References IsEntirelyWhiteSpace(), util::FilePiece::ReadLine(), and UTIL_THROW.
Referenced by ReadARPA().
void lm::ReadNGram | ( | util::FilePiece & | f, | |
const unsigned char | n, | |||
const Voc & | vocab, | |||
Iterator | indices_out, | |||
Weights & | weights, | |||
PositiveProbWarn & | warn | |||
) | [inline] |
Definition at line 71 of file read_arpa.hh.
References index, kARPASpaces, util::FilePiece::Offset(), ReadBackoff(), util::FilePiece::ReadDelimited(), util::FilePiece::ReadFloat(), UTIL_THROW_IF, and lm::PositiveProbWarn::Warn().
Referenced by lm::ngram::LowerRestBuild< Model >::LowerRestBuild().
void lm::ReadNGramHeader | ( | util::FilePiece & | in, | |
unsigned int | length | |||
) |
Definition at line 80 of file read_arpa.cc.
References IsEntirelyWhiteSpace(), util::FilePiece::ReadLine(), and UTIL_THROW.
Referenced by lm::ngram::LowerRestBuild< Model >::LowerRestBuild(), Read1Grams(), and ReadNGrams().
void lm::ReadNGrams | ( | util::FilePiece & | in, | |
unsigned int | length, | |||
uint64_t | number, | |||
Output & | out | |||
) | [inline] |
Definition at line 72 of file arpa_io.hh.
References util::FilePiece::ReadLine(), and ReadNGramHeader().
Referenced by ReadARPA().
size_t lm::SizeNeededForCounts | ( | const std::vector< uint64_t > & | number | ) |
Definition at line 35 of file arpa_io.cc.
References util::StringStream::str(), and WriteCounts().
Referenced by ReadARPA().
boost::program_options::typed_value< std::string > * lm::SizeOption | ( | std::size_t & | to, | |
const char * | default_value | |||
) |
Definition at line 20 of file size_option.cc.
Referenced by main().
void lm::WriteCounts | ( | Stream & | out, | |
const std::vector< uint64_t > & | number | |||
) | [inline] |
Definition at line 27 of file arpa_io.cc.
Referenced by lm::ARPAOutput::Finish(), and SizeNeededForCounts().
const bool lm::kARPASpaces = {0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} |
const WordIndex lm::kMaxWordIndex = UINT_MAX |
Definition at line 9 of file word_index.hh.
Definition at line 10 of file word_index.hh.
Referenced by lm::SpecialVocab::IsSpecial(), lm::builder::CorpusCount::Run(), lm::builder::AdjustCounts::Run(), and lm::SpecialVocab::UNK().