/disk4/html/www/moses/doxygen/mosesdecoder/lm/search_trie.cc File Reference

#include "lm/search_trie.hh"
#include "lm/bhiksha.hh"
#include "lm/binary_format.hh"
#include "lm/blank.hh"
#include "lm/lm_exception.hh"
#include "lm/max_order.hh"
#include "lm/quantize.hh"
#include "lm/trie.hh"
#include "lm/trie_sort.hh"
#include "lm/vocab.hh"
#include "lm/weights.hh"
#include "lm/word_index.hh"
#include "util/ersatz_progress.hh"
#include "util/mmap.hh"
#include "util/proxy_iterator.hh"
#include "util/scoped.hh"
#include "util/sized_iterator.hh"
#include <algorithm>
#include <cstring>
#include <cstdio>
#include <cstdlib>
#include <queue>
#include <limits>
#include <numeric>
#include <vector>

Go to the source code of this file.

Namespaces

namespace  lm
namespace  lm::ngram
namespace  lm::ngram::trie

Functions

template<class Quant , class Bhiksha >
void lm::ngram::trie::BuildTrie (SortedFiles &files, std::vector< uint64_t > &counts, const Config &config, TrieSearch< Quant, Bhiksha > &out, Quant &quant, SortedVocabulary &vocab, BinaryFormat &backing)


Variable Documentation

uint8_t * allocated_

Definition at line 173 of file search_trie.cc.

unsigned char array

Definition at line 54 of file search_trie.cc.

Definition at line 171 of file search_trie.cc.

float basis_[KENLM_MAX_ORDER]

Definition at line 372 of file search_trie.cc.

WordIndex been_[KENLM_MAX_ORDER]

Definition at line 369 of file search_trie.cc.

unsigned char been_length_

Definition at line 370 of file search_trie.cc.

const WordIndex* begin

BitPacked& bigram_pack_

Definition at line 314 of file search_trie.cc.

RecordReader* contexts_

Definition at line 309 of file search_trie.cc.

std::vector<uint64_t> counts_

Definition at line 261 of file search_trie.cc.

uint8_t* current_

Definition at line 173 of file search_trie.cc.

Doing& doing_

Definition at line 374 of file search_trie.cc.

const WordIndex * end

Definition at line 322 of file search_trie.cc.

Referenced by lmtable::add(), Moses::FactorCollection::AddFactor(), MosesTuning::ReferenceSet::AddLine(), Moses::Syntax::S2T::PChart::AddVertex(), lmtable::addwithoffset(), Alignments::Alignments(), as_utf8_end(), as_wide_impl(), Moses::calcNgramExpectations(), Moses::dsgHypothesis::calculateDsgProb(), TERCPPNS_TERCpp::terCalc::calculerPermutations(), lmtable::checkbounds(), lmtable::cpsublm(), TERCPPNS_TERCpp::terCalc::createConcordMots(), Moses::Syntax::S2T::Manager< Parser >::Decode(), utf_decoder< Traits, opt_swap >::decode_utf16_block(), utf_decoder< Traits, opt_swap >::decode_utf32_block(), Moses::DistortionScoreProducer::EmptyHypothesisState(), Moses::Syntax::S2T::Scope3Parser< Callback >::EnumerateHyperedges(), Moses::Syntax::S2T::RecursiveCYKPlusParser< Callback >::EnumerateHyperedges(), Moses::LanguageModelKen< Model >::EvaluateWhenApplied(), Moses::LanguageModelIRST::EvaluateWhenApplied(), Moses::LanguageModelDALM::EvaluateWhenApplied(), Moses::Syntax::S2T::PatternApplicationTrie::Extend(), MosesTuning::StreamingHypPackEnumerator::finished(), Moses::Scope3Parser::GetChartRuleCollection(), Moses::FactorCollection::GetFactor(), Moses::Bitmap::GetID(), Moses::Bitmap::GetIDPlus(), Moses::IncrementIterators(), invertAlignment(), Moses::PhraseDecoder::Load(), Moses::BlockHashIndex::Load(), load_stream_data_seek(), Moses::BlockHashIndex::LoadIndex(), main(), util::MurmurHash64A(), search::NBestList::NBestList(), Moses::VWFeatureSourceWindow::operator()(), Moses::VWFeatureSourceSenseWindow::operator()(), Moses::VWFeatureSourcePhraseInternal::operator()(), Moses::VWFeatureSourceIndicator::operator()(), lm::ngram::trie::EntryCompare::operator()(), outputPhrasePair(), Moses::Hypothesis::PrintHypothesis(), Moses::ProcessAndStripSGML(), MosesTraining::ReadAlignment(), util::stream::LineInput::Run(), SAMT(), MosesTuning::StatisticsBasedScorer::score(), MosesTuning::SentenceLevelScorer::score(), MosesTraining::unescape(), writeLeftHandSideLabelCounts(), and Moses::ChartManager::~ChartManager().

std::size_t entry_size_

Definition at line 175 of file search_trie.cc.

uint64_t index

Definition at line 55 of file search_trie.cc.

Referenced by MosesTraining::PropertiesConsolidator::ActivatePartsOfSpeechProcessing(), MosesTraining::PropertiesConsolidator::ActivateSourceLabelsProcessing(), MosesTraining::PropertiesConsolidator::ActivateTargetSyntacticPreferencesProcessing(), Moses::StackLatticeBuilder::Build(), Moses::calcNgramExpectations(), Moses::BackwardLanguageModel< Model >::CalcScore(), Moses::PrefixTreeMap::ConvertPhrase(), Moses::BackwardLanguageModel< Model >::Evaluate(), Moses::LanguageModelImplementation::EvaluateWhenApplied(), Moses::ApplicableRuleTrie::Extend(), double_conversion::PowersOfTenCache::GetCachedPowerForBinaryExponentRange(), double_conversion::PowersOfTenCache::GetCachedPowerForDecimalExponent(), ngramtable::getfreq(), Moses::Edge::GetPhraseSuffix(), Moses::LexicalReorderingTableCompact::GetScore(), GetScore(), sapt::imTSA< TOKEN >::imTSA(), Moses::SoftSourceSyntacticConstraintsFeature::LoadSourceLabelSet(), Moses::SoftSourceSyntacticConstraintsFeature::LoadTargetSourceLeftHandSideJointCountFile(), MosesTraining::Syntax::GHKM::Node::LowestCommonAncestor(), MosesTraining::Syntax::GHKM::ExtractGHKM::Main(), main(), numberize(), sapt::snt_adder< L2R_Token< SimpleWordId > >::operator()(), MosesTraining::Syntax::Tree< T >::LeafIter< T >::operator++(), MosesTraining::Syntax::Tree< T >::PreOrderIter< T >::operator++(), processPhrasePairs(), lm::ReadNGram(), sapt::imTSA< TOKEN >::save_as_mm_tsa(), ngramtable::setfreq(), sapt::mmTtrack< TKN >::sntEnd(), sapt::mmTtrack< TKN >::sntStart(), sapt::write_mm_2d_table(), and sapt::write_tokenindex_to_disk().

float* it_[KENLM_MAX_ORDER-1]

Definition at line 227 of file search_trie.cc.

BitPackedLongest& longest_

Definition at line 313 of file search_trie.cc.

BackoffMessages messages_[KENLM_MAX_ORDER-1]

Definition at line 225 of file search_trie.cc.

BitPackedMiddle<Bhiksha>* const middle_

Definition at line 312 of file search_trie.cc.

const unsigned char order_

const Quant& quant_

Definition at line 310 of file search_trie.cc.

SRISucks& sri_

Definition at line 265 of file search_trie.cc.

const unsigned char total_order_

Definition at line 367 of file search_trie.cc.

UnigramValue* const unigrams_

Definition at line 263 of file search_trie.cc.

std::vector<float> values_[KENLM_MAX_ORDER-1]

Definition at line 224 of file search_trie.cc.


Generated on Thu Jul 6 00:31:25 2017 for Moses by  doxygen 1.5.9