#include <onlineRLM.h>
Public Member Functions | |
OnlineRLM (uint16_t MBs, int width, int bucketRange, count_t order, Moses::Vocab *v, float qBase=8) | |
OnlineRLM (Moses::FileHandler *fin, count_t order) | |
~OnlineRLM () | |
float | getProb (const wordID_t *ngram, int len, const void **state) |
bool | insert (const std::vector< std::string > &ngram, const int value) |
bool | update (const std::vector< std::string > &ngram, const int value) |
int | query (const wordID_t *IDs, const int len) |
int | sbsqQuery (const std::vector< std::string > &ngram, int *len, bool bStrict=false) |
int | sbsqQuery (const wordID_t *IDs, const int len, int *codes, bool bStrict=false) |
void | remove (const std::vector< std::string > &ngram) |
count_t | heurDelete (count_t num2del, count_t order=5) |
uint64_t | corpusSize () |
void | corpusSize (uint64_t c) |
void | clearCache () |
void | save (Moses::FileHandler *fout) |
void | load (Moses::FileHandler *fin) |
void | randDelete (int num2del) |
int | countHits () |
int | countPrefixes () |
int | cleanUpHPD () |
void | clearMarkings () |
void | removeNonMarked () |
Public Attributes | |
Moses::Vocab * | vocab_ |
Protected Member Functions | |
void | markQueried (const uint64_t &index) |
void | markQueried (hpdEntry_t &value) |
bool | markPrefix (const wordID_t *IDs, const int len, bool bSet) |
Definition at line 21 of file onlineRLM.h.
OnlineRLM< T >::OnlineRLM | ( | uint16_t | MBs, | |
int | width, | |||
int | bucketRange, | |||
count_t | order, | |||
Moses::Vocab * | v, | |||
float | qBase = 8 | |||
) | [inline] |
Definition at line 24 of file onlineRLM.h.
References PerfectHash< T >::cells_, UTIL_THROW_IF2, and OnlineRLM< T >::vocab_.
OnlineRLM< T >::OnlineRLM | ( | Moses::FileHandler * | fin, | |
count_t | order | |||
) | [inline] |
Definition at line 37 of file onlineRLM.h.
References OnlineRLM< T >::load().
Definition at line 391 of file onlineRLM.h.
References PerfectHash< T >::dict_, iterate, and Utils::splitToStr().
Referenced by OnlineRLM< T >::removeNonMarked().
Definition at line 70 of file onlineRLM.h.
References randlm::Cache< T >::clear().
Definition at line 408 of file onlineRLM.h.
References PerfectHash< T >::dict_, PerfectHash< T >::hitMask_, iterate, and randlm::Filter< T >::reset().
Definition at line 67 of file onlineRLM.h.
Definition at line 64 of file onlineRLM.h.
Definition at line 367 of file onlineRLM.h.
References PerfectHash< T >::cells_, PerfectHash< T >::dict_, iterate, and randlm::BitFilter::testBit().
Definition at line 380 of file onlineRLM.h.
References PerfectHash< T >::cells_, and randlm::BitFilter::testBit().
float OnlineRLM< T >::getProb | ( | const wordID_t * | ngram, | |
int | len, | |||
const void ** | state | |||
) | [inline] |
Definition at line 284 of file onlineRLM.h.
References randlm::Cache< T >::checkCacheNgram(), Moses::Vocab::GetkOOVWordID(), OnlineRLM< T >::query(), randlm::Cache< T >::setCacheNgram(), Moses::Vocab::Size(), strict_checks_, UTIL_THROW_IF2, and OnlineRLM< T >::vocab_.
count_t OnlineRLM< T >::heurDelete | ( | count_t | num2del, | |
count_t | order = 5 | |||
) | [inline] |
Definition at line 212 of file onlineRLM.h.
References PerfectHash< T >::bucketRange_, PerfectHash< T >::filter_, PerfectHash< T >::idxTracker_, OnlineRLM< T >::remove(), randlm::BitFilter::testBit(), and PerfectHash< T >::totBuckets_.
bool OnlineRLM< T >::insert | ( | const std::vector< std::string > & | ngram, | |
const int | value | |||
) | [inline] |
Definition at line 98 of file onlineRLM.h.
References PerfectHash< T >::cells_, Moses::Vocab::GetBOSWordID(), Moses::Vocab::GetWordID(), index, OnlineRLM< T >::markPrefix(), OnlineRLM< T >::markQueried(), and OnlineRLM< T >::vocab_.
void OnlineRLM< T >::load | ( | Moses::FileHandler * | fin | ) | [inline] |
Reimplemented from PerfectHash< T >.
Definition at line 435 of file onlineRLM.h.
References UTIL_THROW_IF2, and OnlineRLM< T >::vocab_.
Referenced by OnlineRLM< T >::OnlineRLM().
bool OnlineRLM< T >::markPrefix | ( | const wordID_t * | IDs, | |
const int | len, | |||
bool | bSet | |||
) | [inline, protected] |
Definition at line 160 of file onlineRLM.h.
References PerfectHash< T >::cells_, randlm::Cache< T >::checkCacheNgram(), randlm::Cache< T >::clear(), randlm::BitFilter::clearBit(), PerfectHash< T >::dict_, randlm::Cache< T >::nodes(), NULL, OnlineRLM< T >::query(), randlm::BitFilter::setBit(), randlm::Cache< T >::setCacheNgram(), and UTIL_THROW_IF2.
Referenced by OnlineRLM< T >::insert(), and OnlineRLM< T >::update().
void OnlineRLM< T >::markQueried | ( | hpdEntry_t & | value | ) | [inline, protected, virtual] |
Implements PerfectHash< T >.
Definition at line 196 of file onlineRLM.h.
References PerfectHash< T >::hitMask_.
void OnlineRLM< T >::markQueried | ( | const uint64_t & | index | ) | [inline, protected, virtual] |
Implements PerfectHash< T >.
Definition at line 189 of file onlineRLM.h.
References randlm::BitFilter::setBit().
Referenced by OnlineRLM< T >::insert(), and OnlineRLM< T >::update().
Definition at line 140 of file onlineRLM.h.
References PerfectHash< T >::cells_, PerfectHash< T >::dict_, PerfectHash< T >::hitMask_, and UTIL_THROW_IF2.
Referenced by OnlineRLM< T >::getProb(), OnlineRLM< T >::markPrefix(), and OnlineRLM< T >::sbsqQuery().
Definition at line 354 of file onlineRLM.h.
References PerfectHash< T >::cells_, PerfectHash< T >::filter_, and OnlineRLM< T >::remove().
void OnlineRLM< T >::remove | ( | const std::vector< std::string > & | ngram | ) | [inline] |
Definition at line 203 of file onlineRLM.h.
References Moses::Vocab::GetWordID(), and OnlineRLM< T >::vocab_.
Referenced by OnlineRLM< T >::heurDelete(), OnlineRLM< T >::randDelete(), and OnlineRLM< T >::removeNonMarked().
Definition at line 452 of file onlineRLM.h.
References PerfectHash< T >::cells_, OnlineRLM< T >::cleanUpHPD(), PerfectHash< T >::filter_, OnlineRLM< T >::remove(), and randlm::BitFilter::testBit().
void OnlineRLM< T >::save | ( | Moses::FileHandler * | fout | ) | [inline] |
Reimplemented from PerfectHash< T >.
Definition at line 420 of file onlineRLM.h.
References randlm::BitFilter::save(), Moses::Vocab::Save(), and OnlineRLM< T >::vocab_.
int OnlineRLM< T >::sbsqQuery | ( | const wordID_t * | IDs, | |
const int | len, | |||
int * | codes, | |||
bool | bStrict = false | |||
) | [inline] |
Definition at line 254 of file onlineRLM.h.
References PerfectHash< T >::dict_, PerfectHash< T >::hitMask_, and OnlineRLM< T >::query().
int OnlineRLM< T >::sbsqQuery | ( | const std::vector< std::string > & | ngram, | |
int * | len, | |||
bool | bStrict = false | |||
) | [inline] |
Definition at line 244 of file onlineRLM.h.
References Moses::Vocab::GetWordID(), and OnlineRLM< T >::vocab_.
bool OnlineRLM< T >::update | ( | const std::vector< std::string > & | ngram, | |
const int | value | |||
) | [inline] |
Definition at line 117 of file onlineRLM.h.
References PerfectHash< T >::cells_, PerfectHash< T >::dict_, Moses::Vocab::GetWordID(), index, Moses::Vocab::MakeOpen(), OnlineRLM< T >::markPrefix(), OnlineRLM< T >::markQueried(), PerfectHash< T >::update2(), and OnlineRLM< T >::vocab_.
Moses::Vocab* OnlineRLM< T >::vocab_ |
Definition at line 81 of file onlineRLM.h.
Referenced by OnlineRLM< T >::getProb(), OnlineRLM< T >::insert(), OnlineRLM< T >::load(), OnlineRLM< T >::OnlineRLM(), OnlineRLM< T >::remove(), OnlineRLM< T >::save(), OnlineRLM< T >::sbsqQuery(), OnlineRLM< T >::update(), and OnlineRLM< T >::~OnlineRLM().