#include <lmtable.h>


Public Member Functions | |
| lmtable () | |
| virtual | ~lmtable () |
| table_entry_pos_t | wdprune (float *thr, int aflag=0) |
| table_entry_pos_t | wdprune (float *thr, int aflag, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos, double lk=0, double bo=0, double *ts=0, double *tbs=0) |
| double | lprobx (ngram ong, double *lkp=0, double *bop=0, int *bol=0) |
| table_entry_pos_t | ngcnt (table_entry_pos_t *cnt) |
| table_entry_pos_t | ngcnt (table_entry_pos_t *cnt, ngram ng, int l, table_entry_pos_t ipos, table_entry_pos_t epos) |
| int | pscale (int lev, table_entry_pos_t ipos, table_entry_pos_t epos, double s) |
| void | init_probcache () |
| void | init_statecache () |
| void | init_lmtcaches (int uptolev) |
| void | check_cache_levels () |
| void | reset_caches () |
| void | reset_mmap () |
| bool | is_probcache_active () |
| bool | is_statecache_active () |
| bool | are_lmtcaches_active () |
| bool | is_inverted (const bool flag) |
| bool | is_inverted () |
| void | configure (int n, bool quantized) |
| double | getlogOOVpenalty () const |
| double | setlogOOVpenalty (int dub) |
| double | setlogOOVpenalty2 (double oovp) |
| int | maxlevel () const |
| bool | isQuantized () const |
| void | savetxt (const char *filename) |
| void | savebin (const char *filename) |
| void | dumplm (std::fstream &out, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos) |
| void | load (std::istream &inp, const char *filename=NULL, const char *outfilename=NULL, int mmap=0, OUTFILE_TYPE outtype=NONE) |
| void | loadtxt (std::istream &inp, const char *header, const char *outfilename, int mmap) |
| void | loadtxt (std::istream &inp, const char *header) |
| void | loadtxtmmap (std::istream &inp, const char *header, const char *outfilename) |
| void | loadbin (std::istream &inp, const char *header, const char *filename=NULL, int mmap=0) |
| void | loadbinheader (std::istream &inp, const char *header) |
| void | loadbincodebook (std::istream &inp, int l) |
| void | loadcenters (std::istream &inp, int Order) |
| lmtable * | cpsublm (dictionary *subdict, bool keepunigr=true) |
| int | reload (std::set< string > words) |
| void | filter (const char *lmfile) |
| virtual double | lprob (ngram ng, double *bow=NULL, int *bol=NULL, char **maxsuffptr=NULL, unsigned int *statesize=NULL) |
| virtual double | clprob (ngram ng) |
| void * | search (int lev, table_entry_pos_t offs, table_entry_pos_t n, int sz, int *w, LMT_ACTION action, char **found=(char **) NULL) |
| int | mybsearch (char *ar, table_entry_pos_t n, int size, char *key, table_entry_pos_t *idx) |
| template<typename TA , typename TB > | |
| int | add (ngram &ng, TA prob, TB bow) |
| void | checkbounds (int level) |
| int | get (ngram &ng) |
| int | get (ngram &ng, int n, int lev) |
| int | succscan (ngram &h, ngram &ng, LMT_ACTION action, int lev) |
| virtual const char * | maxsuffptr (ngram ong, unsigned int *size=NULL) |
| virtual const char * | cmaxsuffptr (ngram ong, unsigned int *size=NULL) |
| void | putmem (char *ptr, int value, int offs, int size) |
| void | getmem (char *ptr, int *value, int offs, int size) |
| template<typename T > | |
| void | putmem (char *ptr, T value, int offs) |
| template<typename T > | |
| void | getmem (char *ptr, T *value, int offs) |
| int | nodesize (LMT_TYPE ndt) |
| int | word (node nd, int value=-1) |
| float | prob (node nd, LMT_TYPE ndt) |
| template<typename T > | |
| float | prob (node nd, LMT_TYPE ndt, T value) |
| float | bow (node nd, LMT_TYPE ndt) |
| template<typename T > | |
| T | bow (node nd, LMT_TYPE ndt, T value) |
| table_entry_pos_t | bound (node nd, LMT_TYPE ndt) |
| template<typename T > | |
| T | bound (node nd, LMT_TYPE ndt, T value) |
| void | stat (int lev=0) |
| void | printTable (int level) |
| virtual dictionary * | getDict () const |
| void | setOrderQuery (bool v) |
| bool | isOrderQuery () const |
Public Attributes | |
| dictionary * | dict |
Protected Attributes | |
| char * | table [LMTMAXLEV+1] |
| LMT_TYPE | tbltype [LMTMAXLEV+1] |
| table_entry_pos_t | cursize [LMTMAXLEV+1] |
| table_entry_pos_t | maxsize [LMTMAXLEV+1] |
| table_entry_pos_t * | startpos [LMTMAXLEV+1] |
| int | maxlev |
| char | info [100] |
| int | totget [LMTMAXLEV+1] |
| int | totbsearch [LMTMAXLEV+1] |
| bool | isQtable |
| bool | isItable |
| bool | isInverted |
| bool | isPruned |
| int | NumCenters [LMTMAXLEV+1] |
| float * | Pcenters [LMTMAXLEV+1] |
| float * | Bcenters [LMTMAXLEV+1] |
| double | logOOVpenalty |
| int | dictionary_upperbound |
| int | backoff_state |
| ngramcache * | lmtcache [LMTMAXLEV+1] |
| ngramcache * | probcache |
| ngramcache * | statecache |
| ngramcache * | statesizecache |
| int | max_cache_lev |
| int | memmap |
| int | diskid |
| off_t | tableOffs [LMTMAXLEV+1] |
| off_t | tableGaps [LMTMAXLEV+1] |
| bool | orderQuery |
Definition at line 89 of file lmtable.h.
| lmtable::lmtable | ( | ) |
Definition at line 57 of file lmtable.cpp.
References configure(), cursize, dict, info, isInverted, isPruned, lmtcache, LMTMAXLEV, logOOVpenalty, max_cache_lev, maxsize, memmap, NULL, NumCenters, probcache, setOrderQuery(), statecache, statesizecache, tbltype, totbsearch, and totget.
Referenced by cpsublm().


| virtual lmtable::~lmtable | ( | ) | [inline, virtual] |
| int lmtable::add | ( | ngram & | ng, | |
| TA | prob, | |||
| TB | bow | |||
| ) | [inline] |
Definition at line 760 of file lmtable.cpp.
References bound(), BOUND_EMPTY1, BOUND_EMPTY2, bow(), cursize, LMT_FIND, maxlev, nodesize(), prob(), search(), ngram::size, startpos, tbltype, word(), and ngram::wordp().

| bool lmtable::are_lmtcaches_active | ( | ) | [inline] |
| table_entry_pos_t lmtable::bound | ( | node | nd, | |
| LMT_TYPE | ndt | |||
| ) | [inline] |
Definition at line 416 of file lmtable.h.
References getmem(), LMTCODESIZE, PROBSIZE, QINTERNAL, and QPROBSIZE.
Referenced by add(), checkbounds(), cpsublm(), get(), ngcnt(), succscan(), and wdprune().


| void lmtable::check_cache_levels | ( | ) |
Definition at line 115 of file lmtable.cpp.
References cursize, ngramcache::cursize(), ngramcache::isfull(), lmtcache, max_cache_lev, probcache, ngramcache::reset(), statecache, and statesizecache.
Referenced by main().


| void lmtable::checkbounds | ( | int | level | ) |
Definition at line 697 of file lmtable.cpp.
References bound(), BOUND_EMPTY1, BOUND_EMPTY2, createtempfile(), cursize, nodesize(), removefile(), startpos, and tbltype.

| double lmtable::clprob | ( | ngram | ng | ) | [virtual] |
Reimplemented in lmmacro.
Definition at line 1656 of file lmtable.cpp.
References ngramcache::add(), ngramcache::get(), lprob(), probcache, ngram::size, and ngram::wordp().

Reimplemented in lmmacro.
Definition at line 1468 of file lmtable.cpp.
References ngramcache::add(), ngramcache::get(), maxsuffptr(), NULL, ngram::size, statecache, statesizecache, and ngram::wordp().

| void lmtable::configure | ( | int | n, | |
| bool | quantized | |||
| ) |
| lmtable * lmtable::cpsublm | ( | dictionary * | subdict, | |
| bool | keepunigr = true | |||
| ) |
Definition at line 931 of file lmtable.cpp.
References Bcenters, bound(), configure(), cursize, dictionary::decode(), dict, dictionary::genoovcode(), lmtable(), maxlev, nodesize(), NULL, NumCenters, dictionary::oovcode(), Pcenters, dictionary::size(), table, tbltype, and word().
Referenced by main().


| void lmtable::dumplm | ( | std::fstream & | out, | |
| ngram | ng, | |||
| int | ilev, | |||
| int | elev, | |||
| table_entry_pos_t | ipos, | |||
| table_entry_pos_t | epos | |||
| ) |
| int lmtable::get | ( | ngram & | ng, | |
| int | n, | |||
| int | lev | |||
| ) |
Definition at line 1251 of file lmtable.cpp.
References ngramcache::add(), bound(), BOUND_EMPTY1, bow(), ngram::bow, cursize, error(), ngram::freq, ngram::info, ngram::lev, ngram::link, LMT_FIND, lmtcache, nodesize(), NOPROB, NULL, ngram::path, ngram::prob, prob(), search(), ngram::size, ngram::succ, tbltype, totget, and ngram::wordp().

| virtual dictionary* lmtable::getDict | ( | ) | const [inline, virtual] |
Definition at line 441 of file lmtable.h.
References dict.
Referenced by lmmacro::clprob(), lmmacro::cmaxsuffptr(), Moses::LanguageModelIRST::Load(), lmmacro::loadLexicalClasses(), lmmacro::lprob(), lmmacro::maxsuffptr(), lmmacro::Micro2MacroMapping(), ngcnt(), lmmacro::One2OneMapping(), savebin(), savetxt(), stat(), and wdprune().

| double lmtable::getlogOOVpenalty | ( | ) | const [inline] |
Definition at line 228 of file lmtable.h.
References logOOVpenalty.
Referenced by main().

| void lmtable::getmem | ( | char * | ptr, | |
| T * | value, | |||
| int | offs | |||
| ) | [inline] |
| void lmtable::getmem | ( | char * | ptr, | |
| int * | value, | |||
| int | offs, | |||
| int | size | |||
| ) | [inline] |
| void lmtable::init_lmtcaches | ( | int | uptolev | ) |
| void lmtable::init_probcache | ( | ) |
| void lmtable::init_statecache | ( | ) |
| bool lmtable::is_inverted | ( | ) | [inline] |
| bool lmtable::is_inverted | ( | const bool | flag | ) | [inline] |
Definition at line 222 of file lmtable.h.
References isInverted.
Referenced by main().

| bool lmtable::is_probcache_active | ( | ) | [inline] |
| bool lmtable::is_statecache_active | ( | ) | [inline] |
| bool lmtable::isOrderQuery | ( | ) | const [inline] |
| bool lmtable::isQuantized | ( | ) | const [inline] |
| void lmtable::load | ( | std::istream & | inp, | |
| const char * | filename = NULL, |
|||
| const char * | outfilename = NULL, |
|||
| int | mmap = 0, |
|||
| OUTFILE_TYPE | outtype = NONE | |||
| ) |
Referenced by Moses::LanguageModelIRST::Load(), load_lm(), and main().

| void lmtable::loadbin | ( | std::istream & | inp, | |
| const char * | header, | |||
| const char * | filename = NULL, |
|||
| int | mmap = 0 | |||
| ) |
| void lmtable::loadbincodebook | ( | std::istream & | inp, | |
| int | l | |||
| ) |
| void lmtable::loadbinheader | ( | std::istream & | inp, | |
| const char * | header | |||
| ) |
| void lmtable::loadcenters | ( | std::istream & | inp, | |
| int | Order | |||
| ) |
| void lmtable::loadtxt | ( | std::istream & | inp, | |
| const char * | header | |||
| ) |
| void lmtable::loadtxt | ( | std::istream & | inp, | |
| const char * | header, | |||
| const char * | outfilename, | |||
| int | mmap | |||
| ) |
| void lmtable::loadtxtmmap | ( | std::istream & | inp, | |
| const char * | header, | |||
| const char * | outfilename | |||
| ) |
| double lmtable::lprob | ( | ngram | ng, | |
| double * | bow = NULL, |
|||
| int * | bol = NULL, |
|||
| char ** | maxsuffptr = NULL, |
|||
| unsigned int * | statesize = NULL | |||
| ) | [virtual] |
Definition at line 1558 of file lmtable.cpp.
References Bcenters, ngram::bow, bow(), dict, ngram::invert(), isInverted, ngram::lev, ngram::link, logOOVpenalty, MIN, NULL, dictionary::oovcode(), ngram::path, ngram::prob, ngram::size, tbltype, UNIGRAM_RESOLUTION, and ngram::wordp().
Referenced by clprob(), main(), and wdprune().


| double lmtable::lprobx | ( | ngram | ong, | |
| double * | lkp = 0, |
|||
| double * | bop = 0, |
|||
| int * | bol = 0 | |||
| ) |
Definition at line 1737 of file lmtable.cpp.
References Bcenters, ngram::bow, dict, ngram::prob, ngram::shift(), ngram::size, and UNIGRAM_RESOLUTION.

| int lmtable::maxlevel | ( | ) | const [inline] |
Definition at line 239 of file lmtable.h.
References maxlev.
Referenced by Moses::LanguageModelIRST::Load(), and main().

Reimplemented in lmmacro.
Definition at line 1438 of file lmtable.cpp.
References ngram::link, NULL, ngram::size, and ngram::succ.
Referenced by cmaxsuffptr(), and main().

| int lmtable::mybsearch | ( | char * | ar, | |
| table_entry_pos_t | n, | |||
| int | size, | |||
| char * | key, | |||
| table_entry_pos_t * | idx | |||
| ) |
Definition at line 880 of file lmtable.cpp.
References word().
Referenced by search().


| table_entry_pos_t lmtable::ngcnt | ( | table_entry_pos_t * | cnt, | |
| ngram | ng, | |||
| int | l, | |||
| table_entry_pos_t | ipos, | |||
| table_entry_pos_t | epos | |||
| ) |
Definition at line 1921 of file lmtable.cpp.
References bound(), ngcnt(), nodesize(), NOPROB, prob(), ngram::pushc(), tbltype, word(), and ngram::wordp().

| table_entry_pos_t lmtable::ngcnt | ( | table_entry_pos_t * | cnt | ) |
| int lmtable::nodesize | ( | LMT_TYPE | ndt | ) | [inline] |
Definition at line 317 of file lmtable.h.
References BOUNDSIZE, INTERNAL, LEAF, LMTCODESIZE, PROBSIZE, QINTERNAL, QLEAF, and QPROBSIZE.
Referenced by add(), checkbounds(), cpsublm(), get(), ngcnt(), printTable(), pscale(), reset_mmap(), savebin(), stat(), succscan(), wdprune(), and ~lmtable().

| void lmtable::printTable | ( | int | level | ) |
Definition at line 674 of file lmtable.cpp.
References cursize, nodesize(), prob(), tbltype, and word().

Definition at line 371 of file lmtable.h.
References LMTCODESIZE, and putmem().

Definition at line 345 of file lmtable.h.
References getmem(), INTERNAL, LEAF, LMTCODESIZE, QINTERNAL, and QLEAF.
Referenced by add(), get(), lmmacro::lprob(), ngcnt(), printTable(), pscale(), and wdprune().


| int lmtable::pscale | ( | int | lev, | |
| table_entry_pos_t | ipos, | |||
| table_entry_pos_t | epos, | |||
| double | s | |||
| ) |
TOCHECK: Nicola 18 dicembre 2009)
Definition at line 1890 of file lmtable.cpp.
References nodesize(), NOPROB, prob(), and tbltype.
Referenced by wdprune().


| void lmtable::putmem | ( | char * | ptr, | |
| T | value, | |||
| int | offs | |||
| ) | [inline] |
| void lmtable::putmem | ( | char * | ptr, | |
| int | value, | |||
| int | offs, | |||
| int | size | |||
| ) | [inline] |
| int lmtable::reload | ( | std::set< string > | words | ) |
Definition at line 181 of file lmtable.cpp.
References dict, dictionary::encode(), dictionary::incflag(), and NULL.

| void lmtable::reset_caches | ( | ) |
Definition at line 125 of file lmtable.cpp.
References cursize, ngramcache::cursize(), lmtcache, MAX, max_cache_lev, ngramcache::maxsize(), probcache, ngramcache::reset(), statecache, and statesizecache.
Referenced by Moses::LanguageModelIRST::CleanUpAfterSentenceProcessing().


| void lmtable::reset_mmap | ( | ) |
Definition at line 1717 of file lmtable.cpp.
References cursize, diskid, memmap, MMap(), Munmap(), nodesize(), tableGaps, tableOffs, and tbltype.
Referenced by Moses::LanguageModelIRST::~LanguageModelIRST().


| void lmtable::savebin | ( | const char * | filename | ) |
Definition at line 1100 of file lmtable.cpp.
References Bcenters, cursize, getDict(), isInverted, isPruned, maxlev, nodesize(), dictionary::save(), and tbltype.
Referenced by main().


| void lmtable::savetxt | ( | const char * | filename | ) |
| void * lmtable::search | ( | int | lev, | |
| table_entry_pos_t | offs, | |||
| table_entry_pos_t | n, | |||
| int | sz, | |||
| int * | w, | |||
| LMT_ACTION | action, | |||
| char ** | found = (char **)NULL | |||
| ) |
Definition at line 836 of file lmtable.cpp.
References error(), LMT_FIND, LMTCODESIZE, mybsearch(), NULL, putmem(), and totbsearch.
Referenced by add(), and get().


| double lmtable::setlogOOVpenalty | ( | int | dub | ) | [inline] |
Definition at line 230 of file lmtable.h.
References dict, logOOVpenalty, and dictionary::size().
Referenced by Moses::LanguageModelIRST::Load(), load_lm(), and main().


| double lmtable::setlogOOVpenalty2 | ( | double | oovp | ) | [inline] |
Definition at line 235 of file lmtable.h.
References logOOVpenalty.
Referenced by main().

| void lmtable::setOrderQuery | ( | bool | v | ) | [inline] |
Definition at line 445 of file lmtable.h.
References orderQuery.
Referenced by lmtable().

| void lmtable::stat | ( | int | lev = 0 |
) |
Definition at line 1689 of file lmtable.cpp.
References cursize, getDict(), nodesize(), dictionary::stat(), tbltype, totbsearch, and totget.
Referenced by main().


| int lmtable::succscan | ( | ngram & | h, | |
| ngram & | ng, | |||
| LMT_ACTION | action, | |||
| int | lev | |||
| ) |
Definition at line 1393 of file lmtable.cpp.
References bound(), ngram::lev, ngram::link, LMT_CONT, LMT_INIT, ngram::midx, nodesize(), ngram::size, ngram::succ, ngram::succlink, tbltype, ngram::trans(), word(), and ngram::wordp().

| table_entry_pos_t lmtable::wdprune | ( | float * | thr, | |
| int | aflag, | |||
| ngram | ng, | |||
| int | ilev, | |||
| int | elev, | |||
| table_entry_pos_t | ipos, | |||
| table_entry_pos_t | epos, | |||
| double | lk = 0, |
|||
| double | bo = 0, |
|||
| double * | ts = 0, |
|||
| double * | tbs = 0 | |||
| ) |
TOCHECK: Nicola 18 dicembre 2009)
Definition at line 1803 of file lmtable.cpp.
References bound(), bow(), cursize, k, lprob(), nodesize(), NOPROB, prob(), pscale(), ngram::pushc(), ngram::size, tbltype, wdprune(), word(), and ngram::wordp().

| table_entry_pos_t lmtable::wdprune | ( | float * | thr, | |
| int | aflag = 0 | |||
| ) |
Definition at line 1788 of file lmtable.cpp.
References cursize, getDict(), isPruned, and ngram::size.
Referenced by main(), and wdprune().


| int lmtable::word | ( | node | nd, | |
| int | value = -1 | |||
| ) | [inline] |
Definition at line 333 of file lmtable.h.
References getmem(), LMTCODESIZE, and putmem().
Referenced by add(), cpsublm(), mybsearch(), ngcnt(), printTable(), succscan(), and wdprune().


int lmtable::backoff_state [protected] |
float* lmtable::Bcenters[LMTMAXLEV+1] [protected] |
table_entry_pos_t lmtable::cursize[LMTMAXLEV+1] [protected] |
Definition at line 94 of file lmtable.h.
Referenced by add(), check_cache_levels(), checkbounds(), cpsublm(), get(), lmtable(), ngcnt(), printTable(), reset_caches(), reset_mmap(), savebin(), savetxt(), stat(), wdprune(), and ~lmtable().
int lmtable::dictionary_upperbound [protected] |
int lmtable::diskid [protected] |
char lmtable::info[100] [protected] |
bool lmtable::isInverted [protected] |
bool lmtable::isItable [protected] |
bool lmtable::isPruned [protected] |
bool lmtable::isQtable [protected] |
ngramcache* lmtable::lmtcache[LMTMAXLEV+1] [protected] |
Definition at line 126 of file lmtable.h.
Referenced by are_lmtcaches_active(), check_cache_levels(), get(), init_lmtcaches(), lmtable(), reset_caches(), and ~lmtable().
double lmtable::logOOVpenalty [protected] |
Definition at line 121 of file lmtable.h.
Referenced by getlogOOVpenalty(), lmtable(), lprob(), setlogOOVpenalty(), and setlogOOVpenalty2().
int lmtable::max_cache_lev [protected] |
Definition at line 130 of file lmtable.h.
Referenced by check_cache_levels(), init_lmtcaches(), lmtable(), and reset_caches().
int lmtable::maxlev [protected] |
Definition at line 98 of file lmtable.h.
Referenced by add(), lmmacro::clprob(), configure(), cpsublm(), init_probcache(), init_statecache(), maxlevel(), savebin(), savetxt(), and ~lmtable().
table_entry_pos_t lmtable::maxsize[LMTMAXLEV+1] [protected] |
int lmtable::memmap [protected] |
int lmtable::NumCenters[LMTMAXLEV+1] [protected] |
bool lmtable::orderQuery [protected] |
float* lmtable::Pcenters[LMTMAXLEV+1] [protected] |
ngramcache* lmtable::probcache [protected] |
Definition at line 127 of file lmtable.h.
Referenced by check_cache_levels(), clprob(), lmmacro::clprob(), init_probcache(), is_probcache_active(), lmtable(), reset_caches(), and ~lmtable().
table_entry_pos_t* lmtable::startpos[LMTMAXLEV+1] [protected] |
ngramcache* lmtable::statecache [protected] |
Definition at line 128 of file lmtable.h.
Referenced by check_cache_levels(), cmaxsuffptr(), init_statecache(), is_statecache_active(), lmtable(), reset_caches(), and ~lmtable().
ngramcache* lmtable::statesizecache [protected] |
Definition at line 129 of file lmtable.h.
Referenced by check_cache_levels(), cmaxsuffptr(), init_statecache(), lmtable(), reset_caches(), and ~lmtable().
char* lmtable::table[LMTMAXLEV+1] [protected] |
off_t lmtable::tableGaps[LMTMAXLEV+1] [protected] |
off_t lmtable::tableOffs[LMTMAXLEV+1] [protected] |
LMT_TYPE lmtable::tbltype[LMTMAXLEV+1] [protected] |
Definition at line 93 of file lmtable.h.
Referenced by add(), checkbounds(), configure(), cpsublm(), get(), lmtable(), lprob(), ngcnt(), printTable(), pscale(), reset_mmap(), savebin(), stat(), succscan(), wdprune(), and ~lmtable().
int lmtable::totbsearch[LMTMAXLEV+1] [protected] |
int lmtable::totget[LMTMAXLEV+1] [protected] |
1.5.9