#include <lmtable.h>


Public Member Functions | |
| lmtable () | |
| virtual | ~lmtable () |
| table_entry_pos_t | wdprune (float *thr, int aflag=0) |
| table_entry_pos_t | wdprune (float *thr, int aflag, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos, double lk=0, double bo=0, double *ts=0, double *tbs=0) |
| double | lprobx (ngram ong, double *lkp=0, double *bop=0, int *bol=0) |
| table_entry_pos_t | ngcnt (table_entry_pos_t *cnt) |
| table_entry_pos_t | ngcnt (table_entry_pos_t *cnt, ngram ng, int l, table_entry_pos_t ipos, table_entry_pos_t epos) |
| int | pscale (int lev, table_entry_pos_t ipos, table_entry_pos_t epos, double s) |
| void | init_probcache () |
| void | init_statecache () |
| void | init_lmtcaches (int uptolev) |
| void | check_cache_levels () |
| void | reset_caches () |
| void | reset_mmap () |
| bool | is_probcache_active () |
| bool | is_statecache_active () |
| bool | are_lmtcaches_active () |
| bool | is_inverted (const bool flag) |
| bool | is_inverted () |
| void | configure (int n, bool quantized) |
| double | getlogOOVpenalty () const |
| double | setlogOOVpenalty (int dub) |
| double | setlogOOVpenalty2 (double oovp) |
| int | maxlevel () const |
| bool | isQuantized () const |
| void | savetxt (const char *filename) |
| void | savebin (const char *filename) |
| void | dumplm (std::fstream &out, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos) |
| void | load (std::istream &inp, const char *filename=NULL, const char *outfilename=NULL, int mmap=0, OUTFILE_TYPE outtype=NONE) |
| void | loadtxt (std::istream &inp, const char *header, const char *outfilename, int mmap) |
| void | loadtxt (std::istream &inp, const char *header) |
| void | loadtxtmmap (std::istream &inp, const char *header, const char *outfilename) |
| void | loadbin (std::istream &inp, const char *header, const char *filename=NULL, int mmap=0) |
| void | loadbinheader (std::istream &inp, const char *header) |
| void | loadbincodebook (std::istream &inp, int l) |
| void | loadcenters (std::istream &inp, int Order) |
| lmtable * | cpsublm (dictionary *subdict, bool keepunigr=true) |
| int | reload (std::set< string > words) |
| void | filter (const char *lmfile) |
| virtual double | lprob (ngram ng, double *bow=NULL, int *bol=NULL, char **maxsuffptr=NULL, unsigned int *statesize=NULL) |
| virtual double | clprob (ngram ng) |
| void * | search (int lev, table_entry_pos_t offs, table_entry_pos_t n, int sz, int *w, LMT_ACTION action, char **found=(char **) NULL) |
| int | mybsearch (char *ar, table_entry_pos_t n, int size, char *key, table_entry_pos_t *idx) |
| template<typename TA , typename TB > | |
| int | add (ngram &ng, TA prob, TB bow) |
| void | checkbounds (int level) |
| int | get (ngram &ng) |
| int | get (ngram &ng, int n, int lev) |
| int | succscan (ngram &h, ngram &ng, LMT_ACTION action, int lev) |
| virtual const char * | maxsuffptr (ngram ong, unsigned int *size=NULL) |
| virtual const char * | cmaxsuffptr (ngram ong, unsigned int *size=NULL) |
| void | putmem (char *ptr, int value, int offs, int size) |
| void | getmem (char *ptr, int *value, int offs, int size) |
| template<typename T > | |
| void | putmem (char *ptr, T value, int offs) |
| template<typename T > | |
| void | getmem (char *ptr, T *value, int offs) |
| int | nodesize (LMT_TYPE ndt) |
| int | word (node nd, int value=-1) |
| float | prob (node nd, LMT_TYPE ndt) |
| template<typename T > | |
| float | prob (node nd, LMT_TYPE ndt, T value) |
| float | bow (node nd, LMT_TYPE ndt) |
| template<typename T > | |
| T | bow (node nd, LMT_TYPE ndt, T value) |
| table_entry_pos_t | bound (node nd, LMT_TYPE ndt) |
| template<typename T > | |
| T | bound (node nd, LMT_TYPE ndt, T value) |
| void | stat (int lev=0) |
| void | printTable (int level) |
| virtual dictionary * | getDict () const |
| void | setOrderQuery (bool v) |
| bool | isOrderQuery () const |
Public Attributes | |
| dictionary * | dict |
Protected Attributes | |
| char * | table [LMTMAXLEV+1] |
| LMT_TYPE | tbltype [LMTMAXLEV+1] |
| table_entry_pos_t | cursize [LMTMAXLEV+1] |
| table_entry_pos_t | maxsize [LMTMAXLEV+1] |
| table_entry_pos_t * | startpos [LMTMAXLEV+1] |
| int | maxlev |
| char | info [100] |
| int | totget [LMTMAXLEV+1] |
| int | totbsearch [LMTMAXLEV+1] |
| bool | isQtable |
| bool | isItable |
| bool | isInverted |
| bool | isPruned |
| int | NumCenters [LMTMAXLEV+1] |
| float * | Pcenters [LMTMAXLEV+1] |
| float * | Bcenters [LMTMAXLEV+1] |
| double | logOOVpenalty |
| int | dictionary_upperbound |
| int | backoff_state |
| ngramcache * | lmtcache [LMTMAXLEV+1] |
| ngramcache * | probcache |
| ngramcache * | statecache |
| ngramcache * | statesizecache |
| int | max_cache_lev |
| int | memmap |
| int | diskid |
| off_t | tableOffs [LMTMAXLEV+1] |
| off_t | tableGaps [LMTMAXLEV+1] |
| bool | orderQuery |
| lmtable::lmtable | ( | ) |
References configure(), cursize, dict, info, isInverted, isPruned, lmtcache, LMTMAXLEV, logOOVpenalty, max_cache_lev, maxsize, memmap, NULL, NumCenters, probcache, setOrderQuery(), statecache, statesizecache, tbltype, totbsearch, and totget.
Referenced by cpsublm().


| virtual lmtable::~lmtable | ( | ) | [inline, virtual] |
| int lmtable::add | ( | ngram & | ng, | |
| TA | prob, | |||
| TB | bow | |||
| ) | [inline] |
References bound(), BOUND_EMPTY1, BOUND_EMPTY2, bow(), cursize, LMT_FIND, maxlev, nodesize(), prob(), search(), ngram::size, startpos, tbltype, word(), and ngram::wordp().

References LMTCODESIZE, PROBSIZE, putmem(), QINTERNAL, and QPROBSIZE.

| table_entry_pos_t lmtable::bound | ( | node | nd, | |
| LMT_TYPE | ndt | |||
| ) | [inline] |
References getmem(), LMTCODESIZE, PROBSIZE, QINTERNAL, and QPROBSIZE.
Referenced by add(), checkbounds(), cpsublm(), get(), ngcnt(), succscan(), and wdprune().


References LMTCODESIZE, PROBSIZE, putmem(), QINTERNAL, and QPROBSIZE.

| void lmtable::check_cache_levels | ( | ) |
References cursize, ngramcache::cursize(), ngramcache::isfull(), lmtcache, max_cache_lev, probcache, ngramcache::reset(), statecache, and statesizecache.
Referenced by main().


| void lmtable::checkbounds | ( | int | level | ) |
References bound(), BOUND_EMPTY1, BOUND_EMPTY2, createtempfile(), cursize, nodesize(), removefile(), startpos, and tbltype.

| double lmtable::clprob | ( | ngram | ng | ) | [virtual] |
Reimplemented in lmmacro.
References ngramcache::add(), ngramcache::get(), lprob(), probcache, ngram::size, and ngram::wordp().

Reimplemented in lmmacro.
References ngramcache::add(), ngramcache::get(), maxsuffptr(), NULL, ngram::size, statecache, statesizecache, and ngram::wordp().

| void lmtable::configure | ( | int | n, | |
| bool | quantized | |||
| ) |
| lmtable * lmtable::cpsublm | ( | dictionary * | subdict, | |
| bool | keepunigr = true | |||
| ) |
References Bcenters, bound(), configure(), cursize, dictionary::decode(), dict, dictionary::genoovcode(), lmtable(), maxlev, nodesize(), NULL, NumCenters, dictionary::oovcode(), Pcenters, dictionary::size(), table, tbltype, and word().
Referenced by main().


| void lmtable::dumplm | ( | std::fstream & | out, | |
| ngram | ng, | |||
| int | ilev, | |||
| int | elev, | |||
| table_entry_pos_t | ipos, | |||
| table_entry_pos_t | epos | |||
| ) |
| void lmtable::filter | ( | const char * | lmfile | ) | [inline] |
| int lmtable::get | ( | ngram & | ng, | |
| int | n, | |||
| int | lev | |||
| ) |
References ngramcache::add(), bound(), BOUND_EMPTY1, bow(), ngram::bow, cursize, error(), ngram::freq, ngram::info, ngram::lev, ngram::link, LMT_FIND, lmtcache, nodesize(), NOPROB, NULL, ngram::path, ngram::prob, prob(), search(), ngram::size, ngram::succ, tbltype, totget, and ngram::wordp().

| int lmtable::get | ( | ngram & | ng | ) | [inline] |
| virtual dictionary* lmtable::getDict | ( | ) | const [inline, virtual] |
References dict.
Referenced by lmmacro::clprob(), lmmacro::cmaxsuffptr(), Moses::LanguageModelIRST::CreateFactors(), Moses::LanguageModelIRST::GetLmID(), Moses::LanguageModelIRST::Load(), lmmacro::loadLexicalClasses(), lmmacro::lprob(), lmmacro::maxsuffptr(), lmmacro::Micro2MacroMapping(), ngcnt(), lmmacro::One2OneMapping(), savebin(), savetxt(), stat(), and wdprune().

| double lmtable::getlogOOVpenalty | ( | ) | const [inline] |
| void lmtable::getmem | ( | char * | ptr, | |
| T * | value, | |||
| int | offs | |||
| ) | [inline] |
References NULL.
| void lmtable::getmem | ( | char * | ptr, | |
| int * | value, | |||
| int | offs, | |||
| int | size | |||
| ) | [inline] |
| void lmtable::init_lmtcaches | ( | int | uptolev | ) |
References lmtcache, max_cache_lev, and NULL.
Referenced by Moses::LanguageModelIRST::Load().

| void lmtable::init_probcache | ( | ) |
| void lmtable::init_statecache | ( | ) |
References maxlev, NULL, statecache, and statesizecache.
Referenced by Moses::LanguageModelIRST::Load().

| bool lmtable::is_inverted | ( | ) | [inline] |
References isInverted.
| bool lmtable::is_inverted | ( | const bool | flag | ) | [inline] |
| bool lmtable::is_statecache_active | ( | ) | [inline] |
References NULL, and statecache.
| bool lmtable::isOrderQuery | ( | ) | const [inline] |
References orderQuery.
| bool lmtable::isQuantized | ( | ) | const [inline] |
References isQtable.
| void lmtable::load | ( | std::istream & | inp, | |
| const char * | filename = NULL, |
|||
| const char * | outfilename = NULL, |
|||
| int | mmap = 0, |
|||
| OUTFILE_TYPE | outtype = NONE | |||
| ) |
Referenced by Moses::LanguageModelIRST::Load(), load_lm(), and main().

| void lmtable::loadbin | ( | std::istream & | inp, | |
| const char * | header, | |||
| const char * | filename = NULL, |
|||
| int | mmap = 0 | |||
| ) |
| void lmtable::loadbincodebook | ( | std::istream & | inp, | |
| int | l | |||
| ) |
| void lmtable::loadbinheader | ( | std::istream & | inp, | |
| const char * | header | |||
| ) |
| void lmtable::loadcenters | ( | std::istream & | inp, | |
| int | Order | |||
| ) |
| void lmtable::loadtxt | ( | std::istream & | inp, | |
| const char * | header | |||
| ) |
| void lmtable::loadtxt | ( | std::istream & | inp, | |
| const char * | header, | |||
| const char * | outfilename, | |||
| int | mmap | |||
| ) |
| void lmtable::loadtxtmmap | ( | std::istream & | inp, | |
| const char * | header, | |||
| const char * | outfilename | |||
| ) |
| double lmtable::lprob | ( | ngram | ng, | |
| double * | bow = NULL, |
|||
| int * | bol = NULL, |
|||
| char ** | maxsuffptr = NULL, |
|||
| unsigned int * | statesize = NULL | |||
| ) | [virtual] |
References Bcenters, ngram::bow, bow(), dict, ngram::invert(), isInverted, ngram::lev, ngram::link, logOOVpenalty, MIN, NULL, dictionary::oovcode(), ngram::path, ngram::prob, ngram::size, tbltype, UNIGRAM_RESOLUTION, and ngram::wordp().
Referenced by clprob(), main(), and wdprune().


| double lmtable::lprobx | ( | ngram | ong, | |
| double * | lkp = 0, |
|||
| double * | bop = 0, |
|||
| int * | bol = 0 | |||
| ) |
References Bcenters, ngram::bow, dict, ngram::prob, ngram::shift(), ngram::size, and UNIGRAM_RESOLUTION.

| int lmtable::maxlevel | ( | ) | const [inline] |
References maxlev.
Referenced by Moses::LanguageModelIRST::Load(), and main().

Reimplemented in lmmacro.
References ngram::link, NULL, ngram::size, and ngram::succ.
Referenced by cmaxsuffptr(), and main().

| int lmtable::mybsearch | ( | char * | ar, | |
| table_entry_pos_t | n, | |||
| int | size, | |||
| char * | key, | |||
| table_entry_pos_t * | idx | |||
| ) |
| table_entry_pos_t lmtable::ngcnt | ( | table_entry_pos_t * | cnt, | |
| ngram | ng, | |||
| int | l, | |||
| table_entry_pos_t | ipos, | |||
| table_entry_pos_t | epos | |||
| ) |
References bound(), ngcnt(), nodesize(), NOPROB, prob(), ngram::pushc(), tbltype, word(), and ngram::wordp().

| table_entry_pos_t lmtable::ngcnt | ( | table_entry_pos_t * | cnt | ) |
| int lmtable::nodesize | ( | LMT_TYPE | ndt | ) | [inline] |
References BOUNDSIZE, INTERNAL, LEAF, LMTCODESIZE, PROBSIZE, QINTERNAL, QLEAF, and QPROBSIZE.
Referenced by add(), checkbounds(), cpsublm(), get(), ngcnt(), printTable(), pscale(), reset_mmap(), savebin(), stat(), succscan(), wdprune(), and ~lmtable().

| void lmtable::printTable | ( | int | level | ) |
References cursize, nodesize(), prob(), tbltype, and word().

References getmem(), INTERNAL, LEAF, LMTCODESIZE, QINTERNAL, and QLEAF.
Referenced by add(), get(), lmmacro::lprob(), ngcnt(), printTable(), pscale(), and wdprune().


| int lmtable::pscale | ( | int | lev, | |
| table_entry_pos_t | ipos, | |||
| table_entry_pos_t | epos, | |||
| double | s | |||
| ) |
TOCHECK: Nicola 18 dicembre 2009)
References nodesize(), NOPROB, prob(), and tbltype.
Referenced by wdprune().


| void lmtable::putmem | ( | char * | ptr, | |
| T | value, | |||
| int | offs | |||
| ) | [inline] |
References NULL.
| void lmtable::putmem | ( | char * | ptr, | |
| int | value, | |||
| int | offs, | |||
| int | size | |||
| ) | [inline] |
| int lmtable::reload | ( | std::set< string > | words | ) |
References dict, dictionary::encode(), dictionary::incflag(), and NULL.

| void lmtable::reset_caches | ( | ) |
References cursize, ngramcache::cursize(), lmtcache, MAX, max_cache_lev, ngramcache::maxsize(), probcache, ngramcache::reset(), statecache, and statesizecache.
Referenced by Moses::LanguageModelIRST::CleanUpAfterSentenceProcessing().


| void lmtable::reset_mmap | ( | ) |
References cursize, diskid, memmap, MMap(), Munmap(), nodesize(), tableGaps, tableOffs, and tbltype.
Referenced by Moses::LanguageModelIRST::~LanguageModelIRST().


| void lmtable::savebin | ( | const char * | filename | ) |
References Bcenters, cursize, getDict(), isInverted, isPruned, maxlev, nodesize(), dictionary::save(), and tbltype.
Referenced by main().


| void lmtable::savetxt | ( | const char * | filename | ) |
| void * lmtable::search | ( | int | lev, | |
| table_entry_pos_t | offs, | |||
| table_entry_pos_t | n, | |||
| int | sz, | |||
| int * | w, | |||
| LMT_ACTION | action, | |||
| char ** | found = (char **)NULL | |||
| ) |
References error(), LMT_FIND, LMTCODESIZE, mybsearch(), NULL, putmem(), and totbsearch.
Referenced by add(), and get().


| double lmtable::setlogOOVpenalty | ( | int | dub | ) | [inline] |
References dict, logOOVpenalty, and dictionary::size().
Referenced by Moses::LanguageModelIRST::Load(), load_lm(), and main().


| double lmtable::setlogOOVpenalty2 | ( | double | oovp | ) | [inline] |
| void lmtable::setOrderQuery | ( | bool | v | ) | [inline] |
| void lmtable::stat | ( | int | lev = 0 |
) |
References cursize, getDict(), nodesize(), dictionary::stat(), tbltype, totbsearch, and totget.
Referenced by main().


| int lmtable::succscan | ( | ngram & | h, | |
| ngram & | ng, | |||
| LMT_ACTION | action, | |||
| int | lev | |||
| ) |
References bound(), ngram::lev, ngram::link, LMT_CONT, LMT_INIT, ngram::midx, nodesize(), ngram::size, ngram::succ, ngram::succlink, tbltype, ngram::trans(), word(), and ngram::wordp().

| table_entry_pos_t lmtable::wdprune | ( | float * | thr, | |
| int | aflag, | |||
| ngram | ng, | |||
| int | ilev, | |||
| int | elev, | |||
| table_entry_pos_t | ipos, | |||
| table_entry_pos_t | epos, | |||
| double | lk = 0, |
|||
| double | bo = 0, |
|||
| double * | ts = 0, |
|||
| double * | tbs = 0 | |||
| ) |
TOCHECK: Nicola 18 dicembre 2009)
References bound(), bow(), cursize, k, lprob(), nodesize(), NOPROB, prob(), pscale(), ngram::pushc(), ngram::size, tbltype, wdprune(), word(), and ngram::wordp().

| table_entry_pos_t lmtable::wdprune | ( | float * | thr, | |
| int | aflag = 0 | |||
| ) |
| int lmtable::word | ( | node | nd, | |
| int | value = -1 | |||
| ) | [inline] |
References getmem(), LMTCODESIZE, and putmem().
Referenced by add(), cpsublm(), mybsearch(), ngcnt(), printTable(), succscan(), and wdprune().


int lmtable::backoff_state [protected] |
float* lmtable::Bcenters[LMTMAXLEV+1] [protected] |
table_entry_pos_t lmtable::cursize[LMTMAXLEV+1] [protected] |
Referenced by add(), check_cache_levels(), checkbounds(), cpsublm(), get(), lmtable(), ngcnt(), printTable(), reset_caches(), reset_mmap(), savebin(), savetxt(), stat(), wdprune(), and ~lmtable().
Reimplemented in lmmacro.
Referenced by cpsublm(), getDict(), lmtable(), lprob(), lprobx(), main(), reload(), setlogOOVpenalty(), and ~lmtable().
int lmtable::dictionary_upperbound [protected] |
int lmtable::diskid [protected] |
Referenced by reset_mmap().
char lmtable::info[100] [protected] |
Referenced by lmtable().
bool lmtable::isInverted [protected] |
Referenced by is_inverted(), lmtable(), lprob(), and savebin().
bool lmtable::isItable [protected] |
bool lmtable::isPruned [protected] |
bool lmtable::isQtable [protected] |
Referenced by isQuantized(), and ~lmtable().
ngramcache* lmtable::lmtcache[LMTMAXLEV+1] [protected] |
Referenced by are_lmtcaches_active(), check_cache_levels(), get(), init_lmtcaches(), lmtable(), reset_caches(), and ~lmtable().
double lmtable::logOOVpenalty [protected] |
Referenced by getlogOOVpenalty(), lmtable(), lprob(), setlogOOVpenalty(), and setlogOOVpenalty2().
int lmtable::max_cache_lev [protected] |
Referenced by check_cache_levels(), init_lmtcaches(), lmtable(), and reset_caches().
int lmtable::maxlev [protected] |
Referenced by add(), lmmacro::clprob(), configure(), cpsublm(), init_probcache(), init_statecache(), maxlevel(), savebin(), savetxt(), and ~lmtable().
table_entry_pos_t lmtable::maxsize[LMTMAXLEV+1] [protected] |
Referenced by lmtable().
int lmtable::memmap [protected] |
Referenced by lmtable(), reset_mmap(), and ~lmtable().
int lmtable::NumCenters[LMTMAXLEV+1] [protected] |
bool lmtable::orderQuery [protected] |
Referenced by isOrderQuery(), and setOrderQuery().
float* lmtable::Pcenters[LMTMAXLEV+1] [protected] |
Referenced by cpsublm(), and ~lmtable().
ngramcache* lmtable::probcache [protected] |
Referenced by check_cache_levels(), clprob(), lmmacro::clprob(), init_probcache(), is_probcache_active(), lmtable(), reset_caches(), and ~lmtable().
table_entry_pos_t* lmtable::startpos[LMTMAXLEV+1] [protected] |
Referenced by add(), and checkbounds().
ngramcache* lmtable::statecache [protected] |
Referenced by check_cache_levels(), cmaxsuffptr(), init_statecache(), is_statecache_active(), lmtable(), reset_caches(), and ~lmtable().
ngramcache* lmtable::statesizecache [protected] |
Referenced by check_cache_levels(), cmaxsuffptr(), init_statecache(), lmtable(), reset_caches(), and ~lmtable().
char* lmtable::table[LMTMAXLEV+1] [protected] |
Referenced by cpsublm(), and ~lmtable().
off_t lmtable::tableGaps[LMTMAXLEV+1] [protected] |
Referenced by reset_mmap(), and ~lmtable().
off_t lmtable::tableOffs[LMTMAXLEV+1] [protected] |
Referenced by reset_mmap().
LMT_TYPE lmtable::tbltype[LMTMAXLEV+1] [protected] |
Referenced by add(), checkbounds(), configure(), cpsublm(), get(), lmtable(), lprob(), ngcnt(), printTable(), pscale(), reset_mmap(), savebin(), stat(), succscan(), wdprune(), and ~lmtable().
int lmtable::totbsearch[LMTMAXLEV+1] [protected] |
int lmtable::totget[LMTMAXLEV+1] [protected] |
1.5.9