lmtable Class Reference

#include <lmtable.h>

Inheritance diagram for lmtable:

Inheritance graph
[legend]
Collaboration diagram for lmtable:

Collaboration graph
[legend]

List of all members.

Public Member Functions

 lmtable ()
virtual ~lmtable ()
table_entry_pos_t wdprune (float *thr, int aflag=0)
table_entry_pos_t wdprune (float *thr, int aflag, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos, double lk=0, double bo=0, double *ts=0, double *tbs=0)
double lprobx (ngram ong, double *lkp=0, double *bop=0, int *bol=0)
table_entry_pos_t ngcnt (table_entry_pos_t *cnt)
table_entry_pos_t ngcnt (table_entry_pos_t *cnt, ngram ng, int l, table_entry_pos_t ipos, table_entry_pos_t epos)
int pscale (int lev, table_entry_pos_t ipos, table_entry_pos_t epos, double s)
void init_probcache ()
void init_statecache ()
void init_lmtcaches (int uptolev)
void check_cache_levels ()
void reset_caches ()
void reset_mmap ()
bool is_probcache_active ()
bool is_statecache_active ()
bool are_lmtcaches_active ()
bool is_inverted (const bool flag)
bool is_inverted ()
void configure (int n, bool quantized)
double getlogOOVpenalty () const
double setlogOOVpenalty (int dub)
double setlogOOVpenalty2 (double oovp)
int maxlevel () const
bool isQuantized () const
void savetxt (const char *filename)
void savebin (const char *filename)
void dumplm (std::fstream &out, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos)
void load (std::istream &inp, const char *filename=NULL, const char *outfilename=NULL, int mmap=0, OUTFILE_TYPE outtype=NONE)
void loadtxt (std::istream &inp, const char *header, const char *outfilename, int mmap)
void loadtxt (std::istream &inp, const char *header)
void loadtxtmmap (std::istream &inp, const char *header, const char *outfilename)
void loadbin (std::istream &inp, const char *header, const char *filename=NULL, int mmap=0)
void loadbinheader (std::istream &inp, const char *header)
void loadbincodebook (std::istream &inp, int l)
void loadcenters (std::istream &inp, int Order)
lmtablecpsublm (dictionary *subdict, bool keepunigr=true)
int reload (std::set< string > words)
void filter (const char *lmfile)
virtual double lprob (ngram ng, double *bow=NULL, int *bol=NULL, char **maxsuffptr=NULL, unsigned int *statesize=NULL)
virtual double clprob (ngram ng)
void * search (int lev, table_entry_pos_t offs, table_entry_pos_t n, int sz, int *w, LMT_ACTION action, char **found=(char **) NULL)
int mybsearch (char *ar, table_entry_pos_t n, int size, char *key, table_entry_pos_t *idx)
template<typename TA , typename TB >
int add (ngram &ng, TA prob, TB bow)
void checkbounds (int level)
int get (ngram &ng)
int get (ngram &ng, int n, int lev)
int succscan (ngram &h, ngram &ng, LMT_ACTION action, int lev)
virtual const char * maxsuffptr (ngram ong, unsigned int *size=NULL)
virtual const char * cmaxsuffptr (ngram ong, unsigned int *size=NULL)
void putmem (char *ptr, int value, int offs, int size)
void getmem (char *ptr, int *value, int offs, int size)
template<typename T >
void putmem (char *ptr, T value, int offs)
template<typename T >
void getmem (char *ptr, T *value, int offs)
int nodesize (LMT_TYPE ndt)
int word (node nd, int value=-1)
float prob (node nd, LMT_TYPE ndt)
template<typename T >
float prob (node nd, LMT_TYPE ndt, T value)
float bow (node nd, LMT_TYPE ndt)
template<typename T >
bow (node nd, LMT_TYPE ndt, T value)
table_entry_pos_t bound (node nd, LMT_TYPE ndt)
template<typename T >
bound (node nd, LMT_TYPE ndt, T value)
void stat (int lev=0)
void printTable (int level)
virtual dictionarygetDict () const
void setOrderQuery (bool v)
bool isOrderQuery () const

Public Attributes

dictionarydict

Protected Attributes

char * table [LMTMAXLEV+1]
LMT_TYPE tbltype [LMTMAXLEV+1]
table_entry_pos_t cursize [LMTMAXLEV+1]
table_entry_pos_t maxsize [LMTMAXLEV+1]
table_entry_pos_tstartpos [LMTMAXLEV+1]
int maxlev
char info [100]
int totget [LMTMAXLEV+1]
int totbsearch [LMTMAXLEV+1]
bool isQtable
bool isItable
bool isInverted
bool isPruned
int NumCenters [LMTMAXLEV+1]
float * Pcenters [LMTMAXLEV+1]
float * Bcenters [LMTMAXLEV+1]
double logOOVpenalty
int dictionary_upperbound
int backoff_state
ngramcachelmtcache [LMTMAXLEV+1]
ngramcacheprobcache
ngramcachestatecache
ngramcachestatesizecache
int max_cache_lev
int memmap
int diskid
off_t tableOffs [LMTMAXLEV+1]
off_t tableGaps [LMTMAXLEV+1]
bool orderQuery


Constructor & Destructor Documentation

lmtable::lmtable (  ) 

References configure(), cursize, dict, info, isInverted, isPruned, lmtcache, LMTMAXLEV, logOOVpenalty, max_cache_lev, maxsize, memmap, NULL, NumCenters, probcache, setOrderQuery(), statecache, statesizecache, tbltype, totbsearch, and totget.

Referenced by cpsublm().

Here is the call graph for this function:

Here is the caller graph for this function:

virtual lmtable::~lmtable (  )  [inline, virtual]

References Bcenters, cursize, dict, isQtable, lmtcache, LMTMAXLEV, maxlev, memmap, Munmap(), nodesize(), Pcenters, probcache, statecache, statesizecache, table, tableGaps, and tbltype.

Here is the call graph for this function:


Member Function Documentation

template<typename TA , typename TB >
int lmtable::add ( ngram ng,
TA  prob,
TB  bow 
) [inline]

References bound(), BOUND_EMPTY1, BOUND_EMPTY2, bow(), cursize, LMT_FIND, maxlev, nodesize(), prob(), search(), ngram::size, startpos, tbltype, word(), and ngram::wordp().

Here is the call graph for this function:

bool lmtable::are_lmtcaches_active (  )  [inline]

References lmtcache, and NULL.

template<typename T >
T lmtable::bound ( node  nd,
LMT_TYPE  ndt,
value 
) [inline]

References LMTCODESIZE, PROBSIZE, putmem(), QINTERNAL, and QPROBSIZE.

Here is the call graph for this function:

table_entry_pos_t lmtable::bound ( node  nd,
LMT_TYPE  ndt 
) [inline]

References getmem(), LMTCODESIZE, PROBSIZE, QINTERNAL, and QPROBSIZE.

Referenced by add(), checkbounds(), cpsublm(), get(), ngcnt(), succscan(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >
T lmtable::bow ( node  nd,
LMT_TYPE  ndt,
value 
) [inline]

References LMTCODESIZE, PROBSIZE, putmem(), QINTERNAL, and QPROBSIZE.

Here is the call graph for this function:

float lmtable::bow ( node  nd,
LMT_TYPE  ndt 
) [inline]

References getmem(), INTERNAL, LEAF, LMTCODESIZE, PROBSIZE, QINTERNAL, QLEAF, and QPROBSIZE.

Referenced by add(), get(), lprob(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::check_cache_levels (  ) 

References cursize, ngramcache::cursize(), ngramcache::isfull(), lmtcache, max_cache_lev, probcache, ngramcache::reset(), statecache, and statesizecache.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::checkbounds ( int  level  ) 

References bound(), BOUND_EMPTY1, BOUND_EMPTY2, createtempfile(), cursize, nodesize(), removefile(), startpos, and tbltype.

Here is the call graph for this function:

double lmtable::clprob ( ngram  ng  )  [virtual]

Reimplemented in lmmacro.

References ngramcache::add(), ngramcache::get(), lprob(), probcache, ngram::size, and ngram::wordp().

Here is the call graph for this function:

const char * lmtable::cmaxsuffptr ( ngram  ong,
unsigned int *  size = NULL 
) [virtual]

Reimplemented in lmmacro.

References ngramcache::add(), ngramcache::get(), maxsuffptr(), NULL, ngram::size, statecache, statesizecache, and ngram::wordp().

Here is the call graph for this function:

void lmtable::configure ( int  n,
bool  quantized 
)

References INTERNAL, LEAF, maxlev, QINTERNAL, QLEAF, and tbltype.

Referenced by cpsublm(), and lmtable().

Here is the caller graph for this function:

lmtable * lmtable::cpsublm ( dictionary subdict,
bool  keepunigr = true 
)

References Bcenters, bound(), configure(), cursize, dictionary::decode(), dict, dictionary::genoovcode(), lmtable(), maxlev, nodesize(), NULL, NumCenters, dictionary::oovcode(), Pcenters, dictionary::size(), table, tbltype, and word().

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::dumplm ( std::fstream &  out,
ngram  ng,
int  ilev,
int  elev,
table_entry_pos_t  ipos,
table_entry_pos_t  epos 
)

Referenced by savetxt().

Here is the caller graph for this function:

void lmtable::filter ( const char *  lmfile  )  [inline]

int lmtable::get ( ngram ng,
int  n,
int  lev 
)

int lmtable::get ( ngram ng  )  [inline]

virtual dictionary* lmtable::getDict (  )  const [inline, virtual]

double lmtable::getlogOOVpenalty (  )  const [inline]

References logOOVpenalty.

Referenced by main().

Here is the caller graph for this function:

template<typename T >
void lmtable::getmem ( char *  ptr,
T *  value,
int  offs 
) [inline]

References NULL.

void lmtable::getmem ( char *  ptr,
int *  value,
int  offs,
int  size 
) [inline]

References NULL.

Referenced by bound(), bow(), prob(), and word().

Here is the caller graph for this function:

void lmtable::init_lmtcaches ( int  uptolev  ) 

References lmtcache, max_cache_lev, and NULL.

Referenced by Moses::LanguageModelIRST::Load().

Here is the caller graph for this function:

void lmtable::init_probcache (  ) 

References maxlev, NULL, and probcache.

Referenced by Moses::LanguageModelIRST::Load(), load_lm(), and main().

Here is the caller graph for this function:

void lmtable::init_statecache (  ) 

References maxlev, NULL, statecache, and statesizecache.

Referenced by Moses::LanguageModelIRST::Load().

Here is the caller graph for this function:

bool lmtable::is_inverted (  )  [inline]

References isInverted.

bool lmtable::is_inverted ( const bool  flag  )  [inline]

References isInverted.

Referenced by main().

Here is the caller graph for this function:

bool lmtable::is_probcache_active (  )  [inline]

References NULL, and probcache.

bool lmtable::is_statecache_active (  )  [inline]

References NULL, and statecache.

bool lmtable::isOrderQuery (  )  const [inline]

References orderQuery.

bool lmtable::isQuantized (  )  const [inline]

References isQtable.

void lmtable::load ( std::istream &  inp,
const char *  filename = NULL,
const char *  outfilename = NULL,
int  mmap = 0,
OUTFILE_TYPE  outtype = NONE 
)

Referenced by Moses::LanguageModelIRST::Load(), load_lm(), and main().

Here is the caller graph for this function:

void lmtable::loadbin ( std::istream &  inp,
const char *  header,
const char *  filename = NULL,
int  mmap = 0 
)

void lmtable::loadbincodebook ( std::istream &  inp,
int  l 
)

void lmtable::loadbinheader ( std::istream &  inp,
const char *  header 
)

void lmtable::loadcenters ( std::istream &  inp,
int  Order 
)

void lmtable::loadtxt ( std::istream &  inp,
const char *  header 
)

void lmtable::loadtxt ( std::istream &  inp,
const char *  header,
const char *  outfilename,
int  mmap 
)

void lmtable::loadtxtmmap ( std::istream &  inp,
const char *  header,
const char *  outfilename 
)

double lmtable::lprob ( ngram  ng,
double *  bow = NULL,
int *  bol = NULL,
char **  maxsuffptr = NULL,
unsigned int *  statesize = NULL 
) [virtual]

References Bcenters, ngram::bow, bow(), dict, ngram::invert(), isInverted, ngram::lev, ngram::link, logOOVpenalty, MIN, NULL, dictionary::oovcode(), ngram::path, ngram::prob, ngram::size, tbltype, UNIGRAM_RESOLUTION, and ngram::wordp().

Referenced by clprob(), main(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

double lmtable::lprobx ( ngram  ong,
double *  lkp = 0,
double *  bop = 0,
int *  bol = 0 
)

References Bcenters, ngram::bow, dict, ngram::prob, ngram::shift(), ngram::size, and UNIGRAM_RESOLUTION.

Here is the call graph for this function:

int lmtable::maxlevel (  )  const [inline]

References maxlev.

Referenced by Moses::LanguageModelIRST::Load(), and main().

Here is the caller graph for this function:

const char * lmtable::maxsuffptr ( ngram  ong,
unsigned int *  size = NULL 
) [virtual]

Reimplemented in lmmacro.

References ngram::link, NULL, ngram::size, and ngram::succ.

Referenced by cmaxsuffptr(), and main().

Here is the caller graph for this function:

int lmtable::mybsearch ( char *  ar,
table_entry_pos_t  n,
int  size,
char *  key,
table_entry_pos_t idx 
)

References word().

Referenced by search().

Here is the call graph for this function:

Here is the caller graph for this function:

table_entry_pos_t lmtable::ngcnt ( table_entry_pos_t cnt,
ngram  ng,
int  l,
table_entry_pos_t  ipos,
table_entry_pos_t  epos 
)

References bound(), ngcnt(), nodesize(), NOPROB, prob(), ngram::pushc(), tbltype, word(), and ngram::wordp().

Here is the call graph for this function:

table_entry_pos_t lmtable::ngcnt ( table_entry_pos_t cnt  ) 

References cursize, and getDict().

Referenced by ngcnt(), and savetxt().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::nodesize ( LMT_TYPE  ndt  )  [inline]

References BOUNDSIZE, INTERNAL, LEAF, LMTCODESIZE, PROBSIZE, QINTERNAL, QLEAF, and QPROBSIZE.

Referenced by add(), checkbounds(), cpsublm(), get(), ngcnt(), printTable(), pscale(), reset_mmap(), savebin(), stat(), succscan(), wdprune(), and ~lmtable().

Here is the caller graph for this function:

void lmtable::printTable ( int  level  ) 

References cursize, nodesize(), prob(), tbltype, and word().

Here is the call graph for this function:

template<typename T >
float lmtable::prob ( node  nd,
LMT_TYPE  ndt,
value 
) [inline]

References LMTCODESIZE, and putmem().

Here is the call graph for this function:

float lmtable::prob ( node  nd,
LMT_TYPE  ndt 
) [inline]

References getmem(), INTERNAL, LEAF, LMTCODESIZE, QINTERNAL, and QLEAF.

Referenced by add(), get(), lmmacro::lprob(), ngcnt(), printTable(), pscale(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::pscale ( int  lev,
table_entry_pos_t  ipos,
table_entry_pos_t  epos,
double  s 
)

TOCHECK: Nicola 18 dicembre 2009)

References nodesize(), NOPROB, prob(), and tbltype.

Referenced by wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >
void lmtable::putmem ( char *  ptr,
value,
int  offs 
) [inline]

References NULL.

void lmtable::putmem ( char *  ptr,
int  value,
int  offs,
int  size 
) [inline]

References NULL.

Referenced by bound(), bow(), prob(), search(), and word().

Here is the caller graph for this function:

int lmtable::reload ( std::set< string >  words  ) 

References dict, dictionary::encode(), dictionary::incflag(), and NULL.

Here is the call graph for this function:

void lmtable::reset_caches (  ) 

References cursize, ngramcache::cursize(), lmtcache, MAX, max_cache_lev, ngramcache::maxsize(), probcache, ngramcache::reset(), statecache, and statesizecache.

Referenced by Moses::LanguageModelIRST::CleanUpAfterSentenceProcessing().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::reset_mmap (  ) 

References cursize, diskid, memmap, MMap(), Munmap(), nodesize(), tableGaps, tableOffs, and tbltype.

Referenced by Moses::LanguageModelIRST::~LanguageModelIRST().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::savebin ( const char *  filename  ) 

References Bcenters, cursize, getDict(), isInverted, isPruned, maxlev, nodesize(), dictionary::save(), and tbltype.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::savetxt ( const char *  filename  ) 

References Bcenters, cursize, dumplm(), getDict(), isPruned, MAX_NGRAM, maxlev, ngcnt(), and ngram::size.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void * lmtable::search ( int  lev,
table_entry_pos_t  offs,
table_entry_pos_t  n,
int  sz,
int *  w,
LMT_ACTION  action,
char **  found = (char **)NULL 
)

References error(), LMT_FIND, LMTCODESIZE, mybsearch(), NULL, putmem(), and totbsearch.

Referenced by add(), and get().

Here is the call graph for this function:

Here is the caller graph for this function:

double lmtable::setlogOOVpenalty ( int  dub  )  [inline]

References dict, logOOVpenalty, and dictionary::size().

Referenced by Moses::LanguageModelIRST::Load(), load_lm(), and main().

Here is the call graph for this function:

Here is the caller graph for this function:

double lmtable::setlogOOVpenalty2 ( double  oovp  )  [inline]

References logOOVpenalty.

Referenced by main().

Here is the caller graph for this function:

void lmtable::setOrderQuery ( bool  v  )  [inline]

References orderQuery.

Referenced by lmtable().

Here is the caller graph for this function:

void lmtable::stat ( int  lev = 0  ) 

References cursize, getDict(), nodesize(), dictionary::stat(), tbltype, totbsearch, and totget.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::succscan ( ngram h,
ngram ng,
LMT_ACTION  action,
int  lev 
)

table_entry_pos_t lmtable::wdprune ( float *  thr,
int  aflag,
ngram  ng,
int  ilev,
int  elev,
table_entry_pos_t  ipos,
table_entry_pos_t  epos,
double  lk = 0,
double  bo = 0,
double *  ts = 0,
double *  tbs = 0 
)

TOCHECK: Nicola 18 dicembre 2009)

References bound(), bow(), cursize, k, lprob(), nodesize(), NOPROB, prob(), pscale(), ngram::pushc(), ngram::size, tbltype, wdprune(), word(), and ngram::wordp().

Here is the call graph for this function:

table_entry_pos_t lmtable::wdprune ( float *  thr,
int  aflag = 0 
)

References cursize, getDict(), isPruned, and ngram::size.

Referenced by main(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::word ( node  nd,
int  value = -1 
) [inline]

References getmem(), LMTCODESIZE, and putmem().

Referenced by add(), cpsublm(), mybsearch(), ngcnt(), printTable(), succscan(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:


Member Data Documentation

int lmtable::backoff_state [protected]

float* lmtable::Bcenters[LMTMAXLEV+1] [protected]

table_entry_pos_t lmtable::cursize[LMTMAXLEV+1] [protected]

int lmtable::diskid [protected]

Referenced by reset_mmap().

char lmtable::info[100] [protected]

Referenced by lmtable().

bool lmtable::isInverted [protected]

Referenced by is_inverted(), lmtable(), lprob(), and savebin().

bool lmtable::isItable [protected]

bool lmtable::isPruned [protected]

Referenced by lmtable(), savebin(), savetxt(), and wdprune().

bool lmtable::isQtable [protected]

Referenced by isQuantized(), and ~lmtable().

ngramcache* lmtable::lmtcache[LMTMAXLEV+1] [protected]

double lmtable::logOOVpenalty [protected]

int lmtable::max_cache_lev [protected]

int lmtable::maxlev [protected]

table_entry_pos_t lmtable::maxsize[LMTMAXLEV+1] [protected]

Referenced by lmtable().

int lmtable::memmap [protected]

Referenced by lmtable(), reset_mmap(), and ~lmtable().

int lmtable::NumCenters[LMTMAXLEV+1] [protected]

Referenced by cpsublm(), and lmtable().

bool lmtable::orderQuery [protected]

Referenced by isOrderQuery(), and setOrderQuery().

float* lmtable::Pcenters[LMTMAXLEV+1] [protected]

Referenced by cpsublm(), and ~lmtable().

table_entry_pos_t* lmtable::startpos[LMTMAXLEV+1] [protected]

Referenced by add(), and checkbounds().

char* lmtable::table[LMTMAXLEV+1] [protected]

Referenced by cpsublm(), and ~lmtable().

off_t lmtable::tableGaps[LMTMAXLEV+1] [protected]

Referenced by reset_mmap(), and ~lmtable().

off_t lmtable::tableOffs[LMTMAXLEV+1] [protected]

Referenced by reset_mmap().

LMT_TYPE lmtable::tbltype[LMTMAXLEV+1] [protected]

int lmtable::totbsearch[LMTMAXLEV+1] [protected]

Referenced by lmtable(), search(), and stat().

int lmtable::totget[LMTMAXLEV+1] [protected]

Referenced by get(), lmtable(), and stat().


The documentation for this class was generated from the following files:

Generated on Fri Jul 30 20:05:15 2010 for Moses by  doxygen 1.5.9