lmtable Class Reference

#include <lmtable.h>

Inheritance diagram for lmtable:

Collaboration diagram for lmtable:

List of all members.

Public Member Functions

lmtable ()

virtual ~lmtable ()

table_entry_pos_t wdprune (float *thr, int aflag=0)

table_entry_pos_t wdprune (float *thr, int aflag, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos, double lk=0, double bo=0, double *ts=0, double *tbs=0)

double lprobx (ngram ong, double *lkp=0, double *bop=0, int *bol=0)

table_entry_pos_t ngcnt (table_entry_pos_t *cnt)

table_entry_pos_t ngcnt (table_entry_pos_t *cnt, ngram ng, int l, table_entry_pos_t ipos, table_entry_pos_t epos)

int pscale (int lev, table_entry_pos_t ipos, table_entry_pos_t epos, double s)

void init_probcache ()

void init_statecache ()

void init_lmtcaches (int uptolev)

void check_cache_levels ()

void reset_caches ()

void reset_mmap ()

bool is_probcache_active ()

bool is_statecache_active ()

bool are_lmtcaches_active ()

bool is_inverted (const bool flag)

bool is_inverted ()

void configure (int n, bool quantized)

double getlogOOVpenalty () const

double setlogOOVpenalty (int dub)

double setlogOOVpenalty2 (double oovp)

int maxlevel () const

bool isQuantized () const

void savetxt (const char *filename)

void savebin (const char *filename)

void dumplm (std::fstream &out, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos)

void load (std::istream &inp, const char *filename=NULL, const char *outfilename=NULL, int mmap=0, OUTFILE_TYPE outtype=NONE)

void loadtxt (std::istream &inp, const char *header, const char *outfilename, int mmap)

void loadtxt (std::istream &inp, const char *header)

void loadtxtmmap (std::istream &inp, const char *header, const char *outfilename)

void loadbin (std::istream &inp, const char *header, const char *filename=NULL, int mmap=0)

void loadbinheader (std::istream &inp, const char *header)

void loadbincodebook (std::istream &inp, int l)

void loadcenters (std::istream &inp, int Order)

lmtable * cpsublm (dictionary *subdict, bool keepunigr=true)

int reload (std::set< string > words)

void filter (const char *lmfile)

virtual double lprob (ngram ng, double *bow=NULL, int *bol=NULL, char **maxsuffptr=NULL, unsigned int *statesize=NULL)

virtual double clprob (ngram ng)

void * search (int lev, table_entry_pos_t offs, table_entry_pos_t n, int sz, int *w, LMT_ACTION action, char **found=(char **) NULL)

int mybsearch (char *ar, table_entry_pos_t n, int size, char *key, table_entry_pos_t *idx)

template<typename TA , typename TB >

int add (ngram &ng, TA prob, TB bow)

void checkbounds (int level)

int get (ngram &ng)

int get (ngram &ng, int n, int lev)

int succscan (ngram &h, ngram &ng, LMT_ACTION action, int lev)

virtual const char * maxsuffptr (ngram ong, unsigned int *size=NULL)

virtual const char * cmaxsuffptr (ngram ong, unsigned int *size=NULL)

void putmem (char *ptr, int value, int offs, int size)

void getmem (char *ptr, int *value, int offs, int size)

template<typename T >

void putmem (char *ptr, T value, int offs)

template<typename T >

void getmem (char *ptr, T *value, int offs)

int nodesize (LMT_TYPE ndt)

int word (node nd, int value=-1)

float prob (node nd, LMT_TYPE ndt)

template<typename T >

float prob (node nd, LMT_TYPE ndt, T value)

float bow (node nd, LMT_TYPE ndt)

template<typename T >

T bow (node nd, LMT_TYPE ndt, T value)

table_entry_pos_t bound (node nd, LMT_TYPE ndt)

template<typename T >

T bound (node nd, LMT_TYPE ndt, T value)

void stat (int lev=0)

void printTable (int level)

virtual dictionary * getDict () const

void setOrderQuery (bool v)

bool isOrderQuery () const

Public Attributes

dictionary * dict

Protected Attributes

char * table [LMTMAXLEV+1]

LMT_TYPE tbltype [LMTMAXLEV+1]

table_entry_pos_t cursize [LMTMAXLEV+1]

table_entry_pos_t maxsize [LMTMAXLEV+1]

table_entry_pos_t * startpos [LMTMAXLEV+1]

int maxlev

char info [100]

int totget [LMTMAXLEV+1]

int totbsearch [LMTMAXLEV+1]

bool isQtable

bool isItable

bool isInverted

bool isPruned

int NumCenters [LMTMAXLEV+1]

float * Pcenters [LMTMAXLEV+1]

float * Bcenters [LMTMAXLEV+1]

double logOOVpenalty

int dictionary_upperbound

int backoff_state

ngramcache * lmtcache [LMTMAXLEV+1]

ngramcache * probcache

ngramcache * statecache

ngramcache * statesizecache

int max_cache_lev

int memmap

int diskid

off_t tableOffs [LMTMAXLEV+1]

off_t tableGaps [LMTMAXLEV+1]

bool orderQuery

Detailed Description

Definition at line 89 of file lmtable.h.

Constructor & Destructor Documentation

lmtable::lmtable ( )

Definition at line 57 of file lmtable.cpp.

References configure(), cursize, dict, info, isInverted, isPruned, lmtcache, LMTMAXLEV, logOOVpenalty, max_cache_lev, maxsize, memmap, NULL, NumCenters, probcache, setOrderQuery(), statecache, statesizecache, tbltype, totbsearch, and totget.

Referenced by cpsublm().

Here is the call graph for this function:

Here is the caller graph for this function:

virtual lmtable::~lmtable ( ) [inline, virtual]

Definition at line 153 of file lmtable.h.

References Bcenters, cursize, dict, isQtable, lmtcache, LMTMAXLEV, maxlev, memmap, Munmap(), nodesize(), Pcenters, probcache, statecache, statesizecache, table, tableGaps, and tbltype.

Here is the call graph for this function:

Member Function Documentation

template<typename TA , typename TB >

int lmtable::add	(	ngram &	ng,
		TA	prob,
		TB	bow
	)			`[inline]`

Definition at line 760 of file lmtable.cpp.

References bound(), BOUND_EMPTY1, BOUND_EMPTY2, bow(), cursize, LMT_FIND, maxlev, nodesize(), prob(), search(), ngram::size, startpos, tbltype, word(), and ngram::wordp().

Here is the call graph for this function:

bool lmtable::are_lmtcaches_active ( ) [inline]

Definition at line 220 of file lmtable.h.

References lmtcache, and NULL.

template<typename T >

T lmtable::bound	(	node	nd,
		LMT_TYPE	ndt,
		T	value
	)			`[inline]`

Definition at line 427 of file lmtable.h.

References LMTCODESIZE, PROBSIZE, putmem(), QINTERNAL, and QPROBSIZE.

Here is the call graph for this function:

table_entry_pos_t lmtable::bound	(	node	nd,
		LMT_TYPE	ndt
	)			`[inline]`

Definition at line 416 of file lmtable.h.

References getmem(), LMTCODESIZE, PROBSIZE, QINTERNAL, and QPROBSIZE.

Referenced by add(), checkbounds(), cpsublm(), get(), ngcnt(), succscan(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

T lmtable::bow	(	node	nd,
		LMT_TYPE	ndt,
		T	value
	)			`[inline]`

Definition at line 406 of file lmtable.h.

References LMTCODESIZE, PROBSIZE, putmem(), QINTERNAL, and QPROBSIZE.

Here is the call graph for this function:

float lmtable::bow	(	node	nd,
		LMT_TYPE	ndt
	)			`[inline]`

Definition at line 380 of file lmtable.h.

References getmem(), INTERNAL, LEAF, LMTCODESIZE, PROBSIZE, QINTERNAL, QLEAF, and QPROBSIZE.

Referenced by add(), get(), lprob(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::check_cache_levels ( )

Definition at line 115 of file lmtable.cpp.

References cursize, ngramcache::cursize(), ngramcache::isfull(), lmtcache, max_cache_lev, probcache, ngramcache::reset(), statecache, and statesizecache.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::checkbounds ( int level )

Definition at line 697 of file lmtable.cpp.

References bound(), BOUND_EMPTY1, BOUND_EMPTY2, createtempfile(), cursize, nodesize(), removefile(), startpos, and tbltype.

Here is the call graph for this function:

double lmtable::clprob ( ngram ng ) [virtual]

Reimplemented in lmmacro.

Definition at line 1656 of file lmtable.cpp.

References ngramcache::add(), ngramcache::get(), lprob(), probcache, ngram::size, and ngram::wordp().

Here is the call graph for this function:

const char * lmtable::cmaxsuffptr	(	ngram	ong,
		unsigned int *	size = `NULL`
	)			`[virtual]`

Reimplemented in lmmacro.

Definition at line 1468 of file lmtable.cpp.

References ngramcache::add(), ngramcache::get(), maxsuffptr(), NULL, ngram::size, statecache, statesizecache, and ngram::wordp().

Here is the call graph for this function:

void lmtable::configure	(	int	n,
		bool	quantized
	)

Definition at line 135 of file lmtable.cpp.

References INTERNAL, LEAF, maxlev, QINTERNAL, QLEAF, and tbltype.

Referenced by cpsublm(), and lmtable().

Here is the caller graph for this function:

lmtable * lmtable::cpsublm	(	dictionary *	subdict,
		bool	keepunigr = `true`
	)

Definition at line 931 of file lmtable.cpp.

References Bcenters, bound(), configure(), cursize, dictionary::decode(), dict, dictionary::genoovcode(), lmtable(), maxlev, nodesize(), NULL, NumCenters, dictionary::oovcode(), Pcenters, dictionary::size(), table, tbltype, and word().

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::dumplm	(	std::fstream &	out,
		ngram	ng,
		int	ilev,
		int	elev,
		table_entry_pos_t	ipos,
		table_entry_pos_t	epos
	)

Referenced by savetxt().

Here is the caller graph for this function:

void lmtable::filter ( const char * lmfile ) [inline]

Definition at line 263 of file lmtable.h.

int lmtable::get	(	ngram &	ng,
		int	n,
		int	lev
	)

Definition at line 1251 of file lmtable.cpp.

References ngramcache::add(), bound(), BOUND_EMPTY1, bow(), ngram::bow, cursize, error(), ngram::freq, ngram::info, ngram::lev, ngram::link, LMT_FIND, lmtcache, nodesize(), NOPROB, NULL, ngram::path, ngram::prob, prob(), search(), ngram::size, ngram::succ, tbltype, totget, and ngram::wordp().

Here is the call graph for this function:

int lmtable::get ( ngram & ng ) [inline]

Definition at line 283 of file lmtable.h.

virtual dictionary* lmtable::getDict ( ) const [inline, virtual]

Definition at line 441 of file lmtable.h.

References dict.

Referenced by lmmacro::clprob(), lmmacro::cmaxsuffptr(), Moses::LanguageModelIRST::Load(), lmmacro::loadLexicalClasses(), lmmacro::lprob(), lmmacro::maxsuffptr(), lmmacro::Micro2MacroMapping(), ngcnt(), lmmacro::One2OneMapping(), savebin(), savetxt(), stat(), and wdprune().

Here is the caller graph for this function:

double lmtable::getlogOOVpenalty ( ) const [inline]

Definition at line 228 of file lmtable.h.

References logOOVpenalty.

Referenced by main().

Here is the caller graph for this function:

template<typename T >

void lmtable::getmem	(	char *	ptr,
		T *	value,
		int	offs
	)			`[inline]`

Definition at line 311 of file lmtable.h.

References NULL.

void lmtable::getmem	(	char *	ptr,
		int *	value,
		int	offs,
		int	size
	)			`[inline]`

Definition at line 297 of file lmtable.h.

References NULL.

Referenced by bound(), bow(), prob(), and word().

Here is the caller graph for this function:

void lmtable::init_lmtcaches ( int uptolev )

Definition at line 107 of file lmtable.cpp.

References lmtcache, max_cache_lev, and NULL.

void lmtable::init_probcache ( )

Definition at line 90 of file lmtable.cpp.

References maxlev, NULL, and probcache.

Referenced by load_lm(), and main().

Here is the caller graph for this function:

void lmtable::init_statecache ( )

Definition at line 99 of file lmtable.cpp.

References maxlev, NULL, statecache, and statesizecache.

bool lmtable::is_inverted ( ) [inline]

Definition at line 223 of file lmtable.h.

References isInverted.

bool lmtable::is_inverted ( const bool flag ) [inline]

Definition at line 222 of file lmtable.h.

References isInverted.

Referenced by main().

Here is the caller graph for this function:

bool lmtable::is_probcache_active ( ) [inline]

Definition at line 218 of file lmtable.h.

References NULL, and probcache.

bool lmtable::is_statecache_active ( ) [inline]

Definition at line 219 of file lmtable.h.

References NULL, and statecache.

bool lmtable::isOrderQuery ( ) const [inline]

Definition at line 449 of file lmtable.h.

References orderQuery.

bool lmtable::isQuantized ( ) const [inline]

Definition at line 240 of file lmtable.h.

References isQtable.

void lmtable::load	(	std::istream &	inp,
		const char *	filename = `NULL`,
		const char *	outfilename = `NULL`,
		int	mmap = `0`,
		OUTFILE_TYPE	outtype = `NONE`
	)

Referenced by Moses::LanguageModelIRST::Load(), load_lm(), and main().

Here is the caller graph for this function:

void lmtable::loadbin	(	std::istream &	inp,
		const char *	header,
		const char *	filename = `NULL`,
		int	mmap = `0`
	)

void lmtable::loadbincodebook	(	std::istream &	inp,
		int	l
	)

void lmtable::loadbinheader	(	std::istream &	inp,
		const char *	header
	)

void lmtable::loadcenters	(	std::istream &	inp,
		int	Order
	)

void lmtable::loadtxt	(	std::istream &	inp,
		const char *	header
	)

void lmtable::loadtxt	(	std::istream &	inp,
		const char *	header,
		const char *	outfilename,
		int	mmap
	)

void lmtable::loadtxtmmap	(	std::istream &	inp,
		const char *	header,
		const char *	outfilename
	)

double lmtable::lprob	(	ngram	ng,
		double *	bow = `NULL`,
		int *	bol = `NULL`,
		char **	maxsuffptr = `NULL`,
		unsigned int *	statesize = `NULL`
	)			`[virtual]`

Definition at line 1558 of file lmtable.cpp.

References Bcenters, ngram::bow, bow(), dict, ngram::invert(), isInverted, ngram::lev, ngram::link, logOOVpenalty, MIN, NULL, dictionary::oovcode(), ngram::path, ngram::prob, ngram::size, tbltype, UNIGRAM_RESOLUTION, and ngram::wordp().

Referenced by clprob(), main(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

double lmtable::lprobx	(	ngram	ong,
		double *	lkp = `0`,
		double *	bop = `0`,
		int *	bol = `0`
	)

Definition at line 1737 of file lmtable.cpp.

References Bcenters, ngram::bow, dict, ngram::prob, ngram::shift(), ngram::size, and UNIGRAM_RESOLUTION.

Here is the call graph for this function:

int lmtable::maxlevel ( ) const [inline]

Definition at line 239 of file lmtable.h.

References maxlev.

Referenced by Moses::LanguageModelIRST::Load(), and main().

Here is the caller graph for this function:

const char * lmtable::maxsuffptr	(	ngram	ong,
		unsigned int *	size = `NULL`
	)			`[virtual]`

Reimplemented in lmmacro.

Definition at line 1438 of file lmtable.cpp.

References ngram::link, NULL, ngram::size, and ngram::succ.

Referenced by cmaxsuffptr(), and main().

Here is the caller graph for this function:

int lmtable::mybsearch	(	char *	ar,
		table_entry_pos_t	n,
		int	size,
		char *	key,
		table_entry_pos_t *	idx
	)

Definition at line 880 of file lmtable.cpp.

References word().

Referenced by search().

Here is the call graph for this function:

Here is the caller graph for this function:

table_entry_pos_t lmtable::ngcnt	(	table_entry_pos_t *	cnt,
		ngram	ng,
		int	l,
		table_entry_pos_t	ipos,
		table_entry_pos_t	epos
	)

Definition at line 1921 of file lmtable.cpp.

References bound(), ngcnt(), nodesize(), NOPROB, prob(), ngram::pushc(), tbltype, word(), and ngram::wordp().

Here is the call graph for this function:

table_entry_pos_t lmtable::ngcnt ( table_entry_pos_t * cnt )

Definition at line 1912 of file lmtable.cpp.

References cursize, and getDict().

Referenced by ngcnt(), and savetxt().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::nodesize ( LMT_TYPE ndt ) [inline]

Definition at line 317 of file lmtable.h.

References BOUNDSIZE, INTERNAL, LEAF, LMTCODESIZE, PROBSIZE, QINTERNAL, QLEAF, and QPROBSIZE.

Referenced by add(), checkbounds(), cpsublm(), get(), ngcnt(), printTable(), pscale(), reset_mmap(), savebin(), stat(), succscan(), wdprune(), and ~lmtable().

Here is the caller graph for this function:

void lmtable::printTable ( int level )

Definition at line 674 of file lmtable.cpp.

References cursize, nodesize(), prob(), tbltype, and word().

Here is the call graph for this function:

template<typename T >

float lmtable::prob	(	node	nd,
		LMT_TYPE	ndt,
		T	value
	)			`[inline]`

Definition at line 371 of file lmtable.h.

References LMTCODESIZE, and putmem().

Here is the call graph for this function:

float lmtable::prob	(	node	nd,
		LMT_TYPE	ndt
	)			`[inline]`

Definition at line 345 of file lmtable.h.

References getmem(), INTERNAL, LEAF, LMTCODESIZE, QINTERNAL, and QLEAF.

Referenced by add(), get(), lmmacro::lprob(), ngcnt(), printTable(), pscale(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::pscale	(	int	lev,
		table_entry_pos_t	ipos,
		table_entry_pos_t	epos,
		double	s
	)

TOCHECK: Nicola 18 dicembre 2009)

Definition at line 1890 of file lmtable.cpp.

References nodesize(), NOPROB, prob(), and tbltype.

Referenced by wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

void lmtable::putmem	(	char *	ptr,
		T	value,
		int	offs
	)			`[inline]`

Definition at line 305 of file lmtable.h.

References NULL.

void lmtable::putmem	(	char *	ptr,
		int	value,
		int	offs,
		int	size
	)			`[inline]`

Definition at line 291 of file lmtable.h.

References NULL.

Referenced by bound(), bow(), prob(), search(), and word().

Here is the caller graph for this function:

int lmtable::reload ( std::set< string > words )

Definition at line 181 of file lmtable.cpp.

References dict, dictionary::encode(), dictionary::incflag(), and NULL.

Here is the call graph for this function:

void lmtable::reset_caches ( )

Definition at line 125 of file lmtable.cpp.

References cursize, ngramcache::cursize(), lmtcache, MAX, max_cache_lev, ngramcache::maxsize(), probcache, ngramcache::reset(), statecache, and statesizecache.

Referenced by Moses::LanguageModelIRST::CleanUpAfterSentenceProcessing().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::reset_mmap ( )

Definition at line 1717 of file lmtable.cpp.

References cursize, diskid, memmap, MMap(), Munmap(), nodesize(), tableGaps, tableOffs, and tbltype.

Referenced by Moses::LanguageModelIRST::~LanguageModelIRST().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::savebin ( const char * filename )

Definition at line 1100 of file lmtable.cpp.

References Bcenters, cursize, getDict(), isInverted, isPruned, maxlev, nodesize(), dictionary::save(), and tbltype.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void lmtable::savetxt ( const char * filename )

Definition at line 1051 of file lmtable.cpp.

References Bcenters, cursize, dumplm(), getDict(), isPruned, MAX_NGRAM, maxlev, ngcnt(), and ngram::size.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void * lmtable::search	(	int	lev,
		table_entry_pos_t	offs,
		table_entry_pos_t	n,
		int	sz,
		int *	w,
		LMT_ACTION	action,
		char **	found = `(char **)NULL`
	)

Definition at line 836 of file lmtable.cpp.

References error(), LMT_FIND, LMTCODESIZE, mybsearch(), NULL, putmem(), and totbsearch.

Referenced by add(), and get().

Here is the call graph for this function:

Here is the caller graph for this function:

double lmtable::setlogOOVpenalty ( int dub ) [inline]

Definition at line 230 of file lmtable.h.

References dict, logOOVpenalty, and dictionary::size().

Referenced by Moses::LanguageModelIRST::Load(), load_lm(), and main().

Here is the call graph for this function:

Here is the caller graph for this function:

double lmtable::setlogOOVpenalty2 ( double oovp ) [inline]

Definition at line 235 of file lmtable.h.

References logOOVpenalty.

Referenced by main().

Here is the caller graph for this function:

void lmtable::setOrderQuery ( bool v ) [inline]

Definition at line 445 of file lmtable.h.

References orderQuery.

Referenced by lmtable().

Here is the caller graph for this function:

void lmtable::stat ( int lev = 0 )

Definition at line 1689 of file lmtable.cpp.

References cursize, getDict(), nodesize(), dictionary::stat(), tbltype, totbsearch, and totget.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::succscan	(	ngram &	h,
		ngram &	ng,
		LMT_ACTION	action,
		int	lev
	)

Definition at line 1393 of file lmtable.cpp.

References bound(), ngram::lev, ngram::link, LMT_CONT, LMT_INIT, ngram::midx, nodesize(), ngram::size, ngram::succ, ngram::succlink, tbltype, ngram::trans(), word(), and ngram::wordp().

Here is the call graph for this function:

table_entry_pos_t lmtable::wdprune	(	float *	thr,
		int	aflag,
		ngram	ng,
		int	ilev,
		int	elev,
		table_entry_pos_t	ipos,
		table_entry_pos_t	epos,
		double	lk = `0`,
		double	bo = `0`,
		double *	ts = `0`,
		double *	tbs = `0`
	)

TOCHECK: Nicola 18 dicembre 2009)

Definition at line 1803 of file lmtable.cpp.

References bound(), bow(), cursize, k, lprob(), nodesize(), NOPROB, prob(), pscale(), ngram::pushc(), ngram::size, tbltype, wdprune(), word(), and ngram::wordp().

Here is the call graph for this function:

table_entry_pos_t lmtable::wdprune	(	float *	thr,
		int	aflag = `0`
	)

Definition at line 1788 of file lmtable.cpp.

References cursize, getDict(), isPruned, and ngram::size.

Referenced by main(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

int lmtable::word	(	node	nd,
		int	value = `-1`
	)			`[inline]`

Definition at line 333 of file lmtable.h.

References getmem(), LMTCODESIZE, and putmem().

Referenced by add(), cpsublm(), mybsearch(), ngcnt(), printTable(), succscan(), and wdprune().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

int lmtable::backoff_state [protected]

Definition at line 123 of file lmtable.h.

float* lmtable::Bcenters[LMTMAXLEV+1] [protected]

Definition at line 119 of file lmtable.h.

Referenced by cpsublm(), lprob(), lprobx(), savebin(), savetxt(), and ~lmtable().

table_entry_pos_t lmtable::cursize[LMTMAXLEV+1] [protected]

Definition at line 94 of file lmtable.h.

Referenced by add(), check_cache_levels(), checkbounds(), cpsublm(), get(), lmtable(), ngcnt(), printTable(), reset_caches(), reset_mmap(), savebin(), savetxt(), stat(), wdprune(), and ~lmtable().

dictionary* lmtable::dict

Reimplemented in lmmacro.

Definition at line 149 of file lmtable.h.

Referenced by cpsublm(), getDict(), lmtable(), lprob(), lprobx(), main(), reload(), setlogOOVpenalty(), and ~lmtable().

int lmtable::dictionary_upperbound [protected]

Definition at line 122 of file lmtable.h.

int lmtable::diskid [protected]

Definition at line 134 of file lmtable.h.

Referenced by reset_mmap().

char lmtable::info[100] [protected]

Definition at line 99 of file lmtable.h.

Referenced by lmtable().

bool lmtable::isInverted [protected]

Definition at line 112 of file lmtable.h.

Referenced by is_inverted(), lmtable(), lprob(), and savebin().

bool lmtable::isItable [protected]

Definition at line 109 of file lmtable.h.

bool lmtable::isPruned [protected]

Definition at line 115 of file lmtable.h.

Referenced by lmtable(), savebin(), savetxt(), and wdprune().

bool lmtable::isQtable [protected]

Definition at line 106 of file lmtable.h.

Referenced by isQuantized(), and ~lmtable().

ngramcache* lmtable::lmtcache[LMTMAXLEV+1] [protected]

Definition at line 126 of file lmtable.h.

Referenced by are_lmtcaches_active(), check_cache_levels(), get(), init_lmtcaches(), lmtable(), reset_caches(), and ~lmtable().

double lmtable::logOOVpenalty [protected]

Definition at line 121 of file lmtable.h.

Referenced by getlogOOVpenalty(), lmtable(), lprob(), setlogOOVpenalty(), and setlogOOVpenalty2().

int lmtable::max_cache_lev [protected]

Definition at line 130 of file lmtable.h.

Referenced by check_cache_levels(), init_lmtcaches(), lmtable(), and reset_caches().

int lmtable::maxlev [protected]

Definition at line 98 of file lmtable.h.

Referenced by add(), lmmacro::clprob(), configure(), cpsublm(), init_probcache(), init_statecache(), maxlevel(), savebin(), savetxt(), and ~lmtable().

table_entry_pos_t lmtable::maxsize[LMTMAXLEV+1] [protected]

Definition at line 95 of file lmtable.h.

Referenced by lmtable().

int lmtable::memmap [protected]

Definition at line 133 of file lmtable.h.

Referenced by lmtable(), reset_mmap(), and ~lmtable().

int lmtable::NumCenters[LMTMAXLEV+1] [protected]

Definition at line 117 of file lmtable.h.

Referenced by cpsublm(), and lmtable().

bool lmtable::orderQuery [protected]

Definition at line 140 of file lmtable.h.

Referenced by isOrderQuery(), and setOrderQuery().

float* lmtable::Pcenters[LMTMAXLEV+1] [protected]

Definition at line 118 of file lmtable.h.

Referenced by cpsublm(), and ~lmtable().

ngramcache* lmtable::probcache [protected]

Definition at line 127 of file lmtable.h.

Referenced by check_cache_levels(), clprob(), lmmacro::clprob(), init_probcache(), is_probcache_active(), lmtable(), reset_caches(), and ~lmtable().

table_entry_pos_t* lmtable::startpos[LMTMAXLEV+1] [protected]

Definition at line 96 of file lmtable.h.

Referenced by add(), and checkbounds().

ngramcache* lmtable::statecache [protected]

Definition at line 128 of file lmtable.h.

Referenced by check_cache_levels(), cmaxsuffptr(), init_statecache(), is_statecache_active(), lmtable(), reset_caches(), and ~lmtable().

ngramcache* lmtable::statesizecache [protected]

Definition at line 129 of file lmtable.h.

Referenced by check_cache_levels(), cmaxsuffptr(), init_statecache(), lmtable(), reset_caches(), and ~lmtable().

char* lmtable::table[LMTMAXLEV+1] [protected]

Definition at line 92 of file lmtable.h.

Referenced by cpsublm(), and ~lmtable().

off_t lmtable::tableGaps[LMTMAXLEV+1] [protected]

Definition at line 136 of file lmtable.h.

Referenced by reset_mmap(), and ~lmtable().

off_t lmtable::tableOffs[LMTMAXLEV+1] [protected]

Definition at line 135 of file lmtable.h.

Referenced by reset_mmap().

LMT_TYPE lmtable::tbltype[LMTMAXLEV+1] [protected]

Definition at line 93 of file lmtable.h.

Referenced by add(), checkbounds(), configure(), cpsublm(), get(), lmtable(), lprob(), ngcnt(), printTable(), pscale(), reset_mmap(), savebin(), stat(), succscan(), wdprune(), and ~lmtable().

int lmtable::totbsearch[LMTMAXLEV+1] [protected]

Definition at line 103 of file lmtable.h.

Referenced by lmtable(), search(), and stat().

int lmtable::totget[LMTMAXLEV+1] [protected]

Definition at line 102 of file lmtable.h.

Referenced by get(), lmtable(), and stat().

The documentation for this class was generated from the following files:

irstlm/src/lmtable.h
irstlm/src/lmtable.cpp


Public Member Functions
	lmtable ()
virtual	~lmtable ()
table_entry_pos_t	wdprune (float *thr, int aflag=0)
table_entry_pos_t	wdprune (float thr, int aflag, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos, double lk=0, double bo=0, double ts=0, double *tbs=0)
double	lprobx (ngram ong, double lkp=0, double bop=0, int *bol=0)
table_entry_pos_t	ngcnt (table_entry_pos_t *cnt)
table_entry_pos_t	ngcnt (table_entry_pos_t *cnt, ngram ng, int l, table_entry_pos_t ipos, table_entry_pos_t epos)
int	pscale (int lev, table_entry_pos_t ipos, table_entry_pos_t epos, double s)
void	init_probcache ()
void	init_statecache ()
void	init_lmtcaches (int uptolev)
void	check_cache_levels ()
void	reset_caches ()
void	reset_mmap ()
bool	is_probcache_active ()
bool	is_statecache_active ()
bool	are_lmtcaches_active ()
bool	is_inverted (const bool flag)
bool	is_inverted ()
void	configure (int n, bool quantized)
double	getlogOOVpenalty () const
double	setlogOOVpenalty (int dub)
double	setlogOOVpenalty2 (double oovp)
int	maxlevel () const
bool	isQuantized () const
void	savetxt (const char *filename)
void	savebin (const char *filename)
void	dumplm (std::fstream &out, ngram ng, int ilev, int elev, table_entry_pos_t ipos, table_entry_pos_t epos)
void	load (std::istream &inp, const char filename=NULL, const char outfilename=NULL, int mmap=0, OUTFILE_TYPE outtype=NONE)
void	loadtxt (std::istream &inp, const char header, const char outfilename, int mmap)
void	loadtxt (std::istream &inp, const char *header)
void	loadtxtmmap (std::istream &inp, const char header, const char outfilename)
void	loadbin (std::istream &inp, const char header, const char filename=NULL, int mmap=0)
void	loadbinheader (std::istream &inp, const char *header)
void	loadbincodebook (std::istream &inp, int l)
void	loadcenters (std::istream &inp, int Order)
lmtable *	cpsublm (dictionary *subdict, bool keepunigr=true)
int	reload (std::set< string > words)
void	filter (const char *lmfile)
virtual double	lprob (ngram ng, double bow=NULL, int bol=NULL, char *maxsuffptr=NULL, unsigned int statesize=NULL)
virtual double	clprob (ngram ng)
void *	search (int lev, table_entry_pos_t offs, table_entry_pos_t n, int sz, int w, LMT_ACTION action, char found=(char *) NULL)
int	mybsearch (char ar, table_entry_pos_t n, int size, char key, table_entry_pos_t *idx)
template<typename TA , typename TB >
int	add (ngram &ng, TA prob, TB bow)
void	checkbounds (int level)
int	get (ngram &ng)
int	get (ngram &ng, int n, int lev)
int	succscan (ngram &h, ngram &ng, LMT_ACTION action, int lev)
virtual const char *	maxsuffptr (ngram ong, unsigned int *size=NULL)
virtual const char *	cmaxsuffptr (ngram ong, unsigned int *size=NULL)
void	putmem (char *ptr, int value, int offs, int size)
void	getmem (char ptr, int value, int offs, int size)
template<typename T >
void	putmem (char *ptr, T value, int offs)
template<typename T >
void	getmem (char ptr, T value, int offs)
int	nodesize (LMT_TYPE ndt)
int	word (node nd, int value=-1)
float	prob (node nd, LMT_TYPE ndt)
template<typename T >
float	prob (node nd, LMT_TYPE ndt, T value)
float	bow (node nd, LMT_TYPE ndt)
template<typename T >
T	bow (node nd, LMT_TYPE ndt, T value)
table_entry_pos_t	bound (node nd, LMT_TYPE ndt)
template<typename T >
T	bound (node nd, LMT_TYPE ndt, T value)
void	stat (int lev=0)
void	printTable (int level)
virtual dictionary *	getDict () const
void	setOrderQuery (bool v)
bool	isOrderQuery () const
Public Attributes
dictionary *	dict
Protected Attributes
char *	table [LMTMAXLEV+1]
LMT_TYPE	tbltype [LMTMAXLEV+1]
table_entry_pos_t	cursize [LMTMAXLEV+1]
table_entry_pos_t	maxsize [LMTMAXLEV+1]
table_entry_pos_t *	startpos [LMTMAXLEV+1]
int	maxlev
char	info [100]
int	totget [LMTMAXLEV+1]
int	totbsearch [LMTMAXLEV+1]
bool	isQtable
bool	isItable
bool	isInverted
bool	isPruned
int	NumCenters [LMTMAXLEV+1]
float *	Pcenters [LMTMAXLEV+1]
float *	Bcenters [LMTMAXLEV+1]
double	logOOVpenalty
int	dictionary_upperbound
int	backoff_state
ngramcache *	lmtcache [LMTMAXLEV+1]
ngramcache *	probcache
ngramcache *	statecache
ngramcache *	statesizecache
int	max_cache_lev
int	memmap
int	diskid
off_t	tableOffs [LMTMAXLEV+1]
off_t	tableGaps [LMTMAXLEV+1]
bool	orderQuery