irstlm/src/interplm.h

Go to the documentation of this file.

/******************************************************************************
IrstLM: IRST Language Model Toolkit
Copyright (C) 2006 Marcello Federico, ITC-irst Trento, Italy

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA

******************************************************************************/
// Basic Interpolated LM class


#define SHIFT_BETA   1
#define SHIFT_ONE    2
#define SHIFT_ZERO   3
#define SHIFT_ONE_BETA   4
#define LINEAR_WB    5
#define LINEAR_GT    6
#define MIXTURE      7
#define MOD_SHIFT_BETA   8


class interplm:public ngramtable{
        
        int lms;
        
        double epsilon; //Bayes smoothing
        
        int unismooth; //0 Bayes, 1 Witten Bell
        
        int prune_singletons;
        
        int prune_top_singletons;
        
public: 
        
        int backoff; //0 interpolation, 1 Back-off  
        
        interplm(char* ngtfile,int depth=0,TABLETYPE tt=FULL);
        
        int prunesingletons(int flag=-1){
                return (flag==-1?prune_singletons:prune_singletons=flag);
        }
        
        int prunetopsingletons(int flag=-1){
                return (flag==-1?prune_top_singletons:prune_top_singletons=flag);
        }
        
        void gencorrcounts();
        
        void gensuccstat();
        
        virtual int dub(){return dict->dub();}
        
        virtual int dub(int value){ return (dict->dub(value)); }
        
        int setusmooth(int v=0){return unismooth=v;}
        
        double setepsilon(double v=1.0){return epsilon=v;}
        
        ngramtable *unitbl;
        
        void trainunigr();
        
        double unigr(ngram ng);
        
        double zerofreq(int lev);
        
        inline int lmsize(){return lms;}
        
        inline int obswrd(){return dict->size();}
        
        virtual int train(){return 0;}
        
        virtual void adapt(char* ngtfile, double w){}
        
        virtual double prob(ngram ng,int size){return 0.0;}
        
        virtual double boprob(ngram ng,int size){return 0.0;}
        
        void test_ngt(ngramtable& ngt,int sz=0,int backoff=0,int checkpr=0);
        
        void test_txt(char *filename,int sz=0,int backoff=0,int checkpr=0,char* outpr=NULL);
        
        void test(char* filename,int sz,int backoff=0,int checkpr=0,char* outpr=NULL);
        
        virtual int discount(ngram ng,int size,double& fstar,double& lambda,int cv=0){return 0;}
        
        virtual int savebin(char *filename,int lmsize=2){return 0;}
        
        virtual int netsize(){return 0;}
        
        void lmstat(int level){stat(level);}
        
        virtual ~interplm(){}
        
        
};

---