00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022 #ifdef WIN32
00023 #include <windows.h>
00024 #include <string.h>
00025 #include <io.h>
00026 #else
00027 #include <cstring>
00028 #include <cstdlib>
00029 #include <iostream>
00030 #include <sys/types.h>
00031 #include <sys/mman.h>
00032 #endif
00033
00034 #include "timer.h"
00035 #include "util.h"
00036
00037 using namespace std;
00038
00039 string gettempfolder()
00040 {
00041 #ifdef _WIN32
00042 char *tmpPath = getenv("TMP");
00043 string str(tmpPath);
00044 if (str.substr(str.size() - 1, 1) != "\\")
00045 str += "\\";
00046 return str;
00047 #else
00048 char *tmpPath = getenv("TMP");
00049 if (!tmpPath || !*tmpPath)
00050 return "/tmp/";
00051 string str(tmpPath);
00052 if (str.substr(str.size() - 1, 1) != "/")
00053 str += "/";
00054 return str;
00055 #endif
00056 }
00057
00058 string createtempName()
00059 {
00060 string tmpfolder = gettempfolder();
00061 #ifdef _WIN32
00062 char buffer[BUFSIZ];
00063
00064
00065 ::GetTempFileNameA(tmpfolder.c_str(), "", 0, buffer);
00066 #else
00067 char buffer[tmpfolder.size() + 16];
00068 strcpy(buffer, tmpfolder.c_str());
00069 strcat(buffer, "dskbuff--XXXXXX");
00070 int fd=mkstemp(buffer);
00071 close(fd);
00072 #endif
00073 return (string) buffer;
00074 }
00075
00076 void createtempfile(mfstream &fileStream, string &filePath, std::ios_base::openmode flags)
00077 {
00078 filePath = createtempName();
00079 fileStream.open(filePath.c_str(), flags);
00080 if (fileStream == 0)
00081 {
00082 perror("error creating file");
00083 exit(4);
00084 }
00085 }
00086
00087 void removefile(const std::string &filePath)
00088 {
00089 #ifdef _WIN32
00090 ::DeleteFileA(filePath.c_str());
00091 #else
00092 if (remove(filePath.c_str()) != 0)
00093 {
00094 perror("Error deleting file" );
00095 exit(2);
00096 }
00097 #endif
00098 }
00099
00100 inputfilestream::inputfilestream(const std::string &filePath)
00101 : std::istream(0),
00102 m_streambuf(0)
00103 {
00104
00105 std::filebuf* fb = new std::filebuf();
00106 _good=(fb->open(filePath.c_str(), std::ios::in)!=NULL);
00107
00108 if (filePath.size() > 3 &&
00109 filePath.substr(filePath.size() - 3, 3) == ".gz") {
00110 fb->close();
00111 delete fb;
00112 m_streambuf = new gzfilebuf(filePath.c_str());
00113 } else {
00114 m_streambuf = fb;
00115 }
00116 this->init(m_streambuf);
00117 }
00118
00119 inputfilestream::~inputfilestream()
00120 {
00121 delete m_streambuf;
00122 m_streambuf = 0;
00123 }
00124
00125 void inputfilestream::close()
00126 {
00127 }
00128
00129
00130
00131
00132
00133
00134
00135
00136
00137
00138
00139
00140
00141
00142
00143
00144 void *MMap(int fd, int access, off_t offset, size_t len, off_t *gap)
00145 {
00146 void *p;
00147 int pgsz,g=0;
00148
00149 #ifdef _WIN32
00150
00151
00152
00153
00154
00155
00156
00157
00158
00159
00160
00161
00162
00163
00164
00165
00166
00167
00168
00169
00170
00171
00172
00173
00174 #else
00175 if(offset) {
00176 pgsz = sysconf(_SC_PAGESIZE);
00177 g = *gap = offset%pgsz;
00178 } else if(gap) {
00179 *gap=0;
00180 }
00181 p = mmap((void*)0, len+g, access,
00182 MAP_SHARED|MAP_FILE,
00183 fd, offset-g);
00184 if((long)p==-1L)
00185 {
00186 perror("mmap failed");
00187 p=0;
00188 }
00189 #endif
00190 return p;
00191 }
00192
00193
00194 int Munmap(void *p,size_t len,int sync)
00195 {
00196 int r=0;
00197
00198 #ifdef _WIN32
00199
00200
00201
00202
00203
00204 #else
00205 cerr << "len = " << len << endl;
00206 cerr << "sync = " << sync << endl;
00207 cerr << "running msync..." << endl;
00208 if(sync) msync(p, len, MS_SYNC);
00209 cerr << "done. Running munmap..." << endl;
00210 if((r=munmap((void*)p, len)))
00211 {
00212 perror("munmap() failed");
00213 }
00214 cerr << "done" << endl;
00215
00216 #endif
00217 return r;
00218 }
00219
00220
00221
00222 Timer g_timer;
00223
00224
00225 void ResetUserTime()
00226 {
00227 g_timer.start();
00228 };
00229
00230 void PrintUserTime(const std::string &message)
00231 {
00232 g_timer.check(message.c_str());
00233 }
00234
00235 double GetUserTime()
00236 {
00237 return g_timer.get_elapsed_time();
00238 }
00239
00240
00241
00242
00243 int parseWords(char *sentence, const char **words, int max)
00244 {
00245 char *word;
00246 int i = 0;
00247
00248 const char *const wordSeparators = " \t\r\n";
00249
00250 for (word = strtok(sentence, wordSeparators);
00251 i < max && word != 0;
00252 i++, word = strtok(0, wordSeparators)) {
00253 words[i] = word;
00254 }
00255
00256 if (i < max) {
00257 words[i] = 0;
00258 }
00259
00260 return i;
00261 }
00262
00263
00264
00265
00266
00267
00268
00269
00270
00271
00272 int parseline(istream& inp, int Order,ngram& ng,float& prob,float& bow)
00273 {
00274
00275 const char* words[1+ LMTMAXLEV + 1 + 1];
00276 int howmany;
00277 char line[MAX_LINE];
00278
00279 inp.getline(line,MAX_LINE);
00280 if (strlen(line)==MAX_LINE-1) {
00281 cerr << "parseline: input line exceed MAXLINE ("
00282 << MAX_LINE << ") chars " << line << "\n";
00283 exit(1);
00284 }
00285
00286 howmany = parseWords(line, words, Order + 3);
00287
00288 if (!(howmany == (Order+ 1) || howmany == (Order + 2)))
00289 assert(howmany == (Order+ 1) || howmany == (Order + 2));
00290
00291
00292 ng.size=0;
00293 for (int i=1; i<=Order; i++)
00294 ng.pushw(strcmp(words[i],"<unk>")?words[i]:ng.dict->OOV());
00295
00296
00297 assert(sscanf(words[0],"%f",&prob));
00298 if (howmany==(Order+2))
00299 assert(sscanf(words[Order+1],"%f",&bow));
00300 else
00301 bow=0.0;
00302
00303 return 1;
00304 }