00001 #include "mmsapt.h"
00002 #include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h"
00003 #include "moses/TranslationModel/UG/generic/program_options/ug_splice_arglist.h"
00004 #include <boost/foreach.hpp>
00005 #include <boost/format.hpp>
00006 #include <boost/tokenizer.hpp>
00007 #include <boost/shared_ptr.hpp>
00008 #include <algorithm>
00009 #include <iostream>
00010 #include <sstream>
00011
00012 using namespace Moses;
00013 using namespace sapt;
00014 using namespace std;
00015 using namespace boost;
00016
00017 typedef L2R_Token<SimpleWordId> Token;
00018 typedef mmBitext<Token> mmbitext;
00019 typedef imBitext<Token> imbitext;
00020 typedef Bitext<Token>::iter iter;
00021
00022 mmbitext bg;
00023 vector<string> src,trg,aln;
00024
00025 void
00026 show(ostream& out, iter& f)
00027 {
00028 iter b(bg.I2.get(),f.getToken(0),f.size());
00029 if (b.size() == f.size())
00030 out << setw(12) << int(round(b.approxOccurrenceCount()));
00031 else
00032 out << string(12,' ');
00033 out << " " << setw(5) << int(round(f.approxOccurrenceCount())) << " ";
00034 out << f.str(bg.V1.get()) << endl;
00035 }
00036
00037
00038 void
00039 dump(ostream& out, iter& f)
00040 {
00041 float cnt = f.size() ? f.approxOccurrenceCount() : 0;
00042 if (f.down())
00043 {
00044 cnt = f.approxOccurrenceCount();
00045 do { dump(out,f); }
00046 while (f.over());
00047 f.up();
00048 }
00049 if (f.size() && cnt < f.approxOccurrenceCount() && f.approxOccurrenceCount() > 1)
00050 show(out,f);
00051 }
00052
00053
00054 void
00055 read_data(string fname, vector<string>& dest)
00056 {
00057 ifstream in(fname.c_str());
00058 string line;
00059 while (getline(in,line)) dest.push_back(line);
00060 in.close();
00061 }
00062
00063 void
00064 show_snt(ostream& out, TokenIndex const& V, vector<Token> const& snt,
00065 vector<vector<int> > const& a)
00066 {
00067 for (size_t i = 0; i < snt.size(); ++i)
00068 {
00069 cout << format("%d:%s[") % i % V[snt[i].id()];
00070 for (size_t k = 0; k < a[i].size(); ++k)
00071 cout << (k?",":"") << a[i][k];
00072 cout << "] ";
00073 }
00074 cout << endl;
00075 }
00076
00077
00078 void show_pair(size_t const sid)
00079 {
00080 vector<Token> s,t;
00081 fill_token_seq(*bg.V1,src[sid],s);
00082 fill_token_seq(*bg.V2,trg[sid],t);
00083 vector<vector<int> > a1(s.size()),a2(t.size());
00084 istringstream buf(aln[sid]);
00085 cout << aln[sid] << endl;
00086 int i,k; char c;
00087 while (buf >> i >> c >> k)
00088 {
00089 a1[i].push_back(k);
00090 a2[k].push_back(i);
00091 cout << i << "-" << k << " ";
00092 }
00093 cout << endl;
00094 show_snt(cout,*bg.V1,s,a1);
00095 show_snt(cout,*bg.V2,t,a2);
00096 }
00097
00098 int main(int argc, char* argv[])
00099 {
00100 if (argc < 5)
00101 {
00102 cerr << "usage: " << argv[0]
00103 << " <bg base name> <L1> <L2> <fg base name>"
00104 << endl;
00105 exit(1);
00106 }
00107 bg.open(argv[1],argv[2],argv[3]);
00108 SPTR<imbitext> fg(new imbitext(bg.V1,bg.V2));
00109 string base = argv[4];
00110 if (*base.rbegin() != '.') base += '.';
00111 string srcfile = base + argv[2];
00112 string trgfile = base + argv[3];
00113 string alnfile = base + "symal";
00114 read_data(srcfile,src);
00115 read_data(trgfile,trg);
00116 read_data(alnfile,aln);
00117 fg = fg->add(src,trg,aln);
00118
00119 vector<float> bias(src.size(),1./(src.size()-1));
00120 for (size_t sid = 0; sid < src.size(); ++sid)
00121 {
00122 bias[sid] = 0;
00123
00124
00125 vector<Token> snt;
00126 fill_token_seq(*bg.V1,src[sid],snt);
00127 vector<vector<SPTR<vector<PhrasePair<Token> > > > > FG,BG;
00128 fg->lookup(snt,*fg->I1,FG,NULL,NULL,&bias,true);
00129 bg.lookup(snt,*bg.I1,BG,NULL,NULL,NULL,true);
00130 set<SPTR<vector<PhrasePair<Token> > > > seen;
00131 for (size_t i = 0; i < snt.size(); ++i)
00132 {
00133 Bitext<Token>::iter m0(fg->I1.get());
00134 Bitext<Token>::iter m1(bg.I1.get());
00135 for (size_t k = 0; k < FG[i].size(); ++k)
00136 {
00137 if (!m0.extend(snt[i+k].id())) break;
00138 if (k && m0.approxOccurrenceCount() < 2) break;
00139 if (m1.size() == k && (!m1.extend(snt[i+k].id()) ||
00140 m1.approxOccurrenceCount() < 25))
00141 {
00142 cout << toString((*fg->V1), m0.getToken(0), m0.size()) << " "
00143 << int(m0.approxOccurrenceCount());
00144 if (m1.size() == k + 1)
00145 cout << " "<< int(m1.approxOccurrenceCount());
00146 else if (m1.size())
00147 cout << " ["<< int(m1.approxOccurrenceCount()) << "]";
00148 else
00149 cout << " NEW!";
00150 cout << endl;
00151 }
00152 if (m0.approxOccurrenceCount() < 2) break;
00153 BOOST_FOREACH(PhrasePair<Token> const& pp, *FG[i][k])
00154 {
00155 if (pp.joint < 2) continue;
00156 SPTR<pstats> bgstats;
00157 jstats const* bgjstats = NULL;
00158 Bitext<Token>::iter m2(bg.I2.get(), pp.start2, pp.len2);
00159 if (m1.approxOccurrenceCount() > 5000 ||
00160 m2.approxOccurrenceCount() > 5000)
00161 continue;
00162 if (m1.size() == pp.len1 && m2.size() == pp.len2)
00163 {
00164 bgstats = bg.lookup(m1,NULL);
00165 if (bgstats)
00166 {
00167 pstats::trg_map_t::const_iterator mx;
00168 mx = bgstats->trg.find(m2.getPid());
00169 if (mx != bgstats->trg.end())
00170 bgjstats = &mx->second;
00171 }
00172 }
00173 cout << toString(*fg->V1, pp.start1, pp.len1) << " ::: "
00174 << toString(*fg->V2, pp.start2, pp.len2) << " "
00175 << format("[%u/%u/%u]") % pp.good1 % pp.joint % pp.good2;
00176 if (bgjstats)
00177 cout << " " << (format("[%u/%u/%u]")
00178 % bgstats->good % bgjstats->rcnt()
00179 % (bgjstats->cnt2() * bgstats->good
00180 / bgstats->raw_cnt));
00181 else if (m1.size() == pp.len1)
00182 cout << " " << int(m1.approxOccurrenceCount());
00183 cout << endl;
00184
00185 }
00186 }
00187 }
00188 bias[sid] = 1./(src.size()-1);
00189 }
00190 exit(0);
00191 }
00192
00193
00194