00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035 #ifdef WIN32
00036
00037
00038 #endif
00039
00040 #include <exception>
00041 #include <fstream>
00042 #include "Main.h"
00043 #include "TranslationAnalysis.h"
00044 #include "mbr.h"
00045 #include "IOWrapper.h"
00046
00047 #include "moses/FactorCollection.h"
00048 #include "moses/Manager.h"
00049 #include "moses/Phrase.h"
00050 #include "moses/Util.h"
00051 #include "moses/Timer.h"
00052 #include "moses/Sentence.h"
00053 #include "moses/ConfusionNet.h"
00054 #include "moses/WordLattice.h"
00055 #include "moses/TreeInput.h"
00056 #include "moses/ThreadPool.h"
00057 #include "moses/ChartManager.h"
00058 #include "moses/ChartHypothesis.h"
00059 #include "moses/ChartTrellisPath.h"
00060 #include "moses/ChartTrellisPathList.h"
00061 #include "moses/Incremental.h"
00062
00063 #include "util/usage.hh"
00064
00065 using namespace std;
00066 using namespace Moses;
00067 using namespace MosesChartCmd;
00068
00070 void fix(std::ostream& stream, size_t size)
00071 {
00072 stream.setf(std::ios::fixed);
00073 stream.precision(size);
00074 }
00075
00079 class TranslationTask : public Task
00080 {
00081 public:
00082 TranslationTask(InputType *source, IOWrapper &ioWrapper)
00083 : m_source(source)
00084 , m_ioWrapper(ioWrapper) {
00085 }
00086
00087 ~TranslationTask() {
00088 delete m_source;
00089 }
00090
00091 void Run() {
00092 const StaticData &staticData = StaticData::Instance();
00093 const size_t translationId = m_source->GetTranslationId();
00094
00095 VERBOSE(2,"\nTRANSLATING(" << translationId << "): " << *m_source);
00096
00097 if (staticData.GetSearchAlgorithm() == ChartIncremental) {
00098 Incremental::Manager manager(*m_source);
00099 const std::vector<search::Applied> &nbest = manager.ProcessSentence();
00100 if (!nbest.empty()) {
00101 m_ioWrapper.OutputBestHypo(nbest[0], translationId);
00102 } else {
00103 m_ioWrapper.OutputBestNone(translationId);
00104 }
00105 if (staticData.GetNBestSize() > 0)
00106 m_ioWrapper.OutputNBestList(nbest, translationId);
00107 return;
00108 }
00109
00110 ChartManager manager(*m_source);
00111 manager.ProcessSentence();
00112
00113 CHECK(!staticData.UseMBR());
00114
00115
00116 const ChartHypothesis *bestHypo = manager.GetBestHypothesis();
00117 m_ioWrapper.OutputBestHypo(bestHypo, translationId);
00118 IFVERBOSE(2) {
00119 PrintUserTime("Best Hypothesis Generation Time:");
00120 }
00121
00122 if (!staticData.GetAlignmentOutputFile().empty()) {
00123 m_ioWrapper.OutputAlignment(translationId, bestHypo);
00124 }
00125
00126 if (staticData.IsDetailedTranslationReportingEnabled()) {
00127 const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
00128 m_ioWrapper.OutputDetailedTranslationReport(bestHypo, sentence, translationId);
00129 }
00130
00131
00132 size_t nBestSize = staticData.GetNBestSize();
00133 if (nBestSize > 0) {
00134 VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO " << staticData.GetNBestFilePath() << endl);
00135 ChartTrellisPathList nBestList;
00136 manager.CalcNBest(nBestSize, nBestList,staticData.GetDistinctNBest());
00137 m_ioWrapper.OutputNBestList(nBestList, translationId);
00138 IFVERBOSE(2) {
00139 PrintUserTime("N-Best Hypotheses Generation Time:");
00140 }
00141 }
00142
00143 if (staticData.GetOutputSearchGraph()) {
00144 std::ostringstream out;
00145 manager.GetSearchGraph(translationId, out);
00146 OutputCollector *oc = m_ioWrapper.GetSearchGraphOutputCollector();
00147 CHECK(oc);
00148 oc->Write(translationId, out.str());
00149 }
00150
00151 IFVERBOSE(2) {
00152 PrintUserTime("Sentence Decoding Time:");
00153 }
00154 manager.CalcDecoderStatistics();
00155 }
00156
00157 private:
00158
00159 TranslationTask(const TranslationTask &);
00160 TranslationTask &operator=(const TranslationTask &);
00161
00162 InputType *m_source;
00163 IOWrapper &m_ioWrapper;
00164 };
00165
00166 bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source)
00167 {
00168 delete source;
00169 switch(inputType) {
00170 case SentenceInput:
00171 source = ioWrapper.GetInput(new Sentence);
00172 break;
00173 case ConfusionNetworkInput:
00174 source = ioWrapper.GetInput(new ConfusionNet);
00175 break;
00176 case WordLatticeInput:
00177 source = ioWrapper.GetInput(new WordLattice);
00178 break;
00179 case TreeInputType:
00180 source = ioWrapper.GetInput(new TreeInput);
00181 break;
00182 default:
00183 TRACE_ERR("Unknown input type: " << inputType << "\n");
00184 }
00185 return (source ? true : false);
00186 }
00187 static void PrintFeatureWeight(const FeatureFunction* ff)
00188 {
00189 cout << ff->GetScoreProducerDescription() << "=";
00190 size_t numScoreComps = ff->GetNumScoreComponents();
00191 vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
00192 for (size_t i = 0; i < numScoreComps; ++i) {
00193 cout << " " << values[i];
00194 }
00195 cout << endl;
00196
00197 }
00198
00199 static void ShowWeights()
00200 {
00201 fix(cout,6);
00202 const StaticData& staticData = StaticData::Instance();
00203 const vector<const StatelessFeatureFunction*>& slf = StatelessFeatureFunction::GetStatelessFeatureFunctions();
00204 const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
00205
00206 for (size_t i = 0; i < sff.size(); ++i) {
00207 const StatefulFeatureFunction *ff = sff[i];
00208 if (ff->IsTuneable()) {
00209 PrintFeatureWeight(ff);
00210 }
00211 }
00212 for (size_t i = 0; i < slf.size(); ++i) {
00213 const StatelessFeatureFunction *ff = slf[i];
00214 if (ff->IsTuneable()) {
00215 PrintFeatureWeight(ff);
00216 }
00217 }
00218 }
00219
00220
00221 int main(int argc, char* argv[])
00222 {
00223 try {
00224 IFVERBOSE(1) {
00225 TRACE_ERR("command: ");
00226 for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" ");
00227 TRACE_ERR(endl);
00228 }
00229
00230 IOWrapper::FixPrecision(cout);
00231 IOWrapper::FixPrecision(cerr);
00232
00233
00234 Parameter parameter;
00235 if (!parameter.LoadParam(argc, argv)) {
00236 return EXIT_FAILURE;
00237 }
00238
00239 const StaticData &staticData = StaticData::Instance();
00240 if (!StaticData::LoadDataStatic(¶meter, argv[0]))
00241 return EXIT_FAILURE;
00242
00243 if (parameter.isParamSpecified("show-weights")) {
00244 ShowWeights();
00245 exit(0);
00246 }
00247
00248 CHECK(staticData.IsChart());
00249
00250
00251 IOWrapper *ioWrapper = GetIOWrapper(staticData);
00252
00253
00254 const ScoreComponentCollection& weights = staticData.GetAllWeights();
00255 IFVERBOSE(2) {
00256 TRACE_ERR("The global weight vector looks like this: ");
00257 TRACE_ERR(weights);
00258 TRACE_ERR("\n");
00259 }
00260
00261 if (ioWrapper == NULL)
00262 return EXIT_FAILURE;
00263
00264 #ifdef WITH_THREADS
00265 ThreadPool pool(staticData.ThreadCount());
00266 #endif
00267
00268
00269 InputType *source=0;
00270 while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
00271 IFVERBOSE(1)
00272 ResetUserTime();
00273 TranslationTask *task = new TranslationTask(source, *ioWrapper);
00274 source = NULL;
00275 #ifdef WITH_THREADS
00276 pool.Submit(task);
00277 #else
00278 task->Run();
00279 delete task;
00280 #endif
00281 }
00282
00283 #ifdef WITH_THREADS
00284 pool.Stop(true);
00285 #endif
00286
00287 delete ioWrapper;
00288
00289 IFVERBOSE(1)
00290 PrintUserTime("End.");
00291
00292 } catch (const std::exception &e) {
00293 std::cerr << "Exception: " << e.what() << std::endl;
00294 return EXIT_FAILURE;
00295 }
00296
00297 IFVERBOSE(1) util::PrintUsage(std::cerr);
00298
00299 #ifndef EXIT_RETURN
00300
00301 exit(EXIT_SUCCESS);
00302 #else
00303 return EXIT_SUCCESS;
00304 #endif
00305 }
00306
00307 IOWrapper *GetIOWrapper(const StaticData &staticData)
00308 {
00309 IOWrapper *ioWrapper;
00310 const std::vector<FactorType> &inputFactorOrder = staticData.GetInputFactorOrder()
00311 ,&outputFactorOrder = staticData.GetOutputFactorOrder();
00312 FactorMask inputFactorUsed(inputFactorOrder);
00313
00314
00315 if (staticData.GetParam("input-file").size() == 1) {
00316 VERBOSE(2,"IO from File" << endl);
00317 string filePath = staticData.GetParam("input-file")[0];
00318
00319 ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed
00320 , staticData.GetNBestSize()
00321 , staticData.GetNBestFilePath()
00322 , filePath);
00323 } else {
00324 VERBOSE(1,"IO from STDOUT/STDIN" << endl);
00325 ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed
00326 , staticData.GetNBestSize()
00327 , staticData.GetNBestFilePath());
00328 }
00329 ioWrapper->ResetTranslationId();
00330
00331 IFVERBOSE(1)
00332 PrintUserTime("Created input-output object");
00333
00334 return ioWrapper;
00335 }