#include <cstdlib>
#include <vector>
#include <string>
#include "util/exception.hh"
#include "moses/Util.h"
#include "InputFileStream.h"
#include "OutputFileStream.h"
#include "PropertiesConsolidator.h"
Go to the source code of this file.
Functions | |
void | processFiles (const std::string &, const std::string &, const std::string &, const std::string &, const std::string &, const std::string &, const std::string &) |
void | loadCountOfCounts (const std::string &) |
void | breakdownCoreAndSparse (const std::string &combined, std::string &core, std::string &sparse) |
bool | getLine (Moses::InputFileStream &file, std::vector< std::string > &item) |
float | maybeLogProb (float a) |
bool | isNonTerminal (const std::string &word) |
int | main (int argc, char *argv[]) |
Variables | |
bool | countsProperty = false |
bool | goodTuringFlag = false |
bool | hierarchicalFlag = false |
bool | kneserNeyFlag = false |
bool | logProbFlag = false |
bool | lowCountFlag = false |
bool | onlyDirectFlag = false |
bool | partsOfSpeechFlag = false |
bool | phraseCountFlag = false |
bool | sourceLabelsFlag = false |
bool | targetSyntacticPreferencesFlag = false |
bool | sparseCountBinFeatureFlag = false |
std::vector< int > | countBin |
float | minScore0 = 0 |
float | minScore2 = 0 |
std::vector< float > | countOfCounts |
std::vector< float > | goodTuringDiscount |
float | kneserNey_D1 |
float | kneserNey_D2 |
float | kneserNey_D3 |
float | totalCount = -1 |
void breakdownCoreAndSparse | ( | const std::string & | combined, | |
std::string & | core, | |||
std::string & | sparse | |||
) |
Definition at line 458 of file consolidate-main.cpp.
References Moses::Tokenize().
Referenced by processFiles().
bool getLine | ( | Moses::InputFileStream & | file, | |
std::vector< std::string > & | item | |||
) |
Definition at line 477 of file consolidate-main.cpp.
References Moses::TokenizeMultiCharSeparator().
bool isNonTerminal | ( | const std::string & | word | ) | [inline] |
Definition at line 65 of file consolidate-main.cpp.
Referenced by Moses::FactorCollection::AddFactor(), calcCrossedNonTerm(), Moses::PhraseDictionaryOnDisk::ConvertFromMoses(), Moses::Phrase::CreateFromString(), NumNonTerminal(), outputPhrasePair(), processFiles(), and Moses::Word::Word().
void loadCountOfCounts | ( | const std::string & | fileNameCountOfCounts | ) |
Definition at line 196 of file consolidate-main.cpp.
References Moses::InputFileStream::Close(), countOfCounts, goodTuringDiscount, goodTuringFlag, kneserNey_D1, kneserNey_D2, kneserNey_D3, totalCount, and UTIL_THROW_IF2.
Referenced by processFiles().
int main | ( | int | argc, | |
char * | argv[] | |||
) |
Definition at line 71 of file consolidate-main.cpp.
References countBin, countsProperty, goodTuringFlag, hierarchicalFlag, kneserNeyFlag, logProbFlag, lowCountFlag, minScore0, minScore2, onlyDirectFlag, partsOfSpeechFlag, phraseCountFlag, processFiles(), sourceLabelsFlag, sparseCountBinFeatureFlag, targetSyntacticPreferencesFlag, UTIL_THROW2, and UTIL_THROW_IF2.
float maybeLogProb | ( | float | a | ) | [inline] |
Definition at line 59 of file consolidate-main.cpp.
References logProbFlag.
Referenced by main(), outputPhrasePair(), and processFiles().
void processFiles | ( | const std::string & | fileNameDirect, | |
const std::string & | fileNameIndirect, | |||
const std::string & | fileNameConsolidated, | |||
const std::string & | fileNameCountOfCounts, | |||
const std::string & | fileNameSourceLabelSet, | |||
const std::string & | fileNamePartsOfSpeechVocabulary, | |||
const std::string & | fileNameTargetSyntacticPreferencesLabelSet | |||
) |
Definition at line 236 of file consolidate-main.cpp.
References MosesTraining::PropertiesConsolidator::ActivatePartsOfSpeechProcessing(), MosesTraining::PropertiesConsolidator::ActivateSourceLabelsProcessing(), MosesTraining::PropertiesConsolidator::ActivateTargetSyntacticPreferencesProcessing(), breakdownCoreAndSparse(), Moses::InputFileStream::Close(), countBin, countsProperty, getLine(), MosesTraining::PropertiesConsolidator::GetPOSPropertyValueFromPropertiesString(), goodTuringDiscount, goodTuringFlag, isNonTerminal(), kneserNey_D1, kneserNey_D2, kneserNey_D3, kneserNeyFlag, loadCountOfCounts(), lowCountFlag, maybeLogProb(), minScore0, minScore2, onlyDirectFlag, partsOfSpeechFlag, phraseCountFlag, MosesTraining::PropertiesConsolidator::ProcessPropertiesString(), sourceLabelsFlag, sparseCountBinFeatureFlag, targetSyntacticPreferencesFlag, Moses::Tokenize(), totalCount, and UTIL_THROW_IF2.
Referenced by main().
std::vector< int > countBin |
std::vector< float > countOfCounts |
Definition at line 48 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), main(), outputPhrasePair(), and writeCountOfCounts().
bool countsProperty = false |
std::vector< float > goodTuringDiscount |
Definition at line 49 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), and processFiles().
bool goodTuringFlag = false |
Definition at line 32 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), main(), outputPhrasePair(), and processFiles().
bool hierarchicalFlag = false |
Definition at line 33 of file consolidate-main.cpp.
Referenced by main(), outputPhrasePair(), and processLine().
float kneserNey_D1 |
Definition at line 50 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), and processFiles().
float kneserNey_D2 |
Definition at line 50 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), and processFiles().
float kneserNey_D3 |
Definition at line 50 of file consolidate-main.cpp.
Referenced by loadCountOfCounts(), and processFiles().
bool kneserNeyFlag = false |
Definition at line 34 of file consolidate-main.cpp.
Referenced by main(), outputPhrasePair(), and processFiles().
bool logProbFlag = false |
Definition at line 35 of file consolidate-main.cpp.
Referenced by main(), maybeLogProb(), and processFiles().
bool lowCountFlag = false |
float minScore0 = 0 |
float minScore2 = 0 |
bool onlyDirectFlag = false |
bool partsOfSpeechFlag = false |
Definition at line 38 of file consolidate-main.cpp.
Referenced by main(), outputPhrasePair(), and processFiles().
bool phraseCountFlag = false |
bool sourceLabelsFlag = false |
bool sparseCountBinFeatureFlag = false |
bool targetSyntacticPreferencesFlag = false |
Definition at line 41 of file consolidate-main.cpp.
Referenced by main(), outputPhrasePair(), and processFiles().
float totalCount = -1 |
Definition at line 50 of file consolidate-main.cpp.
Referenced by sapt::Ttrack< TKN >::count_tokens(), Moses::TargetPreferencesFeature::EvaluateWhenApplied(), Moses::SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(), loadCountOfCounts(), processFiles(), processPhrasePairs(), MosesTraining::PropertiesConsolidator::ProcessSourceLabelsPropertyValue(), MosesTraining::PropertiesConsolidator::ProcessTargetSyntacticPreferencesPropertyValue(), and Moses::ConfusionNet::ReadFormat0().