#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
#include <cmath>
#include <cassert>
#include <limits>
#include <map>
#include <cstdlib>
#include <cstring>
#include "util/exception.hh"
#include "util/string_stream.hh"
#include "TypeDef.h"
Go to the source code of this file.
Classes | |
class | Moses::UnorderedComparer< T > |
Namespaces | |
namespace | Moses |
$Id$ | |
Defines | |
#define | TRACE_ERR(str) do {} while (false) |
#define | VERBOSE(level, str) { IFVERBOSE(level) { TRACE_ERR(str); } } |
#define | IFVERBOSE(level) if (Moses::StaticData::Instance().GetVerboseLevel() >= level) |
#define | XVERBOSE(level, str) VERBOSE(level, "[" << HERE << "] " << str) |
#define | HERE __FILE__ << ":" << __LINE__ |
#define | FEATUREVERBOSE(level, str) FEATUREVERBOSE2(level, "[" << GetScoreProducerDescription() << "] " << str) |
#define | FEATUREVERBOSE2(level, str) { IFFEATUREVERBOSE(level) { TRACE_ERR(str); } } |
#define | IFFEATUREVERBOSE(level) if ((m_verbosity == std::numeric_limits<std::size_t>::max() && StaticData::Instance().GetVerboseLevel() >= level) || (m_verbosity != std::numeric_limits<std::size_t>::max() && m_verbosity >= level)) |
#define | NTH_ELEMENT3(begin, middle, end) std::nth_element(begin, middle, end) |
#define | NTH_ELEMENT4(begin, middle, end, orderer) std::nth_element(begin, middle, end, orderer) |
#define | TO_STRING() std::string ToString() const; |
#define | TO_STRING_BODY(CLASS) |
definition of ToString() function to go in .cpp file. Can be used for any class that can be piped to a stream | |
Functions | |
const std::string | Moses::ToLower (const std::string &str) |
std::string | Moses::Trim (const std::string &str, const std::string dropChars=" \t\n\r") |
delete white spaces at beginning and end of string | |
template<typename T > | |
std::string | Moses::SPrint (const T &input) |
get string representation of any object/variable, as long as it can pipe to a stream | |
template<typename T > | |
T | Moses::Scan (const std::string &input) |
convert string to variable of type T. Used to reading floats, int etc from files | |
template<> | |
std::string | Moses::Scan< std::string > (const std::string &input) |
just return input | |
template<> | |
WordAlignmentSort | Moses::Scan< WordAlignmentSort > (const std::string &input) |
template<> | |
InputTypeEnum | Moses::Scan< InputTypeEnum > (const std::string &input) |
template<> | |
SearchAlgorithm | Moses::Scan< SearchAlgorithm > (const std::string &input) |
template<> | |
S2TParsingAlgorithm | Moses::Scan< S2TParsingAlgorithm > (const std::string &input) |
template<> | |
SourceLabelOverlap | Moses::Scan< SourceLabelOverlap > (const std::string &input) |
template<> | |
XmlInputType | Moses::Scan< XmlInputType > (const std::string &input) |
template<> | |
bool | Moses::Scan< bool > (const std::string &input) |
Specialisation to understand yes/no y/n true/false 0/1. | |
template<typename T > | |
std::vector< T > | Moses::Scan (const std::vector< std::string > &input) |
convert vectors of string to vectors of type T variables | |
template<typename T > | |
void | Moses::Scan (std::vector< T > &output, const std::vector< std::string > &input) |
speeded up version of above | |
std::string | Moses::Replace (const std::string &str, const std::string &todelStr, const std::string &toaddStr) |
std::vector< std::string > | Moses::Tokenize (const std::string &input, const std::string &delimiters=" \t") |
tokenise input string to vector of type T | |
void | Moses::Tokenize (std::vector< std::string > &output, const std::string &str, const std::string &delimiters=" \t") |
template<typename T > | |
void | Moses::Tokenize (std::vector< T > &output, const std::string &input, const std::string &delimiters=" \t") |
std::vector< std::string > | Moses::TokenizeMultiCharSeparator (const std::string &str, const std::string &separator) |
void | Moses::TokenizeMultiCharSeparator (std::vector< std::string > &output, const std::string &str, const std::string &separator) |
std::vector< std::string > | Moses::TokenizeFirstOnly (const std::string &str, const std::string &delimiters=" \t") |
template<typename T > | |
std::string | Moses::Join (const std::string &delimiter, const std::vector< T > &items) |
template<typename It > | |
std::string | Moses::Join (const std::string &delim, It begin, It end) |
float | Moses::TransformScore (float prob) |
transform prob to natural log score | |
float | Moses::UntransformScore (float score) |
transform natural log score to prob. Not currently used | |
float | Moses::TransformLMScore (float irstScore) |
irst number are in log 10, transform to natural log | |
float | Moses::UntransformLMScore (float logNScore) |
float | Moses::FloorScore (float logScore) |
make sure score doesn't fall below LOWEST_SCORE | |
float | Moses::CalcTranslationScore (const std::vector< float > &probVector, const std::vector< float > &weightT) |
template<class COLL > | |
void | Moses::RemoveAllInColl (COLL &coll) |
delete and remove every element of a collection object such as set, list etc | |
template<class COLL > | |
void | Moses::RemoveAllInMap (COLL &coll) |
delete and remove every element of map | |
string | Moses::GetTempFolder () |
x-platform reference to temp folder | |
std::string | Moses::GetMD5Hash (const std::string &filePath) |
MD5 hash of a file. | |
template<typename T > | |
void | Moses::ShrinkToFit (T &v) |
save memory by getting rid of spare, unused elements in a collection | |
bool | Moses::FileExists (const std::string &filePath) |
std::vector< std::map < std::string, std::string > > | Moses::ProcessAndStripDLT (std::string &line) |
std::map< std::string, std::string > | Moses::ProcessAndStripSGML (std::string &line) |
std::string | Moses::PassthroughSGML (std::string &line, const std::string tagName, const std::string &lbrackStr, const std::string &rbrackStr) |
std::string | Moses::GetFirstString (const std::string &str, int &first_pos, const std::string &delimiters=" \t") |
template<class T > | |
T | Moses::log_sum (T log_a, T log_b) |
bool | Moses::Equals (float a, float b) |
void | Moses::FixPrecision (std::ostream &stream, size_t size=3) |
void | Moses::PrintFeatureWeight (const FeatureFunction *ff) |
void | Moses::ShowWeights () |
#define FEATUREVERBOSE | ( | level, | |||
str | ) | FEATUREVERBOSE2(level, "[" << GetScoreProducerDescription() << "] " << str) |
Definition at line 78 of file Util.h.
Referenced by Moses::UnalignedWordCountFeature::EvaluateInIsolation(), Moses::PhraseOrientationFeature::EvaluateInIsolation(), Moses::TargetPreferencesFeature::EvaluateWhenApplied(), Moses::TargetConstituentAdjacencyFeature::EvaluateWhenApplied(), Moses::PhraseOrientationFeature::EvaluateWhenApplied(), Moses::SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(), Moses::Model1Feature::EvaluateWithSourceContext(), Moses::PhraseOrientationFeature::GetHeuristicScoreIndex(), Moses::PhraseOrientationFeature::LeftBoundaryL2RScoreRecursive(), Moses::WordTranslationFeature::Load(), Moses::TargetWordInsertionFeature::Load(), Moses::SourceWordDeletionFeature::Load(), Moses::SoftSourceSyntacticConstraintsFeature::LoadCoreSourceLabelSet(), Moses::SoftSourceSyntacticConstraintsFeature::LoadLabelSet(), Moses::SoftSourceSyntacticConstraintsFeature::LoadSourceLabelSet(), Moses::SoftSourceSyntacticConstraintsFeature::LoadTargetSourceLeftHandSideJointCountFile(), Moses::PhraseOrientationFeature::LoadWordList(), Moses::PhraseOrientationFeature::RightBoundaryR2LScoreRecursive(), Moses::PhraseOrientationFeature::SparseNonTerminalL2RScore(), Moses::PhraseOrientationFeature::SparseNonTerminalR2LScore(), Moses::PhraseOrientationFeature::SparseWordL2RScore(), and Moses::PhraseOrientationFeature::SparseWordR2LScore().
Definition at line 79 of file Util.h.
Referenced by Moses::PhraseOrientationFeature::EvaluateInIsolation(), Moses::TargetPreferencesFeature::EvaluateWhenApplied(), Moses::PhraseOrientationFeature::EvaluateWhenApplied(), Moses::PhraseOrientationFeature::GetHeuristicScoreIndex(), Moses::SoftSourceSyntacticConstraintsFeature::LoadCoreSourceLabelSet(), Moses::SoftSourceSyntacticConstraintsFeature::LoadSourceLabelSet(), and Moses::SoftSourceSyntacticConstraintsFeature::LoadTargetSourceLeftHandSideJointCountFile().
#define HERE __FILE__ << ":" << __LINE__ |
Definition at line 77 of file Util.h.
Referenced by sapt::imBitext< TKN >::add(), batch_run(), sapt::TSA_tree_iterator< Token >::ca(), Moses::filter_arguments(), sapt::PhraseScorer< Token >::fname(), Moses::http_client::http_client(), Moses::Mmsapt::Load(), Moses::parse_timespec(), Moses::parseLine(), sapt::PScorePbwd< Token >::PScorePbwd(), sapt::PScorePfwd< Token >::PScorePfwd(), sapt::PScoreUnaligned< Token >::PScoreUnaligned(), Moses::PrefixTreeF< T, D >::read(), and sapt::toString().
#define IFFEATUREVERBOSE | ( | level | ) | if ((m_verbosity == std::numeric_limits<std::size_t>::max() && StaticData::Instance().GetVerboseLevel() >= level) || (m_verbosity != std::numeric_limits<std::size_t>::max() && m_verbosity >= level)) |
Definition at line 80 of file Util.h.
Referenced by Moses::UnalignedWordCountFeature::EvaluateInIsolation(), Moses::PhraseOrientationFeature::EvaluateInIsolation(), Moses::TargetPreferencesFeature::EvaluateWhenApplied(), Moses::PhraseOrientationFeature::EvaluateWhenApplied(), Moses::SoftSourceSyntacticConstraintsFeature::EvaluateWithSourceContext(), and Moses::PhraseOrientationFeature::GetHeuristicScoreIndex().
#define IFVERBOSE | ( | level | ) | if (Moses::StaticData::Instance().GetVerboseLevel() >= level) |
Definition at line 75 of file Util.h.
Referenced by Moses::ChartManager::AddXmlChartOptions(), batch_run(), Moses::Manager::CalcDecoderStatistics(), Moses::TranslationOptionCollection::CalcEstimatedScore(), Moses::calcNgramExpectations(), Moses::DistortionScoreProducer::CalculateDistortionScore(), Moses::PhraseDictionaryDynamicCacheBased::ClearSource(), Moses::SearchNormal::Decode(), Moses::SearchCubePruning::Decode(), Moses::Manager::Decode(), Moses::ChartManager::Decode(), decoder_main(), Moses::BitmapContainer::Enqueue(), Moses::LanguageModelImplementation::EvaluateWhenApplied(), Moses::PhraseDictionaryDynamicCacheBased::Execute(), Moses::SearchNormal::ExpandHypothesis(), Moses::getLatticeMBRNBest(), Moses::WordLattice::InitializeFromPCNDataType(), Moses::PhraseDictionaryDynamicCacheBased::Insert(), Moses::RuleTableLoaderStandard::Load(), Moses::Syntax::T2S::RuleTrieLoader::Load(), Moses::Syntax::S2T::RuleTrieLoader::Load(), Moses::Syntax::F2S::HyperTreeLoader::Load(), Moses::PhraseDictionaryDynamicCacheBased::Load_Single_File(), main(), Moses::Manager::OutputBest(), Moses::ChartManager::OutputNBest(), Moses::BitmapContainer::ProcessBestHypothesis(), Moses::DecodeStepTranslation::ProcessInitialTranslation(), Moses::DecodeStepTranslation::ProcessInitialTransLEGACY(), Moses::SearchNormal::ProcessOneStack(), Moses::pruneLatticeFB(), Moses::HypothesisStackNormal::PruneToSize(), Moses::HypothesisStackCubePruning::PruneToSize(), Moses::ChartHypothesisCollection::PruneToSize(), Moses::TranslationTask::Run(), and Moses::SyntacticLanguageModelState< MY, MX, YS, B >::SyntacticLanguageModelState().
Definition at line 88 of file Util.h.
Referenced by main(), and Moses::PDTAimp::PruneTargetCandidates().
Definition at line 89 of file Util.h.
Referenced by Moses::CompletedRuleCollection::Add(), Moses::ChartTranslationOptionList::Add(), Moses::ChartTranslationOptionList::ApplyThreshold(), Moses::Hypothesis::CleanupArcList(), Moses::ChartHypothesis::CleanupArcList(), Moses::PhraseDictionaryCompact::GetTargetPhraseCollectionNonCacheLEGACY(), search::NBestList::NBestList(), Moses::TargetPhraseCollection::NthElement(), Moses::PartialTranslOptColl::Prune(), and Moses::TranslationOptionList::SelectNBest().
#define TO_STRING | ( | ) | std::string ToString() const; |
declaration of ToString() function to go in header for each class. This function, as well as the operator<< fn for each class, is for debugging purposes only. The output format is likely to change from time-to-time as classes are updated so shouldn't be relied upon for any decoding algorithm
#define TO_STRING_BODY | ( | CLASS | ) |
Value:
std::string CLASS::ToString() const \ { \ std::stringstream out; \ out << *this; \ return out.str(); \ } \
#define TRACE_ERR | ( | str | ) | do {} while (false) |
Outputting debugging/verbose information to stderr. Use TRACE_ENABLE flag to redirect tracing output into oblivion so that you can output your own ad-hoc debugging info. However, if you use stderr directly, please delete calls to it once you finished debugging so that it won't clutter up. Also use TRACE_ENABLE to turn off output of any debugging info when compiling for a gui front-end so that running gui won't generate output on command line
Definition at line 59 of file Util.h.
Referenced by batch_run(), Moses::PDTAimp::CacheSource(), Moses::Manager::CalcDecoderStatistics(), Moses::TranslationOptionCollection::CalcEstimatedScore(), Moses::PrefixTreeF< T, D >::changeData(), MosesTuning::Timer::check(), Moses::LanguageModelIRST::CleanUpAfterSentenceProcessing(), Moses::WordLattice::ComputeDistortionDistance(), Moses::PhraseDictionaryTree::Create(), Moses::PrefixTreeF< T, D >::create(), Moses::LexicalReorderingTableTree::Create(), Moses::Manager::Decode(), decoder_main(), Moses::FeatureFactory::DefaultSetup(), Moses::PhraseDistanceFeature::EvaluateWithSourceContext(), Moses::ConfusionNet::GetStringRep(), Moses::PhraseDictionaryFuzzyMatch::InitializeForInput(), Moses::WordLattice::InitializeFromPCNDataType(), Moses::RuleTableLoaderStandard::Load(), Moses::Syntax::T2S::RuleTrieLoader::Load(), Moses::Syntax::S2T::RuleTrieLoader::Load(), Moses::Parameter::LoadParam(), main(), Moses::SearchNormal::OutputHypoStack(), Moses::SearchCubePruning::OutputHypoStack(), Moses::SearchNormal::OutputHypoStackSize(), Moses::SearchCubePruning::OutputHypoStackSize(), Moses::BaseManager::OutputSearchGraphAsHypergraph(), Moses::Manager::OutputSearchGraphSLF(), Moses::ParseXmlTagAttribute(), Moses::PassthroughSGML(), Moses::Hypothesis::PrintHypothesis(), Moses::PhraseDictionaryTree::PrintTargetCandidates(), Moses::ProcessAndStripDLT(), Moses::ProcessAndStripSGML(), Moses::ProcessAndStripXMLTags(), Moses::BitmapContainer::ProcessBestHypothesis(), Moses::DecodeStepTranslation::ProcessInitialTranslation(), Moses::DecodeStepTranslation::ProcessInitialTransLEGACY(), Moses::HypothesisStackNormal::PruneToSize(), Moses::HypothesisStackCubePruning::PruneToSize(), Moses::ChartHypothesisCollection::PruneToSize(), Moses::PrefixTreeMap::Read(), Moses::ConfusionNet::ReadFormat0(), MosesTuning::Timer::restart(), Moses::AllOptions::sanity_check(), Moses::Parameter::SetBooleanSwitch(), MosesTuning::Timer::start(), Moses::TokenizeXml(), Moses::LanguageModelIRST::~LanguageModelIRST(), and Moses::PDTAimp::~PDTAimp().
verbose macros
Definition at line 69 of file Util.h.
Referenced by Moses::HypothesisStackNormal::Add(), Moses::HypothesisStackCubePruning::Add(), Moses::ChartHypothesisCollection::Add(), Moses::ChartHypothesisCollection::AddHypothesis(), Moses::HypothesisStackNormal::AddPrune(), Moses::HypothesisStackCubePruning::AddPrune(), lmtable::appendbin_level_nommap(), Moses::BackwardsEdge::BackwardsEdge(), batch_run(), Moses::calcNgramExpectations(), Moses::Timer::check(), Moses::ReorderingConstraint::Check(), Timer::check(), Moses::LexicalReorderingTableCompact::CheckAndLoad(), lmtable::checkbounds(), lmclass::checkMap(), Moses::StaticData::CheckWeights(), Moses::PhraseDictionaryDynamicCacheBased::ClearEntries(), Moses::DynamicCacheBasedLanguageModel::ClearEntries(), Moses::PhraseDictionaryDynamicCacheBased::ClearSource(), lmtable::clprob(), lmmacro::clprob(), lmmacro::cmaxsuffptr(), lmtable::compact_single_level(), Moses::WordLattice::ComputeDistortionDistance(), lmtable::concatenate_single_level(), lmtable::configure(), Moses::PDTAimp::Create(), Moses::TranslationOptionCollectionLattice::CreateTranslationOptions(), Moses::TranslationOptionCollection::CreateTranslationOptions(), Moses::PhraseDictionaryDynamicCacheBased::Decay(), Moses::SearchCubePruning::Decode(), Moses::Manager::Decode(), Moses::ChartManager::Decode(), Moses::DecodeFeature::DecodeFeature(), Moses::DecodeStep::DecodeStep(), Moses::FeatureFactory::DefaultSetup(), Moses::doConsensusDecoding(), Moses::DynamicCacheBasedLanguageModel::DynamicCacheBasedLanguageModel(), Moses::OpSequenceModel::EmptyHypothesisState(), Moses::DesegModel::EmptyHypothesisState(), Moses::VW::EvaluateTranslationOptionListWithSourceContext(), Moses::VW::EvaluateWhenApplied(), Moses::LexicalReordering::EvaluateWhenApplied(), Moses::PhraseDictionaryDynamicCacheBased::Execute(), Moses::DynamicCacheBasedLanguageModel::Execute(), Moses::PhraseDictionaryDynamicCacheBased::Execute_Single_Command(), lmtable::expand_level_nommap(), Moses::ReorderingConstraint::FinalizeWalls(), lmContainer::getLanguageModelType(), Moses::getLatticeMBRNBest(), Moses::InMemoryPerSentenceOnDemandLM::GetPerThreadLM(), Moses::InputOptions::init(), Moses::StaticData::initialize_features(), Moses::PhraseDictionaryMemoryPerSentenceOnDemand::InitializeForInput(), Moses::StaticData::InitializeForInput(), Moses::ReloadingLanguageModel< Model >::InitializeForInput(), Moses::PhraseDictionaryDynamicCacheBased::Insert(), Moses::DynamicCacheBasedLanguageModel::Insert(), Moses::IOWrapper::IOWrapper(), Moses::LanguageModelIRST::LanguageModelIRST(), Moses::LexicalReordering::LexicalReordering(), Moses::PhraseDictionaryDynamicCacheBased::Load(), Moses::LanguageModelIRST::Load(), Moses::DynamicCacheBasedLanguageModel::Load(), lmtable::load(), lmmacro::load(), lmInterpolation::load(), lmclass::load(), Moses::PhraseDictionaryDynamicCacheBased::Load_Multiple_Files(), Moses::PhraseDictionaryDynamicCacheBased::Load_Single_File(), Moses::StaticData::LoadAlternateWeightSettings(), Moses::StaticData::LoadDecodeGraphsNew(), Moses::StaticData::LoadFeatureFunctions(), lmclass::loadMapElement(), Moses::LanguageModelKen< Model >::LoadModel(), lmtable::lprob(), lmmacro::lprob(), lmclass::lprob(), lmmacro::map(), lmclass::mapping(), lmmacro::maxsuffptr(), lmmacro::Micro2MacroMapping(), Moses::Model1Feature::Model1Feature(), Moses::Search::out_of_time(), Moses::Syntax::Manager::OutputBest(), Moses::Manager::OutputBest(), Moses::ChartManager::OutputNBest(), Moses::Manager::OutputSearchGraph(), Moses::Manager::OutputSearchGraphAsHypergraph(), Moses::StaticData::OverrideFeatures(), Moses::PhraseOrientationFeature::PhraseOrientationFeature(), Moses::PhrasePairFeature::PhrasePairFeature(), Moses::PhraseDictionaryDynamicCacheBased::Print(), Moses::ProcessAndStripXMLTags(), Moses::DecodeStepTranslation::ProcessInitialTranslation(), Moses::DecodeStepTranslation::ProcessInitialTransLEGACY(), Moses::SearchNormal::ProcessOneStack(), Moses::TranslationOptionCollection::Prune(), Moses::pruneLatticeFB(), Moses::HypothesisStackNormal::PruneToSize(), Moses::HypothesisStackCubePruning::PruneToSize(), Moses::ChartHypothesisCollection::PruneToSize(), Moses::PhraseDictionaryTree::Read(), Moses::PDTimp::Read(), Moses::ConfusionNet::ReadF(), Moses::ConfusionNet::ReadFormat0(), Moses::ConfusionNet::ReadFormat1(), Moses::PhraseDictionary::ReduceCache(), Moses::ScoreComponentCollection::RegisterScoreProducer(), Moses::ReloadingLanguageModel< Model >::ReloadingLanguageModel(), lmtable::resize_level_nommap(), Moses::RulePairUnlexicalizedSource::RulePairUnlexicalizedSource(), Moses::TranslationTask::Run(), MosesServer::Server::run(), mdiadaptlm::saveARPA_per_level(), mdiadaptlm::saveARPA_per_word(), lmtable::savebin(), lmtable::savebin_level_mmap(), lmtable::savebin_level_nommap(), mdiadaptlm::saveBIN_per_level(), mdiadaptlm::saveBIN_per_word(), Moses::SearchNormal::SearchNormal(), Moses::StaticData::SetExecPath(), Moses::PhraseDictionaryDynamicCacheBased::SetMaxAge(), Moses::DynamicCacheBasedLanguageModel::SetMaxAge(), Moses::PhraseDictionaryDynamicCacheBased::SetParameter(), Moses::DynamicCacheBasedLanguageModel::SetParameter(), Moses::PhraseDictionaryDynamicCacheBased::SetPreComputedScores(), Moses::DynamicCacheBasedLanguageModel::SetQueryType(), Moses::PhraseDictionaryDynamicCacheBased::SetScoreType(), Moses::DynamicCacheBasedLanguageModel::SetScoreType(), Moses::ReorderingConstraint::SetWall(), Moses::ReorderingConstraint::SetZone(), Moses::SoftSourceSyntacticConstraintsFeature::SoftSourceSyntacticConstraintsFeature(), Moses::SourceWordDeletionFeature::SourceWordDeletionFeature(), Moses::Timer::start(), Timer::start(), storage::stat(), strstack::stat(), mempool::stat(), Moses::Timer::stop(), Moses::SyntacticLanguageModelState< MY, MX, YS, B >::SyntacticLanguageModelState(), Moses::TargetConstituentAdjacencyFeature::TargetConstituentAdjacencyFeature(), Moses::TargetPreferencesFeature::TargetPreferencesFeature(), Moses::TargetWordInsertionFeature::TargetWordInsertionFeature(), Moses::UnalignedWordCountFeature::UnalignedWordCountFeature(), Moses::PhraseDictionaryDynamicCacheBased::Update(), Moses::VW::VW(), Moses::VWState::VWState(), Moses::WordTranslationFeature::WordTranslationFeature(), Moses::ChartManager::~ChartManager(), Moses::SyntacticLanguageModelFiles< MH, MO >::~SyntacticLanguageModelFiles(), and Moses::SyntacticLanguageModelState< MY, MX, YS, B >::~SyntacticLanguageModelState().
Definition at line 76 of file Util.h.
Referenced by MosesServer::Updater::breakOutParams(), MosesServer::Updater::execute(), and MosesServer::Server::run().