diff --git a/.tc.training.yml b/.tc.training.yml index b7aaa52f..ca780905 100644 --- a/.tc.training.yml +++ b/.tc.training.yml @@ -21,7 +21,7 @@ payload: {{ SYSTEM_ADD_USER }} && echo -e "#!/bin/bash\nset -xe\nexport PATH=/home/build-user/bin:$PATH && env && id && wget https://github.com/git-lfs/git-lfs/releases/download/v2.2.1/git-lfs-linux-amd64-2.2.1.tar.gz -O - | tar -C /tmp -zxf - && PREFIX=/home/build-user/ /tmp/git-lfs-2.2.1/install.sh && mkdir ~/DeepSpeech/ && git clone --quiet {{ GITHUB_HEAD_REPO_URL }} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet {{ GITHUB_HEAD_SHA }}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && {{ SYSTEM_DO_CLONE }} && - sudo -H -u build-user TENSORFLOW_WHEEL=${TENSORFLOW_WHEEL} {{ TASK_ENV_VARS }} /bin/bash /home/build-user/DeepSpeech/ds/tc-train-tests.sh 2.7.13 + sudo -H -u build-user TENSORFLOW_WHEEL=${TENSORFLOW_WHEEL} DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_ROOT} /bin/bash /home/build-user/DeepSpeech/ds/tc-train-tests.sh 2.7.13 artifacts: "public": type: "directory" diff --git a/DeepSpeech.py b/DeepSpeech.py index f3b353db..a1858f6d 100755 --- a/DeepSpeech.py +++ b/DeepSpeech.py @@ -143,6 +143,12 @@ tf.app.flags.DEFINE_float ('estop_std_thresh', 0.5, 'standard deviatio tf.app.flags.DEFINE_string ('decoder_library_path', 'native_client/libctc_decoder_with_kenlm.so', 'path to the libctc_decoder_with_kenlm.so library containing the decoder implementation.') tf.app.flags.DEFINE_string ('alphabet_config_path', 'data/alphabet.txt', 'path to the configuration file specifying the alphabet used by the network. See the comment in data/alphabet.txt for a description of the format.') +tf.app.flags.DEFINE_string ('lm_binary_path', 'data/lm/lm.binary', 'path to the language model binary file created with KenLM') +tf.app.flags.DEFINE_string ('lm_trie_path', 'data/lm/trie', 'path to the language model trie file created with native_client/generate_trie') +tf.app.flags.DEFINE_integer ('beam_width', 1024, 'beam width used in the CTC decoder when building candidate transcriptions') +tf.app.flags.DEFINE_float ('lm_weight', 2.15, 'the alpha hyperparameter of the CTC decoder. Language Model weight.') +tf.app.flags.DEFINE_float ('word_count_weight', -0.10, 'the beta hyperparameter of the CTC decoder. Word insertion weight (penalty).') +tf.app.flags.DEFINE_float ('valid_word_count_weight', 1.10, 'the beta\' hyperparameter of the CTC decoder. Valid word insertion weight.') for var in ['b1', 'h1', 'b2', 'h2', 'b3', 'h3', 'b5', 'h5', 'b6', 'h6']: tf.app.flags.DEFINE_float('%s_stddev' % var, None, 'standard deviation to use when initialising %s' % var) @@ -458,47 +464,12 @@ custom_op_module = tf.load_op_library(FLAGS.decoder_library_path) def decode_with_lm(inputs, sequence_length, beam_width=100, top_paths=1, merge_repeated=True): - """Performs beam search decoding on the logits given in input. - - **Note** The `ctc_greedy_decoder` is a special case of the - `ctc_beam_search_decoder` with `top_paths=1` and `beam_width=1` (but - that decoder is faster for this special case). - - If `merge_repeated` is `True`, merge repeated classes in the output beams. - This means that if consecutive entries in a beam are the same, - only the first of these is emitted. That is, when the top path - is `A B B B B`, the return value is: - - * `A B` if `merge_repeated = True`. - * `A B B B B` if `merge_repeated = False`. - - Args: - inputs: 3-D `float` `Tensor`, size - `[max_time x batch_size x num_classes]`. The logits. - sequence_length: 1-D `int32` vector containing sequence lengths, - having size `[batch_size]`. - beam_width: An int scalar >= 0 (beam search beam width). - top_paths: An int scalar >= 0, <= beam_width (controls output size). - merge_repeated: Boolean. Default: True. - - Returns: - A tuple `(decoded, log_probabilities)` where - decoded: A list of length top_paths, where `decoded[j]` - is a `SparseTensor` containing the decoded outputs: - `decoded[j].indices`: Indices matrix `(total_decoded_outputs[j] x 2)` - The rows store: [batch, time]. - `decoded[j].values`: Values vector, size `(total_decoded_outputs[j])`. - The vector stores the decoded classes for beam j. - `decoded[j].shape`: Shape vector, size `(2)`. - The shape values are: `[batch_size, max_decoded_length[j]]`. - log_probability: A `float` matrix `(batch_size x top_paths)` containing - sequence log-probabilities. - """ - decoded_ixs, decoded_vals, decoded_shapes, log_probabilities = ( custom_op_module.ctc_beam_search_decoder_with_lm( - inputs, sequence_length, model_path="data/lm/lm.binary", trie_path="data/lm/trie", alphabet_path="data/alphabet.txt", - beam_width=beam_width, top_paths=top_paths, merge_repeated=merge_repeated)) + inputs, sequence_length, beam_width=beam_width, + model_path=FLAGS.lm_binary_path, trie_path=FLAGS.lm_trie_path, alphabet_path=FLAGS.alphabet_config_path, + lm_weight=FLAGS.lm_weight, word_count_weight=FLAGS.word_count_weight, valid_word_count_weight=FLAGS.valid_word_count_weight, + top_paths=top_paths, merge_repeated=merge_repeated)) return ( [tf.SparseTensor(ix, val, shape) for (ix, val, shape) @@ -539,7 +510,7 @@ def calculate_mean_edit_distance_and_loss(model_feeder, tower, dropout): avg_loss = tf.reduce_mean(total_loss) # Beam search decode the batch - decoded, _ = decode_with_lm(logits, batch_seq_len, merge_repeated=False, beam_width=1024) + decoded, _ = decode_with_lm(logits, batch_seq_len, merge_repeated=False, beam_width=FLAGS.beam_width) # Compute the edit (Levenshtein) distance distance = tf.edit_distance(tf.cast(decoded[0], tf.int32), batch_y) diff --git a/data/spell/words.txt b/data/lm/vocab.txt similarity index 99% rename from data/spell/words.txt rename to data/lm/vocab.txt index 98516361..d0c47e0e 100644 --- a/data/spell/words.txt +++ b/data/lm/vocab.txt @@ -6876,7 +6876,7 @@ i would like to conclude by reading an email i got from one of them cindy the da as i worked i couldnt help but think about the individuals and the stories represented in the images one in particular a photo of women of all ages from grandmother to little girl gathered around a baby struck a chord because a similar photo from my family my grandmother and mother myself and newborn daughter hangs on our wall across the globe throughout the ages our basic needs are just the same arent they thank you -i along with hundreds of other volunteers knew we couldnt just sit at home so i decided to join them for three weeks on may the thirteenth i made my way to the town of ōfunato its a small fishing town in iwate prefecture +i along with hundreds of other volunteers knew we couldnt just sit at home so i decided to join them for three weeks on may the thirteenth i made my way to the town of ofunato its a small fishing town in iwate prefecture about fifty thousand people one of the first that was hit by the wave the waters here have been recorded at reaching over twenty four meters in height and traveled over two miles inland as you can imagine the town had been devastated @@ -48368,7 +48368,7 @@ what if we thought of fear as an amazing act of the imagination something that c its easiest to see this link between fear and the imagination in young children whose fears are often extraordinarily vivid when i was a child i lived in california which is you know mostly a very nice place to live but at a certain point most of us learn to leave these kinds of visions behind and grow up we learn that there are no monsters hiding under the bed and not every earthquake brings buildings down -but maybe its no coincidence that some of our most creative minds fail to leave these kinds of fears behind as adults the same incredible imaginations that produced the origin of species jane eyre and the remembrance of things past also generated intense worries that haunted the adult lives of charles darwin charlotte brontăť +but maybe its no coincidence that some of our most creative minds fail to leave these kinds of fears behind as adults the same incredible imaginations that produced the origin of species jane eyre and the remembrance of things past also generated intense worries that haunted the adult lives of charles darwin charlotte brontat and marcel proust so the question is what can the rest of us learn lets take a look at the fears that their imaginations were generating as they drifted in the middle of the pacific twenty four hours had now passed since the capsizing of the ship the time had come for the men to make a plan but they had very few options in his fascinating account of the disaster nathaniel philbrick wrote that these men were just about as far from land as it was possible to be anywhere on earth diff --git a/native_client/BUILD b/native_client/BUILD index 9988cef9..00ac8ce9 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -40,7 +40,8 @@ cc_library( name = "ctc_decoder_with_kenlm", srcs = ["beam_search.cc", "alphabet.h", - "trie_node.h"] + + "trie_node.h" + ] + glob(["kenlm/lm/*.cc", "kenlm/util/*.cc", "kenlm/util/double-conversion/*.cc", "kenlm/lm/*.hh", "kenlm/util/*.hh", "kenlm/util/double-conversion/*.h"], exclude = ["kenlm/*/*test.cc", "kenlm/*/*main.cc"]), @@ -59,10 +60,10 @@ cc_binary( "trie_node.h", "alphabet.h", ] + glob(["kenlm/lm/*.cc", "kenlm/util/*.cc", "kenlm/util/double-conversion/*.cc", - "kenlm/lm/*.hh", "kenlm/util/*.hh", "kenlm/util/double-conversion/*.h"], - exclude = ["kenlm/*/*test.cc", "kenlm/*/*main.cc"]), + "kenlm/lm/*.hh", "kenlm/util/*.hh", "kenlm/util/double-conversion/*.h"], + exclude = ["kenlm/*/*test.cc", "kenlm/*/*main.cc"]), includes = ["kenlm"], - copts = ['-std=c++11'], - linkopts = ['-lm'], + copts = ["-std=c++11"], + linkopts = ["-lm"], defines = ["KENLM_MAX_ORDER=6"], ) diff --git a/native_client/alphabet.h b/native_client/alphabet.h index 4534e437..cf34eebe 100644 --- a/native_client/alphabet.h +++ b/native_client/alphabet.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -46,6 +47,7 @@ public: if (it != str_to_label_.end()) { return it->second; } else { + std::cerr << "Invalid label " << string << std::endl; abort(); } } @@ -55,6 +57,7 @@ public: } bool IsSpace(unsigned int label) const { + //TODO: we should probably do something more i18n-aware here const std::string& str = StringFromLabel(label); return str.size() == 1 && str[0] == ' '; } diff --git a/native_client/beam_search.cc b/native_client/beam_search.cc index 9e5c930d..e82043ec 100644 --- a/native_client/beam_search.cc +++ b/native_client/beam_search.cc @@ -13,14 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// This test illustrates how to make use of the CTCBeamSearchDecoder using a -// custom BeamScorer and BeamState based on a dictionary with a few artificial -// words. #include "tensorflow/core/util/ctc/ctc_beam_search.h" #include -#include #include +#include #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" @@ -43,6 +40,9 @@ REGISTER_OP("CTCBeamSearchDecoderWithLM") .Attr("model_path: string") .Attr("trie_path: string") .Attr("alphabet_path: string") + .Attr("lm_weight: float") + .Attr("word_count_weight: float") + .Attr("valid_word_count_weight: float") .Attr("beam_width: int >= 1 = 100") .Attr("top_paths: int >= 1 = 1") .Attr("merge_repeated: bool = true") @@ -91,6 +91,12 @@ returned if merge_repeated = False. inputs: 3-D, shape: `(max_time x batch_size x num_classes)`, the logits. sequence_length: A vector containing sequence lengths, size `(batch)`. +model_path: A string containing the path to the KenLM model file to use. +trie_path: A string containing the path to the trie file built from the vocabulary. +alphabet_path: A string containing the path to the alphabet file (see alphabet.h). +lm_weight: alpha hyperparameter of CTC decoder. LM weight. +word_count_weight: beta hyperparameter of CTC decoder. Word insertion weight. +valid_word_count_weight: beta' hyperparameter of CTC decoder. Valid word insertion weight. beam_width: A scalar >= 0 (beam search beam width). top_paths: A scalar >= 0, <= beam_width (controls output size). merge_repeated: If true, merge repeated classes in output. @@ -107,38 +113,36 @@ log_probability: A matrix, shaped: `(batch_size x top_paths)`. The sequence log-probabilities. )doc"); +typedef lm::ngram::ProbingModel Model; + struct KenLMBeamState { float language_model_score; float score; float delta_score; std::string incomplete_word; TrieNode *incomplete_word_trie_node; - lm::ngram::ProbingModel::State model_state; + Model::State model_state; }; class KenLMBeamScorer : public tf::ctc::BaseBeamScorer { public: - typedef lm::ngram::ProbingModel Model; - - KenLMBeamScorer(const std::string &kenlm_path, const std::string &trie_path, const std::string &alphabet_path) - : lm_weight(1.0f) - , word_count_weight(-0.1f) - , valid_word_count_weight(1.0f) + KenLMBeamScorer(const std::string &kenlm_path, const std::string &trie_path, + const std::string &alphabet_path, float lm_weight, + float word_count_weight, float valid_word_count_weight) + : model(kenlm_path.c_str(), GetLMConfig()) + , alphabet(alphabet_path.c_str()) + , lm_weight_(lm_weight) + , word_count_weight_(word_count_weight) + , valid_word_count_weight_(valid_word_count_weight) { - lm::ngram::Config config; - config.load_method = util::POPULATE_OR_READ; - model = new Model(kenlm_path.c_str(), config); + std::ifstream in(trie_path, std::ios::in); + TrieNode::ReadFromStream(in, trieRoot, alphabet.GetSize()); - alphabet = new Alphabet(alphabet_path.c_str()); - - std::ifstream in; - in.open(trie_path, std::ios::in); - TrieNode::ReadFromStream(in, trieRoot, alphabet->GetSize()); - in.close(); + Model::State out; + oov_score_ = model.FullScore(model.NullContextState(), model.GetVocabulary().NotFound(), out).prob; } virtual ~KenLMBeamScorer() { - delete model; delete trieRoot; } @@ -149,7 +153,7 @@ class KenLMBeamScorer : public tf::ctc::BaseBeamScorer { root->delta_score = 0.0f; root->incomplete_word.clear(); root->incomplete_word_trie_node = trieRoot; - root->model_state = model->BeginSentenceState(); + root->model_state = model.BeginSentenceState(); } // ExpandState is called when expanding a beam to one of its children. // Called at most once per child beam. In the simplest case, no state @@ -158,13 +162,12 @@ class KenLMBeamScorer : public tf::ctc::BaseBeamScorer { KenLMBeamState* to_state, int to_label) const { CopyState(from_state, to_state); - if (!alphabet->IsSpace(to_label)) { - to_state->incomplete_word += alphabet->StringFromLabel(to_label); + if (!alphabet.IsSpace(to_label)) { + to_state->incomplete_word += alphabet.StringFromLabel(to_label); TrieNode *trie_node = from_state.incomplete_word_trie_node; - // TODO replace with OOV unigram prob? // If we have no valid prefix we assume a very low log probability - float min_unigram_score = -10.0f; + float min_unigram_score = oov_score_; // If prefix does exist if (trie_node != nullptr) { trie_node = trie_node->GetChildAt(to_label); @@ -185,9 +188,9 @@ class KenLMBeamScorer : public tf::ctc::BaseBeamScorer { to_state->model_state); // Give fixed word bonus if (!IsOOV(to_state->incomplete_word)) { - to_state->language_model_score += valid_word_count_weight; + to_state->language_model_score += valid_word_count_weight_; } - to_state->language_model_score += word_count_weight; + to_state->language_model_score += word_count_weight_; UpdateWithLMScore(to_state, lm_score_delta); ResetIncompleteWord(to_state); } @@ -205,8 +208,8 @@ class KenLMBeamScorer : public tf::ctc::BaseBeamScorer { ResetIncompleteWord(state); state->model_state = out; } - lm_score_delta += model->FullScore(state->model_state, - model->GetVocabulary().EndSentence(), + lm_score_delta += model.FullScore(state->model_state, + model.GetVocabulary().EndSentence(), out).prob; UpdateWithLMScore(state, lm_score_delta); } @@ -219,7 +222,7 @@ class KenLMBeamScorer : public tf::ctc::BaseBeamScorer { // there's no state expansion logic, the expansion score is zero. float GetStateExpansionScore(const KenLMBeamState& state, float previous_score) const { - return lm_weight * state.delta_score + previous_score; + return lm_weight_ * state.delta_score + previous_score; } // GetStateEndExpansionScore should be an inexpensive method to retrieve the // (cached) expansion score computed within ExpandStateEnd. The score is @@ -227,28 +230,35 @@ class KenLMBeamScorer : public tf::ctc::BaseBeamScorer { // // The score returned should be a log-probability. float GetStateEndExpansionScore(const KenLMBeamState& state) const { - return lm_weight * state.delta_score; + return lm_weight_ * state.delta_score; } void SetLMWeight(float lm_weight) { - this->lm_weight = lm_weight; + this->lm_weight_ = lm_weight; } void SetWordCountWeight(float word_count_weight) { - this->word_count_weight = word_count_weight; + this->word_count_weight_ = word_count_weight; } void SetValidWordCountWeight(float valid_word_count_weight) { - this->valid_word_count_weight = valid_word_count_weight; + this->valid_word_count_weight_ = valid_word_count_weight; } private: - Model *model; - Alphabet *alphabet; + Model model; + Alphabet alphabet; TrieNode *trieRoot; - float lm_weight; - float word_count_weight; - float valid_word_count_weight; + float lm_weight_; + float word_count_weight_; + float valid_word_count_weight_; + float oov_score_; + + lm::ngram::Config GetLMConfig() { + lm::ngram::Config config; + config.load_method = util::POPULATE_OR_READ; + return config; + } void UpdateWithLMScore(KenLMBeamState *state, float lm_score_delta) const { float previous_score = state->score; @@ -263,17 +273,15 @@ class KenLMBeamScorer : public tf::ctc::BaseBeamScorer { } bool IsOOV(const std::string& word) const { - auto &vocabulary = model->GetVocabulary(); + auto &vocabulary = model.GetVocabulary(); return vocabulary.Index(word) == vocabulary.NotFound(); } float ScoreIncompleteWord(const Model::State& model_state, const std::string& word, Model::State& out) const { - lm::FullScoreReturn full_score_return; - lm::WordIndex vocab = model->GetVocabulary().Index(word); - full_score_return = model->FullScore(model_state, vocab, out); - return full_score_return.prob; + lm::WordIndex word_index = model.GetVocabulary().Index(word); + return model.FullScore(model_state, word_index, out).prob; } void CopyState(const KenLMBeamState& from, KenLMBeamState* to) const { @@ -409,56 +417,22 @@ class CTCDecodeHelper { TF_DISALLOW_COPY_AND_ASSIGN(CTCDecodeHelper); }; -// CTC beam search -class CTCBeamSearchDecoderOp : public tf::OpKernel { +class CTCBeamSearchDecoderWithLMOp : public tf::OpKernel { public: - explicit CTCBeamSearchDecoderOp(tf::OpKernelConstruction *ctx) + explicit CTCBeamSearchDecoderWithLMOp(tf::OpKernelConstruction *ctx) : tf::OpKernel(ctx) , beam_scorer_(GetModelPath(ctx), GetTriePath(ctx), - GetAlphabetPath(ctx)) + GetAlphabetPath(ctx), + GetLMWeight(ctx), + GetWordCountWeight(ctx), + GetValidWordCountWeight(ctx)) { OP_REQUIRES_OK(ctx, ctx->GetAttr("merge_repeated", &merge_repeated_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("beam_width", &beam_width_)); int top_paths; OP_REQUIRES_OK(ctx, ctx->GetAttr("top_paths", &top_paths)); decode_helper_.SetTopPaths(top_paths); - - // const tf::Tensor* model_tensor; - // tf::Status status = ctx->input("model_path", &model_tensor); - // if (!status.ok()) return status; - // auto model_vec = model_tensor->flat(); - // *model_path = model_vec(0); - - // const tf::Tensor* trie_tensor; - // status = ctx->input("trie_path", &trie_tensor); - // if (!status.ok()) return status; - // auto trie_vec = trie_tensor->flat(); - // *trie_path = model_vec(0); - - // const tf::Tensor* alphabet_tensor; - // status = ctx->input("alphabet_path", &alphabet_tensor); - // if (!status.ok()) return status; - // auto alphabet_vec = alphabet_tensor->flat(); - // *alphabet_path = alphabet_vec(0); - } - - std::string GetModelPath(tf::OpKernelConstruction *ctx) { - std::string model_path; - ctx->GetAttr("model_path", &model_path); - return model_path; - } - - std::string GetTriePath(tf::OpKernelConstruction *ctx) { - std::string trie_path; - ctx->GetAttr("trie_path", &trie_path); - return trie_path; - } - - std::string GetAlphabetPath(tf::OpKernelConstruction *ctx) { - std::string alphabet_path; - ctx->GetAttr("alphabet_path", &alphabet_path); - return alphabet_path; } void Compute(tf::OpKernelContext *ctx) override { @@ -539,8 +513,44 @@ class CTCBeamSearchDecoderOp : public tf::OpKernel { KenLMBeamScorer beam_scorer_; bool merge_repeated_; int beam_width_; - TF_DISALLOW_COPY_AND_ASSIGN(CTCBeamSearchDecoderOp); + TF_DISALLOW_COPY_AND_ASSIGN(CTCBeamSearchDecoderWithLMOp); + + std::string GetModelPath(tf::OpKernelConstruction *ctx) { + std::string model_path; + ctx->GetAttr("model_path", &model_path); + return model_path; + } + + std::string GetTriePath(tf::OpKernelConstruction *ctx) { + std::string trie_path; + ctx->GetAttr("trie_path", &trie_path); + return trie_path; + } + + std::string GetAlphabetPath(tf::OpKernelConstruction *ctx) { + std::string alphabet_path; + ctx->GetAttr("alphabet_path", &alphabet_path); + return alphabet_path; + } + + float GetLMWeight(tf::OpKernelConstruction *ctx) { + float lm_weight; + ctx->GetAttr("lm_weight", &lm_weight); + return lm_weight; + } + + float GetWordCountWeight(tf::OpKernelConstruction *ctx) { + float word_count_weight; + ctx->GetAttr("word_count_weight", &word_count_weight); + return word_count_weight; + } + + float GetValidWordCountWeight(tf::OpKernelConstruction *ctx) { + float valid_word_count_weight; + ctx->GetAttr("valid_word_count_weight", &valid_word_count_weight); + return valid_word_count_weight; + } }; REGISTER_KERNEL_BUILDER(Name("CTCBeamSearchDecoderWithLM").Device(tf::DEVICE_CPU), - CTCBeamSearchDecoderOp); + CTCBeamSearchDecoderWithLMOp); diff --git a/native_client/generate_trie.cpp b/native_client/generate_trie.cpp index ce3b9e65..33c5e2d9 100644 --- a/native_client/generate_trie.cpp +++ b/native_client/generate_trie.cpp @@ -10,17 +10,13 @@ using namespace std; typedef lm::ngram::ProbingModel Model; lm::WordIndex GetWordIndex(const Model& model, const std::string& word) { - lm::WordIndex vocab; - vocab = model.GetVocabulary().Index(word); - return vocab; + return model.GetVocabulary().Index(word); } -float ScoreWord(const Model& model, lm::WordIndex vocab) { - Model::State in_state = model.NullContextState(); +float ScoreWord(const Model& model, lm::WordIndex word_index) { + // We don't need to keep state here as we're scoring the words individually. Model::State out; - lm::FullScoreReturn full_score_return; - full_score_return = model.FullScore(in_state, vocab, out); - return full_score_return.prob; + return model.FullScore(model.NullContextState(), word_index, out).prob; } int generate_trie(const char* alphabet_path, const char* kenlm_path, const char* vocab_path, const char* trie_path) { @@ -31,36 +27,38 @@ int generate_trie(const char* alphabet_path, const char* kenlm_path, const char* Model model(kenlm_path, config); TrieNode root(a.GetSize()); - std::ifstream ifs; - ifs.open(vocab_path, std::ifstream::in); - - if (!ifs.is_open()) { - std::cout << "unable to open vocabulary" << std::endl; + std::ifstream ifs(vocab_path, std::ifstream::in); + if (!ifs) { + std::cerr << "unable to open vocabulary file " << vocab_path << std::endl; return -1; } - std::ofstream ofs; - ofs.open(trie_path); + std::ofstream ofs(trie_path); + if (!ofs) { + std::cerr << "unable to open output file " << trie_path << std::endl; + return -1; + } std::string word; while (ifs >> word) { - for_each(word.begin(), word.end(), [](char& a) { a = tolower(a); }); - lm::WordIndex vocab = GetWordIndex(model, word); - float unigram_score = ScoreWord(model, vocab); - root.Insert(word.c_str(), [&a](char c) { - return a.LabelFromString(string(1, c)); - }, vocab, unigram_score); + lm::WordIndex word_index = GetWordIndex(model, word); + float unigram_score = ScoreWord(model, word_index); + root.Insert(word.c_str(), + [&a](const std::string& c) { + return a.LabelFromString(c); + }, + word_index, unigram_score); } root.WriteToStream(ofs); - ifs.close(); - ofs.close(); return 0; } -int main(void) { - return generate_trie("/Users/remorais/Development/DeepSpeech/data/alphabet.txt", - "/Users/remorais/Development/DeepSpeech/data/lm/lm.binary", - "/Users/remorais/Development/DeepSpeech/data/lm/vocab.txt", - "/Users/remorais/Development/DeepSpeech/data/lm/trie"); +int main(int argc, char** argv) { + if (argc != 5) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return -1; + } + + return generate_trie(argv[1], argv[2], argv[3], argv[4]); } diff --git a/native_client/kenlm/README.mozilla b/native_client/kenlm/README.mozilla index 29504b33..3fb17741 100644 --- a/native_client/kenlm/README.mozilla +++ b/native_client/kenlm/README.mozilla @@ -6,4 +6,4 @@ This corresponds to https://github.com/kpu/kenlm/commit/cdd794598ea15dc23a7daaf7 The following procedure was run to remove unneeded files: cd kenlm -rm -rf windows include lm/filter lm/builder util/stream +rm -rf windows include lm/filter lm/builder util/stream util/getopt.* python diff --git a/native_client/kenlm/python/_kenlm.pxd b/native_client/kenlm/python/_kenlm.pxd deleted file mode 100644 index 57b056fa..00000000 --- a/native_client/kenlm/python/_kenlm.pxd +++ /dev/null @@ -1,50 +0,0 @@ -cdef extern from "lm/word_index.hh" namespace "lm": - ctypedef unsigned WordIndex - -cdef extern from "lm/return.hh" namespace "lm": - cdef struct FullScoreReturn: - float prob - unsigned char ngram_length - -cdef extern from "lm/state.hh" namespace "lm::ngram": - cdef cppclass State : - int Compare(const State &other) const - - int hash_value(const State &state) - -cdef extern from "lm/virtual_interface.hh" namespace "lm::base": - cdef cppclass Vocabulary: - WordIndex Index(char*) - WordIndex BeginSentence() - WordIndex EndSentence() - WordIndex NotFound() - - ctypedef Vocabulary const_Vocabulary "const lm::base::Vocabulary" - - cdef cppclass Model: - void BeginSentenceWrite(void *) - void NullContextWrite(void *) - unsigned int Order() - const_Vocabulary& BaseVocabulary() - float BaseScore(void *in_state, WordIndex new_word, void *out_state) - FullScoreReturn BaseFullScore(void *in_state, WordIndex new_word, void *out_state) - -cdef extern from "util/mmap.hh" namespace "util": - cdef enum LoadMethod: - LAZY - POPULATE_OR_LAZY - POPULATE_OR_READ - READ - PARALLEL_READ - -cdef extern from "lm/config.hh" namespace "lm::ngram": - cdef cppclass Config: - Config() - float probing_multiplier - LoadMethod load_method - -cdef extern from "lm/model.hh" namespace "lm::ngram": - cdef Model *LoadVirtual(char *, Config &config) except + - #default constructor - cdef Model *LoadVirtual(char *) except + - diff --git a/native_client/kenlm/python/example.py b/native_client/kenlm/python/example.py deleted file mode 100755 index 7cfc153c..00000000 --- a/native_client/kenlm/python/example.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -import os -import kenlm - -LM = os.path.join(os.path.dirname(__file__), '..', 'lm', 'test.arpa') -model = kenlm.Model(LM) -print('{0}-gram model'.format(model.order)) - -sentence = 'language modeling is fun .' -print(sentence) -print(model.score(sentence)) - -# Check that total full score = direct score -def score(s): - return sum(prob for prob, _, _ in model.full_scores(s)) - -assert (abs(score(sentence) - model.score(sentence)) < 1e-3) - -# Show scores and n-gram matches -words = [''] + sentence.split() + [''] -for i, (prob, length, oov) in enumerate(model.full_scores(sentence)): - print('{0} {1}: {2}'.format(prob, length, ' '.join(words[i+2-length:i+2]))) - if oov: - print('\t"{0}" is an OOV'.format(words[i+1])) - -# Find out-of-vocabulary words -for w in words: - if not w in model: - print('"{0}" is an OOV'.format(w)) - -#Stateful query -state = kenlm.State() -state2 = kenlm.State() -#Use as context. If you don't want , use model.NullContextWrite(state). -model.BeginSentenceWrite(state) -accum = 0.0 -accum += model.BaseScore(state, "a", state2) -accum += model.BaseScore(state2, "sentence", state) -#score defaults to bos = True and eos = True. Here we'll check without the end -#of sentence marker. -assert (abs(accum - model.score("a sentence", eos = False)) < 1e-3) -accum += model.BaseScore(state, "", state2) -assert (abs(accum - model.score("a sentence")) < 1e-3) diff --git a/native_client/kenlm/python/kenlm.cpp b/native_client/kenlm/python/kenlm.cpp deleted file mode 100644 index 14e97990..00000000 --- a/native_client/kenlm/python/kenlm.cpp +++ /dev/null @@ -1,7157 +0,0 @@ -/* Generated by Cython 0.22 */ - -#define PY_SSIZE_T_CLEAN -#ifndef CYTHON_USE_PYLONG_INTERNALS -#ifdef PYLONG_BITS_IN_DIGIT -#define CYTHON_USE_PYLONG_INTERNALS 0 -#else -#include "pyconfig.h" -#ifdef PYLONG_BITS_IN_DIGIT -#define CYTHON_USE_PYLONG_INTERNALS 1 -#else -#define CYTHON_USE_PYLONG_INTERNALS 0 -#endif -#endif -#endif -#include "Python.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) - #error Cython requires Python 2.6+ or Python 3.2+. -#else -#define CYTHON_ABI "0_22" -#include -#ifndef offsetof -#define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#endif -#ifndef DL_IMPORT - #define DL_IMPORT(t) t -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef Py_HUGE_VAL - #define Py_HUGE_VAL HUGE_VAL -#endif -#ifdef PYPY_VERSION -#define CYTHON_COMPILING_IN_PYPY 1 -#define CYTHON_COMPILING_IN_CPYTHON 0 -#else -#define CYTHON_COMPILING_IN_PYPY 0 -#define CYTHON_COMPILING_IN_CPYTHON 1 -#endif -#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) -#define Py_OptimizeFlag 0 -#endif -#define __PYX_BUILD_PY_SSIZE_T "n" -#define CYTHON_FORMAT_SSIZE_T "z" -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyClass_Type -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ - PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) - #define __Pyx_DefaultClassType PyType_Type -#endif -#if PY_MAJOR_VERSION >= 3 - #define Py_TPFLAGS_CHECKTYPES 0 - #define Py_TPFLAGS_HAVE_INDEX 0 - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#if PY_VERSION_HEX < 0x030400a1 && !defined(Py_TPFLAGS_HAVE_FINALIZE) - #define Py_TPFLAGS_HAVE_FINALIZE 0 -#endif -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) - #define CYTHON_PEP393_ENABLED 1 - #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ - 0 : _PyUnicode_Ready((PyObject *)(op))) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) - #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) - #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) -#else - #define CYTHON_PEP393_ENABLED 0 - #define __Pyx_PyUnicode_READY(op) (0) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) - #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) - #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) -#endif -#if CYTHON_COMPILING_IN_PYPY - #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) - #define __Pyx_PyFrozenSet_Size(s) PyObject_Size(s) -#else - #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) - #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \ - PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) - #define __Pyx_PyFrozenSet_Size(s) PySet_Size(s) -#endif -#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) -#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) -#else - #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyStringObject PyUnicodeObject - #define PyString_Type PyUnicode_Type - #define PyString_Check PyUnicode_Check - #define PyString_CheckExact PyUnicode_CheckExact -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) - #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) -#else - #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) - #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) -#endif -#ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) -#endif -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) -#if PY_MAJOR_VERSION >= 3 - #define PyIntObject PyLongObject - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define PyNumber_Int PyNumber_Long -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBoolObject PyLongObject -#endif -#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY - #ifndef PyUnicode_InternFromString - #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) - #endif -#endif -#if PY_VERSION_HEX < 0x030200A4 - typedef long Py_hash_t; - #define __Pyx_PyInt_FromHash_t PyInt_FromLong - #define __Pyx_PyInt_AsHash_t PyInt_AsLong -#else - #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t - #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t -#endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) -#endif -#ifndef CYTHON_INLINE - #if defined(__GNUC__) - #define CYTHON_INLINE __inline__ - #elif defined(_MSC_VER) - #define CYTHON_INLINE __inline - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_INLINE inline - #else - #define CYTHON_INLINE - #endif -#endif -#ifndef CYTHON_RESTRICT - #if defined(__GNUC__) - #define CYTHON_RESTRICT __restrict__ - #elif defined(_MSC_VER) && _MSC_VER >= 1400 - #define CYTHON_RESTRICT __restrict - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_RESTRICT restrict - #else - #define CYTHON_RESTRICT - #endif -#endif -#ifdef NAN -#define __PYX_NAN() ((float) NAN) -#else -static CYTHON_INLINE float __PYX_NAN() { - /* Initialize NaN. The sign is irrelevant, an exponent with all bits 1 and - a nonzero mantissa means NaN. If the first bit in the mantissa is 1, it is - a quiet NaN. */ - float value; - memset(&value, 0xFF, sizeof(value)); - return value; -} -#endif -#define __Pyx_void_to_None(void_result) (void_result, Py_INCREF(Py_None), Py_None) -#ifdef __cplusplus -template -void __Pyx_call_destructor(T* x) { - x->~T(); -} -template -class __Pyx_FakeReference { - public: - __Pyx_FakeReference() : ptr(NULL) { } - __Pyx_FakeReference(T& ref) : ptr(&ref) { } - T *operator->() { return ptr; } - operator T&() { return *ptr; } - private: - T *ptr; -}; -#endif - - -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) -#endif - -#ifndef __PYX_EXTERN_C - #ifdef __cplusplus - #define __PYX_EXTERN_C extern "C" - #else - #define __PYX_EXTERN_C extern - #endif -#endif - -#if defined(WIN32) || defined(MS_WINDOWS) -#define _USE_MATH_DEFINES -#endif -#include -#define __PYX_HAVE__kenlm -#define __PYX_HAVE_API__kenlm -#include "lm/word_index.hh" -#include "lm/return.hh" -#include "lm/state.hh" -#include "lm/virtual_interface.hh" -#include "util/mmap.hh" -#include "lm/config.hh" -#include "lm/model.hh" -#include "ios" -#include "new" -#include "stdexcept" -#include "typeinfo" -#ifdef _OPENMP -#include -#endif /* _OPENMP */ - -#ifdef PYREX_WITHOUT_ASSERTIONS -#define CYTHON_WITHOUT_ASSERTIONS -#endif - -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -typedef struct {PyObject **p; char *s; const Py_ssize_t n; const char* encoding; - const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; - -#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 -#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 -#define __PYX_DEFAULT_STRING_ENCODING "" -#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString -#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#define __Pyx_fits_Py_ssize_t(v, type, is_signed) ( \ - (sizeof(type) < sizeof(Py_ssize_t)) || \ - (sizeof(type) > sizeof(Py_ssize_t) && \ - likely(v < (type)PY_SSIZE_T_MAX || \ - v == (type)PY_SSIZE_T_MAX) && \ - (!is_signed || likely(v > (type)PY_SSIZE_T_MIN || \ - v == (type)PY_SSIZE_T_MIN))) || \ - (sizeof(type) == sizeof(Py_ssize_t) && \ - (is_signed || likely(v < (type)PY_SSIZE_T_MAX || \ - v == (type)PY_SSIZE_T_MAX))) ) -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); -#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) -#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); -#if PY_MAJOR_VERSION < 3 - #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize -#else - #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString - #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize -#endif -#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) -#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) -#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) -#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) -#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) -#if PY_MAJOR_VERSION < 3 -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) -{ - const Py_UNICODE *u_end = u; - while (*u_end++) ; - return (size_t)(u_end - u - 1); -} -#else -#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen -#endif -#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) -#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode -#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode -#define __Pyx_Owned_Py_None(b) (Py_INCREF(Py_None), Py_None) -#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -#if CYTHON_COMPILING_IN_CPYTHON -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) -#else -#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) -#endif -#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII -static int __Pyx_sys_getdefaultencoding_not_ascii; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - PyObject* ascii_chars_u = NULL; - PyObject* ascii_chars_b = NULL; - const char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - if (strcmp(default_encoding_c, "ascii") == 0) { - __Pyx_sys_getdefaultencoding_not_ascii = 0; - } else { - char ascii_chars[128]; - int c; - for (c = 0; c < 128; c++) { - ascii_chars[c] = c; - } - __Pyx_sys_getdefaultencoding_not_ascii = 1; - ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); - if (!ascii_chars_u) goto bad; - ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); - if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { - PyErr_Format( - PyExc_ValueError, - "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", - default_encoding_c); - goto bad; - } - Py_DECREF(ascii_chars_u); - Py_DECREF(ascii_chars_b); - } - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - Py_XDECREF(ascii_chars_u); - Py_XDECREF(ascii_chars_b); - return -1; -} -#endif -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) -#else -#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) -#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT -static char* __PYX_DEFAULT_STRING_ENCODING; -static int __Pyx_init_sys_getdefaultencoding_params(void) { - PyObject* sys; - PyObject* default_encoding = NULL; - char* default_encoding_c; - sys = PyImport_ImportModule("sys"); - if (!sys) goto bad; - default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); - Py_DECREF(sys); - if (!default_encoding) goto bad; - default_encoding_c = PyBytes_AsString(default_encoding); - if (!default_encoding_c) goto bad; - __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); - if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; - strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); - Py_DECREF(default_encoding); - return 0; -bad: - Py_XDECREF(default_encoding); - return -1; -} -#endif -#endif - - -/* Test for GCC > 2.95 */ -#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) - #define likely(x) __builtin_expect(!!(x), 1) - #define unlikely(x) __builtin_expect(!!(x), 0) -#else /* !__GNUC__ or GCC < 2.95 */ - #define likely(x) (x) - #define unlikely(x) (x) -#endif /* __GNUC__ */ - -static PyObject *__pyx_m; -static PyObject *__pyx_d; -static PyObject *__pyx_b; -static PyObject *__pyx_empty_tuple; -static PyObject *__pyx_empty_bytes; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; - - -static const char *__pyx_f[] = { - "kenlm.pyx", -}; - -/*--- Type declarations ---*/ -struct __pyx_obj_5kenlm_FullScoreReturn; -struct __pyx_obj_5kenlm_State; -struct __pyx_obj_5kenlm_Config; -struct __pyx_obj_5kenlm_Model; -struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores; - -/* "kenlm.pyx":11 - * raise TypeError('Cannot convert %s to string' % type(data)) - * - * cdef class FullScoreReturn: # <<<<<<<<<<<<<< - * """ - * Wrapper around FullScoreReturn. - */ -struct __pyx_obj_5kenlm_FullScoreReturn { - PyObject_HEAD - float log_prob; - int ngram_length; - int oov; -}; - - -/* "kenlm.pyx":44 - * return self.oov - * - * cdef class State: # <<<<<<<<<<<<<< - * """ - * Wrapper around lm::ngram::State so that python code can make incremental queries. - */ -struct __pyx_obj_5kenlm_State { - PyObject_HEAD - lm::ngram::State _c_state; -}; - - -/* "kenlm.pyx":80 - * PARALLEL_READ = _kenlm.PARALLEL_READ - * - * cdef class Config: # <<<<<<<<<<<<<< - * """ - * Wrapper around lm::ngram::Config. - */ -struct __pyx_obj_5kenlm_Config { - PyObject_HEAD - lm::ngram::Config _c_config; -}; - - -/* "kenlm.pyx":96 - * self._c_config.load_method = to - * - * cdef class Model: # <<<<<<<<<<<<<< - * """ - * Wrapper around lm::ngram::Model. - */ -struct __pyx_obj_5kenlm_Model { - PyObject_HEAD - lm::base::Model *model; - PyObject *path; - const lm::base::Vocabulary *vocab; -}; - - -/* "kenlm.pyx":190 - * return 10.0**(-self.score(sentence) / words) - * - * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< - * """ - * full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov) - */ -struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores { - PyObject_HEAD - PyObject *__pyx_v_bos; - PyObject *__pyx_v_eos; - lm::ngram::State __pyx_v_out_state; - struct lm::FullScoreReturn __pyx_v_ret; - struct __pyx_obj_5kenlm_Model *__pyx_v_self; - PyObject *__pyx_v_sentence; - lm::ngram::State __pyx_v_state; - float __pyx_v_total; - lm::WordIndex __pyx_v_wid; - PyObject *__pyx_v_word; - PyObject *__pyx_v_words; - PyObject *__pyx_t_0; - Py_ssize_t __pyx_t_1; -}; - - -/* --- Runtime support code (head) --- */ -#ifndef CYTHON_REFNANNY - #define CYTHON_REFNANNY 0 -#endif -#if CYTHON_REFNANNY - typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); - void (*FinishContext)(void**); - } __Pyx_RefNannyAPIStruct; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; - static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); - #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; -#ifdef WITH_THREAD - #define __Pyx_RefNannySetupContext(name, acquire_gil) \ - if (acquire_gil) { \ - PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ - PyGILState_Release(__pyx_gilstate_save); \ - } else { \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ - } -#else - #define __Pyx_RefNannySetupContext(name, acquire_gil) \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) -#endif - #define __Pyx_RefNannyFinishContext() \ - __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) -#else - #define __Pyx_RefNannyDeclarations - #define __Pyx_RefNannySetupContext(name, acquire_gil) - #define __Pyx_RefNannyFinishContext() - #define __Pyx_INCREF(r) Py_INCREF(r) - #define __Pyx_DECREF(r) Py_DECREF(r) - #define __Pyx_GOTREF(r) - #define __Pyx_GIVEREF(r) - #define __Pyx_XINCREF(r) Py_XINCREF(r) - #define __Pyx_XDECREF(r) Py_XDECREF(r) - #define __Pyx_XGOTREF(r) - #define __Pyx_XGIVEREF(r) -#endif -#define __Pyx_XDECREF_SET(r, v) do { \ - PyObject *tmp = (PyObject *) r; \ - r = v; __Pyx_XDECREF(tmp); \ - } while (0) -#define __Pyx_DECREF_SET(r, v) do { \ - PyObject *tmp = (PyObject *) r; \ - r = v; __Pyx_DECREF(tmp); \ - } while (0) -#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) -#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) - -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { - PyTypeObject* tp = Py_TYPE(obj); - if (likely(tp->tp_getattro)) - return tp->tp_getattro(obj, attr_name); -#if PY_MAJOR_VERSION < 3 - if (likely(tp->tp_getattr)) - return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); -#endif - return PyObject_GetAttr(obj, attr_name); -} -#else -#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) -#endif - -static PyObject *__Pyx_GetBuiltinName(PyObject *name); - -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); -#else -#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) -#endif - -static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); -static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); - -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); - -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); - -static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); - -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \ - const char* function_name); - -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact); - -static CYTHON_INLINE int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); - -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); - -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); -#endif - -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); - -static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb); -static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb); - -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); - -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); -#else -#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) -#endif - -#include - -static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases); - -static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, - PyObject *mkw, PyObject *modname, PyObject *doc); -static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, - PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); - -typedef struct { - int code_line; - PyCodeObject* code_object; -} __Pyx_CodeObjectCacheEntry; -struct __Pyx_CodeObjectCache { - int count; - int max_count; - __Pyx_CodeObjectCacheEntry* entries; -}; -static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); -static PyCodeObject *__pyx_find_code_object(int code_line); -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); - -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename); - -#include - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); - -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); - -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); - -#ifndef __Pyx_CppExn2PyErr -#include -#include -#include -#include -static void __Pyx_CppExn2PyErr() { - try { - if (PyErr_Occurred()) - ; // let the latest Python exn pass through and ignore the current one - else - throw; - } catch (const std::bad_alloc& exn) { - PyErr_SetString(PyExc_MemoryError, exn.what()); - } catch (const std::bad_cast& exn) { - PyErr_SetString(PyExc_TypeError, exn.what()); - } catch (const std::domain_error& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::invalid_argument& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::ios_base::failure& exn) { - PyErr_SetString(PyExc_IOError, exn.what()); - } catch (const std::out_of_range& exn) { - PyErr_SetString(PyExc_IndexError, exn.what()); - } catch (const std::overflow_error& exn) { - PyErr_SetString(PyExc_OverflowError, exn.what()); - } catch (const std::range_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::underflow_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::exception& exn) { - PyErr_SetString(PyExc_RuntimeError, exn.what()); - } - catch (...) - { - PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); - } -} -#endif - -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value); - -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_char(unsigned char value); - -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); - -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); - -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); - -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); - -static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); - -#define __Pyx_Generator_USED -#include -#include -typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *); -typedef struct { - PyObject_HEAD - __pyx_generator_body_t body; - PyObject *closure; - PyObject *exc_type; - PyObject *exc_value; - PyObject *exc_traceback; - PyObject *gi_weakreflist; - PyObject *classobj; - PyObject *yieldfrom; - PyObject *gi_name; - PyObject *gi_qualname; - int resume_label; - char is_running; -} __pyx_GeneratorObject; -static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body, - PyObject *closure, PyObject *name, PyObject *qualname); -static int __pyx_Generator_init(void); -static int __Pyx_Generator_clear(PyObject* self); -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); -#else -#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) -#endif - -static int __Pyx_check_binary_version(void); - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); - - -/* Module declarations from '_kenlm' */ - -/* Module declarations from 'kenlm' */ -static PyTypeObject *__pyx_ptype_5kenlm_FullScoreReturn = 0; -static PyTypeObject *__pyx_ptype_5kenlm_State = 0; -static PyTypeObject *__pyx_ptype_5kenlm_Config = 0; -static PyTypeObject *__pyx_ptype_5kenlm_Model = 0; -static PyTypeObject *__pyx_ptype_5kenlm___pyx_scope_struct__full_scores = 0; -static PyObject *__pyx_f_5kenlm_as_str(PyObject *); /*proto*/ -#define __Pyx_MODULE_NAME "kenlm" -int __pyx_module_is_main_kenlm = 0; - -/* Implementation of 'kenlm' */ -static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_builtin_RuntimeError; -static PyObject *__pyx_builtin_IOError; -static int __pyx_pf_5kenlm_15FullScoreReturn___cinit__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, PyObject *__pyx_v_log_prob, PyObject *__pyx_v_ngram_length, PyObject *__pyx_v_oov); /* proto */ -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5kenlm_5State___richcmp__(struct __pyx_obj_5kenlm_State *__pyx_v_qa, struct __pyx_obj_5kenlm_State *__pyx_v_qb, int __pyx_v_op); /* proto */ -static Py_hash_t __pyx_pf_5kenlm_5State_2__hash__(struct __pyx_obj_5kenlm_State *__pyx_v_self); /* proto */ -static int __pyx_pf_5kenlm_6Config___init__(struct __pyx_obj_5kenlm_Config *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5kenlm_6Config_11load_method___get__(struct __pyx_obj_5kenlm_Config *__pyx_v_self); /* proto */ -static int __pyx_pf_5kenlm_6Config_11load_method_2__set__(struct __pyx_obj_5kenlm_Config *__pyx_v_self, PyObject *__pyx_v_to); /* proto */ -static int __pyx_pf_5kenlm_5Model___init__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_path, struct __pyx_obj_5kenlm_Config *__pyx_v_config); /* proto */ -static void __pyx_pf_5kenlm_5Model_2__dealloc__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_5order___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_4score(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_6perplexity(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_8full_scores(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_11BeginSentenceWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_13NullContextWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_15BaseScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_17BaseFullScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state); /* proto */ -static int __pyx_pf_5kenlm_5Model_19__contains__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_word); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_21__repr__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_23__reduce__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5kenlm_5Model_4path___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ -static int __pyx_pf_5kenlm_5Model_4path_2__set__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_value); /* proto */ -static int __pyx_pf_5kenlm_5Model_4path_4__del__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */ -static PyObject *__pyx_tp_new_5kenlm_FullScoreReturn(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5kenlm_State(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5kenlm_Config(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5kenlm_Model(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static PyObject *__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ -static char __pyx_k__3[] = "\n"; -static char __pyx_k__4[] = " "; -static char __pyx_k_os[] = "os"; -static char __pyx_k_bos[] = "bos"; -static char __pyx_k_doc[] = "__doc__"; -static char __pyx_k_eos[] = "eos"; -static char __pyx_k_oov[] = "oov"; -static char __pyx_k_LAZY[] = "LAZY"; -static char __pyx_k_READ[] = "READ"; -static char __pyx_k_args[] = "args"; -static char __pyx_k_main[] = "__main__"; -static char __pyx_k_name[] = "__name__"; -static char __pyx_k_path[] = "path"; -static char __pyx_k_send[] = "send"; -static char __pyx_k_test[] = "__test__"; -static char __pyx_k_utf8[] = "utf8"; -static char __pyx_k_word[] = "word"; -static char __pyx_k_class[] = "__class__"; -static char __pyx_k_close[] = "close"; -static char __pyx_k_kenlm[] = "_kenlm"; -static char __pyx_k_score[] = "score"; -static char __pyx_k_split[] = "split"; -static char __pyx_k_throw[] = "throw"; -static char __pyx_k_config[] = "config"; -static char __pyx_k_encode[] = "encode"; -static char __pyx_k_format[] = "format"; -static char __pyx_k_import[] = "__import__"; -static char __pyx_k_module[] = "__module__"; -static char __pyx_k_0_1_2_3[] = "{0}({1}, {2}, {3})"; -static char __pyx_k_IOError[] = "IOError"; -static char __pyx_k_abspath[] = "abspath"; -static char __pyx_k_kenlm_2[] = "kenlm"; -static char __pyx_k_prepare[] = "__prepare__"; -static char __pyx_k_replace[] = "replace"; -static char __pyx_k_basename[] = "basename"; -static char __pyx_k_in_state[] = "in_state"; -static char __pyx_k_log_prob[] = "log_prob"; -static char __pyx_k_qualname[] = "__qualname__"; -static char __pyx_k_sentence[] = "sentence"; -static char __pyx_k_TypeError[] = "TypeError"; -static char __pyx_k_metaclass[] = "__metaclass__"; -static char __pyx_k_out_state[] = "out_state"; -static char __pyx_k_LoadMethod[] = "LoadMethod"; -static char __pyx_k_full_scores[] = "full_scores"; -static char __pyx_k_Model_from_0[] = ""; -static char __pyx_k_RuntimeError[] = "RuntimeError"; -static char __pyx_k_ngram_length[] = "ngram_length"; -static char __pyx_k_LanguageModel[] = "LanguageModel"; -static char __pyx_k_PARALLEL_READ[] = "PARALLEL_READ"; -static char __pyx_k_POPULATE_OR_LAZY[] = "POPULATE_OR_LAZY"; -static char __pyx_k_POPULATE_OR_READ[] = "POPULATE_OR_READ"; -static char __pyx_k_Cannot_read_model[] = "Cannot read model '{}' ({})"; -static char __pyx_k_Model_full_scores[] = "Model.full_scores"; -static char __pyx_k_Cannot_convert_s_to_string[] = "Cannot convert %s to string"; -static char __pyx_k_Backwards_compatability_stub_Use[] = "Backwards compatability stub. Use Model."; -static PyObject *__pyx_kp_s_0_1_2_3; -static PyObject *__pyx_kp_s_Backwards_compatability_stub_Use; -static PyObject *__pyx_kp_s_Cannot_convert_s_to_string; -static PyObject *__pyx_kp_s_Cannot_read_model; -static PyObject *__pyx_n_s_IOError; -static PyObject *__pyx_n_s_LAZY; -static PyObject *__pyx_n_s_LanguageModel; -static PyObject *__pyx_n_s_LoadMethod; -static PyObject *__pyx_kp_s_Model_from_0; -static PyObject *__pyx_n_s_Model_full_scores; -static PyObject *__pyx_n_s_PARALLEL_READ; -static PyObject *__pyx_n_s_POPULATE_OR_LAZY; -static PyObject *__pyx_n_s_POPULATE_OR_READ; -static PyObject *__pyx_n_s_READ; -static PyObject *__pyx_n_s_RuntimeError; -static PyObject *__pyx_n_s_TypeError; -static PyObject *__pyx_kp_s__3; -static PyObject *__pyx_kp_s__4; -static PyObject *__pyx_n_s_abspath; -static PyObject *__pyx_n_s_args; -static PyObject *__pyx_n_s_basename; -static PyObject *__pyx_n_s_bos; -static PyObject *__pyx_n_s_class; -static PyObject *__pyx_n_s_close; -static PyObject *__pyx_n_s_config; -static PyObject *__pyx_n_s_doc; -static PyObject *__pyx_n_s_encode; -static PyObject *__pyx_n_s_eos; -static PyObject *__pyx_n_s_format; -static PyObject *__pyx_n_s_full_scores; -static PyObject *__pyx_n_s_import; -static PyObject *__pyx_n_s_in_state; -static PyObject *__pyx_n_s_kenlm; -static PyObject *__pyx_n_s_kenlm_2; -static PyObject *__pyx_n_s_log_prob; -static PyObject *__pyx_n_s_main; -static PyObject *__pyx_n_s_metaclass; -static PyObject *__pyx_n_s_module; -static PyObject *__pyx_n_s_name; -static PyObject *__pyx_n_s_ngram_length; -static PyObject *__pyx_n_s_oov; -static PyObject *__pyx_n_s_os; -static PyObject *__pyx_n_s_out_state; -static PyObject *__pyx_n_s_path; -static PyObject *__pyx_n_s_prepare; -static PyObject *__pyx_n_s_qualname; -static PyObject *__pyx_n_s_replace; -static PyObject *__pyx_n_s_score; -static PyObject *__pyx_n_s_send; -static PyObject *__pyx_n_s_sentence; -static PyObject *__pyx_n_s_split; -static PyObject *__pyx_n_s_test; -static PyObject *__pyx_n_s_throw; -static PyObject *__pyx_n_s_utf8; -static PyObject *__pyx_n_s_word; -static PyObject *__pyx_float_10_0; -static struct __pyx_obj_5kenlm_Config *__pyx_k__2; -static PyObject *__pyx_tuple_; -static PyObject *__pyx_tuple__5; - -/* "kenlm.pyx":4 - * cimport _kenlm - * - * cdef bytes as_str(data): # <<<<<<<<<<<<<< - * if isinstance(data, bytes): - * return data - */ - -static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - int __pyx_t_1; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("as_str", 0); - - /* "kenlm.pyx":5 - * - * cdef bytes as_str(data): - * if isinstance(data, bytes): # <<<<<<<<<<<<<< - * return data - * elif isinstance(data, unicode): - */ - __pyx_t_1 = PyBytes_Check(__pyx_v_data); - __pyx_t_2 = (__pyx_t_1 != 0); - if (__pyx_t_2) { - - /* "kenlm.pyx":6 - * cdef bytes as_str(data): - * if isinstance(data, bytes): - * return data # <<<<<<<<<<<<<< - * elif isinstance(data, unicode): - * return data.encode('utf8') - */ - __Pyx_XDECREF(__pyx_r); - if (!(likely(PyBytes_CheckExact(__pyx_v_data))||((__pyx_v_data) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_v_data)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_INCREF(__pyx_v_data); - __pyx_r = ((PyObject*)__pyx_v_data); - goto __pyx_L0; - } - - /* "kenlm.pyx":7 - * if isinstance(data, bytes): - * return data - * elif isinstance(data, unicode): # <<<<<<<<<<<<<< - * return data.encode('utf8') - * raise TypeError('Cannot convert %s to string' % type(data)) - */ - __pyx_t_2 = PyUnicode_Check(__pyx_v_data); - __pyx_t_1 = (__pyx_t_2 != 0); - if (__pyx_t_1) { - - /* "kenlm.pyx":8 - * return data - * elif isinstance(data, unicode): - * return data.encode('utf8') # <<<<<<<<<<<<<< - * raise TypeError('Cannot convert %s to string' % type(data)) - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_data, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_4)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = ((PyObject*)__pyx_t_4); - __pyx_t_4 = 0; - goto __pyx_L0; - } - - /* "kenlm.pyx":9 - * elif isinstance(data, unicode): - * return data.encode('utf8') - * raise TypeError('Cannot convert %s to string' % type(data)) # <<<<<<<<<<<<<< - * - * cdef class FullScoreReturn: - */ - __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Cannot_convert_s_to_string, ((PyObject *)Py_TYPE(__pyx_v_data))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - - /* "kenlm.pyx":4 - * cimport _kenlm - * - * cdef bytes as_str(data): # <<<<<<<<<<<<<< - * if isinstance(data, bytes): - * return data - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("kenlm.as_str", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":24 - * cdef bint oov - * - * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<< - * self.log_prob = log_prob - * self.ngram_length = ngram_length - */ - -/* Python wrapper */ -static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_log_prob = 0; - PyObject *__pyx_v_ngram_length = 0; - PyObject *__pyx_v_oov = 0; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_log_prob,&__pyx_n_s_ngram_length,&__pyx_n_s_oov,0}; - PyObject* values[3] = {0,0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_log_prob)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_ngram_length)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_oov)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - } - __pyx_v_log_prob = values[0]; - __pyx_v_ngram_length = values[1]; - __pyx_v_oov = values[2]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn___cinit__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self), __pyx_v_log_prob, __pyx_v_ngram_length, __pyx_v_oov); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5kenlm_15FullScoreReturn___cinit__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, PyObject *__pyx_v_log_prob, PyObject *__pyx_v_ngram_length, PyObject *__pyx_v_oov) { - int __pyx_r; - __Pyx_RefNannyDeclarations - float __pyx_t_1; - int __pyx_t_2; - int __pyx_t_3; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__cinit__", 0); - - /* "kenlm.pyx":25 - * - * def __cinit__(self, log_prob, ngram_length, oov): - * self.log_prob = log_prob # <<<<<<<<<<<<<< - * self.ngram_length = ngram_length - * self.oov = oov - */ - __pyx_t_1 = __pyx_PyFloat_AsFloat(__pyx_v_log_prob); if (unlikely((__pyx_t_1 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 25; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_self->log_prob = __pyx_t_1; - - /* "kenlm.pyx":26 - * def __cinit__(self, log_prob, ngram_length, oov): - * self.log_prob = log_prob - * self.ngram_length = ngram_length # <<<<<<<<<<<<<< - * self.oov = oov - * - */ - __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_ngram_length); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_self->ngram_length = __pyx_t_2; - - /* "kenlm.pyx":27 - * self.log_prob = log_prob - * self.ngram_length = ngram_length - * self.oov = oov # <<<<<<<<<<<<<< - * - * def __repr__(self): - */ - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v_oov); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_self->oov = __pyx_t_3; - - /* "kenlm.pyx":24 - * cdef bint oov - * - * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<< - * self.log_prob = log_prob - * self.ngram_length = ngram_length - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":29 - * self.oov = oov - * - * def __repr__(self): # <<<<<<<<<<<<<< - * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_2__repr__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - Py_ssize_t __pyx_t_8; - PyObject *__pyx_t_9 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__repr__", 0); - - /* "kenlm.pyx":30 - * - * def __repr__(self): - * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) # <<<<<<<<<<<<<< - * - * property log_prob: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_0_1_2_3, __pyx_n_s_format); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_class); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_name); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_6 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_7 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = NULL; - __pyx_t_8 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); - if (likely(__pyx_t_3)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); - __Pyx_INCREF(__pyx_t_3); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_2, function); - __pyx_t_8 = 1; - } - } - __pyx_t_9 = PyTuple_New(4+__pyx_t_8); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_9); - if (__pyx_t_3) { - PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = NULL; - } - PyTuple_SET_ITEM(__pyx_t_9, 0+__pyx_t_8, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_9, 1+__pyx_t_8, __pyx_t_5); - __Pyx_GIVEREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_9, 2+__pyx_t_8, __pyx_t_6); - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_9, 3+__pyx_t_8, __pyx_t_7); - __Pyx_GIVEREF(__pyx_t_7); - __pyx_t_4 = 0; - __pyx_t_5 = 0; - __pyx_t_6 = 0; - __pyx_t_7 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_9, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":29 - * self.oov = oov - * - * def __repr__(self): # <<<<<<<<<<<<<< - * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_9); - __Pyx_AddTraceback("kenlm.FullScoreReturn.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":33 - * - * property log_prob: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.log_prob - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "kenlm.pyx":34 - * property log_prob: - * def __get__(self): - * return self.log_prob # <<<<<<<<<<<<<< - * - * property ngram_length: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":33 - * - * property log_prob: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.log_prob - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.FullScoreReturn.log_prob.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":37 - * - * property ngram_length: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.ngram_length - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "kenlm.pyx":38 - * property ngram_length: - * def __get__(self): - * return self.ngram_length # <<<<<<<<<<<<<< - * - * property oov: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":37 - * - * property ngram_length: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.ngram_length - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.FullScoreReturn.ngram_length.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":41 - * - * property oov: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.oov - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "kenlm.pyx":42 - * property oov: - * def __get__(self): - * return self.oov # <<<<<<<<<<<<<< - * - * cdef class State: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":41 - * - * property oov: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.oov - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.FullScoreReturn.oov.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":55 - * cdef _kenlm.State _c_state - * - * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<< - * r = qa._c_state.Compare(qb._c_state) - * if op == 0: # < - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op) { - CYTHON_UNUSED int __pyx_lineno = 0; - CYTHON_UNUSED const char *__pyx_filename = NULL; - CYTHON_UNUSED int __pyx_clineno = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qa), __pyx_ptype_5kenlm_State, 1, "qa", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qb), __pyx_ptype_5kenlm_State, 1, "qb", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = __pyx_pf_5kenlm_5State___richcmp__(((struct __pyx_obj_5kenlm_State *)__pyx_v_qa), ((struct __pyx_obj_5kenlm_State *)__pyx_v_qb), ((int)__pyx_v_op)); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5State___richcmp__(struct __pyx_obj_5kenlm_State *__pyx_v_qa, struct __pyx_obj_5kenlm_State *__pyx_v_qb, int __pyx_v_op) { - int __pyx_v_r; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__richcmp__", 0); - - /* "kenlm.pyx":56 - * - * def __richcmp__(State qa, State qb, int op): - * r = qa._c_state.Compare(qb._c_state) # <<<<<<<<<<<<<< - * if op == 0: # < - * return r < 0 - */ - __pyx_v_r = __pyx_v_qa->_c_state.Compare(__pyx_v_qb->_c_state); - - /* "kenlm.pyx":65 - * elif op == 3: # != - * return r != 0 - * elif op == 4: # > # <<<<<<<<<<<<<< - * return r > 0 - * else: # >= - */ - switch (__pyx_v_op) { - - /* "kenlm.pyx":57 - * def __richcmp__(State qa, State qb, int op): - * r = qa._c_state.Compare(qb._c_state) - * if op == 0: # < # <<<<<<<<<<<<<< - * return r < 0 - * elif op == 1: # <= - */ - case 0: - - /* "kenlm.pyx":58 - * r = qa._c_state.Compare(qb._c_state) - * if op == 0: # < - * return r < 0 # <<<<<<<<<<<<<< - * elif op == 1: # <= - * return r <= 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r < 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - break; - - /* "kenlm.pyx":59 - * if op == 0: # < - * return r < 0 - * elif op == 1: # <= # <<<<<<<<<<<<<< - * return r <= 0 - * elif op == 2: # == - */ - case 1: - - /* "kenlm.pyx":60 - * return r < 0 - * elif op == 1: # <= - * return r <= 0 # <<<<<<<<<<<<<< - * elif op == 2: # == - * return r == 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r <= 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - break; - - /* "kenlm.pyx":61 - * elif op == 1: # <= - * return r <= 0 - * elif op == 2: # == # <<<<<<<<<<<<<< - * return r == 0 - * elif op == 3: # != - */ - case 2: - - /* "kenlm.pyx":62 - * return r <= 0 - * elif op == 2: # == - * return r == 0 # <<<<<<<<<<<<<< - * elif op == 3: # != - * return r != 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r == 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - break; - - /* "kenlm.pyx":63 - * elif op == 2: # == - * return r == 0 - * elif op == 3: # != # <<<<<<<<<<<<<< - * return r != 0 - * elif op == 4: # > - */ - case 3: - - /* "kenlm.pyx":64 - * return r == 0 - * elif op == 3: # != - * return r != 0 # <<<<<<<<<<<<<< - * elif op == 4: # > - * return r > 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r != 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - break; - - /* "kenlm.pyx":65 - * elif op == 3: # != - * return r != 0 - * elif op == 4: # > # <<<<<<<<<<<<<< - * return r > 0 - * else: # >= - */ - case 4: - - /* "kenlm.pyx":66 - * return r != 0 - * elif op == 4: # > - * return r > 0 # <<<<<<<<<<<<<< - * else: # >= - * return r >= 0 - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r > 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - break; - default: - - /* "kenlm.pyx":68 - * return r > 0 - * else: # >= - * return r >= 0 # <<<<<<<<<<<<<< - * - * def __hash__(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r >= 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - break; - } - - /* "kenlm.pyx":55 - * cdef _kenlm.State _c_state - * - * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<< - * r = qa._c_state.Compare(qb._c_state) - * if op == 0: # < - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.State.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":70 - * return r >= 0 - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return _kenlm.hash_value(self._c_state) - * - */ - -/* Python wrapper */ -static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self); /*proto*/ -static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self) { - Py_hash_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5State_2__hash__(((struct __pyx_obj_5kenlm_State *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static Py_hash_t __pyx_pf_5kenlm_5State_2__hash__(struct __pyx_obj_5kenlm_State *__pyx_v_self) { - Py_hash_t __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__hash__", 0); - - /* "kenlm.pyx":71 - * - * def __hash__(self): - * return _kenlm.hash_value(self._c_state) # <<<<<<<<<<<<<< - * - * class LoadMethod: - */ - __pyx_r = lm::ngram::hash_value(__pyx_v_self->_c_state); - goto __pyx_L0; - - /* "kenlm.pyx":70 - * return r >= 0 - * - * def __hash__(self): # <<<<<<<<<<<<<< - * return _kenlm.hash_value(self._c_state) - * - */ - - /* function exit code */ - __pyx_L0:; - if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":87 - * cdef _kenlm.Config _c_config - * - * def __init__(self): # <<<<<<<<<<<<<< - * self._c_config = _kenlm.Config() - * - */ - -/* Python wrapper */ -static int __pyx_pw_5kenlm_6Config_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pw_5kenlm_6Config_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); - if (unlikely(PyTuple_GET_SIZE(__pyx_args) > 0)) { - __Pyx_RaiseArgtupleInvalid("__init__", 1, 0, 0, PyTuple_GET_SIZE(__pyx_args)); return -1;} - if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "__init__", 0))) return -1; - __pyx_r = __pyx_pf_5kenlm_6Config___init__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5kenlm_6Config___init__(struct __pyx_obj_5kenlm_Config *__pyx_v_self) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__init__", 0); - - /* "kenlm.pyx":88 - * - * def __init__(self): - * self._c_config = _kenlm.Config() # <<<<<<<<<<<<<< - * - * property load_method: - */ - __pyx_v_self->_c_config = lm::ngram::Config(); - - /* "kenlm.pyx":87 - * cdef _kenlm.Config _c_config - * - * def __init__(self): # <<<<<<<<<<<<<< - * self._c_config = _kenlm.Config() - * - */ - - /* function exit code */ - __pyx_r = 0; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":91 - * - * property load_method: - * def __get__(self): # <<<<<<<<<<<<<< - * return self._c_config.load_method - * def __set__(self, to): - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_6Config_11load_method_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_6Config_11load_method_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_6Config_11load_method___get__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_6Config_11load_method___get__(struct __pyx_obj_5kenlm_Config *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "kenlm.pyx":92 - * property load_method: - * def __get__(self): - * return self._c_config.load_method # <<<<<<<<<<<<<< - * def __set__(self, to): - * self._c_config.load_method = to - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyInt_FromLong(__pyx_v_self->_c_config.load_method); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":91 - * - * property load_method: - * def __get__(self): # <<<<<<<<<<<<<< - * return self._c_config.load_method - * def __set__(self, to): - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Config.load_method.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":93 - * def __get__(self): - * return self._c_config.load_method - * def __set__(self, to): # <<<<<<<<<<<<<< - * self._c_config.load_method = to - * - */ - -/* Python wrapper */ -static int __pyx_pw_5kenlm_6Config_11load_method_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_to); /*proto*/ -static int __pyx_pw_5kenlm_6Config_11load_method_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_to) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__set__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_6Config_11load_method_2__set__(((struct __pyx_obj_5kenlm_Config *)__pyx_v_self), ((PyObject *)__pyx_v_to)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5kenlm_6Config_11load_method_2__set__(struct __pyx_obj_5kenlm_Config *__pyx_v_self, PyObject *__pyx_v_to) { - int __pyx_r; - __Pyx_RefNannyDeclarations - enum util::LoadMethod __pyx_t_1; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__set__", 0); - - /* "kenlm.pyx":94 - * return self._c_config.load_method - * def __set__(self, to): - * self._c_config.load_method = to # <<<<<<<<<<<<<< - * - * cdef class Model: - */ - __pyx_t_1 = ((enum util::LoadMethod)PyInt_AsLong(__pyx_v_to)); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_self->_c_config.load_method = __pyx_t_1; - - /* "kenlm.pyx":93 - * def __get__(self): - * return self._c_config.load_method - * def __set__(self, to): # <<<<<<<<<<<<<< - * self._c_config.load_method = to - * - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("kenlm.Config.load_method.__set__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":105 - * cdef _kenlm.const_Vocabulary* vocab - * - * def __init__(self, path, Config config = Config()): # <<<<<<<<<<<<<< - * """ - * Load the language model. - */ - -/* Python wrapper */ -static int __pyx_pw_5kenlm_5Model_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model___init__[] = "\n Load the language model.\n\n :param path: path to an arpa file or a kenlm binary file.\n :param config: configuration options (see lm/config.hh for documentation)\n "; -#if CYTHON_COMPILING_IN_CPYTHON -struct wrapperbase __pyx_wrapperbase_5kenlm_5Model___init__; -#endif -static int __pyx_pw_5kenlm_5Model_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_path = 0; - struct __pyx_obj_5kenlm_Config *__pyx_v_config = 0; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_path,&__pyx_n_s_config,0}; - PyObject* values[2] = {0,0}; - values[1] = (PyObject *)__pyx_k__2; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_config); - if (value) { values[1] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - __pyx_v_path = values[0]; - __pyx_v_config = ((struct __pyx_obj_5kenlm_Config *)values[1]); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return -1; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_config), __pyx_ptype_5kenlm_Config, 1, "config", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = __pyx_pf_5kenlm_5Model___init__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_path, __pyx_v_config); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5kenlm_5Model___init__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_path, struct __pyx_obj_5kenlm_Config *__pyx_v_config) { - PyObject *__pyx_v_exception = NULL; - PyObject *__pyx_v_exception_message = NULL; - int __pyx_r; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - char *__pyx_t_9; - lm::base::Model *__pyx_t_10; - int __pyx_t_11; - PyObject *__pyx_t_12 = NULL; - Py_ssize_t __pyx_t_13; - PyObject *__pyx_t_14 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__init__", 0); - - /* "kenlm.pyx":112 - * :param config: configuration options (see lm/config.hh for documentation) - * """ - * self.path = os.path.abspath(as_str(path)) # <<<<<<<<<<<<<< - * try: - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - */ - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_os); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_abspath); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __pyx_f_5kenlm_as_str(__pyx_v_path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); - if (likely(__pyx_t_4)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); - __Pyx_INCREF(__pyx_t_4); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_2, function); - } - } - if (!__pyx_t_4) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else { - __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = NULL; - PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GIVEREF(__pyx_t_1); - __Pyx_GOTREF(__pyx_v_self->path); - __Pyx_DECREF(__pyx_v_self->path); - __pyx_v_self->path = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; - - /* "kenlm.pyx":113 - * """ - * self.path = os.path.abspath(as_str(path)) - * try: # <<<<<<<<<<<<<< - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - * except RuntimeError as exception: - */ - { - __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); - __Pyx_XGOTREF(__pyx_t_6); - __Pyx_XGOTREF(__pyx_t_7); - __Pyx_XGOTREF(__pyx_t_8); - /*try:*/ { - - /* "kenlm.pyx":114 - * self.path = os.path.abspath(as_str(path)) - * try: - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) # <<<<<<<<<<<<<< - * except RuntimeError as exception: - * exception_message = str(exception).replace('\n', ' ') - */ - __pyx_t_9 = __Pyx_PyObject_AsString(__pyx_v_self->path); if (unlikely((!__pyx_t_9) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - try { - __pyx_t_10 = lm::ngram::LoadVirtual(__pyx_t_9, __pyx_v_config->_c_config); - } catch(...) { - __Pyx_CppExn2PyErr(); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_self->model = __pyx_t_10; - } - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - goto __pyx_L10_try_end; - __pyx_L3_error:; - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "kenlm.pyx":115 - * try: - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - * except RuntimeError as exception: # <<<<<<<<<<<<<< - * exception_message = str(exception).replace('\n', ' ') - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ - */ - __pyx_t_11 = PyErr_ExceptionMatches(__pyx_builtin_RuntimeError); - if (__pyx_t_11) { - __Pyx_AddTraceback("kenlm.Model.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_1, &__pyx_t_2, &__pyx_t_5) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GOTREF(__pyx_t_5); - __Pyx_INCREF(__pyx_t_2); - __pyx_v_exception = __pyx_t_2; - - /* "kenlm.pyx":116 - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - * except RuntimeError as exception: - * exception_message = str(exception).replace('\n', ' ') # <<<<<<<<<<<<<< - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ - * from exception - */ - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_INCREF(__pyx_v_exception); - PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_exception); - __Pyx_GIVEREF(__pyx_v_exception); - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_replace); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_v_exception_message = __pyx_t_4; - __pyx_t_4 = 0; - - /* "kenlm.pyx":117 - * except RuntimeError as exception: - * exception_message = str(exception).replace('\n', ' ') - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ # <<<<<<<<<<<<<< - * from exception - * self.vocab = &self.model.BaseVocabulary() - */ - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_Cannot_read_model, __pyx_n_s_format); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_12 = NULL; - __pyx_t_13 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_12 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_12)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_12); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - __pyx_t_13 = 1; - } - } - __pyx_t_14 = PyTuple_New(2+__pyx_t_13); if (unlikely(!__pyx_t_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_14); - if (__pyx_t_12) { - PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_12); __Pyx_GIVEREF(__pyx_t_12); __pyx_t_12 = NULL; - } - __Pyx_INCREF(__pyx_v_path); - PyTuple_SET_ITEM(__pyx_t_14, 0+__pyx_t_13, __pyx_v_path); - __Pyx_GIVEREF(__pyx_v_path); - __Pyx_INCREF(__pyx_v_exception_message); - PyTuple_SET_ITEM(__pyx_t_14, 1+__pyx_t_13, __pyx_v_exception_message); - __Pyx_GIVEREF(__pyx_v_exception_message); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_14, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_IOError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "kenlm.pyx":118 - * exception_message = str(exception).replace('\n', ' ') - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ - * from exception # <<<<<<<<<<<<<< - * self.vocab = &self.model.BaseVocabulary() - * - */ - __Pyx_Raise(__pyx_t_4, 0, 0, __pyx_v_exception); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - } - goto __pyx_L5_except_error; - __pyx_L5_except_error:; - __Pyx_XGIVEREF(__pyx_t_6); - __Pyx_XGIVEREF(__pyx_t_7); - __Pyx_XGIVEREF(__pyx_t_8); - __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); - goto __pyx_L1_error; - __pyx_L10_try_end:; - } - - /* "kenlm.pyx":119 - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ - * from exception - * self.vocab = &self.model.BaseVocabulary() # <<<<<<<<<<<<<< - * - * def __dealloc__(self): - */ - __pyx_v_self->vocab = (&__pyx_v_self->model->BaseVocabulary()); - - /* "kenlm.pyx":105 - * cdef _kenlm.const_Vocabulary* vocab - * - * def __init__(self, path, Config config = Config()): # <<<<<<<<<<<<<< - * """ - * Load the language model. - */ - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_12); - __Pyx_XDECREF(__pyx_t_14); - __Pyx_AddTraceback("kenlm.Model.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_exception); - __Pyx_XDECREF(__pyx_v_exception_message); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":121 - * self.vocab = &self.model.BaseVocabulary() - * - * def __dealloc__(self): # <<<<<<<<<<<<<< - * del self.model - * - */ - -/* Python wrapper */ -static void __pyx_pw_5kenlm_5Model_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ -static void __pyx_pw_5kenlm_5Model_3__dealloc__(PyObject *__pyx_v_self) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); - __pyx_pf_5kenlm_5Model_2__dealloc__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} - -static void __pyx_pf_5kenlm_5Model_2__dealloc__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__dealloc__", 0); - - /* "kenlm.pyx":122 - * - * def __dealloc__(self): - * del self.model # <<<<<<<<<<<<<< - * - * property order: - */ - delete __pyx_v_self->model; - - /* "kenlm.pyx":121 - * self.vocab = &self.model.BaseVocabulary() - * - * def __dealloc__(self): # <<<<<<<<<<<<<< - * del self.model - * - */ - - /* function exit code */ - __Pyx_RefNannyFinishContext(); -} - -/* "kenlm.pyx":125 - * - * property order: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.model.Order() - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_5order_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5Model_5order_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_5order___get__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_5order___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__get__", 0); - - /* "kenlm.pyx":126 - * property order: - * def __get__(self): - * return self.model.Order() # <<<<<<<<<<<<<< - * - * def score(self, sentence, bos = True, eos = True): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_unsigned_int(__pyx_v_self->model->Order()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":125 - * - * property order: - * def __get__(self): # <<<<<<<<<<<<<< - * return self.model.Order() - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Model.order.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":128 - * return self.model.Order() - * - * def score(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< - * """ - * Return the log10 probability of a string. By default, the string is - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model_4score[] = "\n Return the log10 probability of a string. By default, the string is\n treated as a sentence. \n return log10 p(sentence | )\n\n If you do not want to condition on the beginning of sentence, pass\n bos = False\n Never include as part of the string. That would be predicting the\n beginning of sentence. Language models are only supposed to condition\n on it as context.\n\n Similarly, the end of sentence token can be omitted with\n eos = False\n Since language models explicitly predict , it can be part of the\n string.\n\n Examples:\n\n #Good: returns log10 p(this is a sentence . | )\n model.score(\"this is a sentence .\")\n #Good: same as the above but more explicit\n model.score(\"this is a sentence .\", bos = True, eos = True)\n\n #Bad: never include \n model.score(\" this is a sentence\")\n #Bad: never include , even if bos = False.\n model.score(\" this is a sentence\", bos = False)\n\n #Good: returns log10 p(a fragment)\n model.score(\"a fragment\", bos = False, eos = False)\n\n #Good: returns log10 p(a fragment )\n model.score(\"a fragment\", bos = False, eos = True)\n\n #Ok, but bad practice: returns log10 p(a fragment )\n #Unlike , the end of sentence token can appear explicitly.\n model.score(\"a fragment \", bos = False, eos = False)\n "; -static PyObject *__pyx_pw_5kenlm_5Model_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_sentence = 0; - PyObject *__pyx_v_bos = 0; - PyObject *__pyx_v_eos = 0; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("score (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_sentence,&__pyx_n_s_bos,&__pyx_n_s_eos,0}; - PyObject* values[3] = {0,0,0}; - values[1] = ((PyObject *)Py_True); - values[2] = ((PyObject *)Py_True); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_bos); - if (value) { values[1] = value; kw_args--; } - } - case 2: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_eos); - if (value) { values[2] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "score") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - __pyx_v_sentence = values[0]; - __pyx_v_bos = values[1]; - __pyx_v_eos = values[2]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("score", 0, 1, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.score", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5kenlm_5Model_4score(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_sentence, __pyx_v_bos, __pyx_v_eos); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_4score(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos) { - PyObject *__pyx_v_words = 0; - lm::ngram::State __pyx_v_state; - lm::ngram::State __pyx_v_out_state; - float __pyx_v_total; - PyObject *__pyx_v_word = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - int __pyx_t_4; - Py_ssize_t __pyx_t_5; - char *__pyx_t_6; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("score", 0); - - /* "kenlm.pyx":167 - * model.score("a fragment ", bos = False, eos = False) - * """ - * cdef list words = as_str(sentence).split() # <<<<<<<<<<<<<< - * cdef _kenlm.State state - * if bos: - */ - __pyx_t_2 = __pyx_f_5kenlm_as_str(__pyx_v_sentence); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_split); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_2); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "list", Py_TYPE(__pyx_t_1)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_words = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; - - /* "kenlm.pyx":169 - * cdef list words = as_str(sentence).split() - * cdef _kenlm.State state - * if bos: # <<<<<<<<<<<<<< - * self.model.BeginSentenceWrite(&state) - * else: - */ - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_bos); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 169; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_4) { - - /* "kenlm.pyx":170 - * cdef _kenlm.State state - * if bos: - * self.model.BeginSentenceWrite(&state) # <<<<<<<<<<<<<< - * else: - * self.model.NullContextWrite(&state) - */ - __pyx_v_self->model->BeginSentenceWrite((&__pyx_v_state)); - goto __pyx_L3; - } - /*else*/ { - - /* "kenlm.pyx":172 - * self.model.BeginSentenceWrite(&state) - * else: - * self.model.NullContextWrite(&state) # <<<<<<<<<<<<<< - * cdef _kenlm.State out_state - * cdef float total = 0 - */ - __pyx_v_self->model->NullContextWrite((&__pyx_v_state)); - } - __pyx_L3:; - - /* "kenlm.pyx":174 - * self.model.NullContextWrite(&state) - * cdef _kenlm.State out_state - * cdef float total = 0 # <<<<<<<<<<<<<< - * for word in words: - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - */ - __pyx_v_total = 0.0; - - /* "kenlm.pyx":175 - * cdef _kenlm.State out_state - * cdef float total = 0 - * for word in words: # <<<<<<<<<<<<<< - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - * state = out_state - */ - if (unlikely(__pyx_v_words == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __pyx_t_1 = __pyx_v_words; __Pyx_INCREF(__pyx_t_1); __pyx_t_5 = 0; - for (;;) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_3 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_3); __pyx_t_5++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_1, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif - __Pyx_XDECREF_SET(__pyx_v_word, __pyx_t_3); - __pyx_t_3 = 0; - - /* "kenlm.pyx":176 - * cdef float total = 0 - * for word in words: - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) # <<<<<<<<<<<<<< - * state = out_state - * if eos: - */ - __pyx_t_6 = __Pyx_PyObject_AsString(__pyx_v_word); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 176; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->BaseScore((&__pyx_v_state), __pyx_v_self->vocab->Index(__pyx_t_6), (&__pyx_v_out_state))); - - /* "kenlm.pyx":177 - * for word in words: - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - * state = out_state # <<<<<<<<<<<<<< - * if eos: - * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) - */ - __pyx_v_state = __pyx_v_out_state; - - /* "kenlm.pyx":175 - * cdef _kenlm.State out_state - * cdef float total = 0 - * for word in words: # <<<<<<<<<<<<<< - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - * state = out_state - */ - } - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "kenlm.pyx":178 - * total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - * state = out_state - * if eos: # <<<<<<<<<<<<<< - * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) - * return total - */ - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_eos); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_4) { - - /* "kenlm.pyx":179 - * state = out_state - * if eos: - * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) # <<<<<<<<<<<<<< - * return total - * - */ - __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->BaseScore((&__pyx_v_state), __pyx_v_self->vocab->EndSentence(), (&__pyx_v_out_state))); - goto __pyx_L6; - } - __pyx_L6:; - - /* "kenlm.pyx":180 - * if eos: - * total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) - * return total # <<<<<<<<<<<<<< - * - * def perplexity(self, sentence): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_total); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 180; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":128 - * return self.model.Order() - * - * def score(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< - * """ - * Return the log10 probability of a string. By default, the string is - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("kenlm.Model.score", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_words); - __Pyx_XDECREF(__pyx_v_word); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":182 - * return total - * - * def perplexity(self, sentence): # <<<<<<<<<<<<<< - * """ - * Compute perplexity of a sentence. - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_7perplexity(PyObject *__pyx_v_self, PyObject *__pyx_v_sentence); /*proto*/ -static char __pyx_doc_5kenlm_5Model_6perplexity[] = "\n Compute perplexity of a sentence.\n @param sentence One full sentence to score. Do not include or .\n "; -static PyObject *__pyx_pw_5kenlm_5Model_7perplexity(PyObject *__pyx_v_self, PyObject *__pyx_v_sentence) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("perplexity (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_6perplexity(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((PyObject *)__pyx_v_sentence)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_6perplexity(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence) { - PyObject *__pyx_v_words = NULL; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - Py_ssize_t __pyx_t_4; - PyObject *__pyx_t_5 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("perplexity", 0); - - /* "kenlm.pyx":187 - * @param sentence One full sentence to score. Do not include or . - * """ - * words = len(as_str(sentence).split()) + 1 # For # <<<<<<<<<<<<<< - * return 10.0**(-self.score(sentence) / words) - * - */ - __pyx_t_2 = __pyx_f_5kenlm_as_str(__pyx_v_sentence); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_split); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_2); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_4 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_4 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyInt_FromSsize_t((__pyx_t_4 + 1)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_words = __pyx_t_1; - __pyx_t_1 = 0; - - /* "kenlm.pyx":188 - * """ - * words = len(as_str(sentence).split()) + 1 # For - * return 10.0**(-self.score(sentence) / words) # <<<<<<<<<<<<<< - * - * def full_scores(self, sentence, bos = True, eos = True): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_score); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_2); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - if (!__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_v_sentence); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - } else { - __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_2); __Pyx_GIVEREF(__pyx_t_2); __pyx_t_2 = NULL; - __Pyx_INCREF(__pyx_v_sentence); - PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_v_sentence); - __Pyx_GIVEREF(__pyx_v_sentence); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - } - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyNumber_Negative(__pyx_t_1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyNumber_Divide(__pyx_t_3, __pyx_v_words); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyNumber_Power(__pyx_float_10_0, __pyx_t_1, Py_None); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":182 - * return total - * - * def perplexity(self, sentence): # <<<<<<<<<<<<<< - * """ - * Compute perplexity of a sentence. - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_AddTraceback("kenlm.Model.perplexity", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_words); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} -static PyObject *__pyx_gb_5kenlm_5Model_10generator(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ - -/* "kenlm.pyx":190 - * return 10.0**(-self.score(sentence) / words) - * - * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< - * """ - * full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_9full_scores(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model_8full_scores[] = "\n full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov)\n @param sentence is a string (do not use boundary symbols)\n @param bos should kenlm add a bos state\n @param eos should kenlm add an eos state\n "; -static PyObject *__pyx_pw_5kenlm_5Model_9full_scores(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_sentence = 0; - PyObject *__pyx_v_bos = 0; - PyObject *__pyx_v_eos = 0; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("full_scores (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_sentence,&__pyx_n_s_bos,&__pyx_n_s_eos,0}; - PyObject* values[3] = {0,0,0}; - values[1] = ((PyObject *)Py_True); - values[2] = ((PyObject *)Py_True); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_sentence)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_bos); - if (value) { values[1] = value; kw_args--; } - } - case 2: - if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_eos); - if (value) { values[2] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "full_scores") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } else { - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - __pyx_v_sentence = values[0]; - __pyx_v_bos = values[1]; - __pyx_v_eos = values[2]; - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("full_scores", 0, 1, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.full_scores", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5kenlm_5Model_8full_scores(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_sentence, __pyx_v_bos, __pyx_v_eos); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_8full_scores(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos) { - struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *__pyx_cur_scope; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("full_scores", 0); - __pyx_cur_scope = (struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(__pyx_ptype_5kenlm___pyx_scope_struct__full_scores, __pyx_empty_tuple, NULL); - if (unlikely(!__pyx_cur_scope)) { - __Pyx_RefNannyFinishContext(); - return NULL; - } - __Pyx_GOTREF(__pyx_cur_scope); - __pyx_cur_scope->__pyx_v_self = __pyx_v_self; - __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - __pyx_cur_scope->__pyx_v_sentence = __pyx_v_sentence; - __Pyx_INCREF(__pyx_cur_scope->__pyx_v_sentence); - __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_sentence); - __pyx_cur_scope->__pyx_v_bos = __pyx_v_bos; - __Pyx_INCREF(__pyx_cur_scope->__pyx_v_bos); - __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_bos); - __pyx_cur_scope->__pyx_v_eos = __pyx_v_eos; - __Pyx_INCREF(__pyx_cur_scope->__pyx_v_eos); - __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_eos); - { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5kenlm_5Model_10generator, (PyObject *) __pyx_cur_scope, __pyx_n_s_full_scores, __pyx_n_s_Model_full_scores); if (unlikely(!gen)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_cur_scope); - __Pyx_RefNannyFinishContext(); - return (PyObject *) gen; - } - - /* function exit code */ - __pyx_L1_error:; - __Pyx_AddTraceback("kenlm.Model.full_scores", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_gb_5kenlm_5Model_10generator(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ -{ - struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *__pyx_cur_scope = ((struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)__pyx_generator->closure); - PyObject *__pyx_r = NULL; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - int __pyx_t_4; - Py_ssize_t __pyx_t_5; - char *__pyx_t_6; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("None", 0); - switch (__pyx_generator->resume_label) { - case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L7_resume_from_yield; - case 2: goto __pyx_L9_resume_from_yield; - default: /* CPython raises the right error here */ - __Pyx_RefNannyFinishContext(); - return NULL; - } - __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - - /* "kenlm.pyx":197 - * @param eos should kenlm add an eos state - * """ - * cdef list words = as_str(sentence).split() # <<<<<<<<<<<<<< - * cdef _kenlm.State state - * if bos: - */ - __pyx_t_2 = __pyx_f_5kenlm_as_str(__pyx_cur_scope->__pyx_v_sentence); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_split); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_3))) { - __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); - if (likely(__pyx_t_2)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); - __Pyx_INCREF(__pyx_t_2); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_3, function); - } - } - if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "list", Py_TYPE(__pyx_t_1)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GIVEREF(__pyx_t_1); - __pyx_cur_scope->__pyx_v_words = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; - - /* "kenlm.pyx":199 - * cdef list words = as_str(sentence).split() - * cdef _kenlm.State state - * if bos: # <<<<<<<<<<<<<< - * self.model.BeginSentenceWrite(&state) - * else: - */ - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_bos); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_4) { - - /* "kenlm.pyx":200 - * cdef _kenlm.State state - * if bos: - * self.model.BeginSentenceWrite(&state) # <<<<<<<<<<<<<< - * else: - * self.model.NullContextWrite(&state) - */ - __pyx_cur_scope->__pyx_v_self->model->BeginSentenceWrite((&__pyx_cur_scope->__pyx_v_state)); - goto __pyx_L4; - } - /*else*/ { - - /* "kenlm.pyx":202 - * self.model.BeginSentenceWrite(&state) - * else: - * self.model.NullContextWrite(&state) # <<<<<<<<<<<<<< - * cdef _kenlm.State out_state - * cdef _kenlm.FullScoreReturn ret - */ - __pyx_cur_scope->__pyx_v_self->model->NullContextWrite((&__pyx_cur_scope->__pyx_v_state)); - } - __pyx_L4:; - - /* "kenlm.pyx":205 - * cdef _kenlm.State out_state - * cdef _kenlm.FullScoreReturn ret - * cdef float total = 0 # <<<<<<<<<<<<<< - * cdef _kenlm.WordIndex wid - * for word in words: - */ - __pyx_cur_scope->__pyx_v_total = 0.0; - - /* "kenlm.pyx":207 - * cdef float total = 0 - * cdef _kenlm.WordIndex wid - * for word in words: # <<<<<<<<<<<<<< - * wid = self.vocab.Index(word) - * ret = self.model.BaseFullScore(&state, wid, &out_state) - */ - if (unlikely(__pyx_cur_scope->__pyx_v_words == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 207; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __pyx_t_1 = __pyx_cur_scope->__pyx_v_words; __Pyx_INCREF(__pyx_t_1); __pyx_t_5 = 0; - for (;;) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_3 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_3); __pyx_t_5++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 207; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_1, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 207; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif - __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_word); - __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_word, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - - /* "kenlm.pyx":208 - * cdef _kenlm.WordIndex wid - * for word in words: - * wid = self.vocab.Index(word) # <<<<<<<<<<<<<< - * ret = self.model.BaseFullScore(&state, wid, &out_state) - * yield (ret.prob, ret.ngram_length, wid == 0) - */ - __pyx_t_6 = __Pyx_PyObject_AsString(__pyx_cur_scope->__pyx_v_word); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 208; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_cur_scope->__pyx_v_wid = __pyx_cur_scope->__pyx_v_self->vocab->Index(__pyx_t_6); - - /* "kenlm.pyx":209 - * for word in words: - * wid = self.vocab.Index(word) - * ret = self.model.BaseFullScore(&state, wid, &out_state) # <<<<<<<<<<<<<< - * yield (ret.prob, ret.ngram_length, wid == 0) - * state = out_state - */ - __pyx_cur_scope->__pyx_v_ret = __pyx_cur_scope->__pyx_v_self->model->BaseFullScore((&__pyx_cur_scope->__pyx_v_state), __pyx_cur_scope->__pyx_v_wid, (&__pyx_cur_scope->__pyx_v_out_state)); - - /* "kenlm.pyx":210 - * wid = self.vocab.Index(word) - * ret = self.model.BaseFullScore(&state, wid, &out_state) - * yield (ret.prob, ret.ngram_length, wid == 0) # <<<<<<<<<<<<<< - * state = out_state - * if eos: - */ - __pyx_t_3 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret.prob); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 210; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = __Pyx_PyInt_From_unsigned_char(__pyx_cur_scope->__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 210; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyBool_FromLong((__pyx_cur_scope->__pyx_v_wid == 0)); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 210; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 210; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_2); - __Pyx_GIVEREF(__pyx_t_2); - PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_7); - __Pyx_GIVEREF(__pyx_t_7); - __pyx_t_3 = 0; - __pyx_t_2 = 0; - __pyx_t_7 = 0; - __pyx_r = __pyx_t_8; - __pyx_t_8 = 0; - __Pyx_XGIVEREF(__pyx_t_1); - __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_5; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L7_resume_from_yield:; - __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; - __pyx_cur_scope->__pyx_t_0 = 0; - __Pyx_XGOTREF(__pyx_t_1); - __pyx_t_5 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 210; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - - /* "kenlm.pyx":211 - * ret = self.model.BaseFullScore(&state, wid, &out_state) - * yield (ret.prob, ret.ngram_length, wid == 0) - * state = out_state # <<<<<<<<<<<<<< - * if eos: - * ret = self.model.BaseFullScore(&state, - */ - __pyx_cur_scope->__pyx_v_state = __pyx_cur_scope->__pyx_v_out_state; - - /* "kenlm.pyx":207 - * cdef float total = 0 - * cdef _kenlm.WordIndex wid - * for word in words: # <<<<<<<<<<<<<< - * wid = self.vocab.Index(word) - * ret = self.model.BaseFullScore(&state, wid, &out_state) - */ - } - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "kenlm.pyx":212 - * yield (ret.prob, ret.ngram_length, wid == 0) - * state = out_state - * if eos: # <<<<<<<<<<<<<< - * ret = self.model.BaseFullScore(&state, - * self.vocab.EndSentence(), &out_state) - */ - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_cur_scope->__pyx_v_eos); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 212; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_4) { - - /* "kenlm.pyx":213 - * state = out_state - * if eos: - * ret = self.model.BaseFullScore(&state, # <<<<<<<<<<<<<< - * self.vocab.EndSentence(), &out_state) - * yield (ret.prob, ret.ngram_length, False) - */ - __pyx_cur_scope->__pyx_v_ret = __pyx_cur_scope->__pyx_v_self->model->BaseFullScore((&__pyx_cur_scope->__pyx_v_state), __pyx_cur_scope->__pyx_v_self->vocab->EndSentence(), (&__pyx_cur_scope->__pyx_v_out_state)); - - /* "kenlm.pyx":215 - * ret = self.model.BaseFullScore(&state, - * self.vocab.EndSentence(), &out_state) - * yield (ret.prob, ret.ngram_length, False) # <<<<<<<<<<<<<< - * - * - */ - __pyx_t_1 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret.prob); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 215; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_8 = __Pyx_PyInt_From_unsigned_char(__pyx_cur_scope->__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 215; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_7 = PyTuple_New(3); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 215; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_8); - __Pyx_GIVEREF(__pyx_t_8); - __Pyx_INCREF(Py_False); - PyTuple_SET_ITEM(__pyx_t_7, 2, Py_False); - __Pyx_GIVEREF(Py_False); - __pyx_t_1 = 0; - __pyx_t_8 = 0; - __pyx_r = __pyx_t_7; - __pyx_t_7 = 0; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 2; - return __pyx_r; - __pyx_L9_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 215; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - goto __pyx_L8; - } - __pyx_L8:; - - /* "kenlm.pyx":190 - * return 10.0**(-self.score(sentence) / words) - * - * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<< - * """ - * full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov) - */ - - /* function exit code */ - PyErr_SetNone(PyExc_StopIteration); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("full_scores", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_L0:; - __Pyx_XDECREF(__pyx_r); - __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); - __Pyx_RefNannyFinishContext(); - return NULL; -} - -/* "kenlm.pyx":218 - * - * - * def BeginSentenceWrite(self, State state): # <<<<<<<<<<<<<< - * """Change the given state to a BOS state.""" - * self.model.BeginSentenceWrite(&state._c_state) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_12BeginSentenceWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state); /*proto*/ -static char __pyx_doc_5kenlm_5Model_11BeginSentenceWrite[] = "Change the given state to a BOS state."; -static PyObject *__pyx_pw_5kenlm_5Model_12BeginSentenceWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state) { - CYTHON_UNUSED int __pyx_lineno = 0; - CYTHON_UNUSED const char *__pyx_filename = NULL; - CYTHON_UNUSED int __pyx_clineno = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("BeginSentenceWrite (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_state), __pyx_ptype_5kenlm_State, 1, "state", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = __pyx_pf_5kenlm_5Model_11BeginSentenceWrite(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((struct __pyx_obj_5kenlm_State *)__pyx_v_state)); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_11BeginSentenceWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("BeginSentenceWrite", 0); - - /* "kenlm.pyx":220 - * def BeginSentenceWrite(self, State state): - * """Change the given state to a BOS state.""" - * self.model.BeginSentenceWrite(&state._c_state) # <<<<<<<<<<<<<< - * - * def NullContextWrite(self, State state): - */ - __pyx_v_self->model->BeginSentenceWrite((&__pyx_v_state->_c_state)); - - /* "kenlm.pyx":218 - * - * - * def BeginSentenceWrite(self, State state): # <<<<<<<<<<<<<< - * """Change the given state to a BOS state.""" - * self.model.BeginSentenceWrite(&state._c_state) - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":222 - * self.model.BeginSentenceWrite(&state._c_state) - * - * def NullContextWrite(self, State state): # <<<<<<<<<<<<<< - * """Change the given state to a NULL state.""" - * self.model.NullContextWrite(&state._c_state) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_14NullContextWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state); /*proto*/ -static char __pyx_doc_5kenlm_5Model_13NullContextWrite[] = "Change the given state to a NULL state."; -static PyObject *__pyx_pw_5kenlm_5Model_14NullContextWrite(PyObject *__pyx_v_self, PyObject *__pyx_v_state) { - CYTHON_UNUSED int __pyx_lineno = 0; - CYTHON_UNUSED const char *__pyx_filename = NULL; - CYTHON_UNUSED int __pyx_clineno = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("NullContextWrite (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_state), __pyx_ptype_5kenlm_State, 1, "state", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 222; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = __pyx_pf_5kenlm_5Model_13NullContextWrite(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((struct __pyx_obj_5kenlm_State *)__pyx_v_state)); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_13NullContextWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("NullContextWrite", 0); - - /* "kenlm.pyx":224 - * def NullContextWrite(self, State state): - * """Change the given state to a NULL state.""" - * self.model.NullContextWrite(&state._c_state) # <<<<<<<<<<<<<< - * - * def BaseScore(self, State in_state, str word, State out_state): - */ - __pyx_v_self->model->NullContextWrite((&__pyx_v_state->_c_state)); - - /* "kenlm.pyx":222 - * self.model.BeginSentenceWrite(&state._c_state) - * - * def NullContextWrite(self, State state): # <<<<<<<<<<<<<< - * """Change the given state to a NULL state.""" - * self.model.NullContextWrite(&state._c_state) - */ - - /* function exit code */ - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":226 - * self.model.NullContextWrite(&state._c_state) - * - * def BaseScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< - * """ - * Return p(word|in_state) and update the output state. - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_16BaseScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model_15BaseScore[] = "\n Return p(word|in_state) and update the output state.\n Wrapper around model.BaseScore(in_state, Index(word), out_state)\n\n :param word: the suffix\n :param state: the context (defaults to NullContext)\n :returns: p(word|state)\n "; -static PyObject *__pyx_pw_5kenlm_5Model_16BaseScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - struct __pyx_obj_5kenlm_State *__pyx_v_in_state = 0; - PyObject *__pyx_v_word = 0; - struct __pyx_obj_5kenlm_State *__pyx_v_out_state = 0; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("BaseScore (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_in_state,&__pyx_n_s_word,&__pyx_n_s_out_state,0}; - PyObject* values[3] = {0,0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_in_state)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("BaseScore", 1, 3, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 226; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_out_state)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("BaseScore", 1, 3, 3, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 226; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "BaseScore") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 226; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - } - __pyx_v_in_state = ((struct __pyx_obj_5kenlm_State *)values[0]); - __pyx_v_word = ((PyObject*)values[1]); - __pyx_v_out_state = ((struct __pyx_obj_5kenlm_State *)values[2]); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("BaseScore", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 226; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.BaseScore", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_in_state), __pyx_ptype_5kenlm_State, 1, "in_state", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 226; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_word), (&PyString_Type), 1, "word", 1))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 226; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_out_state), __pyx_ptype_5kenlm_State, 1, "out_state", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 226; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = __pyx_pf_5kenlm_5Model_15BaseScore(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_in_state, __pyx_v_word, __pyx_v_out_state); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_15BaseScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state) { - float __pyx_v_total; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - char *__pyx_t_2; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("BaseScore", 0); - - /* "kenlm.pyx":235 - * :returns: p(word|state) - * """ - * cdef float total = self.model.BaseScore(&in_state._c_state, self.vocab.Index(as_str(word)), &out_state._c_state) # <<<<<<<<<<<<<< - * return total - * - */ - __pyx_t_1 = __pyx_f_5kenlm_as_str(__pyx_v_word); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 235; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_t_1); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 235; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_total = __pyx_v_self->model->BaseScore((&__pyx_v_in_state->_c_state), __pyx_v_self->vocab->Index(__pyx_t_2), (&__pyx_v_out_state->_c_state)); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "kenlm.pyx":236 - * """ - * cdef float total = self.model.BaseScore(&in_state._c_state, self.vocab.Index(as_str(word)), &out_state._c_state) - * return total # <<<<<<<<<<<<<< - * - * def BaseFullScore(self, State in_state, str word, State out_state): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_total); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 236; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":226 - * self.model.NullContextWrite(&state._c_state) - * - * def BaseScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< - * """ - * Return p(word|in_state) and update the output state. - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Model.BaseScore", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":238 - * return total - * - * def BaseFullScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< - * """ - * Wrapper around model.BaseScore(in_state, Index(word), out_state) - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_18BaseFullScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5kenlm_5Model_17BaseFullScore[] = "\n Wrapper around model.BaseScore(in_state, Index(word), out_state)\n\n :param word: the suffix\n :param state: the context (defaults to NullContext)\n :returns: FullScoreReturn(word|state)\n "; -static PyObject *__pyx_pw_5kenlm_5Model_18BaseFullScore(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - struct __pyx_obj_5kenlm_State *__pyx_v_in_state = 0; - PyObject *__pyx_v_word = 0; - struct __pyx_obj_5kenlm_State *__pyx_v_out_state = 0; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("BaseFullScore (wrapper)", 0); - { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_in_state,&__pyx_n_s_word,&__pyx_n_s_out_state,0}; - PyObject* values[3] = {0,0,0}; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args; - const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); - switch (pos_args) { - case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - kw_args = PyDict_Size(__pyx_kwds); - switch (pos_args) { - case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_in_state)) != 0)) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_word)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("BaseFullScore", 1, 3, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 238; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_out_state)) != 0)) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("BaseFullScore", 1, 3, 3, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 238; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "BaseFullScore") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 238; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { - goto __pyx_L5_argtuple_error; - } else { - values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - values[2] = PyTuple_GET_ITEM(__pyx_args, 2); - } - __pyx_v_in_state = ((struct __pyx_obj_5kenlm_State *)values[0]); - __pyx_v_word = ((PyObject*)values[1]); - __pyx_v_out_state = ((struct __pyx_obj_5kenlm_State *)values[2]); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("BaseFullScore", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 238; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("kenlm.Model.BaseFullScore", __pyx_clineno, __pyx_lineno, __pyx_filename); - __Pyx_RefNannyFinishContext(); - return NULL; - __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_in_state), __pyx_ptype_5kenlm_State, 1, "in_state", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 238; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_word), (&PyString_Type), 1, "word", 1))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 238; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_out_state), __pyx_ptype_5kenlm_State, 1, "out_state", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 238; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = __pyx_pf_5kenlm_5Model_17BaseFullScore(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), __pyx_v_in_state, __pyx_v_word, __pyx_v_out_state); - - /* function exit code */ - goto __pyx_L0; - __pyx_L1_error:; - __pyx_r = NULL; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_17BaseFullScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state) { - lm::WordIndex __pyx_v_wid; - struct lm::FullScoreReturn __pyx_v_ret; - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - char *__pyx_t_2; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("BaseFullScore", 0); - - /* "kenlm.pyx":246 - * :returns: FullScoreReturn(word|state) - * """ - * cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) # <<<<<<<<<<<<<< - * cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) - */ - __pyx_t_1 = __pyx_f_5kenlm_as_str(__pyx_v_word); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 246; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_t_1); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 246; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_wid = __pyx_v_self->vocab->Index(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "kenlm.pyx":247 - * """ - * cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) - * cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) # <<<<<<<<<<<<<< - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) - * - */ - __pyx_v_ret = __pyx_v_self->model->BaseFullScore((&__pyx_v_in_state->_c_state), __pyx_v_wid, (&__pyx_v_out_state->_c_state)); - - /* "kenlm.pyx":248 - * cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) - * cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) # <<<<<<<<<<<<<< - * - * def __contains__(self, word): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_ret.prob); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 248; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_PyInt_From_unsigned_char(__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 248; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyBool_FromLong((__pyx_v_wid == 0)); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 248; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 248; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_4); - __pyx_t_1 = 0; - __pyx_t_3 = 0; - __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5kenlm_FullScoreReturn)), __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 248; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_r = __pyx_t_4; - __pyx_t_4 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":238 - * return total - * - * def BaseFullScore(self, State in_state, str word, State out_state): # <<<<<<<<<<<<<< - * """ - * Wrapper around model.BaseScore(in_state, Index(word), out_state) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_AddTraceback("kenlm.Model.BaseFullScore", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":250 - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) - * - * def __contains__(self, word): # <<<<<<<<<<<<<< - * cdef bytes w = as_str(word) - * return (self.vocab.Index(w) != 0) - */ - -/* Python wrapper */ -static int __pyx_pw_5kenlm_5Model_20__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_word); /*proto*/ -static int __pyx_pw_5kenlm_5Model_20__contains__(PyObject *__pyx_v_self, PyObject *__pyx_v_word) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__contains__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_19__contains__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((PyObject *)__pyx_v_word)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5kenlm_5Model_19__contains__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_word) { - PyObject *__pyx_v_w = 0; - int __pyx_r; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - char *__pyx_t_2; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__contains__", 0); - - /* "kenlm.pyx":251 - * - * def __contains__(self, word): - * cdef bytes w = as_str(word) # <<<<<<<<<<<<<< - * return (self.vocab.Index(w) != 0) - * - */ - __pyx_t_1 = __pyx_f_5kenlm_as_str(__pyx_v_word); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 251; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_v_w = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; - - /* "kenlm.pyx":252 - * def __contains__(self, word): - * cdef bytes w = as_str(word) - * return (self.vocab.Index(w) != 0) # <<<<<<<<<<<<<< - * - * def __repr__(self): - */ - __pyx_t_2 = __Pyx_PyObject_AsString(__pyx_v_w); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 252; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = (__pyx_v_self->vocab->Index(__pyx_t_2) != 0); - goto __pyx_L0; - - /* "kenlm.pyx":250 - * return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) - * - * def __contains__(self, word): # <<<<<<<<<<<<<< - * cdef bytes w = as_str(word) - * return (self.vocab.Index(w) != 0) - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Model.__contains__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_XDECREF(__pyx_v_w); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":254 - * return (self.vocab.Index(w) != 0) - * - * def __repr__(self): # <<<<<<<<<<<<<< - * return ''.format(os.path.basename(self.path)) - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_22__repr__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5Model_22__repr__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_21__repr__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_21__repr__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__repr__", 0); - - /* "kenlm.pyx":255 - * - * def __repr__(self): - * return ''.format(os.path.basename(self.path)) # <<<<<<<<<<<<<< - * - * def __reduce__(self): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_Model_from_0, __pyx_n_s_format); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_os); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_path); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_basename); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_4))) { - __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); - if (likely(__pyx_t_5)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); - __Pyx_INCREF(__pyx_t_5); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_4, function); - } - } - if (!__pyx_t_5) { - __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_self->path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - } else { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __Pyx_GIVEREF(__pyx_t_5); __pyx_t_5 = NULL; - __Pyx_INCREF(__pyx_v_self->path); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_v_self->path); - __Pyx_GIVEREF(__pyx_v_self->path); - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); - if (likely(__pyx_t_4)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); - __Pyx_INCREF(__pyx_t_4); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_2, function); - } - } - if (!__pyx_t_4) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = NULL; - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 255; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":254 - * return (self.vocab.Index(w) != 0) - * - * def __repr__(self): # <<<<<<<<<<<<<< - * return ''.format(os.path.basename(self.path)) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_AddTraceback("kenlm.Model.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":257 - * return ''.format(os.path.basename(self.path)) - * - * def __reduce__(self): # <<<<<<<<<<<<<< - * return (_kenlm.LanguageModel, (self.path,)) - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_24__reduce__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5Model_24__reduce__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__reduce__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_23__reduce__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_23__reduce__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__reduce__", 0); - - /* "kenlm.pyx":258 - * - * def __reduce__(self): - * return (_kenlm.LanguageModel, (self.path,)) # <<<<<<<<<<<<<< - * - * class LanguageModel(Model): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_kenlm); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 258; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_LanguageModel); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 258; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 258; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_INCREF(__pyx_v_self->path); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->path); - __Pyx_GIVEREF(__pyx_v_self->path); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 258; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); - __Pyx_GIVEREF(__pyx_t_2); - PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_1); - __Pyx_GIVEREF(__pyx_t_1); - __pyx_t_2 = 0; - __pyx_t_1 = 0; - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - /* "kenlm.pyx":257 - * return ''.format(os.path.basename(self.path)) - * - * def __reduce__(self): # <<<<<<<<<<<<<< - * return (_kenlm.LanguageModel, (self.path,)) - * - */ - - /* function exit code */ - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("kenlm.Model.__reduce__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* "kenlm.pyx":102 - * - * cdef _kenlm.Model* model - * cdef public bytes path # <<<<<<<<<<<<<< - * cdef _kenlm.const_Vocabulary* vocab - * - */ - -/* Python wrapper */ -static PyObject *__pyx_pw_5kenlm_5Model_4path_1__get__(PyObject *__pyx_v_self); /*proto*/ -static PyObject *__pyx_pw_5kenlm_5Model_4path_1__get__(PyObject *__pyx_v_self) { - PyObject *__pyx_r = 0; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_4path___get__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_pf_5kenlm_5Model_4path___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(__pyx_v_self->path); - __pyx_r = __pyx_v_self->path; - goto __pyx_L0; - - /* function exit code */ - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* Python wrapper */ -static int __pyx_pw_5kenlm_5Model_4path_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value); /*proto*/ -static int __pyx_pw_5kenlm_5Model_4path_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__set__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_4path_2__set__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self), ((PyObject *)__pyx_v_value)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5kenlm_5Model_4path_2__set__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_value) { - int __pyx_r; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__set__", 0); - if (!(likely(PyBytes_CheckExact(__pyx_v_value))||((__pyx_v_value) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_v_value)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_1 = __pyx_v_value; - __Pyx_INCREF(__pyx_t_1); - __Pyx_GIVEREF(__pyx_t_1); - __Pyx_GOTREF(__pyx_v_self->path); - __Pyx_DECREF(__pyx_v_self->path); - __pyx_v_self->path = ((PyObject*)__pyx_t_1); - __pyx_t_1 = 0; - - /* function exit code */ - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("kenlm.Model.path.__set__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = -1; - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -/* Python wrapper */ -static int __pyx_pw_5kenlm_5Model_4path_5__del__(PyObject *__pyx_v_self); /*proto*/ -static int __pyx_pw_5kenlm_5Model_4path_5__del__(PyObject *__pyx_v_self) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__del__ (wrapper)", 0); - __pyx_r = __pyx_pf_5kenlm_5Model_4path_4__del__(((struct __pyx_obj_5kenlm_Model *)__pyx_v_self)); - - /* function exit code */ - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static int __pyx_pf_5kenlm_5Model_4path_4__del__(struct __pyx_obj_5kenlm_Model *__pyx_v_self) { - int __pyx_r; - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__del__", 0); - __Pyx_INCREF(Py_None); - __Pyx_GIVEREF(Py_None); - __Pyx_GOTREF(__pyx_v_self->path); - __Pyx_DECREF(__pyx_v_self->path); - __pyx_v_self->path = ((PyObject*)Py_None); - - /* function exit code */ - __pyx_r = 0; - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_tp_new_5kenlm_FullScoreReturn(PyTypeObject *t, PyObject *a, PyObject *k) { - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - if (unlikely(__pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(o, a, k) < 0)) { - Py_DECREF(o); o = 0; - } - return o; -} - -static void __pyx_tp_dealloc_5kenlm_FullScoreReturn(PyObject *o) { - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - (*Py_TYPE(o)->tp_free)(o); -} - -static PyObject *__pyx_getprop_5kenlm_15FullScoreReturn_log_prob(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(o); -} - -static PyObject *__pyx_getprop_5kenlm_15FullScoreReturn_ngram_length(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(o); -} - -static PyObject *__pyx_getprop_5kenlm_15FullScoreReturn_oov(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(o); -} - -static PyMethodDef __pyx_methods_5kenlm_FullScoreReturn[] = { - {0, 0, 0, 0} -}; - -static struct PyGetSetDef __pyx_getsets_5kenlm_FullScoreReturn[] = { - {(char *)"log_prob", __pyx_getprop_5kenlm_15FullScoreReturn_log_prob, 0, 0, 0}, - {(char *)"ngram_length", __pyx_getprop_5kenlm_15FullScoreReturn_ngram_length, 0, 0, 0}, - {(char *)"oov", __pyx_getprop_5kenlm_15FullScoreReturn_oov, 0, 0, 0}, - {0, 0, 0, 0, 0} -}; - -static PyTypeObject __pyx_type_5kenlm_FullScoreReturn = { - PyVarObject_HEAD_INIT(0, 0) - "kenlm.FullScoreReturn", /*tp_name*/ - sizeof(struct __pyx_obj_5kenlm_FullScoreReturn), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5kenlm_FullScoreReturn, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #else - 0, /*reserved*/ - #endif - __pyx_pw_5kenlm_15FullScoreReturn_3__repr__, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - "\n Wrapper around FullScoreReturn.\n\n Notes:\n `prob` has been renamed to `log_prob`\n `oov` has been added to flag whether the word is OOV\n ", /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5kenlm_FullScoreReturn, /*tp_methods*/ - 0, /*tp_members*/ - __pyx_getsets_5kenlm_FullScoreReturn, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5kenlm_FullScoreReturn, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyObject *__pyx_tp_new_5kenlm_State(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5kenlm_State *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_5kenlm_State *)o); - new((void*)&(p->_c_state)) lm::ngram::State(); - return o; -} - -static void __pyx_tp_dealloc_5kenlm_State(PyObject *o) { - struct __pyx_obj_5kenlm_State *p = (struct __pyx_obj_5kenlm_State *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - __Pyx_call_destructor(&p->_c_state); - (*Py_TYPE(o)->tp_free)(o); -} - -static PyMethodDef __pyx_methods_5kenlm_State[] = { - {0, 0, 0, 0} -}; - -static PyTypeObject __pyx_type_5kenlm_State = { - PyVarObject_HEAD_INIT(0, 0) - "kenlm.State", /*tp_name*/ - sizeof(struct __pyx_obj_5kenlm_State), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5kenlm_State, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #else - 0, /*reserved*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - __pyx_pw_5kenlm_5State_3__hash__, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - "\n Wrapper around lm::ngram::State so that python code can make incremental queries.\n\n Notes:\n * rich comparisons \n * hashable\n ", /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - __pyx_pw_5kenlm_5State_1__richcmp__, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5kenlm_State, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5kenlm_State, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyObject *__pyx_tp_new_5kenlm_Config(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5kenlm_Config *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_5kenlm_Config *)o); - new((void*)&(p->_c_config)) lm::ngram::Config(); - return o; -} - -static void __pyx_tp_dealloc_5kenlm_Config(PyObject *o) { - struct __pyx_obj_5kenlm_Config *p = (struct __pyx_obj_5kenlm_Config *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - __Pyx_call_destructor(&p->_c_config); - (*Py_TYPE(o)->tp_free)(o); -} - -static PyObject *__pyx_getprop_5kenlm_6Config_load_method(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5kenlm_6Config_11load_method_1__get__(o); -} - -static int __pyx_setprop_5kenlm_6Config_load_method(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { - if (v) { - return __pyx_pw_5kenlm_6Config_11load_method_3__set__(o, v); - } - else { - PyErr_SetString(PyExc_NotImplementedError, "__del__"); - return -1; - } -} - -static PyMethodDef __pyx_methods_5kenlm_Config[] = { - {0, 0, 0, 0} -}; - -static struct PyGetSetDef __pyx_getsets_5kenlm_Config[] = { - {(char *)"load_method", __pyx_getprop_5kenlm_6Config_load_method, __pyx_setprop_5kenlm_6Config_load_method, 0, 0}, - {0, 0, 0, 0, 0} -}; - -static PyTypeObject __pyx_type_5kenlm_Config = { - PyVarObject_HEAD_INIT(0, 0) - "kenlm.Config", /*tp_name*/ - sizeof(struct __pyx_obj_5kenlm_Config), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5kenlm_Config, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #else - 0, /*reserved*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - "\n Wrapper around lm::ngram::Config.\n Pass this to Model's constructor to set the load_method.\n ", /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5kenlm_Config, /*tp_methods*/ - 0, /*tp_members*/ - __pyx_getsets_5kenlm_Config, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - __pyx_pw_5kenlm_6Config_1__init__, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5kenlm_Config, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyObject *__pyx_tp_new_5kenlm_Model(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5kenlm_Model *p; - PyObject *o; - if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { - o = (*t->tp_alloc)(t, 0); - } else { - o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); - } - if (unlikely(!o)) return 0; - p = ((struct __pyx_obj_5kenlm_Model *)o); - p->path = ((PyObject*)Py_None); Py_INCREF(Py_None); - return o; -} - -static void __pyx_tp_dealloc_5kenlm_Model(PyObject *o) { - struct __pyx_obj_5kenlm_Model *p = (struct __pyx_obj_5kenlm_Model *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { - if (PyObject_CallFinalizerFromDealloc(o)) return; - } - #endif - { - PyObject *etype, *eval, *etb; - PyErr_Fetch(&etype, &eval, &etb); - ++Py_REFCNT(o); - __pyx_pw_5kenlm_5Model_3__dealloc__(o); - --Py_REFCNT(o); - PyErr_Restore(etype, eval, etb); - } - Py_CLEAR(p->path); - (*Py_TYPE(o)->tp_free)(o); -} - -static PyObject *__pyx_getprop_5kenlm_5Model_order(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5kenlm_5Model_5order_1__get__(o); -} - -static PyObject *__pyx_getprop_5kenlm_5Model_path(PyObject *o, CYTHON_UNUSED void *x) { - return __pyx_pw_5kenlm_5Model_4path_1__get__(o); -} - -static int __pyx_setprop_5kenlm_5Model_path(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { - if (v) { - return __pyx_pw_5kenlm_5Model_4path_3__set__(o, v); - } - else { - return __pyx_pw_5kenlm_5Model_4path_5__del__(o); - } -} - -static PyMethodDef __pyx_methods_5kenlm_Model[] = { - {"score", (PyCFunction)__pyx_pw_5kenlm_5Model_5score, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5kenlm_5Model_4score}, - {"perplexity", (PyCFunction)__pyx_pw_5kenlm_5Model_7perplexity, METH_O, __pyx_doc_5kenlm_5Model_6perplexity}, - {"full_scores", (PyCFunction)__pyx_pw_5kenlm_5Model_9full_scores, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5kenlm_5Model_8full_scores}, - {"BeginSentenceWrite", (PyCFunction)__pyx_pw_5kenlm_5Model_12BeginSentenceWrite, METH_O, __pyx_doc_5kenlm_5Model_11BeginSentenceWrite}, - {"NullContextWrite", (PyCFunction)__pyx_pw_5kenlm_5Model_14NullContextWrite, METH_O, __pyx_doc_5kenlm_5Model_13NullContextWrite}, - {"BaseScore", (PyCFunction)__pyx_pw_5kenlm_5Model_16BaseScore, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5kenlm_5Model_15BaseScore}, - {"BaseFullScore", (PyCFunction)__pyx_pw_5kenlm_5Model_18BaseFullScore, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5kenlm_5Model_17BaseFullScore}, - {"__reduce__", (PyCFunction)__pyx_pw_5kenlm_5Model_24__reduce__, METH_NOARGS, 0}, - {0, 0, 0, 0} -}; - -static struct PyGetSetDef __pyx_getsets_5kenlm_Model[] = { - {(char *)"order", __pyx_getprop_5kenlm_5Model_order, 0, 0, 0}, - {(char *)"path", __pyx_getprop_5kenlm_5Model_path, __pyx_setprop_5kenlm_5Model_path, 0, 0}, - {0, 0, 0, 0, 0} -}; - -static PySequenceMethods __pyx_tp_as_sequence_Model = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - __pyx_pw_5kenlm_5Model_20__contains__, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyTypeObject __pyx_type_5kenlm_Model = { - PyVarObject_HEAD_INIT(0, 0) - "kenlm.Model", /*tp_name*/ - sizeof(struct __pyx_obj_5kenlm_Model), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5kenlm_Model, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #else - 0, /*reserved*/ - #endif - __pyx_pw_5kenlm_5Model_22__repr__, /*tp_repr*/ - 0, /*tp_as_number*/ - &__pyx_tp_as_sequence_Model, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - "\n Wrapper around lm::ngram::Model.\n ", /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_5kenlm_Model, /*tp_methods*/ - 0, /*tp_members*/ - __pyx_getsets_5kenlm_Model, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - __pyx_pw_5kenlm_5Model_1__init__, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5kenlm_Model, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *__pyx_freelist_5kenlm___pyx_scope_struct__full_scores[8]; -static int __pyx_freecount_5kenlm___pyx_scope_struct__full_scores = 0; - -static PyObject *__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *p; - PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5kenlm___pyx_scope_struct__full_scores > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores)))) { - o = (PyObject*)__pyx_freelist_5kenlm___pyx_scope_struct__full_scores[--__pyx_freecount_5kenlm___pyx_scope_struct__full_scores]; - memset(o, 0, sizeof(struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores)); - (void) PyObject_INIT(o, t); - PyObject_GC_Track(o); - } else { - o = (*t->tp_alloc)(t, 0); - if (unlikely(!o)) return 0; - } - p = ((struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)o); - new((void*)&(p->__pyx_v_out_state)) lm::ngram::State(); - new((void*)&(p->__pyx_v_state)) lm::ngram::State(); - return o; -} - -static void __pyx_tp_dealloc_5kenlm___pyx_scope_struct__full_scores(PyObject *o) { - struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *p = (struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)o; - PyObject_GC_UnTrack(o); - __Pyx_call_destructor(&p->__pyx_v_out_state); - __Pyx_call_destructor(&p->__pyx_v_state); - Py_CLEAR(p->__pyx_v_bos); - Py_CLEAR(p->__pyx_v_eos); - Py_CLEAR(p->__pyx_v_self); - Py_CLEAR(p->__pyx_v_sentence); - Py_CLEAR(p->__pyx_v_word); - Py_CLEAR(p->__pyx_v_words); - Py_CLEAR(p->__pyx_t_0); - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5kenlm___pyx_scope_struct__full_scores < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores)))) { - __pyx_freelist_5kenlm___pyx_scope_struct__full_scores[__pyx_freecount_5kenlm___pyx_scope_struct__full_scores++] = ((struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)o); - } else { - (*Py_TYPE(o)->tp_free)(o); - } -} - -static int __pyx_tp_traverse_5kenlm___pyx_scope_struct__full_scores(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *p = (struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)o; - if (p->__pyx_v_bos) { - e = (*v)(p->__pyx_v_bos, a); if (e) return e; - } - if (p->__pyx_v_eos) { - e = (*v)(p->__pyx_v_eos, a); if (e) return e; - } - if (p->__pyx_v_self) { - e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; - } - if (p->__pyx_v_sentence) { - e = (*v)(p->__pyx_v_sentence, a); if (e) return e; - } - if (p->__pyx_v_word) { - e = (*v)(p->__pyx_v_word, a); if (e) return e; - } - if (p->__pyx_v_words) { - e = (*v)(p->__pyx_v_words, a); if (e) return e; - } - if (p->__pyx_t_0) { - e = (*v)(p->__pyx_t_0, a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_5kenlm___pyx_scope_struct__full_scores(PyObject *o) { - PyObject* tmp; - struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *p = (struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)o; - tmp = ((PyObject*)p->__pyx_v_bos); - p->__pyx_v_bos = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_eos); - p->__pyx_v_eos = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5kenlm_Model *)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_sentence); - p->__pyx_v_sentence = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_word); - p->__pyx_v_word = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_v_words); - p->__pyx_v_words = ((PyObject*)Py_None); Py_INCREF(Py_None); - Py_XDECREF(tmp); - tmp = ((PyObject*)p->__pyx_t_0); - p->__pyx_t_0 = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static PyTypeObject __pyx_type_5kenlm___pyx_scope_struct__full_scores = { - PyVarObject_HEAD_INIT(0, 0) - "kenlm.__pyx_scope_struct__full_scores", /*tp_name*/ - sizeof(struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5kenlm___pyx_scope_struct__full_scores, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - #if PY_MAJOR_VERSION < 3 - 0, /*tp_compare*/ - #else - 0, /*reserved*/ - #endif - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - __pyx_tp_traverse_5kenlm___pyx_scope_struct__full_scores, /*tp_traverse*/ - __pyx_tp_clear_5kenlm___pyx_scope_struct__full_scores, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_5kenlm___pyx_scope_struct__full_scores, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ - 0, /*tp_del*/ - 0, /*tp_version_tag*/ - #if PY_VERSION_HEX >= 0x030400a1 - 0, /*tp_finalize*/ - #endif -}; - -static PyMethodDef __pyx_methods[] = { - {0, 0, 0, 0} -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef __pyx_moduledef = { - #if PY_VERSION_HEX < 0x03020000 - { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, - #else - PyModuleDef_HEAD_INIT, - #endif - "kenlm", - 0, /* m_doc */ - -1, /* m_size */ - __pyx_methods /* m_methods */, - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp_s_0_1_2_3, __pyx_k_0_1_2_3, sizeof(__pyx_k_0_1_2_3), 0, 0, 1, 0}, - {&__pyx_kp_s_Backwards_compatability_stub_Use, __pyx_k_Backwards_compatability_stub_Use, sizeof(__pyx_k_Backwards_compatability_stub_Use), 0, 0, 1, 0}, - {&__pyx_kp_s_Cannot_convert_s_to_string, __pyx_k_Cannot_convert_s_to_string, sizeof(__pyx_k_Cannot_convert_s_to_string), 0, 0, 1, 0}, - {&__pyx_kp_s_Cannot_read_model, __pyx_k_Cannot_read_model, sizeof(__pyx_k_Cannot_read_model), 0, 0, 1, 0}, - {&__pyx_n_s_IOError, __pyx_k_IOError, sizeof(__pyx_k_IOError), 0, 0, 1, 1}, - {&__pyx_n_s_LAZY, __pyx_k_LAZY, sizeof(__pyx_k_LAZY), 0, 0, 1, 1}, - {&__pyx_n_s_LanguageModel, __pyx_k_LanguageModel, sizeof(__pyx_k_LanguageModel), 0, 0, 1, 1}, - {&__pyx_n_s_LoadMethod, __pyx_k_LoadMethod, sizeof(__pyx_k_LoadMethod), 0, 0, 1, 1}, - {&__pyx_kp_s_Model_from_0, __pyx_k_Model_from_0, sizeof(__pyx_k_Model_from_0), 0, 0, 1, 0}, - {&__pyx_n_s_Model_full_scores, __pyx_k_Model_full_scores, sizeof(__pyx_k_Model_full_scores), 0, 0, 1, 1}, - {&__pyx_n_s_PARALLEL_READ, __pyx_k_PARALLEL_READ, sizeof(__pyx_k_PARALLEL_READ), 0, 0, 1, 1}, - {&__pyx_n_s_POPULATE_OR_LAZY, __pyx_k_POPULATE_OR_LAZY, sizeof(__pyx_k_POPULATE_OR_LAZY), 0, 0, 1, 1}, - {&__pyx_n_s_POPULATE_OR_READ, __pyx_k_POPULATE_OR_READ, sizeof(__pyx_k_POPULATE_OR_READ), 0, 0, 1, 1}, - {&__pyx_n_s_READ, __pyx_k_READ, sizeof(__pyx_k_READ), 0, 0, 1, 1}, - {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, - {&__pyx_n_s_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 0, 0, 1, 1}, - {&__pyx_kp_s__3, __pyx_k__3, sizeof(__pyx_k__3), 0, 0, 1, 0}, - {&__pyx_kp_s__4, __pyx_k__4, sizeof(__pyx_k__4), 0, 0, 1, 0}, - {&__pyx_n_s_abspath, __pyx_k_abspath, sizeof(__pyx_k_abspath), 0, 0, 1, 1}, - {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, - {&__pyx_n_s_basename, __pyx_k_basename, sizeof(__pyx_k_basename), 0, 0, 1, 1}, - {&__pyx_n_s_bos, __pyx_k_bos, sizeof(__pyx_k_bos), 0, 0, 1, 1}, - {&__pyx_n_s_class, __pyx_k_class, sizeof(__pyx_k_class), 0, 0, 1, 1}, - {&__pyx_n_s_close, __pyx_k_close, sizeof(__pyx_k_close), 0, 0, 1, 1}, - {&__pyx_n_s_config, __pyx_k_config, sizeof(__pyx_k_config), 0, 0, 1, 1}, - {&__pyx_n_s_doc, __pyx_k_doc, sizeof(__pyx_k_doc), 0, 0, 1, 1}, - {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, - {&__pyx_n_s_eos, __pyx_k_eos, sizeof(__pyx_k_eos), 0, 0, 1, 1}, - {&__pyx_n_s_format, __pyx_k_format, sizeof(__pyx_k_format), 0, 0, 1, 1}, - {&__pyx_n_s_full_scores, __pyx_k_full_scores, sizeof(__pyx_k_full_scores), 0, 0, 1, 1}, - {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, - {&__pyx_n_s_in_state, __pyx_k_in_state, sizeof(__pyx_k_in_state), 0, 0, 1, 1}, - {&__pyx_n_s_kenlm, __pyx_k_kenlm, sizeof(__pyx_k_kenlm), 0, 0, 1, 1}, - {&__pyx_n_s_kenlm_2, __pyx_k_kenlm_2, sizeof(__pyx_k_kenlm_2), 0, 0, 1, 1}, - {&__pyx_n_s_log_prob, __pyx_k_log_prob, sizeof(__pyx_k_log_prob), 0, 0, 1, 1}, - {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, - {&__pyx_n_s_metaclass, __pyx_k_metaclass, sizeof(__pyx_k_metaclass), 0, 0, 1, 1}, - {&__pyx_n_s_module, __pyx_k_module, sizeof(__pyx_k_module), 0, 0, 1, 1}, - {&__pyx_n_s_name, __pyx_k_name, sizeof(__pyx_k_name), 0, 0, 1, 1}, - {&__pyx_n_s_ngram_length, __pyx_k_ngram_length, sizeof(__pyx_k_ngram_length), 0, 0, 1, 1}, - {&__pyx_n_s_oov, __pyx_k_oov, sizeof(__pyx_k_oov), 0, 0, 1, 1}, - {&__pyx_n_s_os, __pyx_k_os, sizeof(__pyx_k_os), 0, 0, 1, 1}, - {&__pyx_n_s_out_state, __pyx_k_out_state, sizeof(__pyx_k_out_state), 0, 0, 1, 1}, - {&__pyx_n_s_path, __pyx_k_path, sizeof(__pyx_k_path), 0, 0, 1, 1}, - {&__pyx_n_s_prepare, __pyx_k_prepare, sizeof(__pyx_k_prepare), 0, 0, 1, 1}, - {&__pyx_n_s_qualname, __pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 0, 1, 1}, - {&__pyx_n_s_replace, __pyx_k_replace, sizeof(__pyx_k_replace), 0, 0, 1, 1}, - {&__pyx_n_s_score, __pyx_k_score, sizeof(__pyx_k_score), 0, 0, 1, 1}, - {&__pyx_n_s_send, __pyx_k_send, sizeof(__pyx_k_send), 0, 0, 1, 1}, - {&__pyx_n_s_sentence, __pyx_k_sentence, sizeof(__pyx_k_sentence), 0, 0, 1, 1}, - {&__pyx_n_s_split, __pyx_k_split, sizeof(__pyx_k_split), 0, 0, 1, 1}, - {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, - {&__pyx_n_s_throw, __pyx_k_throw, sizeof(__pyx_k_throw), 0, 0, 1, 1}, - {&__pyx_n_s_utf8, __pyx_k_utf8, sizeof(__pyx_k_utf8), 0, 0, 1, 1}, - {&__pyx_n_s_word, __pyx_k_word, sizeof(__pyx_k_word), 0, 0, 1, 1}, - {0, 0, 0, 0, 0, 0, 0} -}; -static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_TypeError = __Pyx_GetBuiltinName(__pyx_n_s_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_IOError = __Pyx_GetBuiltinName(__pyx_n_s_IOError); if (!__pyx_builtin_IOError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - return 0; - __pyx_L1_error:; - return -1; -} - -static int __Pyx_InitCachedConstants(void) { - __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - - /* "kenlm.pyx":8 - * return data - * elif isinstance(data, unicode): - * return data.encode('utf8') # <<<<<<<<<<<<<< - * raise TypeError('Cannot convert %s to string' % type(data)) - * - */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_n_s_utf8); if (unlikely(!__pyx_tuple_)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple_); - __Pyx_GIVEREF(__pyx_tuple_); - - /* "kenlm.pyx":116 - * self.model = _kenlm.LoadVirtual(self.path, config._c_config) - * except RuntimeError as exception: - * exception_message = str(exception).replace('\n', ' ') # <<<<<<<<<<<<<< - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ - * from exception - */ - __pyx_tuple__5 = PyTuple_Pack(2, __pyx_kp_s__3, __pyx_kp_s__4); if (unlikely(!__pyx_tuple__5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_tuple__5); - __Pyx_GIVEREF(__pyx_tuple__5); - __Pyx_RefNannyFinishContext(); - return 0; - __pyx_L1_error:; - __Pyx_RefNannyFinishContext(); - return -1; -} - -static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - __pyx_float_10_0 = PyFloat_FromDouble(10.0); if (unlikely(!__pyx_float_10_0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - return 0; - __pyx_L1_error:; - return -1; -} - -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC initkenlm(void); /*proto*/ -PyMODINIT_FUNC initkenlm(void) -#else -PyMODINIT_FUNC PyInit_kenlm(void); /*proto*/ -PyMODINIT_FUNC PyInit_kenlm(void) -#endif -{ - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannyDeclarations - #if CYTHON_REFNANNY - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); - if (!__Pyx_RefNanny) { - PyErr_Clear(); - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); - if (!__Pyx_RefNanny) - Py_FatalError("failed to import 'refnanny' module"); - } - #endif - __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_kenlm(void)", 0); - if ( __Pyx_check_binary_version() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #ifdef __Pyx_CyFunction_USED - if (__Pyx_CyFunction_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif - #ifdef __Pyx_FusedFunction_USED - if (__pyx_FusedFunction_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif - #ifdef __Pyx_Generator_USED - if (__pyx_Generator_init() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif - /*--- Library function declarations ---*/ - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Module creation code ---*/ - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4("kenlm", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (unlikely(!__pyx_m)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - Py_INCREF(__pyx_d); - __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #if CYTHON_COMPILING_IN_PYPY - Py_INCREF(__pyx_b); - #endif - if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - /*--- Initialize various global constants etc. ---*/ - if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if (__Pyx_init_sys_getdefaultencoding_params() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif - if (__pyx_module_is_main_kenlm) { - if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - } - #if PY_MAJOR_VERSION >= 3 - { - PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (!PyDict_GetItemString(modules, "kenlm")) { - if (unlikely(PyDict_SetItemString(modules, "kenlm", __pyx_m) < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - } - #endif - /*--- Builtin init code ---*/ - if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Constants init code ---*/ - if (unlikely(__Pyx_InitCachedConstants() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Global init code ---*/ - /*--- Variable export code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5kenlm_FullScoreReturn) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 11; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_type_5kenlm_FullScoreReturn.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "FullScoreReturn", (PyObject *)&__pyx_type_5kenlm_FullScoreReturn) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 11; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5kenlm_FullScoreReturn = &__pyx_type_5kenlm_FullScoreReturn; - if (PyType_Ready(&__pyx_type_5kenlm_State) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_type_5kenlm_State.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "State", (PyObject *)&__pyx_type_5kenlm_State) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5kenlm_State = &__pyx_type_5kenlm_State; - if (PyType_Ready(&__pyx_type_5kenlm_Config) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 80; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_type_5kenlm_Config.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Config", (PyObject *)&__pyx_type_5kenlm_Config) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 80; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5kenlm_Config = &__pyx_type_5kenlm_Config; - if (PyType_Ready(&__pyx_type_5kenlm_Model) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_type_5kenlm_Model.tp_print = 0; - #if CYTHON_COMPILING_IN_CPYTHON - { - PyObject *wrapper = PyObject_GetAttrString((PyObject *)&__pyx_type_5kenlm_Model, "__init__"); if (unlikely(!wrapper)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) { - __pyx_wrapperbase_5kenlm_5Model___init__ = *((PyWrapperDescrObject *)wrapper)->d_base; - __pyx_wrapperbase_5kenlm_5Model___init__.doc = __pyx_doc_5kenlm_5Model___init__; - ((PyWrapperDescrObject *)wrapper)->d_base = &__pyx_wrapperbase_5kenlm_5Model___init__; - } - } - #endif - if (PyObject_SetAttrString(__pyx_m, "Model", (PyObject *)&__pyx_type_5kenlm_Model) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5kenlm_Model = &__pyx_type_5kenlm_Model; - if (PyType_Ready(&__pyx_type_5kenlm___pyx_scope_struct__full_scores) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_type_5kenlm___pyx_scope_struct__full_scores.tp_print = 0; - __pyx_ptype_5kenlm___pyx_scope_struct__full_scores = &__pyx_type_5kenlm___pyx_scope_struct__full_scores; - /*--- Type import code ---*/ - /*--- Variable import code ---*/ - /*--- Function import code ---*/ - /*--- Execution code ---*/ - - /* "kenlm.pyx":1 - * import os # <<<<<<<<<<<<<< - * cimport _kenlm - * - */ - __pyx_t_1 = __Pyx_Import(__pyx_n_s_os, 0, -1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_os, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "kenlm.pyx":73 - * return _kenlm.hash_value(self._c_state) - * - * class LoadMethod: # <<<<<<<<<<<<<< - * LAZY = _kenlm.LAZY - * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY - */ - __pyx_t_1 = __Pyx_Py3MetaclassPrepare((PyObject *) NULL, __pyx_empty_tuple, __pyx_n_s_LoadMethod, __pyx_n_s_LoadMethod, (PyObject *) NULL, __pyx_n_s_kenlm_2, (PyObject *) NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 73; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - - /* "kenlm.pyx":74 - * - * class LoadMethod: - * LAZY = _kenlm.LAZY # <<<<<<<<<<<<<< - * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY - * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ - */ - __pyx_t_2 = PyInt_FromLong(util::LAZY); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_LAZY, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "kenlm.pyx":75 - * class LoadMethod: - * LAZY = _kenlm.LAZY - * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY # <<<<<<<<<<<<<< - * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ - * READ = _kenlm.READ - */ - __pyx_t_2 = PyInt_FromLong(util::POPULATE_OR_LAZY); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_POPULATE_OR_LAZY, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "kenlm.pyx":76 - * LAZY = _kenlm.LAZY - * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY - * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ # <<<<<<<<<<<<<< - * READ = _kenlm.READ - * PARALLEL_READ = _kenlm.PARALLEL_READ - */ - __pyx_t_2 = PyInt_FromLong(util::POPULATE_OR_READ); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_POPULATE_OR_READ, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "kenlm.pyx":77 - * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY - * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ - * READ = _kenlm.READ # <<<<<<<<<<<<<< - * PARALLEL_READ = _kenlm.PARALLEL_READ - * - */ - __pyx_t_2 = PyInt_FromLong(util::READ); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_READ, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "kenlm.pyx":78 - * POPULATE_OR_READ = _kenlm.POPULATE_OR_READ - * READ = _kenlm.READ - * PARALLEL_READ = _kenlm.PARALLEL_READ # <<<<<<<<<<<<<< - * - * cdef class Config: - */ - __pyx_t_2 = PyInt_FromLong(util::PARALLEL_READ); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - if (PyObject_SetItem(__pyx_t_1, __pyx_n_s_PARALLEL_READ, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - - /* "kenlm.pyx":73 - * return _kenlm.hash_value(self._c_state) - * - * class LoadMethod: # <<<<<<<<<<<<<< - * LAZY = _kenlm.LAZY - * POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY - */ - __pyx_t_2 = __Pyx_Py3ClassCreate(((PyObject*)&__Pyx_DefaultClassType), __pyx_n_s_LoadMethod, __pyx_empty_tuple, __pyx_t_1, NULL, 0, 1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 73; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_LoadMethod, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 73; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "kenlm.pyx":105 - * cdef _kenlm.const_Vocabulary* vocab - * - * def __init__(self, path, Config config = Config()): # <<<<<<<<<<<<<< - * """ - * Load the language model. - */ - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5kenlm_Config)), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_k__2 = ((struct __pyx_obj_5kenlm_Config *)__pyx_t_1); - __Pyx_GIVEREF(__pyx_t_1); - __pyx_t_1 = 0; - - /* "kenlm.pyx":260 - * return (_kenlm.LanguageModel, (self.path,)) - * - * class LanguageModel(Model): # <<<<<<<<<<<<<< - * """Backwards compatability stub. Use Model.""" - */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 260; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_INCREF(((PyObject *)((PyObject*)__pyx_ptype_5kenlm_Model))); - PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)((PyObject*)__pyx_ptype_5kenlm_Model))); - __Pyx_GIVEREF(((PyObject *)((PyObject*)__pyx_ptype_5kenlm_Model))); - __pyx_t_2 = __Pyx_CalculateMetaclass(NULL, __pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 260; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_Py3MetaclassPrepare(__pyx_t_2, __pyx_t_1, __pyx_n_s_LanguageModel, __pyx_n_s_LanguageModel, (PyObject *) NULL, __pyx_n_s_kenlm_2, __pyx_kp_s_Backwards_compatability_stub_Use); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 260; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_Py3ClassCreate(__pyx_t_2, __pyx_n_s_LanguageModel, __pyx_t_1, __pyx_t_3, NULL, 0, 1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 260; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_LanguageModel, __pyx_t_4) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 260; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "kenlm.pyx":1 - * import os # <<<<<<<<<<<<<< - * cimport _kenlm - * - */ - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /*--- Wrapped vars code ---*/ - - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - if (__pyx_m) { - if (__pyx_d) { - __Pyx_AddTraceback("init kenlm", __pyx_clineno, __pyx_lineno, __pyx_filename); - } - Py_DECREF(__pyx_m); __pyx_m = 0; - } else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ImportError, "init kenlm"); - } - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else - return __pyx_m; - #endif -} - -/* --- Runtime support code --- */ -#if CYTHON_REFNANNY -static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { - PyObject *m = NULL, *p = NULL; - void *r = NULL; - m = PyImport_ImportModule((char *)modname); - if (!m) goto end; - p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); - if (!p) goto end; - r = PyLong_AsVoidPtr(p); -end: - Py_XDECREF(p); - Py_XDECREF(m); - return (__Pyx_RefNannyAPIStruct *)r; -} -#endif - -static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); - if (unlikely(!result)) { - PyErr_Format(PyExc_NameError, -#if PY_MAJOR_VERSION >= 3 - "name '%U' is not defined", name); -#else - "name '%.200s' is not defined", PyString_AS_STRING(name)); -#endif - } - return result; -} - -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { - PyObject *result; - ternaryfunc call = func->ob_type->tp_call; - if (unlikely(!call)) - return PyObject_Call(func, arg, kw); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = (*call)(func, arg, kw); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { -#if CYTHON_COMPILING_IN_CPYTHON - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyThreadState *tstate = PyThreadState_GET(); - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -#else - PyErr_Restore(type, value, tb); -#endif -} -static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { -#if CYTHON_COMPILING_IN_CPYTHON - PyThreadState *tstate = PyThreadState_GET(); - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -#else - PyErr_Fetch(type, value, tb); -#endif -} - -#if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { - Py_XINCREF(type); - if (!value || value == Py_None) - value = NULL; - else - Py_INCREF(value); - if (!tb || tb == Py_None) - tb = NULL; - else { - Py_INCREF(tb); - if (!PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - } - if (PyType_Check(type)) { -#if CYTHON_COMPILING_IN_PYPY - if (!value) { - Py_INCREF(Py_None); - value = Py_None; - } -#endif - PyErr_NormalizeException(&type, &value, &tb); - } else { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - value = type; - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } - } - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; -} -#else -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { - PyObject* owned_instance = NULL; - if (tb == Py_None) { - tb = 0; - } else if (tb && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto bad; - } - if (value == Py_None) - value = 0; - if (PyExceptionInstance_Check(type)) { - if (value) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto bad; - } - value = type; - type = (PyObject*) Py_TYPE(value); - } else if (PyExceptionClass_Check(type)) { - PyObject *instance_class = NULL; - if (value && PyExceptionInstance_Check(value)) { - instance_class = (PyObject*) Py_TYPE(value); - if (instance_class != type) { - if (PyObject_IsSubclass(instance_class, type)) { - type = instance_class; - } else { - instance_class = NULL; - } - } - } - if (!instance_class) { - PyObject *args; - if (!value) - args = PyTuple_New(0); - else if (PyTuple_Check(value)) { - Py_INCREF(value); - args = value; - } else - args = PyTuple_Pack(1, value); - if (!args) - goto bad; - owned_instance = PyObject_Call(type, args, NULL); - Py_DECREF(args); - if (!owned_instance) - goto bad; - value = owned_instance; - if (!PyExceptionInstance_Check(value)) { - PyErr_Format(PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto bad; - } - } - } else { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto bad; - } -#if PY_VERSION_HEX >= 0x03030000 - if (cause) { -#else - if (cause && cause != Py_None) { -#endif - PyObject *fixed_cause; - if (cause == Py_None) { - fixed_cause = NULL; - } else if (PyExceptionClass_Check(cause)) { - fixed_cause = PyObject_CallObject(cause, NULL); - if (fixed_cause == NULL) - goto bad; - } else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - Py_INCREF(fixed_cause); - } else { - PyErr_SetString(PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto bad; - } - PyException_SetCause(value, fixed_cause); - } - PyErr_SetObject(type, value); - if (tb) { -#if CYTHON_COMPILING_IN_PYPY - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_Fetch(tmp_type, tmp_value, tmp_tb); - Py_INCREF(tb); - PyErr_Restore(tmp_type, tmp_value, tb); - Py_XDECREF(tmp_tb); -#else - PyThreadState *tstate = PyThreadState_GET(); - PyObject* tmp_tb = tstate->curexc_traceback; - if (tb != tmp_tb) { - Py_INCREF(tb); - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_tb); - } -#endif - } -bad: - Py_XDECREF(owned_instance); - return; -} -#endif - -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *more_or_less; - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - PyErr_Format(PyExc_TypeError, - "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", - func_name, more_or_less, num_expected, - (num_expected == 1) ? "" : "s", num_found); -} - -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AsString(kw_name)); - #endif -} - -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - continue; - } - name = first_kw_arg; - #if PY_MAJOR_VERSION < 3 - if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { - while (*name) { - if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) - && _PyString_Eq(**name, key)) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - if ((**argname == key) || ( - (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) - && _PyString_Eq(**argname, key))) { - goto arg_passed_twice; - } - argname++; - } - } - } else - #endif - if (likely(PyUnicode_Check(key))) { - while (*name) { - int cmp = (**name == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**name, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) { - values[name-argnames] = value; - break; - } - name++; - } - if (*name) continue; - else { - PyObject*** argname = argnames; - while (argname != first_kw_arg) { - int cmp = (**argname == key) ? 0 : - #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : - #endif - PyUnicode_Compare(**argname, key); - if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; - if (cmp == 0) goto arg_passed_twice; - argname++; - } - } - } else - goto invalid_keyword_type; - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, key); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); -} -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact) -{ - if (unlikely(!type)) { - PyErr_SetString(PyExc_SystemError, "Missing type object"); - return 0; - } - if (none_allowed && obj == Py_None) return 1; - else if (exact) { - if (likely(Py_TYPE(obj) == type)) return 1; - #if PY_MAJOR_VERSION == 2 - else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; - #endif - } - else { - if (likely(PyObject_TypeCheck(obj, type))) return 1; - } - __Pyx_RaiseArgumentTypeInvalid(name, obj, type); - return 0; -} - -static CYTHON_INLINE int __Pyx_CheckKeywordStrings( - PyObject *kwdict, - const char* function_name, - int kw_allowed) -{ - PyObject* key = 0; - Py_ssize_t pos = 0; -#if CYTHON_COMPILING_IN_PYPY - if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) - goto invalid_keyword; - return 1; -#else - while (PyDict_Next(kwdict, &pos, &key, 0)) { - #if PY_MAJOR_VERSION < 3 - if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) - #endif - if (unlikely(!PyUnicode_Check(key))) - goto invalid_keyword_type; - } - if ((!kw_allowed) && unlikely(key)) - goto invalid_keyword; - return 1; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", function_name); - return 0; -#endif -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%.200s() got an unexpected keyword argument '%.200s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif - return 0; -} - -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { - PyObject *result; -#if CYTHON_COMPILING_IN_CPYTHON - result = PyDict_GetItem(__pyx_d, name); - if (likely(result)) { - Py_INCREF(result); - } else { -#else - result = PyObject_GetItem(__pyx_d, name); - if (!result) { - PyErr_Clear(); -#endif - result = __Pyx_GetBuiltinName(name); - } - return result; -} - -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); - } - } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject* args = PyTuple_Pack(1, arg); - return (likely(args)) ? __Pyx_PyObject_Call(func, args, NULL) : NULL; -} -#endif - -static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb) { -#if CYTHON_COMPILING_IN_CPYTHON - PyThreadState *tstate = PyThreadState_GET(); - *type = tstate->exc_type; - *value = tstate->exc_value; - *tb = tstate->exc_traceback; - Py_XINCREF(*type); - Py_XINCREF(*value); - Py_XINCREF(*tb); -#else - PyErr_GetExcInfo(type, value, tb); -#endif -} -static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb) { -#if CYTHON_COMPILING_IN_CPYTHON - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyThreadState *tstate = PyThreadState_GET(); - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = type; - tstate->exc_value = value; - tstate->exc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -#else - PyErr_SetExcInfo(type, value, tb); -#endif -} - -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { - PyObject *local_type, *local_value, *local_tb; -#if CYTHON_COMPILING_IN_CPYTHON - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyThreadState *tstate = PyThreadState_GET(); - local_type = tstate->curexc_type; - local_value = tstate->curexc_value; - local_tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -#else - PyErr_Fetch(&local_type, &local_value, &local_tb); -#endif - PyErr_NormalizeException(&local_type, &local_value, &local_tb); -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(tstate->curexc_type)) -#else - if (unlikely(PyErr_Occurred())) -#endif - goto bad; - #if PY_MAJOR_VERSION >= 3 - if (local_tb) { - if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) - goto bad; - } - #endif - Py_XINCREF(local_tb); - Py_XINCREF(local_type); - Py_XINCREF(local_value); - *type = local_type; - *value = local_value; - *tb = local_tb; -#if CYTHON_COMPILING_IN_CPYTHON - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = local_type; - tstate->exc_value = local_value; - tstate->exc_traceback = local_tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -#else - PyErr_SetExcInfo(local_type, local_value, local_tb); -#endif - return 0; -bad: - *type = 0; - *value = 0; - *tb = 0; - Py_XDECREF(local_type); - Py_XDECREF(local_value); - Py_XDECREF(local_tb); - return -1; -} - -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { - return __Pyx_PyObject_CallMethO(func, NULL); - } - } - return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); -} -#endif - -static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases) { - Py_ssize_t i, nbases = PyTuple_GET_SIZE(bases); - for (i=0; i < nbases; i++) { - PyTypeObject *tmptype; - PyObject *tmp = PyTuple_GET_ITEM(bases, i); - tmptype = Py_TYPE(tmp); -#if PY_MAJOR_VERSION < 3 - if (tmptype == &PyClass_Type) - continue; -#endif - if (!metaclass) { - metaclass = tmptype; - continue; - } - if (PyType_IsSubtype(metaclass, tmptype)) - continue; - if (PyType_IsSubtype(tmptype, metaclass)) { - metaclass = tmptype; - continue; - } - PyErr_SetString(PyExc_TypeError, - "metaclass conflict: " - "the metaclass of a derived class " - "must be a (non-strict) subclass " - "of the metaclasses of all its bases"); - return NULL; - } - if (!metaclass) { -#if PY_MAJOR_VERSION < 3 - metaclass = &PyClass_Type; -#else - metaclass = &PyType_Type; -#endif - } - Py_INCREF((PyObject*) metaclass); - return (PyObject*) metaclass; -} - -static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, - PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { - PyObject *ns; - if (metaclass) { - PyObject *prep = __Pyx_PyObject_GetAttrStr(metaclass, __pyx_n_s_prepare); - if (prep) { - PyObject *pargs = PyTuple_Pack(2, name, bases); - if (unlikely(!pargs)) { - Py_DECREF(prep); - return NULL; - } - ns = PyObject_Call(prep, pargs, mkw); - Py_DECREF(prep); - Py_DECREF(pargs); - } else { - if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) - return NULL; - PyErr_Clear(); - ns = PyDict_New(); - } - } else { - ns = PyDict_New(); - } - if (unlikely(!ns)) - return NULL; - if (unlikely(PyObject_SetItem(ns, __pyx_n_s_module, modname) < 0)) goto bad; - if (unlikely(PyObject_SetItem(ns, __pyx_n_s_qualname, qualname) < 0)) goto bad; - if (unlikely(doc && PyObject_SetItem(ns, __pyx_n_s_doc, doc) < 0)) goto bad; - return ns; -bad: - Py_DECREF(ns); - return NULL; -} -static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, - PyObject *dict, PyObject *mkw, - int calculate_metaclass, int allow_py2_metaclass) { - PyObject *result, *margs; - PyObject *owned_metaclass = NULL; - if (allow_py2_metaclass) { - owned_metaclass = PyObject_GetItem(dict, __pyx_n_s_metaclass); - if (owned_metaclass) { - metaclass = owned_metaclass; - } else if (likely(PyErr_ExceptionMatches(PyExc_KeyError))) { - PyErr_Clear(); - } else { - return NULL; - } - } - if (calculate_metaclass && (!metaclass || PyType_Check(metaclass))) { - metaclass = __Pyx_CalculateMetaclass((PyTypeObject*) metaclass, bases); - Py_XDECREF(owned_metaclass); - if (unlikely(!metaclass)) - return NULL; - owned_metaclass = metaclass; - } - margs = PyTuple_Pack(3, name, bases, dict); - if (unlikely(!margs)) { - result = NULL; - } else { - result = PyObject_Call(metaclass, margs, mkw); - Py_DECREF(margs); - } - Py_XDECREF(owned_metaclass); - return result; -} - -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { - int start = 0, mid = 0, end = count - 1; - if (end >= 0 && code_line > entries[end].code_line) { - return count; - } - while (start < end) { - mid = (start + end) / 2; - if (code_line < entries[mid].code_line) { - end = mid; - } else if (code_line > entries[mid].code_line) { - start = mid + 1; - } else { - return mid; - } - } - if (code_line <= entries[mid].code_line) { - return mid; - } else { - return mid + 1; - } -} -static PyCodeObject *__pyx_find_code_object(int code_line) { - PyCodeObject* code_object; - int pos; - if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { - return NULL; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { - return NULL; - } - code_object = __pyx_code_cache.entries[pos].code_object; - Py_INCREF(code_object); - return code_object; -} -static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { - int pos, i; - __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; - if (unlikely(!code_line)) { - return; - } - if (unlikely(!entries)) { - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); - if (likely(entries)) { - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = 64; - __pyx_code_cache.count = 1; - entries[0].code_line = code_line; - entries[0].code_object = code_object; - Py_INCREF(code_object); - } - return; - } - pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); - if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { - PyCodeObject* tmp = entries[pos].code_object; - entries[pos].code_object = code_object; - Py_DECREF(tmp); - return; - } - if (__pyx_code_cache.count == __pyx_code_cache.max_count) { - int new_max = __pyx_code_cache.max_count + 64; - entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( - __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); - if (unlikely(!entries)) { - return; - } - __pyx_code_cache.entries = entries; - __pyx_code_cache.max_count = new_max; - } - for (i=__pyx_code_cache.count; i>pos; i--) { - entries[i] = entries[i-1]; - } - entries[pos].code_line = code_line; - entries[pos].code_object = code_object; - __pyx_code_cache.count++; - Py_INCREF(code_object); -} - -#include "compile.h" -#include "frameobject.h" -#include "traceback.h" -static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( - const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(filename); - #else - py_srcfile = PyUnicode_FromString(filename); - #endif - if (!py_srcfile) goto bad; - if (c_line) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_code = __Pyx_PyCode_New( - 0, - 0, - 0, - 0, - 0, - __pyx_empty_bytes, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - py_line, - __pyx_empty_bytes /*PyObject *lnotab*/ - ); - Py_DECREF(py_srcfile); - Py_DECREF(py_funcname); - return py_code; -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - return NULL; -} -static void __Pyx_AddTraceback(const char *funcname, int c_line, - int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - py_code = __pyx_find_code_object(c_line ? c_line : py_line); - if (!py_code) { - py_code = __Pyx_CreateCodeObjectForTraceback( - funcname, c_line, py_line, filename); - if (!py_code) goto bad; - __pyx_insert_code_object(c_line ? c_line : py_line, py_code); - } - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - __pyx_d, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - py_frame->f_lineno = py_line; - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_import; - py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); - if (!py_import) - goto bad; - #endif - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - { - #if PY_MAJOR_VERSION >= 3 - if (level == -1) { - if (strchr(__Pyx_MODULE_NAME, '.')) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(1); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); - #endif - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - goto bad; - PyErr_Clear(); - } - } - level = 0; - } - #endif - if (!module) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(level); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else - module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); - #endif - } - } -bad: - #if PY_VERSION_HEX < 0x03030000 - Py_XDECREF(py_import); - #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); - return module; -} - -#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value) \ - { \ - func_type value = func_value; \ - if (sizeof(target_type) < sizeof(func_type)) { \ - if (unlikely(value != (func_type) (target_type) value)) { \ - func_type zero = 0; \ - if (is_unsigned && unlikely(value < zero)) \ - goto raise_neg_overflow; \ - else \ - goto raise_overflow; \ - } \ - } \ - return (target_type) value; \ - } - -#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 - #if CYTHON_USE_PYLONG_INTERNALS - #include "longintrepr.h" - #endif -#endif - -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { - const int neg_one = (int) -1, const_zero = 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(int) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (int) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 - #if CYTHON_USE_PYLONG_INTERNALS - switch (Py_SIZE(x)) { - case 0: return 0; - case 1: __PYX_VERIFY_RETURN_INT(int, digit, ((PyLongObject*)x)->ob_digit[0]); - } - #endif -#endif - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } - if (sizeof(int) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT(int, unsigned long, PyLong_AsUnsignedLong(x)) - } else if (sizeof(int) <= sizeof(unsigned long long)) { - __PYX_VERIFY_RETURN_INT(int, unsigned long long, PyLong_AsUnsignedLongLong(x)) - } - } else { -#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 - #if CYTHON_USE_PYLONG_INTERNALS - switch (Py_SIZE(x)) { - case 0: return 0; - case 1: __PYX_VERIFY_RETURN_INT(int, digit, +(((PyLongObject*)x)->ob_digit[0])); - case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, -(sdigit) ((PyLongObject*)x)->ob_digit[0]); - } - #endif -#endif - if (sizeof(int) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT(int, long, PyLong_AsLong(x)) - } else if (sizeof(int) <= sizeof(long long)) { - __PYX_VERIFY_RETURN_INT(int, long long, PyLong_AsLongLong(x)) - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - int val; - PyObject *v = __Pyx_PyNumber_Int(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (int) -1; - } - } else { - int val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (int) -1; - val = __Pyx_PyInt_As_int(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to int"); - return (int) -1; -} - -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { - const int neg_one = (int) -1, const_zero = 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(int) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(int) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); - } else if (sizeof(int) <= sizeof(unsigned long long)) { - return PyLong_FromUnsignedLongLong((unsigned long long) value); - } - } else { - if (sizeof(int) <= sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(int) <= sizeof(long long)) { - return PyLong_FromLongLong((long long) value); - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(int), - little, !is_unsigned); - } -} - -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) { - const unsigned int neg_one = (unsigned int) -1, const_zero = 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(unsigned int) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(unsigned int) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); - } else if (sizeof(unsigned int) <= sizeof(unsigned long long)) { - return PyLong_FromUnsignedLongLong((unsigned long long) value); - } - } else { - if (sizeof(unsigned int) <= sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(unsigned int) <= sizeof(long long)) { - return PyLong_FromLongLong((long long) value); - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(unsigned int), - little, !is_unsigned); - } -} - -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_char(unsigned char value) { - const unsigned char neg_one = (unsigned char) -1, const_zero = 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(unsigned char) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(unsigned char) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); - } else if (sizeof(unsigned char) <= sizeof(unsigned long long)) { - return PyLong_FromUnsignedLongLong((unsigned long long) value); - } - } else { - if (sizeof(unsigned char) <= sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(unsigned char) <= sizeof(long long)) { - return PyLong_FromLongLong((long long) value); - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(unsigned char), - little, !is_unsigned); - } -} - -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) -1, const_zero = 0; - const int is_unsigned = neg_one > const_zero; - if (is_unsigned) { - if (sizeof(long) < sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(unsigned long)) { - return PyLong_FromUnsignedLong((unsigned long) value); - } else if (sizeof(long) <= sizeof(unsigned long long)) { - return PyLong_FromUnsignedLongLong((unsigned long long) value); - } - } else { - if (sizeof(long) <= sizeof(long)) { - return PyInt_FromLong((long) value); - } else if (sizeof(long) <= sizeof(long long)) { - return PyLong_FromLongLong((long long) value); - } - } - { - int one = 1; int little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&value; - return _PyLong_FromByteArray(bytes, sizeof(long), - little, !is_unsigned); - } -} - -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { - const long neg_one = (long) -1, const_zero = 0; - const int is_unsigned = neg_one > const_zero; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_Check(x))) { - if (sizeof(long) < sizeof(long)) { - __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) - } else { - long val = PyInt_AS_LONG(x); - if (is_unsigned && unlikely(val < 0)) { - goto raise_neg_overflow; - } - return (long) val; - } - } else -#endif - if (likely(PyLong_Check(x))) { - if (is_unsigned) { -#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 - #if CYTHON_USE_PYLONG_INTERNALS - switch (Py_SIZE(x)) { - case 0: return 0; - case 1: __PYX_VERIFY_RETURN_INT(long, digit, ((PyLongObject*)x)->ob_digit[0]); - } - #endif -#endif - if (unlikely(Py_SIZE(x) < 0)) { - goto raise_neg_overflow; - } - if (sizeof(long) <= sizeof(unsigned long)) { - __PYX_VERIFY_RETURN_INT(long, unsigned long, PyLong_AsUnsignedLong(x)) - } else if (sizeof(long) <= sizeof(unsigned long long)) { - __PYX_VERIFY_RETURN_INT(long, unsigned long long, PyLong_AsUnsignedLongLong(x)) - } - } else { -#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 - #if CYTHON_USE_PYLONG_INTERNALS - switch (Py_SIZE(x)) { - case 0: return 0; - case 1: __PYX_VERIFY_RETURN_INT(long, digit, +(((PyLongObject*)x)->ob_digit[0])); - case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, -(sdigit) ((PyLongObject*)x)->ob_digit[0]); - } - #endif -#endif - if (sizeof(long) <= sizeof(long)) { - __PYX_VERIFY_RETURN_INT(long, long, PyLong_AsLong(x)) - } else if (sizeof(long) <= sizeof(long long)) { - __PYX_VERIFY_RETURN_INT(long, long long, PyLong_AsLongLong(x)) - } - } - { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else - long val; - PyObject *v = __Pyx_PyNumber_Int(x); - #if PY_MAJOR_VERSION < 3 - if (likely(v) && !PyLong_Check(v)) { - PyObject *tmp = v; - v = PyNumber_Long(tmp); - Py_DECREF(tmp); - } - #endif - if (likely(v)) { - int one = 1; int is_little = (int)*(unsigned char *)&one; - unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); - Py_DECREF(v); - if (likely(!ret)) - return val; - } -#endif - return (long) -1; - } - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (long) -1; - val = __Pyx_PyInt_As_long(tmp); - Py_DECREF(tmp); - return val; - } -raise_overflow: - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to long"); - return (long) -1; -raise_neg_overflow: - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to long"); - return (long) -1; -} - -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); - if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); - goto bad; - } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; - } -done: - Py_DECREF(fake_module); - return cached_type; -bad: - Py_XDECREF(cached_type); - cached_type = NULL; - goto done; -} - -static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; -#if CYTHON_COMPILING_IN_CPYTHON - PyThreadState *tstate = PyThreadState_GET(); - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = *type; - tstate->exc_value = *value; - tstate->exc_traceback = *tb; -#else - PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); - PyErr_SetExcInfo(*type, *value, *tb); -#endif - *type = tmp_type; - *value = tmp_value; - *tb = tmp_tb; -} - -static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { - PyObject *method, *result = NULL; - method = __Pyx_PyObject_GetAttrStr(obj, method_name); - if (unlikely(!method)) goto bad; -#if CYTHON_COMPILING_IN_CPYTHON - if (likely(PyMethod_Check(method))) { - PyObject *self = PyMethod_GET_SELF(method); - if (likely(self)) { - PyObject *args; - PyObject *function = PyMethod_GET_FUNCTION(method); - args = PyTuple_New(2); - if (unlikely(!args)) goto bad; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 1, arg); - Py_INCREF(function); - Py_DECREF(method); method = NULL; - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); - return result; - } - } -#endif - result = __Pyx_PyObject_CallOneArg(method, arg); -bad: - Py_XDECREF(method); - return result; -} - -static PyObject *__Pyx_Generator_Next(PyObject *self); -static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value); -static PyObject *__Pyx_Generator_Close(PyObject *self); -static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args); -static PyTypeObject *__pyx_GeneratorType = 0; -#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) -#define __Pyx_Generator_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { - PyObject *et, *ev, *tb; - PyObject *value = NULL; - __Pyx_ErrFetch(&et, &ev, &tb); - if (!et) { - Py_XDECREF(tb); - Py_XDECREF(ev); - Py_INCREF(Py_None); - *pvalue = Py_None; - return 0; - } - if (unlikely(et != PyExc_StopIteration) && - unlikely(!PyErr_GivenExceptionMatches(et, PyExc_StopIteration))) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - if (likely(et == PyExc_StopIteration)) { - if (likely(!ev) || !PyObject_IsInstance(ev, PyExc_StopIteration)) { - if (!ev) { - Py_INCREF(Py_None); - ev = Py_None; - } - Py_XDECREF(tb); - Py_DECREF(et); - *pvalue = ev; - return 0; - } - } - PyErr_NormalizeException(&et, &ev, &tb); - if (unlikely(!PyObject_IsInstance(ev, PyExc_StopIteration))) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - Py_XDECREF(tb); - Py_DECREF(et); -#if PY_VERSION_HEX >= 0x030300A0 - value = ((PyStopIterationObject *)ev)->value; - Py_INCREF(value); - Py_DECREF(ev); -#else - { - PyObject* args = PyObject_GetAttr(ev, __pyx_n_s_args); - Py_DECREF(ev); - if (likely(args)) { - value = PyObject_GetItem(args, 0); - Py_DECREF(args); - } - if (unlikely(!value)) { - __Pyx_ErrRestore(NULL, NULL, NULL); - Py_INCREF(Py_None); - value = Py_None; - } - } -#endif - *pvalue = value; - return 0; -} -#endif -static CYTHON_INLINE -void __Pyx_Generator_ExceptionClear(__pyx_GeneratorObject *self) { - PyObject *exc_type = self->exc_type; - PyObject *exc_value = self->exc_value; - PyObject *exc_traceback = self->exc_traceback; - self->exc_type = NULL; - self->exc_value = NULL; - self->exc_traceback = NULL; - Py_XDECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_traceback); -} -static CYTHON_INLINE -int __Pyx_Generator_CheckRunning(__pyx_GeneratorObject *gen) { - if (unlikely(gen->is_running)) { - PyErr_SetString(PyExc_ValueError, - "generator already executing"); - return 1; - } - return 0; -} -static CYTHON_INLINE -PyObject *__Pyx_Generator_SendEx(__pyx_GeneratorObject *self, PyObject *value) { - PyObject *retval; - assert(!self->is_running); - if (unlikely(self->resume_label == 0)) { - if (unlikely(value && value != Py_None)) { - PyErr_SetString(PyExc_TypeError, - "can't send non-None value to a " - "just-started generator"); - return NULL; - } - } - if (unlikely(self->resume_label == -1)) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - if (value) { -#if CYTHON_COMPILING_IN_PYPY -#else - if (self->exc_traceback) { - PyThreadState *tstate = PyThreadState_GET(); - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_XINCREF(tstate->frame); - assert(f->f_back == NULL); - f->f_back = tstate->frame; - } -#endif - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); - } else { - __Pyx_Generator_ExceptionClear(self); - } - self->is_running = 1; - retval = self->body((PyObject *) self, value); - self->is_running = 0; - if (retval) { - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); -#if CYTHON_COMPILING_IN_PYPY -#else - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_CLEAR(f->f_back); - } -#endif - } else { - __Pyx_Generator_ExceptionClear(self); - } - return retval; -} -static CYTHON_INLINE -PyObject *__Pyx_Generator_FinishDelegation(__pyx_GeneratorObject *gen) { - PyObject *ret; - PyObject *val = NULL; - __Pyx_Generator_Undelegate(gen); - __Pyx_PyGen_FetchStopIterationValue(&val); - ret = __Pyx_Generator_SendEx(gen, val); - Py_XDECREF(val); - return ret; -} -static PyObject *__Pyx_Generator_Next(PyObject *self) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Generator_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - ret = Py_TYPE(yf)->tp_iternext(yf); - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - return __Pyx_Generator_FinishDelegation(gen); - } - return __Pyx_Generator_SendEx(gen, Py_None); -} -static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Generator_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Generator_Send(yf, value); - } else { - if (value == Py_None) - ret = PyIter_Next(yf); - else - ret = __Pyx_PyObject_CallMethod1(yf, __pyx_n_s_send, value); - } - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - return __Pyx_Generator_FinishDelegation(gen); - } - return __Pyx_Generator_SendEx(gen, value); -} -static int __Pyx_Generator_CloseIter(__pyx_GeneratorObject *gen, PyObject *yf) { - PyObject *retval = NULL; - int err = 0; - if (__Pyx_Generator_CheckExact(yf)) { - retval = __Pyx_Generator_Close(yf); - if (!retval) - return -1; - } else { - PyObject *meth; - gen->is_running = 1; - meth = PyObject_GetAttr(yf, __pyx_n_s_close); - if (unlikely(!meth)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_WriteUnraisable(yf); - } - PyErr_Clear(); - } else { - retval = PyObject_CallFunction(meth, NULL); - Py_DECREF(meth); - if (!retval) - err = -1; - } - gen->is_running = 0; - } - Py_XDECREF(retval); - return err; -} -static PyObject *__Pyx_Generator_Close(PyObject *self) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; - PyObject *retval, *raised_exception; - PyObject *yf = gen->yieldfrom; - int err = 0; - if (unlikely(__Pyx_Generator_CheckRunning(gen))) - return NULL; - if (yf) { - Py_INCREF(yf); - err = __Pyx_Generator_CloseIter(gen, yf); - __Pyx_Generator_Undelegate(gen); - Py_DECREF(yf); - } - if (err == 0) - PyErr_SetNone(PyExc_GeneratorExit); - retval = __Pyx_Generator_SendEx(gen, NULL); - if (retval) { - Py_DECREF(retval); - PyErr_SetString(PyExc_RuntimeError, - "generator ignored GeneratorExit"); - return NULL; - } - raised_exception = PyErr_Occurred(); - if (!raised_exception - || raised_exception == PyExc_StopIteration - || raised_exception == PyExc_GeneratorExit - || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) - || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) - { - if (raised_exception) PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - return NULL; -} -static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; - PyObject *typ; - PyObject *tb = NULL; - PyObject *val = NULL; - PyObject *yf = gen->yieldfrom; - if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) - return NULL; - if (unlikely(__Pyx_Generator_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - Py_INCREF(yf); - if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { - int err = __Pyx_Generator_CloseIter(gen, yf); - Py_DECREF(yf); - __Pyx_Generator_Undelegate(gen); - if (err < 0) - return __Pyx_Generator_SendEx(gen, NULL); - goto throw_here; - } - gen->is_running = 1; - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Generator_Throw(yf, args); - } else { - PyObject *meth = PyObject_GetAttr(yf, __pyx_n_s_throw); - if (unlikely(!meth)) { - Py_DECREF(yf); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - gen->is_running = 0; - return NULL; - } - PyErr_Clear(); - __Pyx_Generator_Undelegate(gen); - gen->is_running = 0; - goto throw_here; - } - ret = PyObject_CallObject(meth, args); - Py_DECREF(meth); - } - gen->is_running = 0; - Py_DECREF(yf); - if (!ret) { - ret = __Pyx_Generator_FinishDelegation(gen); - } - return ret; - } -throw_here: - __Pyx_Raise(typ, val, tb, NULL); - return __Pyx_Generator_SendEx(gen, NULL); -} -static int __Pyx_Generator_traverse(PyObject *self, visitproc visit, void *arg) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; - Py_VISIT(gen->closure); - Py_VISIT(gen->classobj); - Py_VISIT(gen->yieldfrom); - Py_VISIT(gen->exc_type); - Py_VISIT(gen->exc_value); - Py_VISIT(gen->exc_traceback); - return 0; -} -static int __Pyx_Generator_clear(PyObject *self) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; - Py_CLEAR(gen->closure); - Py_CLEAR(gen->classobj); - Py_CLEAR(gen->yieldfrom); - Py_CLEAR(gen->exc_type); - Py_CLEAR(gen->exc_value); - Py_CLEAR(gen->exc_traceback); - Py_CLEAR(gen->gi_name); - Py_CLEAR(gen->gi_qualname); - return 0; -} -static void __Pyx_Generator_dealloc(PyObject *self) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; - PyObject_GC_UnTrack(gen); - if (gen->gi_weakreflist != NULL) - PyObject_ClearWeakRefs(self); - if (gen->resume_label > 0) { - PyObject_GC_Track(self); -#if PY_VERSION_HEX >= 0x030400a1 - if (PyObject_CallFinalizerFromDealloc(self)) -#else - Py_TYPE(gen)->tp_del(self); - if (self->ob_refcnt > 0) -#endif - { - return; - } - PyObject_GC_UnTrack(self); - } - __Pyx_Generator_clear(self); - PyObject_GC_Del(gen); -} -static void __Pyx_Generator_del(PyObject *self) { - PyObject *res; - PyObject *error_type, *error_value, *error_traceback; - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; - if (gen->resume_label <= 0) - return ; -#if PY_VERSION_HEX < 0x030400a1 - assert(self->ob_refcnt == 0); - self->ob_refcnt = 1; -#endif - __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); - res = __Pyx_Generator_Close(self); - if (res == NULL) - PyErr_WriteUnraisable(self); - else - Py_DECREF(res); - __Pyx_ErrRestore(error_type, error_value, error_traceback); -#if PY_VERSION_HEX < 0x030400a1 - assert(self->ob_refcnt > 0); - if (--self->ob_refcnt == 0) { - return; - } - { - Py_ssize_t refcnt = self->ob_refcnt; - _Py_NewReference(self); - self->ob_refcnt = refcnt; - } -#if CYTHON_COMPILING_IN_CPYTHON - assert(PyType_IS_GC(self->ob_type) && - _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); - _Py_DEC_REFTOTAL; -#endif -#ifdef COUNT_ALLOCS - --Py_TYPE(self)->tp_frees; - --Py_TYPE(self)->tp_allocs; -#endif -#endif -} -static PyObject * -__Pyx_Generator_get_name(__pyx_GeneratorObject *self) -{ - Py_INCREF(self->gi_name); - return self->gi_name; -} -static int -__Pyx_Generator_set_name(__pyx_GeneratorObject *self, PyObject *value) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { -#else - if (unlikely(value == NULL || !PyString_Check(value))) { -#endif - PyErr_SetString(PyExc_TypeError, - "__name__ must be set to a string object"); - return -1; - } - tmp = self->gi_name; - Py_INCREF(value); - self->gi_name = value; - Py_XDECREF(tmp); - return 0; -} -static PyObject * -__Pyx_Generator_get_qualname(__pyx_GeneratorObject *self) -{ - Py_INCREF(self->gi_qualname); - return self->gi_qualname; -} -static int -__Pyx_Generator_set_qualname(__pyx_GeneratorObject *self, PyObject *value) -{ - PyObject *tmp; -#if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { -#else - if (unlikely(value == NULL || !PyString_Check(value))) { -#endif - PyErr_SetString(PyExc_TypeError, - "__qualname__ must be set to a string object"); - return -1; - } - tmp = self->gi_qualname; - Py_INCREF(value); - self->gi_qualname = value; - Py_XDECREF(tmp); - return 0; -} -static PyGetSetDef __pyx_Generator_getsets[] = { - {(char *) "__name__", (getter)__Pyx_Generator_get_name, (setter)__Pyx_Generator_set_name, - (char*) PyDoc_STR("name of the generator"), 0}, - {(char *) "__qualname__", (getter)__Pyx_Generator_get_qualname, (setter)__Pyx_Generator_set_qualname, - (char*) PyDoc_STR("qualified name of the generator"), 0}, - {0, 0, 0, 0, 0} -}; -static PyMemberDef __pyx_Generator_memberlist[] = { - {(char *) "gi_running", T_BOOL, offsetof(__pyx_GeneratorObject, is_running), READONLY, NULL}, - {0, 0, 0, 0, 0} -}; -static PyMethodDef __pyx_Generator_methods[] = { - {"send", (PyCFunction) __Pyx_Generator_Send, METH_O, 0}, - {"throw", (PyCFunction) __Pyx_Generator_Throw, METH_VARARGS, 0}, - {"close", (PyCFunction) __Pyx_Generator_Close, METH_NOARGS, 0}, - {0, 0, 0, 0} -}; -static PyTypeObject __pyx_GeneratorType_type = { - PyVarObject_HEAD_INIT(0, 0) - "generator", - sizeof(__pyx_GeneratorObject), - 0, - (destructor) __Pyx_Generator_dealloc, - 0, - 0, - 0, -#if PY_MAJOR_VERSION < 3 - 0, -#else - 0, -#endif - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, - 0, - (traverseproc) __Pyx_Generator_traverse, - 0, - 0, - offsetof(__pyx_GeneratorObject, gi_weakreflist), - 0, - (iternextfunc) __Pyx_Generator_Next, - __pyx_Generator_methods, - __pyx_Generator_memberlist, - __pyx_Generator_getsets, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, -#if PY_VERSION_HEX >= 0x030400a1 - 0, -#else - __Pyx_Generator_del, -#endif - 0, -#if PY_VERSION_HEX >= 0x030400a1 - __Pyx_Generator_del, -#endif -}; -static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body, - PyObject *closure, PyObject *name, PyObject *qualname) { - __pyx_GeneratorObject *gen = - PyObject_GC_New(__pyx_GeneratorObject, &__pyx_GeneratorType_type); - if (gen == NULL) - return NULL; - gen->body = body; - gen->closure = closure; - Py_XINCREF(closure); - gen->is_running = 0; - gen->resume_label = 0; - gen->classobj = NULL; - gen->yieldfrom = NULL; - gen->exc_type = NULL; - gen->exc_value = NULL; - gen->exc_traceback = NULL; - gen->gi_weakreflist = NULL; - Py_XINCREF(qualname); - gen->gi_qualname = qualname; - Py_XINCREF(name); - gen->gi_name = name; - PyObject_GC_Track(gen); - return gen; -} -static int __pyx_Generator_init(void) { - __pyx_GeneratorType_type.tp_getattro = PyObject_GenericGetAttr; - __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; - __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); - if (__pyx_GeneratorType == NULL) { - return -1; - } - return 0; -} - -static int __Pyx_check_binary_version(void) { - char ctversion[4], rtversion[4]; - PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); - PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); - if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { - char message[200]; - PyOS_snprintf(message, sizeof(message), - "compiletime version %s of module '%.100s' " - "does not match runtime version %s", - ctversion, __Pyx_MODULE_NAME, rtversion); - return PyErr_WarnEx(NULL, message, 1); - } - return 0; -} - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { - while (t->p) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - ++t; - } - return 0; -} - -static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { - return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { - Py_ssize_t ignore; - return __Pyx_PyObject_AsStringAndSize(o, &ignore); -} -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT - if ( -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - __Pyx_sys_getdefaultencoding_not_ascii && -#endif - PyUnicode_Check(o)) { -#if PY_VERSION_HEX < 0x03030000 - char* defenc_c; - PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); - if (!defenc) return NULL; - defenc_c = PyBytes_AS_STRING(defenc); -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - { - char* end = defenc_c + PyBytes_GET_SIZE(defenc); - char* c; - for (c = defenc_c; c < end; c++) { - if ((unsigned char) (*c) >= 128) { - PyUnicode_AsASCIIString(o); - return NULL; - } - } - } -#endif - *length = PyBytes_GET_SIZE(defenc); - return defenc_c; -#else - if (__Pyx_PyUnicode_READY(o) == -1) return NULL; -#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - if (PyUnicode_IS_ASCII(o)) { - *length = PyUnicode_GET_LENGTH(o); - return PyUnicode_AsUTF8(o); - } else { - PyUnicode_AsASCIIString(o); - return NULL; - } -#else - return PyUnicode_AsUTF8AndSize(o, length); -#endif -#endif - } else -#endif -#if !CYTHON_COMPILING_IN_PYPY - if (PyByteArray_Check(o)) { - *length = PyByteArray_GET_SIZE(o); - return PyByteArray_AS_STRING(o); - } else -#endif - { - char* result; - int r = PyBytes_AsStringAndSize(o, &result, length); - if (unlikely(r < 0)) { - return NULL; - } else { - return result; - } - } -} -static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - int is_true = x == Py_True; - if (is_true | (x == Py_False) | (x == Py_None)) return is_true; - else return PyObject_IsTrue(x); -} -static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { - PyNumberMethods *m; - const char *name = NULL; - PyObject *res = NULL; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(x) || PyLong_Check(x)) -#else - if (PyLong_Check(x)) -#endif - return Py_INCREF(x), x; - m = Py_TYPE(x)->tp_as_number; -#if PY_MAJOR_VERSION < 3 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } -#else - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Long(x); - } -#endif - if (res) { -#if PY_MAJOR_VERSION < 3 - if (!PyInt_Check(res) && !PyLong_Check(res)) { -#else - if (!PyLong_Check(res)) { -#endif - PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} -static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject *x; -#if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(b))) - return PyInt_AS_LONG(b); -#endif - if (likely(PyLong_CheckExact(b))) { - #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 - #if CYTHON_USE_PYLONG_INTERNALS - switch (Py_SIZE(b)) { - case -1: return -(sdigit)((PyLongObject*)b)->ob_digit[0]; - case 0: return 0; - case 1: return ((PyLongObject*)b)->ob_digit[0]; - } - #endif - #endif - return PyLong_AsSsize_t(b); - } - x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} -static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { - return PyInt_FromSize_t(ival); -} - - -#endif /* Py_PYTHON_H */ diff --git a/native_client/kenlm/python/kenlm.pyx b/native_client/kenlm/python/kenlm.pyx deleted file mode 100644 index 549a7803..00000000 --- a/native_client/kenlm/python/kenlm.pyx +++ /dev/null @@ -1,261 +0,0 @@ -import os -cimport _kenlm - -cdef bytes as_str(data): - if isinstance(data, bytes): - return data - elif isinstance(data, unicode): - return data.encode('utf8') - raise TypeError('Cannot convert %s to string' % type(data)) - -cdef class FullScoreReturn: - """ - Wrapper around FullScoreReturn. - - Notes: - `prob` has been renamed to `log_prob` - `oov` has been added to flag whether the word is OOV - """ - - cdef float log_prob - cdef int ngram_length - cdef bint oov - - def __cinit__(self, log_prob, ngram_length, oov): - self.log_prob = log_prob - self.ngram_length = ngram_length - self.oov = oov - - def __repr__(self): - return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) - - property log_prob: - def __get__(self): - return self.log_prob - - property ngram_length: - def __get__(self): - return self.ngram_length - - property oov: - def __get__(self): - return self.oov - -cdef class State: - """ - Wrapper around lm::ngram::State so that python code can make incremental queries. - - Notes: - * rich comparisons - * hashable - """ - - cdef _kenlm.State _c_state - - def __richcmp__(State qa, State qb, int op): - r = qa._c_state.Compare(qb._c_state) - if op == 0: # < - return r < 0 - elif op == 1: # <= - return r <= 0 - elif op == 2: # == - return r == 0 - elif op == 3: # != - return r != 0 - elif op == 4: # > - return r > 0 - else: # >= - return r >= 0 - - def __hash__(self): - return _kenlm.hash_value(self._c_state) - -class LoadMethod: - LAZY = _kenlm.LAZY - POPULATE_OR_LAZY = _kenlm.POPULATE_OR_LAZY - POPULATE_OR_READ = _kenlm.POPULATE_OR_READ - READ = _kenlm.READ - PARALLEL_READ = _kenlm.PARALLEL_READ - -cdef class Config: - """ - Wrapper around lm::ngram::Config. - Pass this to Model's constructor to set the load_method. - """ - cdef _kenlm.Config _c_config - - def __init__(self): - self._c_config = _kenlm.Config() - - property load_method: - def __get__(self): - return self._c_config.load_method - def __set__(self, to): - self._c_config.load_method = to - -cdef class Model: - """ - Wrapper around lm::ngram::Model. - """ - - cdef _kenlm.Model* model - cdef public bytes path - cdef _kenlm.const_Vocabulary* vocab - - def __init__(self, path, Config config = Config()): - """ - Load the language model. - - :param path: path to an arpa file or a kenlm binary file. - :param config: configuration options (see lm/config.hh for documentation) - """ - self.path = os.path.abspath(as_str(path)) - try: - self.model = _kenlm.LoadVirtual(self.path, config._c_config) - except RuntimeError as exception: - exception_message = str(exception).replace('\n', ' ') - raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ - from exception - self.vocab = &self.model.BaseVocabulary() - - def __dealloc__(self): - del self.model - - property order: - def __get__(self): - return self.model.Order() - - def score(self, sentence, bos = True, eos = True): - """ - Return the log10 probability of a string. By default, the string is - treated as a sentence. - return log10 p(sentence | ) - - If you do not want to condition on the beginning of sentence, pass - bos = False - Never include as part of the string. That would be predicting the - beginning of sentence. Language models are only supposed to condition - on it as context. - - Similarly, the end of sentence token can be omitted with - eos = False - Since language models explicitly predict , it can be part of the - string. - - Examples: - - #Good: returns log10 p(this is a sentence . | ) - model.score("this is a sentence .") - #Good: same as the above but more explicit - model.score("this is a sentence .", bos = True, eos = True) - - #Bad: never include - model.score(" this is a sentence") - #Bad: never include , even if bos = False. - model.score(" this is a sentence", bos = False) - - #Good: returns log10 p(a fragment) - model.score("a fragment", bos = False, eos = False) - - #Good: returns log10 p(a fragment ) - model.score("a fragment", bos = False, eos = True) - - #Ok, but bad practice: returns log10 p(a fragment ) - #Unlike , the end of sentence token can appear explicitly. - model.score("a fragment ", bos = False, eos = False) - """ - cdef list words = as_str(sentence).split() - cdef _kenlm.State state - if bos: - self.model.BeginSentenceWrite(&state) - else: - self.model.NullContextWrite(&state) - cdef _kenlm.State out_state - cdef float total = 0 - for word in words: - total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state) - state = out_state - if eos: - total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state) - return total - - def perplexity(self, sentence): - """ - Compute perplexity of a sentence. - @param sentence One full sentence to score. Do not include or . - """ - words = len(as_str(sentence).split()) + 1 # For - return 10.0**(-self.score(sentence) / words) - - def full_scores(self, sentence, bos = True, eos = True): - """ - full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov) - @param sentence is a string (do not use boundary symbols) - @param bos should kenlm add a bos state - @param eos should kenlm add an eos state - """ - cdef list words = as_str(sentence).split() - cdef _kenlm.State state - if bos: - self.model.BeginSentenceWrite(&state) - else: - self.model.NullContextWrite(&state) - cdef _kenlm.State out_state - cdef _kenlm.FullScoreReturn ret - cdef float total = 0 - cdef _kenlm.WordIndex wid - for word in words: - wid = self.vocab.Index(word) - ret = self.model.BaseFullScore(&state, wid, &out_state) - yield (ret.prob, ret.ngram_length, wid == 0) - state = out_state - if eos: - ret = self.model.BaseFullScore(&state, - self.vocab.EndSentence(), &out_state) - yield (ret.prob, ret.ngram_length, False) - - - def BeginSentenceWrite(self, State state): - """Change the given state to a BOS state.""" - self.model.BeginSentenceWrite(&state._c_state) - - def NullContextWrite(self, State state): - """Change the given state to a NULL state.""" - self.model.NullContextWrite(&state._c_state) - - def BaseScore(self, State in_state, str word, State out_state): - """ - Return p(word|in_state) and update the output state. - Wrapper around model.BaseScore(in_state, Index(word), out_state) - - :param word: the suffix - :param state: the context (defaults to NullContext) - :returns: p(word|state) - """ - cdef float total = self.model.BaseScore(&in_state._c_state, self.vocab.Index(as_str(word)), &out_state._c_state) - return total - - def BaseFullScore(self, State in_state, str word, State out_state): - """ - Wrapper around model.BaseScore(in_state, Index(word), out_state) - - :param word: the suffix - :param state: the context (defaults to NullContext) - :returns: FullScoreReturn(word|state) - """ - cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word)) - cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state) - return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0) - - def __contains__(self, word): - cdef bytes w = as_str(word) - return (self.vocab.Index(w) != 0) - - def __repr__(self): - return ''.format(os.path.basename(self.path)) - - def __reduce__(self): - return (_kenlm.LanguageModel, (self.path,)) - -class LanguageModel(Model): - """Backwards compatability stub. Use Model.""" diff --git a/native_client/kenlm/util/getopt.c b/native_client/kenlm/util/getopt.c deleted file mode 100644 index 50eef42c..00000000 --- a/native_client/kenlm/util/getopt.c +++ /dev/null @@ -1,78 +0,0 @@ -/* -POSIX getopt for Windows - -AT&T Public License - -Code given out at the 1985 UNIFORUM conference in Dallas. -*/ - -#ifndef __GNUC__ - -#include "getopt.hh" -#include -#include - -#define NULL 0 -#define EOF (-1) -#define ERR(s, c) if(opterr){\ - char errbuf[2];\ - errbuf[0] = c; errbuf[1] = '\n';\ - fputs(argv[0], stderr);\ - fputs(s, stderr);\ - fputc(c, stderr);} - //(void) write(2, argv[0], (unsigned)strlen(argv[0]));\ - //(void) write(2, s, (unsigned)strlen(s));\ - //(void) write(2, errbuf, 2);} - -int opterr = 1; -int optind = 1; -int optopt; -char *optarg; - -int -getopt(argc, argv, opts) -int argc; -char **argv, *opts; -{ - static int sp = 1; - register int c; - register char *cp; - - if(sp == 1) - if(optind >= argc || - argv[optind][0] != '-' || argv[optind][1] == '\0') - return(EOF); - else if(strcmp(argv[optind], "--") == NULL) { - optind++; - return(EOF); - } - optopt = c = argv[optind][sp]; - if(c == ':' || (cp=strchr(opts, c)) == NULL) { - ERR(": illegal option -- ", c); - if(argv[optind][++sp] == '\0') { - optind++; - sp = 1; - } - return('?'); - } - if(*++cp == ':') { - if(argv[optind][sp+1] != '\0') - optarg = &argv[optind++][sp+1]; - else if(++optind >= argc) { - ERR(": option requires an argument -- ", c); - sp = 1; - return('?'); - } else - optarg = argv[optind++]; - sp = 1; - } else { - if(argv[optind][++sp] == '\0') { - sp = 1; - optind++; - } - optarg = NULL; - } - return(c); -} - -#endif /* __GNUC__ */ diff --git a/native_client/kenlm/util/getopt.hh b/native_client/kenlm/util/getopt.hh deleted file mode 100644 index 9b0792b0..00000000 --- a/native_client/kenlm/util/getopt.hh +++ /dev/null @@ -1,33 +0,0 @@ -/* -POSIX getopt for Windows - -AT&T Public License - -Code given out at the 1985 UNIFORUM conference in Dallas. -*/ - -#ifdef __GNUC__ -#include -#endif -#ifndef __GNUC__ - -#ifndef UTIL_GETOPT_H -#define UTIL_GETOPT_H - -#ifdef __cplusplus -extern "C" { -#endif - -extern int opterr; -extern int optind; -extern int optopt; -extern char *optarg; -extern int getopt(int argc, char **argv, char *opts); - -#ifdef __cplusplus -} -#endif - -#endif /* UTIL_GETOPT_H */ -#endif /* __GNUC__ */ - diff --git a/native_client/trie_node.h b/native_client/trie_node.h index 988a0f2e..93ed9b0f 100644 --- a/native_client/trie_node.h +++ b/native_client/trie_node.h @@ -15,37 +15,40 @@ limitations under the License. #include "lm/model.hh" +#include #include -#include #include +#include #include +#include +#include class TrieNode { public: TrieNode(int vocab_size) - : vocab_size(vocab_size) - , prefixCount(0) - , min_score_word(0) - , min_unigram_score(std::numeric_limits::max()) + : vocab_size_(vocab_size) + , prefixCount_(0) + , min_score_word_(0) + , min_unigram_score_(std::numeric_limits::max()) { - children = new TrieNode*[vocab_size](); + children_ = new TrieNode*[vocab_size_](); } ~TrieNode() { - for (int i = 0; i < vocab_size; i++) { - delete children[i]; + for (int i = 0; i < vocab_size_; i++) { + delete children_[i]; } - delete children; + delete children_; } - void WriteToStream(std::ostream& os) { + void WriteToStream(std::ostream& os) const { WriteNode(os); - for (int i = 0; i < vocab_size; i++) { - if (children[i] == nullptr) { + for (int i = 0; i < vocab_size_; i++) { + if (children_[i] == nullptr) { os << -1 << std::endl; } else { // Recursive call - children[i]->WriteToStream(os); + children_[i]->WriteToStream(os); } } } @@ -63,61 +66,80 @@ public: obj = new TrieNode(vocab_size); obj->ReadNode(is, prefixCount); for (int i = 0; i < vocab_size; i++) { - // Recursive call - ReadFromStream(is, obj->children[i], vocab_size); + ReadFromStream(is, obj->children_[i], vocab_size); } } - void Insert(const char* word, std::function translator, + void Insert(const char* word, std::function translator, lm::WordIndex lm_word, float unigram_score) { - char wordCharacter = *word; - prefixCount++; - if (unigram_score < min_unigram_score) { - min_unigram_score = unigram_score; - min_score_word = lm_word; + // All strings are UTF-8 encoded at the API boundaries. We need to iterate + // on codepoints in order to support multi-byte characters, so we convert + // to UCS-4 to extract the first codepoint, then the codepoint back to + // UTF-8 to translate it into a vocabulary index. + + //TODO We should normalize the input first, and possibly iterate by grapheme + // instead of codepoint for languages that don't have composed versions + // of multi-codepoint characters. This requires extra dependencies so + // leaving as a future improvement when the need arises. + std::wstring_convert, char32_t> ucs4conv; + std::u32string codepoints = ucs4conv.from_bytes(word); + Insert(codepoints.begin(), translator, lm_word, unigram_score); + } + + void Insert(const std::u32string::iterator& codepoints, + std::function translator, + lm::WordIndex lm_word, float unigram_score) { + std::wstring_convert, char32_t> ucs4conv; + char32_t firstCodepoint = *codepoints; + std::string firstCodepoint_utf8 = ucs4conv.to_bytes(firstCodepoint); + + prefixCount_++; + if (unigram_score < min_unigram_score_) { + min_unigram_score_ = unigram_score; + min_score_word_ = lm_word; } - if (wordCharacter != '\0') { - int vocabIndex = translator(wordCharacter); - TrieNode *child = children[vocabIndex]; - if (child == nullptr) - child = children[vocabIndex] = new TrieNode(vocab_size); - child->Insert(word + 1, translator, lm_word, unigram_score); + if (firstCodepoint != 0) { + int vocabIndex = translator(firstCodepoint_utf8); + if (children_[vocabIndex] == nullptr) { + children_[vocabIndex] = new TrieNode(vocab_size_); + } + children_[vocabIndex]->Insert(codepoints+1, translator, lm_word, unigram_score); } } - int GetFrequency() { - return prefixCount; + int GetPrefixCount() const { + return prefixCount_; } - lm::WordIndex GetMinScoreWordIndex() { - return min_score_word; + lm::WordIndex GetMinScoreWordIndex() const { + return min_score_word_; } - float GetMinUnigramScore() { - return min_unigram_score; + float GetMinUnigramScore() const { + return min_unigram_score_; } TrieNode *GetChildAt(int vocabIndex) { - return children[vocabIndex]; + return children_[vocabIndex]; } private: - int vocab_size; - int prefixCount; - lm::WordIndex min_score_word; - float min_unigram_score; - TrieNode **children; + int vocab_size_; + int prefixCount_; + lm::WordIndex min_score_word_; + float min_unigram_score_; + TrieNode **children_; void WriteNode(std::ostream& os) const { - os << prefixCount << std::endl; - os << min_score_word << std::endl; - os << min_unigram_score << std::endl; + os << prefixCount_ << std::endl; + os << min_score_word_ << std::endl; + os << min_unigram_score_ << std::endl; } void ReadNode(std::istream& is, int first_input) { - prefixCount = first_input; - is >> min_score_word; - is >> min_unigram_score; + prefixCount_ = first_input; + is >> min_score_word_; + is >> min_unigram_score_; } };