File Coverage

udpipe/udpipe.cpp

Criterion	Covered	Total	%
statement	1870	9263	20.1
branch	1707	16906	10.1
condition			n/a
subroutine			n/a
pod			n/a
total	3577	26169	13.6

// This file is part of UDPipe .
//
// This file is a bundle of all sources and headers of UDPipe library.
// Comments and copyrights of all individual files are kept.

#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include

namespace ufal {
namespace udpipe {

/////////
// File: utils/common.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Headers available in all sources

namespace utils {

using namespace std;

// Assert that int is at least 4B
static_assert(sizeof(int) >= sizeof(int32_t), "Int must be at least 4B wide!");

// Assert that we are on a little endian system
#ifdef __BYTE_ORDER__
static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "Only little endian systems are supported!");
#endif

#define runtime_failure(message) exit((cerr << message << endl, 1))

} // namespace utils

/////////
// File: utils/string_piece.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

struct string_piece {
  const char* str;
  size_t len;

  string_piece() : str(nullptr), len(0) {}
  string_piece(const char* str) : str(str), len(strlen(str)) {}
  string_piece(const char* str, size_t len) : str(str), len(len) {}
  string_piece(const string& str) : str(str.c_str()), len(str.size()) {}
};

inline ostream& operator<<(ostream& os, const string_piece& str) {
  return os.write(str.str, str.len);
}

inline bool operator==(const string_piece& a, const string_piece& b) {
  return a.len == b.len && memcmp(a.str, b.str, a.len) == 0;
}

inline bool operator!=(const string_piece& a, const string_piece& b) {
  return a.len != b.len || memcmp(a.str, b.str, a.len) != 0;
}

} // namespace utils

/////////
// File: common.h
/////////

// This file is part of UDPipe .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

using namespace utils;

/////////
// File: sentence/empty_node.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class empty_node {
 public:
  int id;         // 0 is root, >0 is sentence word, <0 is undefined
  int index;      // index for the current id, should be numbered from 1, 0=undefined
  string form;    // form
  string lemma;   // lemma
  string upostag; // universal part-of-speech tag
  string xpostag; // language-specific part-of-speech tag
  string feats;   // list of morphological features
  string deps;    // secondary dependencies
  string misc;    // miscellaneous information

  empty_node(int id = -1, int index = 0) : id(id), index(index) {}
};

/////////
// File: sentence/token.h
/////////

// This file is part of UDPipe .
//
// Copyright 2017 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class token {
 public:
  string form;
  string misc;

  token(string_piece form = string_piece(), string_piece misc = string_piece());

  // CoNLL-U defined SpaceAfter=No feature
  bool get_space_after() const;
  void set_space_after(bool space_after);

  // UDPipe-specific all-spaces-preserving SpacesBefore and SpacesAfter features
  void get_spaces_before(string& spaces_before) const;
  void set_spaces_before(string_piece spaces_before);
  void get_spaces_after(string& spaces_after) const;
  void set_spaces_after(string_piece spaces_after);
  void get_spaces_in_token(string& spaces_in_token) const;
  void set_spaces_in_token(string_piece spaces_in_token);

  // UDPipe-specific TokenRange feature
  bool get_token_range(size_t& start, size_t& end) const;
  void set_token_range(size_t start, size_t end);

 private:
  bool get_misc_field(string_piece name, string_piece& value) const;
  void remove_misc_field(string_piece name);
  string& start_misc_field(string_piece name);

  void append_escaped_spaces(string_piece spaces, string& escaped_spaces) const;
  void unescape_spaces(string_piece escaped_spaces, string& spaces) const;
};

/////////
// File: sentence/multiword_token.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class multiword_token : public token {
 public:
  // form and misc are inherited from token
  int id_first, id_last;

  multiword_token(int id_first = -1, int id_last = -1, string_piece form = string_piece(), string_piece misc = string_piece())
      : token(form, misc), id_first(id_first), id_last(id_last) {}
};

/////////
// File: sentence/word.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class word : public token {
 public:
  // form and misc are inherited from token
  int id;         // 0 is root, >0 is sentence word, <0 is undefined
  string lemma;   // lemma
  string upostag; // universal part-of-speech tag
  string xpostag; // language-specific part-of-speech tag
  string feats;   // list of morphological features
  int head;       // head, 0 is root, <0 is undefined
  string deprel;  // dependency relation to the head
  string deps;    // secondary dependencies

  vector children;

  word(int id = -1, string_piece form = string_piece()) : token(form), id(id), head(-1) {}
};

/////////
// File: sentence/sentence.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class sentence {
 public:
  sentence();

  vector words;
  vector multiword_tokens;
  vector empty_nodes;
  vector comments;
  static const string root_form;

  // Basic sentence modifications
  bool empty();
  void clear();
  word& add_word(string_piece form = string_piece());
  void set_head(int id, int head, const string& deprel);
  void unlink_all_words();

  // CoNLL-U defined comments
  bool get_new_doc(string* id = nullptr) const;
  void set_new_doc(bool new_doc, string_piece id = string_piece());
  bool get_new_par(string* id = nullptr) const;
  void set_new_par(bool new_par, string_piece id = string_piece());
  bool get_sent_id(string& id) const;
  void set_sent_id(string_piece id);
  bool get_text(string& text) const;
  void set_text(string_piece text);

 private:
  bool get_comment(string_piece name, string* value) const;
  void remove_comment(string_piece name);
  void set_comment(string_piece name, string_piece value = string_piece());
};

/////////
// File: sentence/input_format.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class input_format {
 public:
  virtual ~input_format() {}

  virtual bool read_block(istream& is, string& block) const = 0;
  virtual void reset_document(string_piece id = string_piece()) = 0;
  virtual void set_text(string_piece text, bool make_copy = false) = 0;
  virtual bool next_sentence(sentence& s, string& error) = 0;

  // Static factory methods
  static input_format* new_input_format(const string& name);
  static input_format* new_conllu_input_format(const string& options = string());
  static input_format* new_generic_tokenizer_input_format(const string& options = string());
  static input_format* new_horizontal_input_format(const string& options = string());
  static input_format* new_vertical_input_format(const string& options = string());

  static input_format* new_presegmented_tokenizer(input_format* tokenizer);

  static const string CONLLU_V1;
  static const string CONLLU_V2;
  static const string GENERIC_TOKENIZER_NORMALIZED_SPACES;
  static const string GENERIC_TOKENIZER_PRESEGMENTED;
  static const string GENERIC_TOKENIZER_RANGES;
};

/////////
// File: model/model.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class model {
 public:
  virtual ~model() {}

  static model* load(const char* fname);
  static model* load(istream& is);

  virtual input_format* new_tokenizer(const string& options) const = 0;
  virtual bool tag(sentence& s, const string& options, string& error) const = 0;
  virtual bool parse(sentence& s, const string& options, string& error) const = 0;

  static const string DEFAULT;
  static const string TOKENIZER_NORMALIZED_SPACES;
  static const string TOKENIZER_PRESEGMENTED;
  static const string TOKENIZER_RANGES;
};

/////////
// File: model/evaluator.h
/////////

// This file is part of UDPipe .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class evaluator {
 public:
  evaluator(const model* m, const string& tokenizer, const string& tagger, const string& parser);

  void set_model(const model* m);
  void set_tokenizer(const string& tokenizer);
  void set_tagger(const string& tagger);
  void set_parser(const string& parser);

  bool evaluate(istream& is, ostream& os, string& error) const;

  static const string DEFAULT;
  static const string NONE;

 private:
  const model* m;
  string tokenizer, tagger, parser;

  struct f1_info { size_t total_system, total_gold; double precision, recall, f1; };
  template
  static f1_info evaluate_f1(const vector>& system, const vector>& gold);

  class evaluation_data {
   public:
    struct word_data {
      size_t start, end;
      bool is_multiword;
      word w;

      word_data(size_t start, size_t end, int id, bool is_multiword, const word& w);
    };

    void add_sentence(const sentence& s);

    u32string chars;
    vector> sentences, tokens;
    vector> multiwords;
    vector words;
  };

  class word_alignment {
   public:
    struct pair_system_gold {
      word system; const word& gold;
      pair_system_gold(const word& system, const word& gold) : system(system), gold(gold) {}
    };
    vector matched;
    size_t total_system, total_gold;

    template
    f1_info evaluate_f1(Equals equals);

    static bool perfect_alignment(const evaluation_data& system, const evaluation_data& gold, word_alignment& alignment);
    static void best_alignment(const evaluation_data& system, const evaluation_data& gold, word_alignment& alignment);
  };
};

/////////
// File: unilib/unicode.h
/////////

// This file is part of UniLib .
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.3.0
// Unicode version: 15.0.0

namespace unilib {

class unicode {
  enum : uint8_t {
    _Lu = 1, _Ll = 2, _Lt = 3, _Lm = 4, _Lo = 5,
    _Mn = 6, _Mc = 7, _Me = 8,
    _Nd = 9, _Nl = 10, _No = 11,
    _Pc = 12, _Pd = 13, _Ps = 14, _Pe = 15, _Pi = 16, _Pf = 17, _Po = 18,
    _Sm = 19, _Sc = 20, _Sk = 21, _So = 22,
    _Zs = 23, _Zl = 24, _Zp = 25,
    _Cc = 26, _Cf = 27, _Cs = 28, _Co = 29, _Cn = 30
  };

 public:
  typedef uint32_t category_t;
  enum : category_t {
    Lu = 1 << _Lu, Ll = 1 << _Ll, Lt = 1 << _Lt, Lut = Lu | Lt, LC = Lu | Ll | Lt,
      Lm = 1 << _Lm, Lo = 1 << _Lo, L = Lu | Ll | Lt | Lm | Lo,
    Mn = 1 << _Mn, Mc = 1 << _Mc, Me = 1 << _Me, M = Mn | Mc | Me,
    Nd = 1 << _Nd, Nl = 1 << _Nl, No = 1 << _No, N = Nd | Nl | No,
    Pc = 1 << _Pc, Pd = 1 << _Pd, Ps = 1 << _Ps, Pe = 1 << _Pe, Pi = 1 << _Pi,
      Pf = 1 << _Pf, Po = 1 << _Po, P = Pc | Pd | Ps | Pe | Pi | Pf | Po,
    Sm = 1 << _Sm, Sc = 1 << _Sc, Sk = 1 << _Sk, So = 1 << _So, S = Sm | Sc | Sk | So,
    Zs = 1 << _Zs, Zl = 1 << _Zl, Zp = 1 << _Zp, Z = Zs | Zl | Zp,
    Cc = 1 << _Cc, Cf = 1 << _Cf, Cs = 1 << _Cs, Co = 1 << _Co, Cn = 1 << _Cn, C = Cc | Cf | Cs | Co | Cn
  };

  static inline category_t category(char32_t chr);

  static inline char32_t lowercase(char32_t chr);
  static inline char32_t uppercase(char32_t chr);
  static inline char32_t titlecase(char32_t chr);

 private:
  static const char32_t CHARS = 0x110000;
  static const int32_t DEFAULT_CAT = Cn;

  static const uint8_t category_index[CHARS >> 8];
  static const uint8_t category_block[][256];
  static const uint8_t othercase_index[CHARS >> 8];
  static const char32_t othercase_block[][256];

  enum othercase_type { LOWER_ONLY = 1, UPPERTITLE_ONLY = 2, UPPER_ONLY = 3, LOWER_THEN_UPPER = 4, UPPER_THEN_TITLE = 5, TITLE_THEN_LOWER = 6 };
};

unicode::category_t unicode::category(char32_t chr) {
  return chr < CHARS ? 1 << category_block[category_index[chr >> 8]][chr & 0xFF] : DEFAULT_CAT;
}

char32_t unicode::lowercase(char32_t chr) {
  if (chr < CHARS) {
    char32_t othercase = othercase_block[othercase_index[chr >> 8]][chr & 0xFF];
    if ((othercase & 0xFF) == othercase_type::LOWER_ONLY) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::LOWER_THEN_UPPER) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::TITLE_THEN_LOWER) return othercase_block[othercase_index[(othercase >> 8) >> 8]][(othercase >> 8) & 0xFF] >> 8;
  }
  return chr;
}

char32_t unicode::uppercase(char32_t chr) {
  if (chr < CHARS) {
    char32_t othercase = othercase_block[othercase_index[chr >> 8]][chr & 0xFF];
    if ((othercase & 0xFF) == othercase_type::UPPERTITLE_ONLY) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::UPPER_ONLY) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::UPPER_THEN_TITLE) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::LOWER_THEN_UPPER) return othercase_block[othercase_index[(othercase >> 8) >> 8]][(othercase >> 8) & 0xFF] >> 8;
  }
  return chr;
}

char32_t unicode::titlecase(char32_t chr) {
  if (chr < CHARS) {
    char32_t othercase = othercase_block[othercase_index[chr >> 8]][chr & 0xFF];
    if ((othercase & 0xFF) == othercase_type::UPPERTITLE_ONLY) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::TITLE_THEN_LOWER) return othercase >> 8;
    if ((othercase & 0xFF) == othercase_type::UPPER_THEN_TITLE) return othercase_block[othercase_index[(othercase >> 8) >> 8]][(othercase >> 8) & 0xFF] >> 8;
  }
  return chr;
}

} // namespace unilib

/////////
// File: unilib/utf8.h
/////////

// This file is part of UniLib .
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.3.0
// Unicode version: 15.0.0

namespace unilib {

class utf8 {
 public:
  static bool valid(const char* str);
  static bool valid(const char* str, size_t len);
  static inline bool valid(const std::string& str);

  static inline char32_t decode(const char*& str);
  static inline char32_t decode(const char*& str, size_t& len);
  static inline char32_t first(const char* str);
  static inline char32_t first(const char* str, size_t len);
  static inline char32_t first(const std::string& str);

  static void decode(const char* str, std::u32string& decoded);
  static void decode(const char* str, size_t len, std::u32string& decoded);
  static inline void decode(const std::string& str, std::u32string& decoded);

  class string_decoder {
   public:
    class iterator;
    inline iterator begin();
    inline iterator end();
   private:
    inline string_decoder(const char* str);
    const char* str;
    friend class utf8;
  };
  static inline string_decoder decoder(const char* str);
  static inline string_decoder decoder(const std::string& str);

  class buffer_decoder {
   public:
    class iterator;
    inline iterator begin();
    inline iterator end();
   private:
    inline buffer_decoder(const char* str, size_t len);
    const char* str;
    size_t len;
    friend class utf8;
  };
  static inline buffer_decoder decoder(const char* str, size_t len);

  static inline void append(char*& str, char32_t chr);
  static inline void append(std::string& str, char32_t chr);
  static void encode(const std::u32string& str, std::string& encoded);

  template static void map(F f, const char* str, std::string& result);
  template static void map(F f, const char* str, size_t len, std::string& result);
  template static void map(F f, const std::string& str, std::string& result);

 private:
  static const char REPLACEMENT_CHAR = '?';
};

bool utf8::valid(const std::string& str) {
  return valid(str.c_str());
}

char32_t utf8::decode(const char*& str) {
  if (((unsigned char)*str) < 0x80) return (unsigned char)*str++;
  else if (((unsigned char)*str) < 0xC0) return ++str, REPLACEMENT_CHAR;
  else if (((unsigned char)*str) < 0xE0) {
    char32_t res = (((unsigned char)*str++) & 0x1F) << 6;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + (((unsigned char)*str++) & 0x3F);
  } else if (((unsigned char)*str) < 0xF0) {
    char32_t res = (((unsigned char)*str++) & 0x0F) << 12;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += (((unsigned char)*str++) & 0x3F) << 6;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + (((unsigned char)*str++) & 0x3F);
  } else if (((unsigned char)*str) < 0xF8) {
    char32_t res = (((unsigned char)*str++) & 0x07) << 18;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += (((unsigned char)*str++) & 0x3F) << 12;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += (((unsigned char)*str++) & 0x3F) << 6;
    if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + (((unsigned char)*str++) & 0x3F);
  } else return ++str, REPLACEMENT_CHAR;
}

char32_t utf8::decode(const char*& str, size_t& len) {
  if (!len) return 0;
  --len;
  if (((unsigned char)*str) < 0x80) return (unsigned char)*str++;
  else if (((unsigned char)*str) < 0xC0) return ++str, REPLACEMENT_CHAR;
  else if (((unsigned char)*str) < 0xE0) {
    char32_t res = (((unsigned char)*str++) & 0x1F) << 6;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + ((--len, ((unsigned char)*str++)) & 0x3F);
  } else if (((unsigned char)*str) < 0xF0) {
    char32_t res = (((unsigned char)*str++) & 0x0F) << 12;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += ((--len, ((unsigned char)*str++)) & 0x3F) << 6;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + ((--len, ((unsigned char)*str++)) & 0x3F);
  } else if (((unsigned char)*str) < 0xF8) {
    char32_t res = (((unsigned char)*str++) & 0x07) << 18;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += ((--len, ((unsigned char)*str++)) & 0x3F) << 12;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    res += ((--len, ((unsigned char)*str++)) & 0x3F) << 6;
    if (len <= 0 || ((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) return REPLACEMENT_CHAR;
    return res + ((--len, ((unsigned char)*str++)) & 0x3F);
  } else return ++str, REPLACEMENT_CHAR;
}

char32_t utf8::first(const char* str) {
  return decode(str);
}

char32_t utf8::first(const char* str, size_t len) {
  return decode(str, len);
}

char32_t utf8::first(const std::string& str) {
  return first(str.c_str());
}

void utf8::decode(const std::string& str, std::u32string& decoded) {
  decode(str.c_str(), decoded);
}

class utf8::string_decoder::iterator : public std::iterator {
 public:
  iterator(const char* str) : codepoint(0), next(str) { operator++(); }
  iterator(const iterator& it) : codepoint(it.codepoint), next(it.next) {}
  iterator& operator++() { if (next) { codepoint = decode(next); if (!codepoint) next = nullptr; } return *this; }
  iterator operator++(int) { iterator tmp(*this); operator++(); return tmp; }
  bool operator==(const iterator& other) const { return next == other.next; }
  bool operator!=(const iterator& other) const { return next != other.next; }
  const char32_t& operator*() { return codepoint; }
 private:
  char32_t codepoint;
  const char* next;
};

utf8::string_decoder::string_decoder(const char* str) : str(str) {}

utf8::string_decoder::iterator utf8::string_decoder::begin() {
  return iterator(str);
}

utf8::string_decoder::iterator utf8::string_decoder::end() {
  return iterator(nullptr);
}

utf8::string_decoder utf8::decoder(const char* str) {
  return string_decoder(str);
}

utf8::string_decoder utf8::decoder(const std::string& str) {
  return string_decoder(str.c_str());
}

class utf8::buffer_decoder::iterator : public std::iterator {
 public:
  iterator(const char* str, size_t len) : codepoint(0), next(str), len(len) { operator++(); }
  iterator(const iterator& it) : codepoint(it.codepoint), next(it.next), len(it.len) {}
  iterator& operator++() { if (!len) next = nullptr; if (next) codepoint = decode(next, len); return *this; }
  iterator operator++(int) { iterator tmp(*this); operator++(); return tmp; }
  bool operator==(const iterator& other) const { return next == other.next; }
  bool operator!=(const iterator& other) const { return next != other.next; }
  const char32_t& operator*() { return codepoint; }
 private:
  char32_t codepoint;
  const char* next;
  size_t len;
};

utf8::buffer_decoder::buffer_decoder(const char* str, size_t len) : str(str), len(len) {}

utf8::buffer_decoder::iterator utf8::buffer_decoder::begin() {
  return iterator(str, len);
}

utf8::buffer_decoder::iterator utf8::buffer_decoder::end() {
  return iterator(nullptr, 0);
}

utf8::buffer_decoder utf8::decoder(const char* str, size_t len) {
  return buffer_decoder(str, len);
}

void utf8::append(char*& str, char32_t chr) {
  if (chr < 0x80) *str++ = chr;
  else if (chr < 0x800) { *str++ = 0xC0 + (chr >> 6); *str++ = 0x80 + (chr & 0x3F); }
  else if (chr < 0x10000) { *str++ = 0xE0 + (chr >> 12); *str++ = 0x80 + ((chr >> 6) & 0x3F); *str++ = 0x80 + (chr & 0x3F); }
  else if (chr < 0x200000) { *str++ = 0xF0 + (chr >> 18); *str++ = 0x80 + ((chr >> 12) & 0x3F); *str++ = 0x80 + ((chr >> 6) & 0x3F); *str++ = 0x80 + (chr & 0x3F); }
  else *str++ = REPLACEMENT_CHAR;
}

void utf8::append(std::string& str, char32_t chr) {
  if (chr < 0x80) str += chr;
  else if (chr < 0x800) { str += 0xC0 + (chr >> 6); str += 0x80 + (chr & 0x3F); }
  else if (chr < 0x10000) { str += 0xE0 + (chr >> 12); str += 0x80 + ((chr >> 6) & 0x3F); str += 0x80 + (chr & 0x3F); }
  else if (chr < 0x200000) { str += 0xF0 + (chr >> 18); str += 0x80 + ((chr >> 12) & 0x3F); str += 0x80 + ((chr >> 6) & 0x3F); str += 0x80 + (chr & 0x3F); }
  else str += REPLACEMENT_CHAR;
}

template void utf8::map(F f, const char* str, std::string& result) {
  result.clear();

  for (char32_t chr; (chr = decode(str)); )
    append(result, f(chr));
}

template void utf8::map(F f, const char* str, size_t len, std::string& result) {
  result.clear();

  while (len)
    append(result, f(decode(str, len)));
}

template void utf8::map(F f, const std::string& str, std::string& result) {
  map(f, str.c_str(), result);
}

} // namespace unilib

/////////
// File: model/evaluator.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string evaluator::DEFAULT;
const string evaluator::NONE = "none";

evaluator::evaluator(const model* m, const string& tokenizer, const string& tagger, const string& parser) {
  set_model(m);
  set_tokenizer(tokenizer);
  set_tagger(tagger);
  set_parser(parser);
}

void evaluator::set_model(const model* m) {
  this->m = m;
}

void evaluator::set_tokenizer(const string& tokenizer) {
  this->tokenizer = tokenizer;
}

void evaluator::set_tagger(const string& tagger) {
  this->tagger = tagger;
}

void evaluator::set_parser(const string& parser) {
  this->parser = parser;
}

bool evaluator::evaluate(istream& is, ostream& os, string& error) const {
  error.clear();

  unique_ptr conllu_input(input_format::new_conllu_input_format());
  if (!conllu_input) return error.assign("Cannot allocate CoNLL-U input format instance!"), false;

  vector plain_text_paragraphs(1); unsigned space_after_nos = 0;
  sentence system, gold;
  evaluation_data gold_data, system_goldtok_data, system_goldtok_goldtags_data, system_plaintext_data;

  string block;
  while (conllu_input->read_block(is, block)) {
    conllu_input->set_text(block);
    while (conllu_input->next_sentence(gold, error)) {
      gold_data.add_sentence(gold);

      // Detokenize the input when tokenizing
      if (tokenizer != NONE) {
        if (gold.get_new_doc() || gold.get_new_par()) {
          plain_text_paragraphs.back().append("\n\n");
          plain_text_paragraphs.emplace_back();
        }

        for (size_t i = 1, j = 0; i < gold.words.size(); i++) {
          const token& tok = j < gold.multiword_tokens.size() && gold.multiword_tokens[j].id_first == int(i) ? (const token&)gold.multiword_tokens[j] : (const token&)gold.words[i];
          plain_text_paragraphs.back().append(tok.form);
          if (tok.get_space_after())
            plain_text_paragraphs.back().push_back(' ');
          else
            space_after_nos += 1;
          if (j < gold.multiword_tokens.size() && gold.multiword_tokens[j].id_first == int(i))
            i = gold.multiword_tokens[j++].id_last;
        }
      }

      // Goldtok data
      if (tokenizer == NONE && tagger != NONE) {
        system.clear();
        for (size_t i = 1; i < gold.words.size(); i++)
          system.add_word(gold.words[i].form);

        if (tagger != NONE) {
          if (!m->tag(system, tagger, error))
            return false;
          if (parser != NONE)
            if (!m->parse(system, parser, error))
              return false;
        }
        system_goldtok_data.add_sentence(system);
      }

      // Goldtok_goldtags data
      if (tokenizer == NONE && tagger == NONE && parser != NONE) {
        system.clear();
        for (size_t i = 1; i < gold.words.size(); i++) {
          system.add_word(gold.words[i].form);
          system.words[i].upostag = gold.words[i].upostag;
          system.words[i].xpostag = gold.words[i].xpostag;
          system.words[i].feats = gold.words[i].feats;
          system.words[i].lemma = gold.words[i].lemma;
        }
        if (parser != NONE)
          if (!m->parse(system, parser, error))
            return false;
        system_goldtok_goldtags_data.add_sentence(system);
      }
    }
    if (!error.empty()) return false;
  }

  // Tokenize, tag and parse plaintext input
  if (tokenizer != NONE) {
    unique_ptr t(m->new_tokenizer(tokenizer));
    if (!t) return error.assign("Cannot allocate new tokenizer!"), false;

    for (auto&& plain_text : plain_text_paragraphs) {
      t->set_text(plain_text);
      while (t->next_sentence(system, error)) {
        if (tagger != NONE) {
          if (!m->tag(system, tagger, error))
            return false;

          if (parser != NONE)
            if (!m->parse(system, parser, error))
              return false;
        }
        system_plaintext_data.add_sentence(system);
      }
      if (!error.empty()) return false;
    }
  }

  // Evaluate from plain text
  if (tokenizer != NONE) {
    if (system_plaintext_data.chars != gold_data.chars) {
      os << "Cannot evaluate tokenizer, it returned different sequence of token characters!" << endl;
    } else {
      word_alignment plaintext_alignment;
      word_alignment::best_alignment(system_plaintext_data, gold_data, plaintext_alignment);

      os << "Number of SpaceAfter=No features in gold data: " << space_after_nos << endl;

      auto tokens = evaluate_f1(system_plaintext_data.tokens, gold_data.tokens);
      auto multiwords = evaluate_f1(system_plaintext_data.multiwords, gold_data.multiwords);
      auto sentences = evaluate_f1(system_plaintext_data.sentences, gold_data.sentences);
      auto words = plaintext_alignment.evaluate_f1([](const word&, const word&) {return true;});
      if (multiwords.total_gold || multiwords.total_system)
        os << "Tokenizer tokens - system: " << tokens.total_system << ", gold: " << tokens.total_gold
           << ", precision: " << fixed << setprecision(2) << 100. * tokens.precision
           << "%, recall: " << 100. * tokens.recall << "%, f1: " << 100. * tokens.f1 << "%" << endl
           << "Tokenizer multiword tokens - system: " << multiwords.total_system << ", gold: " << multiwords.total_gold
           << ", precision: " << fixed << setprecision(2) << 100. * multiwords.precision
           << "%, recall: " << 100. * multiwords.recall << "%, f1: " << 100. * multiwords.f1 << "%" << endl;
      os << "Tokenizer words - system: " << words.total_system << ", gold: " << words.total_gold
         << ", precision: " << fixed << setprecision(2) << 100. * words.precision
         << "%, recall: " << 100. * words.recall << "%, f1: " << 100. * words.f1 << "%" << endl
         << "Tokenizer sentences - system: " << sentences.total_system << ", gold: " << sentences.total_gold
         << ", precision: " << fixed << setprecision(2) << 100. * sentences.precision
         << "%, recall: " << 100. * sentences.recall << "%, f1: " << 100. * sentences.f1 << "%" << endl;

      if (tagger != NONE) {
        auto upostags = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.upostag == u.upostag; });
        auto xpostags = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.xpostag == u.xpostag; });
        auto feats = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.feats == u.feats; });
        auto alltags = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.upostag == u.upostag && w.xpostag == u.xpostag && w.feats == u.feats; });
        auto lemmas = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.lemma == u.lemma; });
        os << "Tagging from plain text (CoNLL17 F1 score) - gold forms: " << upostags.total_gold << ", upostag: "
           << fixed << setprecision(2) << 100. * upostags.f1 << "%, xpostag: "
           << 100. * xpostags.f1 << "%, feats: " << 100. * feats.f1 << "%, alltags: "
           << 100. * alltags.f1 << "%, lemmas: " << 100. * lemmas.f1 << '%' << endl;
      }

      if (tagger != NONE && parser != NONE) {
        auto uas = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head; });
        auto las = plaintext_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head && w.deprel == u.deprel; });
        os << "Parsing from plain text with computed tags (CoNLL17 F1 score) - gold forms: " << uas.total_gold
           << ", UAS: " << fixed << setprecision(2) << 100. * uas.f1 << "%, LAS: " << 100. * las.f1 << '%' << endl;
      }
    }
  }

  // Evaluate tagger from gold tokenization
  if (tokenizer == NONE && tagger != NONE) {
    word_alignment goldtok_alignment;
    if (!word_alignment::perfect_alignment(system_goldtok_data, gold_data, goldtok_alignment))
      return error.assign("Internal UDPipe error (the words of the gold data do not match)!"), false;

    auto upostags = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.upostag == u.upostag; });
    auto xpostags = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.xpostag == u.xpostag; });
    auto feats = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.feats == u.feats; });
    auto alltags = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.upostag == u.upostag && w.xpostag == u.xpostag && w.feats == u.feats; });
    auto lemmas = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.lemma == u.lemma; });
    os << "Tagging from gold tokenization - forms: " << upostags.total_gold << ", upostag: "
       << fixed << setprecision(2) << 100. * upostags.f1 << "%, xpostag: "
       << 100. * xpostags.f1 << "%, feats: " << 100. * feats.f1 << "%, alltags: "
       << 100. * alltags.f1 << "%, lemmas: " << 100. * lemmas.f1 << '%' << endl;

    if (parser != NONE) {
      auto uas = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head; });
      auto las = goldtok_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head && w.deprel == u.deprel; });
      os << "Parsing from gold tokenization with computed tags - forms: " << uas.total_gold
         << ", UAS: " << fixed << setprecision(2) << 100. * uas.f1 << "%, LAS: " << 100. * las.f1 << '%' << endl;
    }
  }

  // Evaluate parser from gold tokenization
  if (tokenizer == NONE && tagger == NONE && parser != NONE) {
    word_alignment goldtok_goldtags_alignment;
    if (!word_alignment::perfect_alignment(system_goldtok_goldtags_data, gold_data, goldtok_goldtags_alignment))
      return error.assign("Internal UDPipe error (the words of the goldtok data do not match)!"), false;

    auto uas = goldtok_goldtags_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head; });
    auto las = goldtok_goldtags_alignment.evaluate_f1([](const word& w, const word& u) { return w.head == u.head && w.deprel == u.deprel; });
    os << "Parsing from gold tokenization with gold tags - forms: " << uas.total_gold
       << ", UAS: " << fixed << setprecision(2) << 100. * uas.f1 << "%, LAS: " << 100. * las.f1 << '%' << endl;
  }

  return true;
}

template
evaluator::f1_info evaluator::evaluate_f1(const vector>& system, const vector>& gold) {
  size_t both = 0;
  for (size_t si = 0, gi = 0; si < system.size() || gi < gold.size(); )
    if (si < system.size() && (gi == gold.size() || system[si].first < gold[gi].first))
      si++;
    else if (gi < gold.size() && (si == system.size() || gold[gi].first < system[si].first))
      gi++;
    else
      both += system[si++].second == gold[gi++].second;

  return {system.size(), gold.size(), system.size() ? both / double(system.size()) : 0.,
    gold.size() ? both / double(gold.size()) : 0., system.size()+gold.size() ? 2 * both / double(system.size() + gold.size()) : 0. };
}

evaluator::evaluation_data::word_data::word_data(size_t start, size_t end, int id, bool is_multiword, const word& w)
  : start(start), end(end), is_multiword(is_multiword), w(w)
{
  // Use absolute ids for words and heads
  this->w.id = id;
  this->w.head = w.head ? id + (w.head - w.id) : 0;

  // Forms in MWTs are compares case-insensitively in LCS, therefore
  // we lowercase them here.
  unilib::utf8::map(unilib::unicode::lowercase, w.form, this->w.form);

  // During evaluation, only universal part of DEPREL (up to a colon) is used.
  auto colon = w.deprel.find(':');
  if (colon != string::npos)
    this->w.deprel.erase(colon);
}

void evaluator::evaluation_data::add_sentence(const sentence& s) {
  sentences.emplace_back(chars.size(), chars.size());
  for (size_t i = 1, j = 0; i < s.words.size(); i++) {
    tokens.emplace_back(chars.size(), chars.size());
    const string& form = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? s.multiword_tokens[j].form : s.words[i].form;
    for (auto&& chr : unilib::utf8::decoder(form))
      if (chr != ' ')
        chars.push_back(chr);
    tokens.back().second = chars.size();

    if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i)) {
      multiwords.emplace_back(tokens.back().first, form);
      for (size_t k = i; int(k) <= s.multiword_tokens[j].id_last; k++) {
        words.emplace_back(tokens.back().first, tokens.back().second, (int)words.size() + 1, true, s.words[k]);
        multiwords.back().second.append(" ").append(words.back().w.form);
      }
      i = s.multiword_tokens[j++].id_last;
    } else {
      words.emplace_back(tokens.back().first, tokens.back().second, (int)words.size() + 1, false, s.words[i]);
    }
  }
  sentences.back().second = chars.size();
}

template
evaluator::f1_info evaluator::word_alignment::evaluate_f1(Equals equals) {
  size_t both = 0;
  for (auto&& match : matched)
    if (equals(match.system, match.gold))
      both++;

  return {total_system, total_gold, total_system ? both / double(total_system) : 0.,
    total_gold ? both / double(total_gold) : 0., total_system+total_gold ? 2 * both / double(total_system + total_gold) : 0. };
}

bool evaluator::word_alignment::perfect_alignment(const evaluation_data& system, const evaluation_data& gold, word_alignment& alignment) {
  alignment.total_system = system.words.size();
  alignment.total_gold = gold.words.size();
  if (alignment.total_system != alignment.total_gold) return false;

  alignment.matched.clear();
  alignment.matched.reserve(alignment.total_system);
  for (size_t i = 0; i < system.words.size(); i++) {
    if (system.words[i].w.form != gold.words[i].w.form)
      return false;
    alignment.matched.emplace_back(system.words[i].w, gold.words[i].w);
  }

  return true;
}

void evaluator::word_alignment::best_alignment(const evaluation_data& system, const evaluation_data& gold, word_alignment& alignment) {
  alignment.total_system = system.words.size();
  alignment.total_gold = gold.words.size();
  alignment.matched.clear();

  for (size_t si = 0, gi = 0; si < system.words.size() && gi < gold.words.size(); )
    if ((system.words[si].start > gold.words[gi].start || !system.words[si].is_multiword) &&
        (gold.words[gi].start > system.words[si].start || !gold.words[gi].is_multiword)) {
      // No multiword, align using start+end indices
      if (system.words[si].start == gold.words[gi].start && system.words[si].end == gold.words[gi].end)
        alignment.matched.emplace_back(system.words[si++].w, gold.words[gi++].w);
      else if (system.words[si].start <= gold.words[gi].start)
        si++;
      else
        gi++;
    } else {
      // We have a multiword
      size_t ss = si, gs = gi, multiword_range_end = system.words[si].is_multiword ? system.words[si].end : gold.words[gi].end;

      // Find all words in the multiword range
      while ((si < system.words.size() && (system.words[si].is_multiword ? system.words[si].start < multiword_range_end :
                                           system.words[si].end <= multiword_range_end)) ||
             (gi < gold.words.size() && (gold.words[gi].is_multiword ? gold.words[gi].start < multiword_range_end :
                                         gold.words[gi].end <= multiword_range_end))) {
        // Extend the multiword range
        if (si < system.words.size() && (gi >= gold.words.size() || system.words[si].start <= gold.words[gi].start)) {
          if (system.words[si].is_multiword) multiword_range_end = max(multiword_range_end, system.words[si].end);
          si++;
        } else {
          if (gold.words[gi].is_multiword) multiword_range_end = max(multiword_range_end, gold.words[gi].end);
          gi++;
        }
      }

      // LCS on the chosen words
      vector> lcs(si - ss);
      for (unsigned s = si - ss; s--; ) {
        lcs[s].resize(gi - gs);
        for (unsigned g = gi - gs; g--; ) {
          lcs[s][g] = max(lcs[s][g], s+1 < lcs.size() ? lcs[s+1][g] : 0);
          lcs[s][g] = max(lcs[s][g], g+1 < lcs[s].size() ? lcs[s][g+1] : 0);
          if (system.words[ss + s].w.form == gold.words[gs + g].w.form)
            lcs[s][g] = max(lcs[s][g], 1 + (s+1 < lcs.size() && g+1 < lcs[s].size() ? lcs[s+1][g+1] : 0));
        }
      }

      for (unsigned s = 0, g = 0; s < si - ss && g < gi - gs; ) {
        if (system.words[ss + s].w.form == gold.words[gs + g].w.form)
          alignment.matched.emplace_back(system.words[ss + s++].w, gold.words[gs + g++].w);
        else if (lcs[s][g] == (s+1 < lcs.size() ? lcs[s+1][g] : 0))
          s++;
        else /* if (lcs[s][g] == (g+1 < lcs[s].size() ? lcs[s][g+1] : 0)) */
          g++;
      }
    }

  // Reindex HEAD pointers in system to use gold indices
  vector gold_aligned(system.words.size(), -1);
  for (auto&& match : alignment.matched)
    gold_aligned[match.system.id - 1] = match.gold.id;
  for (auto&& match : alignment.matched)
    if (match.system.head > 0)
      match.system.head = gold_aligned[match.system.head - 1];
}

/////////
// File: morphodita/derivator/derivator.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

struct derivated_lemma {
  string lemma;
};

class derivator {
 public:
  virtual ~derivator() {}

  // For given lemma, return the parent in the derivation graph.
  // The lemma is assumed to be lemma id and any lemma comments are ignored.
  virtual bool parent(string_piece lemma, derivated_lemma& parent) const = 0;

  // For given lemma, return the children in the derivation graph.
  // The lemma is assumed to be lemma id and any lemma comments are ignored.
  virtual bool children(string_piece lemma, vector& children) const = 0;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/tokenizer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Range of a token, measured in Unicode characters, not UTF8 bytes.
struct token_range {
  size_t start;
  size_t length;

  token_range() {}
  token_range(size_t start, size_t length) : start(start), length(length) {}
};

class tokenizer {
 public:
  virtual ~tokenizer() {}

  virtual void set_text(string_piece text, bool make_copy = false) = 0;
  virtual bool next_sentence(vector* forms, vector* tokens) = 0;

  // Static factory methods
  static tokenizer* new_vertical_tokenizer();

  static tokenizer* new_czech_tokenizer();
  static tokenizer* new_english_tokenizer();
  static tokenizer* new_generic_tokenizer();
};

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

struct tagged_form {
  string form;
  string tag;

  tagged_form() {}
  tagged_form(const string& form, const string& tag) : form(form), tag(tag) {}
};

struct tagged_lemma {
  string lemma;
  string tag;

  tagged_lemma() {}
  tagged_lemma(const string& lemma, const string& tag) : lemma(lemma), tag(tag) {}
};

struct tagged_lemma_forms {
  string lemma;
  vector forms;

  tagged_lemma_forms() {}
  tagged_lemma_forms(const string& lemma) : lemma(lemma) {}
};

class morpho {
 public:
  virtual ~morpho() {}

  static morpho* load(istream& is);
  static morpho* load(const char* fname);

  enum guesser_mode { NO_GUESSER = 0, GUESSER = 1, GUESSER_UNSPECIFIED = -1 };

  // Perform morphologic analysis of a form. The form is given by a pointer and
  // length and therefore does not need to be '\0' terminated.  The guesser
  // parameter specifies whether a guesser can be used if the form is not found
  // in the dictionary. Output is assigned to the lemmas vector.
  //
  // If the form is found in the dictionary, analyses are assigned to lemmas
  // and NO_GUESSER returned. If guesser == GUESSER and the form analyses are
  // found using a guesser, they are assigned to lemmas and GUESSER is
  // returned.  Otherwise <0 is returned and lemmas are filled with one
  // analysis containing given form as lemma and a tag for unknown word.
  virtual int analyze(string_piece form, guesser_mode guesser, vector& lemmas) const = 0;

  // Perform morphologic generation of a lemma. The lemma is given by a pointer
  // and length and therefore does not need to be '\0' terminated. Optionally
  // a tag_wildcard can be specified (or be NULL) and if so, results are
  // filtered using this wildcard. The guesser parameter speficies whether
  // a guesser can be used if the lemma is not found in the dictionary. Output
  // is assigned to the forms vector.
  //
  // Tag_wildcard can be either NULL or a wildcard applied to the results.
  // A ? in the wildcard matches any character, [bytes] matches any of the
  // bytes and [^bytes] matches any byte different from the specified ones.
  // A - has no special meaning inside the bytes and if ] is first in bytes, it
  // does not end the bytes group.
  //
  // If the given lemma is only a raw lemma, all lemma ids with this raw lemma
  // are returned. Otherwise only matching lemma ids are returned, ignoring any
  // lemma comments. For every found lemma, matching forms are filtered using
  // the tag_wildcard. If at least one lemma is found in the dictionary,
  // NO_GUESSER is returned. If guesser == GUESSER and the lemma is found by
  // the guesser, GUESSER is returned. Otherwise, forms are cleared and <0 is
  // returned.
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector& forms) const = 0;

  // Rawlemma and lemma id identification
  virtual int raw_lemma_len(string_piece lemma) const = 0;
  virtual int lemma_id_len(string_piece lemma) const = 0;

  // Rawform identification
  virtual int raw_form_len(string_piece form) const = 0;

  // Construct a new tokenizer instance appropriate for this morphology.
  // Can return NULL if no such tokenizer exists.
  virtual tokenizer* new_tokenizer() const = 0;

  // Return a derivator for this morphology, or NULL if it does not exist.
  // The returned instance is owned by the morphology and should not be deleted.
  virtual const derivator* get_derivator() const;

 protected:
  unique_ptr derinet;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/tokenizer_factory.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tokenizer_factory {
 public:
  virtual ~tokenizer_factory() {}

  static tokenizer_factory* load(istream& is);
  static tokenizer_factory* load(const char* fname);

  // Construct a new tokenizer instance.
  virtual tokenizer* new_tokenizer(const morpho* m) const = 0;
};

} // namespace morphodita

/////////
// File: morphodita/tagger/tagger.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tagger {
 public:
  virtual ~tagger() {}

  static tagger* load(const char* fname);
  static tagger* load(istream& is);

  // Return morpho associated with the tagger. Do not delete the pointer, it is
  // owned by the tagger instance and deleted in the tagger destructor.
  virtual const morpho* get_morpho() const = 0;

  // Perform morphologic analysis and subsequent disambiguation.
  virtual void tag(const vector& forms, vector& tags, morpho::guesser_mode guesser = morpho::GUESSER_UNSPECIFIED) const = 0;

  // Perform disambiguation only on given analyses.
  virtual void tag_analyzed(const vector& forms, const vector>& analyses, vector& tags) const = 0;

  // Construct a new tokenizer instance appropriate for this tagger.
  // Can return NULL if no such tokenizer exists.
  // Is equal to get_morpho()->new_tokenizer.
  tokenizer* new_tokenizer() const;
};

} // namespace morphodita

/////////
// File: parsito/tree/node.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class node {
 public:
  int id;         // 0 is root, >0 is sentence node, <0 is undefined
  string form;    // form
  string lemma;   // lemma
  string upostag; // universal part-of-speech tag
  string xpostag; // language-specific part-of-speech tag
  string feats;   // list of morphological features
  int head;       // head, 0 is root, <0 is without parent
  string deprel;  // dependency relation to the head
  string deps;    // secondary dependencies
  string misc;    // miscellaneous information

  vector children;

  node(int id = -1, const string& form = string()) : id(id), form(form), head(-1) {}
};

} // namespace parsito

/////////
// File: parsito/tree/tree.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class tree {
 public:
  tree();

  vector nodes;

  bool empty();
  void clear();
  node& add_node(const string& form);
  void set_head(int id, int head, const string& deprel);
  void unlink_all_nodes();

  static const string root_form;
};

} // namespace parsito

/////////
// File: parsito/configuration/configuration.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class configuration {
 public:
  configuration(bool single_root) : single_root(single_root) {}

  void init(tree* t);
  bool final();

  tree* t;
  vector stack;
  vector buffer;

  bool single_root;
};

} // namespace parsito

/////////
// File: utils/binary_decoder.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

class binary_decoder_error : public runtime_error {
 public:
  explicit binary_decoder_error(const char* description) : runtime_error(description) {}
};

class binary_decoder {
 public:
  inline unsigned char* fill(unsigned len);

  inline unsigned next_1B();
  inline unsigned next_2B();
  inline unsigned next_4B();
  inline void next_str(string& str);
  template  inline const T* next(unsigned elements);

  inline bool is_end();
  inline unsigned tell();
  inline void seek(unsigned pos);

 private:
  vector buffer;
  const unsigned char* data;
  const unsigned char* data_end;
};

//
// Definitions
//

unsigned char* binary_decoder::fill(unsigned len) {
  buffer.resize(len);
  data = buffer.data();
  data_end = buffer.data() + len;

  return buffer.data();
}

unsigned binary_decoder::next_1B() {
  if (data + 1 > data_end) throw binary_decoder_error("No more data in binary_decoder");
  return *data++;
}

unsigned binary_decoder::next_2B() {
  if (data + sizeof(uint16_t) > data_end) throw binary_decoder_error("No more data in binary_decoder");
  uint16_t result;
  memcpy(&result, data, sizeof(uint16_t));
  data += sizeof(uint16_t);
  return result;
}

unsigned binary_decoder::next_4B() {
  if (data + sizeof(uint32_t) > data_end) throw binary_decoder_error("No more data in binary_decoder");
  uint32_t result;
  memcpy(&result, data, sizeof(uint32_t));
  data += sizeof(uint32_t);
  return result;
}

void binary_decoder::next_str(string& str) {
  unsigned len = next_1B();
  if (len == 255) len = next_4B();
  str.assign(next(len), len);
}

template  const T* binary_decoder::next(unsigned elements) {
  if (data + sizeof(T) * elements > data_end) throw binary_decoder_error("No more data in binary_decoder");
  const T* result = (const T*) data;
  data += sizeof(T) * elements;
  return result;
}

bool binary_decoder::is_end() {
  return data >= data_end;
}

unsigned binary_decoder::tell() {
  return data - buffer.data();
}

void binary_decoder::seek(unsigned pos) {
  if (pos > buffer.size()) throw binary_decoder_error("Cannot seek past end of binary_decoder");
  data = buffer.data() + pos;
}

} // namespace utils

/////////
// File: parsito/parser/parser.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Parser
class parser {
 public:
  virtual ~parser() {};

  virtual void parse(tree& t, unsigned beam_size = 0, double* cost = nullptr) const = 0;

  enum { NO_CACHE = 0, FULL_CACHE = 2147483647};
  static parser* load(const char* file, unsigned cache = 1000);
  static parser* load(istream& in, unsigned cache = 1000);

 protected:
  virtual void load(binary_decoder& data, unsigned cache) = 0;
  static parser* create(const string& name);
};

} // namespace parsito

/////////
// File: tokenizer/multiword_splitter.h
/////////

// This file is part of UDPipe .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class multiword_splitter {
 public:
  void append_token(string_piece token, string_piece misc, sentence& s) const;

  static multiword_splitter* load(istream& is);

 private:
  multiword_splitter(unsigned version) : version(version) {}
  unsigned version;
  enum { VERSION_LATEST = 2 };
  friend class multiword_splitter_trainer;

  struct suffix_info {
    vector words;
  };
  unordered_map full_rules, suffix_rules;
};

/////////
// File: utils/parse_int.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Try to parse an int from given string. If the int cannot be parsed or does
// not fit into int, false is returned and the error string is filled using the
// value_name argument.
inline bool parse_int(string_piece str, const char* value_name, int& value, string& error);

// Try to parse an int from given string. If the int cannot be parsed or does
// not fit into int, an error is displayed and program exits.
inline int parse_int(string_piece str, const char* value_name);

//
// Definitions
//

bool parse_int(string_piece str, const char* value_name, int& value, string& error) {
  string_piece original = str;

  // Skip spaces
  while (str.len && (str.str[0] == ' ' || str.str[0] == '\f' || str.str[0] == '\n' || str.str[0] == '\r' || str.str[0] == '\t' || str.str[0] == '\v'))
    str.str++, str.len--;

  // Allow minus
  bool positive = true;
  if (str.len && (str.str[0] == '+' || str.str[0] == '-')) {
    positive = str.str[0] == '+';
    str.str++, str.len--;
  }

  // Parse value, checking for overflow/underflow
  if (!str.len) return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': empty string."), false;
  if (!(str.str[0] >= '0' || str.str[0] <= '9')) return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': non-digit character found."), false;

  value = 0;
  while (str.len && str.str[0] >= '0' && str.str[0] <= '9') {
    if (positive) {
      if (value > (numeric_limits::max() - (str.str[0] - '0')) / 10)
        return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': overflow occured."), false;
      value = 10 * value + (str.str[0] - '0');
    } else {
      if (value < (numeric_limits::min() + (str.str[0] - '0')) / 10)
        return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': underflow occured."), false;
      value = 10 * value - (str.str[0] - '0');
    }
    str.str++, str.len--;
  }

  // Skip spaces
  while (str.len && (str.str[0] == ' ' || str.str[0] == '\f' || str.str[0] == '\n' || str.str[0] == '\r' || str.str[0] == '\t' || str.str[0] == '\v'))
    str.str++, str.len--;

  // Check for remaining characters
  if (str.len) return error.assign("Cannot parse ").append(value_name).append(" int value '").append(original.str, original.len).append("': non-digit character found."), false;

  return true;
}

int parse_int(string_piece str, const char* value_name) {
  int result;
  string error;

  if (!parse_int(str, value_name, result, error))
    runtime_failure(error);

  return result;
}

} // namespace utils

/////////
// File: utils/path_from_utf8.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2022 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

#ifdef _WIN32
inline wstring path_from_utf8(const char* str);
inline wstring path_from_utf8(const string& str);
#else
inline string path_from_utf8(const char* str);
inline const string& path_from_utf8(const string& str);
#endif

//
// Definitions
//

#ifdef _WIN32

inline wstring path_from_utf8(const char* str) {
  // We could implement this using codecvt_utf8_utf16, but it is not available
  // in GCC 4.9, which we still use. We could also use MultiByteToWideChar,
  // but using it would require changing our build infrastructure -- hence
  // we implement the conversion manually.
  wstring wstr;
  while (*str) {
    char32_t chr;
    if (((unsigned char)*str) < 0x80) chr = (unsigned char)*str++;
    else if (((unsigned char)*str) < 0xC0) chr = '?', ++str;
    else if (((unsigned char)*str) < 0xE0) {
      chr = (((unsigned char)*str++) & 0x1F) << 6;
      if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) chr = '?';
      else chr += ((unsigned char)*str++) & 0x3F;
    } else if (((unsigned char)*str) < 0xF0) {
      chr = (((unsigned char)*str++) & 0x0F) << 12;
      if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) chr = '?';
      else {
        chr += (((unsigned char)*str++) & 0x3F) << 6;
        if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) chr = '?';
        else chr += ((unsigned char)*str++) & 0x3F;
      }
    } else if (((unsigned char)*str) < 0xF8) {
      chr = (((unsigned char)*str++) & 0x07) << 18;
      if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) chr = '?';
      else {
        chr += (((unsigned char)*str++) & 0x3F) << 12;
        if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) chr = '?';
        else {
          chr += (((unsigned char)*str++) & 0x3F) << 6;
          if (((unsigned char)*str) < 0x80 || ((unsigned char)*str) >= 0xC0) chr = '?';
          else chr += ((unsigned char)*str++) & 0x3F;
        }
      }
    } else chr = '?', ++str;

    if (chr <= 0xFFFF) wstr.push_back(chr);
    else if (chr <= 0x10FFFF) {
      wstr.push_back(0xD800 + ((chr - 0x10000) >> 10));
      wstr.push_back(0xDC00 + ((chr - 0x10000) & 0x3FF));
    } else {
      wstr.push_back('?');
    }
  }
  return wstr;
}

inline wstring path_from_utf8(const string& str) {
  return path_from_utf8(str.c_str());
}

#else

inline string path_from_utf8(const char* str) {
  return str;
}

inline const string& path_from_utf8(const string& str) {
  return str;
}

#endif

} // namespace utils

/////////
// File: utils/named_values.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

class named_values {
 public:
  typedef unordered_map map;

  inline static bool parse(const string& values, map& parsed_values, string& error);
};

//
// Definitions
//

bool named_values::parse(const string& values, map& parsed_values, string& error) {
  error.clear();
  parsed_values.clear();

  string name, file;
  for (size_t start = 0; start < values.size(); ) {
    while (start < values.size() && values[start] == ';') start++;
    if (start >= values.size()) break;

    size_t name_end = values.find_first_of("=;", start);
    name.assign(values, start, name_end - start);
    string& value = parsed_values[name];

    if (name_end == string::npos) {
      start = name_end;
    } else if (values[name_end] == ';') {
      start = name_end + 1;
    } else /* if (values[name_end] == '=') */ {
      size_t equal_sign = name_end;

      if (equal_sign + 1 + 5 <= values.size() && values.compare(equal_sign + 1, 5, "file:") == 0) {
        // Value of type file:
        size_t file_name = equal_sign + 1 + 5;
        size_t semicolon = min(values.find(';', file_name), values.size());

        file.assign(values, file_name, semicolon - file_name);
        ifstream is(path_from_utf8(file).c_str());
        if (!is.is_open()) return error.assign("Cannot open file '").append(file).append("'!"), false;

        char buffer[1024];
        for (value.clear(); is.read(buffer, sizeof(buffer)); )
          value.append(buffer, sizeof(buffer));
        value.append(buffer, is.gcount());

        start = semicolon + 1;
      } else if (equal_sign + 1 + 5 <= values.size() && values.compare(equal_sign + 1, 5, "data:") == 0) {
        // Value of type data:
        size_t data_size_start = equal_sign + 1 + 5;
        size_t data_size_end = values.find(':', data_size_start);
        if (data_size_end == string::npos) return error.assign("Cannot parse named values, data size of value '").append(name).append("' not terminated!"), false;

        int data_size;
        if (!parse_int(string_piece(values.c_str() + data_size_start, data_size_end - data_size_start), "data_size", data_size, error)) return false;

        size_t data_start = data_size_end + 1, data_end = data_start + data_size;
        if (data_end > values.size()) return error.assign("Cannot parse named values, value '").append(name).append("' shorter than specified length!"), false;
        if (data_end < values.size() && values[data_end] != ';') return error.assign("Cannot parse named values, value '").append(name).append("' not terminated by semicolon!"), false;

        value.assign(values, data_start, data_end - data_start);
        start = data_end + 1;
      } else {
        // Value of string type
        size_t semicolon = min(values.find(';', equal_sign), values.size());
        value.assign(values, equal_sign + 1, semicolon - equal_sign - 1);
        start = semicolon + 1;
      }
    }
  }

  return true;
}

} // namespace utils

/////////
// File: utils/threadsafe_stack.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

template
class threadsafe_stack {
 public:
  inline void push(T* t);
  inline T* pop();

 private:
  vector> stack;
  atomic_flag lock = ATOMIC_FLAG_INIT;
};

//
// Definitions
//

template
void threadsafe_stack::push(T* t) {
  while (lock.test_and_set(memory_order_acquire)) {}
  stack.emplace_back(t);
  lock.clear(memory_order_release);
}

template
T* threadsafe_stack::pop() {
  T* res = nullptr;

  while (lock.test_and_set(memory_order_acquire)) {}
  if (!stack.empty()) {
    res = stack.back().release();
    stack.pop_back();
  }
  lock.clear(memory_order_release);

  return res;
}

} // namespace utils

/////////
// File: model/model_morphodita_parsito.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class model_morphodita_parsito : public model {
 public:
  virtual input_format* new_tokenizer(const string& options) const override;
  virtual bool tag(sentence& s, const string& options, string& error) const override;
  virtual bool parse(sentence& s, const string& options, string& error) const override;

  static model* load(istream& is);

 private:
  model_morphodita_parsito(unsigned version);
  unsigned version;
  enum { VERSION_LATEST = 3 };

  unique_ptr tokenizer_factory;
  unique_ptr splitter;
  struct tagger_model {
    bool raw; bool upostag; int lemma; bool xpostag, feats;
    unique_ptr tagger;

    tagger_model(bool raw, bool upostag, int lemma, bool xpostag, bool feats, morphodita::tagger* tagger)
        : raw(raw), upostag(upostag), lemma(lemma), xpostag(xpostag), feats(feats), tagger(tagger) {}
  };
  vector taggers;
  unique_ptr parser;

  struct tagger_cache {
    vector forms_normalized;
    vector forms_string_pieces;
    vector lemmas;
  };
  mutable threadsafe_stack tagger_caches;

  struct parser_cache {
    parsito::tree tree;
    named_values::map options;
  };
  mutable threadsafe_stack parser_caches;

  bool parse(sentence& s, const string& options, string& error, double* cost) const;

  class joint_with_parsing_tokenizer : public input_format {
   public:
    joint_with_parsing_tokenizer(input_format* tokenizer, const model_morphodita_parsito& model,
                                 int max_sentence_len, double change_boundary_logprob, double sentence_logprob)
        : tokenizer(tokenizer), model(model), max_sentence_len(max_sentence_len),
          change_boundary_logprob(change_boundary_logprob), sentence_logprob(sentence_logprob) {}

    virtual bool read_block(istream& is, string& block) const override;
    virtual void reset_document(string_piece id) override;
    virtual void set_text(string_piece text, bool make_copy = false) override;
    virtual bool next_sentence(sentence& s, string& error) override;

   private:
    bool parse_paragraph(vector& paragraph, string& error);

    unique_ptr tokenizer;
    const model_morphodita_parsito& model;
    int max_sentence_len;
    double change_boundary_logprob;
    double sentence_logprob;

    string_piece text;
    string text_copy;
    bool new_document = true;
    string document_id;
    unsigned sentence_id = 1;
    vector sentences;
    size_t sentences_index = 0;
  };

  void fill_word_analysis(const morphodita::tagged_lemma& analysis, bool raw, bool upostag, int lemma, bool xpostag, bool feats, word& word) const;
  const string& normalize_form(string_piece form, string& output) const;
  const string& normalize_lemma(string_piece lemma, string& output) const;
  friend class trainer_morphodita_parsito;
};

/////////
// File: model/model.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string model::DEFAULT;
const string model::TOKENIZER_NORMALIZED_SPACES = "normalized_spaces";
const string model::TOKENIZER_PRESEGMENTED = "presegmented";
const string model::TOKENIZER_RANGES = "ranges";

model* model::load(const char* fname) {
  ifstream in(path_from_utf8(fname).c_str(), ifstream::in | ifstream::binary);
  if (!in.is_open()) return nullptr;
  return load(in);
}

model* model::load(istream& is) {
  char len;
  if (!is.get(len)) return nullptr;
  string name(len, ' ');
  if (!is.read(&name[0], len)) return nullptr;

  if (name == "morphodita_parsito") return model_morphodita_parsito::load(is);

  return nullptr;
}

/////////
// File: morphodita/tagger/tagger_ids.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tagger_ids {
 public:
  enum tagger_id {
    CZECH2 = 0, CZECH3 = 1, CZECH2_3 = 6,
    /* 2 was used internally for ENGLISH3, but never released publicly */
    GENERIC2 = 3, GENERIC3 = 4, GENERIC4 = 5, GENERIC2_3 = 7,
    CONLLU2 = 8, CONLLU2_3 = 9, CONLLU3 = 10,
  };

  static bool parse(const string& str, tagger_id& id) {
    if (str == "czech2") return id = CZECH2, true;
    if (str == "czech2_3") return id = CZECH2_3, true;
    if (str == "czech3") return id = CZECH3, true;
    if (str == "generic2") return id = GENERIC2, true;
    if (str == "generic2_3") return id = GENERIC2_3, true;
    if (str == "generic3") return id = GENERIC3, true;
    if (str == "generic4") return id = GENERIC4, true;
    if (str == "conllu2") return id = CONLLU2, true;
    if (str == "conllu2_3") return id = CONLLU2_3, true;
    if (str == "conllu3") return id = CONLLU3, true;
    return false;
  }

  static int decoding_order(tagger_id id) {
    switch (id) {
      case CZECH2: return 2;
      case CZECH2_3: return 2;
      case CZECH3: return 3;
      case GENERIC2: return 2;
      case GENERIC2_3: return 2;
      case GENERIC3: return 3;
      case GENERIC4: return 4;
      case CONLLU2: return 2;
      case CONLLU2_3: return 2;
      case CONLLU3: return 3;
    }
    return 0;
  }

  static int window_size(tagger_id id) {
    switch (id) {
      case CZECH2_3: return 3;
      case GENERIC2_3: return 3;
      case CONLLU2_3: return 3;
      default: break;
    }
    return decoding_order(id);
  }
};

typedef tagger_ids::tagger_id tagger_id;

} // namespace morphodita

/////////
// File: tokenizer/morphodita_tokenizer_wrapper.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class morphodita_tokenizer_wrapper : public input_format {
 public:
  morphodita_tokenizer_wrapper(morphodita::tokenizer* tokenizer, const multiword_splitter* splitter, bool normalized_spaces, bool token_ranges);

  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  unique_ptr tokenizer;
  const multiword_splitter* splitter;
  bool normalized_spaces, token_ranges;

  bool new_document = true;
  string document_id;
  unsigned preceeding_newlines = 2;
  unsigned sentence_id = 1;

  string_piece text;
  string text_copy;
  size_t unicode_offset = 0, text_unicode_length = 0;
  string saved_spaces;
  vector forms;
  vector tokens;
  token tok;
};

/////////
// File: utils/getpara.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Read paragraph until EOF or end line. All encountered \n are stored.
inline istream& getpara(istream& is, string& para);

//
// Definitions
//

istream& getpara(istream& is, string& para) {
  para.clear();

  for (string line; getline(is, line); ) {
    para.append(line);
    para.push_back('\n');

    if (line.empty()) break;
  }

  if (is.eof() && !para.empty()) is.clear(istream::eofbit);
  return is;
}

} // namespace utils

/////////
// File: utils/parse_double.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Try to parse an double from given string. If the double cannot be parsed or does
// not fit doubleo double, false is returned and the error string is filled using the
// value_name argument.
inline bool parse_double(string_piece str, const char* value_name, double& value, string& error);

// Try to parse an double from given string. If the double cannot be parsed or does
// not fit doubleo double, an error is displayed and program exits.
inline double parse_double(string_piece str, const char* value_name);

//
// Definitions
//

bool parse_double(string_piece str, const char* value_name, double& value, string& error) {
  string_piece original = str;

  // Skip spaces
  while (str.len && (str.str[0] == ' ' || str.str[0] == '\f' || str.str[0] == '\n' || str.str[0] == '\r' || str.str[0] == '\t' || str.str[0] == '\v'))
    str.str++, str.len--;

  // Allow plus/minus
  bool negative = false;
  if (str.len && (str.str[0] == '+' || str.str[0] == '-')) {
    negative = str.str[0] == '-';
    str.str++, str.len--;
  }

  // Parse value, checking for overflow/underflow
  if (!str.len) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': empty string."), false;
  if (!(str.str[0] >= '0' || str.str[0] <= '9')) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': non-digit character found."), false;

  value = 0;
  while (str.len && str.str[0] >= '0' && str.str[0] <= '9') {
    value = 10 * value + (str.str[0] - '0');
    str.str++, str.len--;
  }

  // If there is a decimal point, parse the rest of the
  if (str.len && str.str[0] == '.') {
    double divider = 1;

    str.str++, str.len--;
    while (str.len && str.str[0] >= '0' && str.str[0] <= '9') {
      value = 10 * value + (str.str[0] - '0');
      divider *= 10.;
      str.str++, str.len--;
    }

    value /= divider;
  }
  if (!isfinite(value)) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': overflow occured."), false;

  // Optionally parse an exponent
  if (str.len && (str.str[0] == 'e' || str.str[0] == 'E')) {
    str.str++, str.len--;

    double exponent = 0;
    bool exponent_negative = false;
    if (str.len && (str.str[0] == '+' || str.str[0] == '-')) {
      exponent_negative = str.str[0] == '-';
      str.str++, str.len--;
    }

    while (str.len && str.str[0] >= '0' && str.str[0] <= '9') {
      exponent = 10 * exponent + (str.str[0] - '0');
      str.str++, str.len--;
    }

    exponent = pow(10., exponent_negative ? -exponent : exponent);
    if (!isfinite(exponent)) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': exponent overflow occured."), false;
    if (exponent == 0) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': exponent underflow occured."), false;

    if (value) {
      value *= exponent;
      if (!isfinite(value)) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': overflow occured."), false;
      if (value == 0) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': underflow occured."), false;
    }
  }

  // Apply initial minus
  if (negative) value *= -1;

  // Skip spaces
  while (str.len && (str.str[0] == ' ' || str.str[0] == '\f' || str.str[0] == '\n' || str.str[0] == '\r' || str.str[0] == '\t' || str.str[0] == '\v'))
    str.str++, str.len--;

  // Check for remaining characters
  if (str.len) return error.assign("Cannot parse ").append(value_name).append(" double value '").append(original.str, original.len).append("': non-digit character found."), false;

  return true;
}

double parse_double(string_piece str, const char* value_name) {
  double result;
  string error;

  if (!parse_double(str, value_name, result, error))
    runtime_failure(error);

  return result;
}

} // namespace utils

/////////
// File: model/model_morphodita_parsito.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Versions:
// 1 - initial version
// 2 - add absolute lemmas (tagger_model::lemma == 2)
//   - use Arabic and space normalization

input_format* model_morphodita_parsito::new_tokenizer(const string& options) const {
  if (!tokenizer_factory)
    return nullptr;

  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  bool normalized_spaces = parsed_options.count("normalized_spaces");
  bool token_ranges = parsed_options.count("ranges");

  const auto* morpho = !taggers.empty() ? taggers[0].tagger->get_morpho() : nullptr;
  unique_ptr result(new morphodita_tokenizer_wrapper(tokenizer_factory->new_tokenizer(morpho), splitter.get(), normalized_spaces, token_ranges));

  // Presegmented
  if (parsed_options.count("presegmented") && result)
    result.reset(input_format::new_presegmented_tokenizer(result.release()));

  // Joint with parsing
  if (parsed_options.count("joint_with_parsing") && result) {
    int max_sentence_len = 20;
    if (parsed_options.count("joint_max_sentence_len") && !parse_int(parsed_options["joint_max_sentence_len"], "joint max sentence len", max_sentence_len, parse_error))
      return nullptr;

    double change_boundary_logprob = -0.5;
    if (parsed_options.count("joint_change_boundary_logprob") && !parse_double(parsed_options["joint_change_boundary_logprob"], "joint change boundary logprob", change_boundary_logprob, parse_error))
      return nullptr;

    double sentence_logprob = -0.5;
    if (parsed_options.count("joint_sentence_logprob") && !parse_double(parsed_options["joint_sentence_logprob"], "joint sentence logprob", sentence_logprob, parse_error))
      return nullptr;

    result.reset(new joint_with_parsing_tokenizer(result.release(), *this, max_sentence_len, change_boundary_logprob, sentence_logprob));
  }

  return result.release();
}

bool model_morphodita_parsito::tag(sentence& s, const string& /*options*/, string& error) const {
  error.clear();

  if (taggers.empty()) return error.assign("No tagger defined for the UDPipe model!"), false;
  if (s.empty()) return true;

  tagger_cache* c = tagger_caches.pop();
  if (!c) c = new tagger_cache();

  // Prepare input forms
  c->forms_normalized.resize(s.words.size() - 1);
  c->forms_string_pieces.resize(s.words.size() - 1);
  for (size_t i = 1; i < s.words.size(); i++)
    c->forms_string_pieces[i - 1] = normalize_form(s.words[i].form, c->forms_normalized[i - 1]);

  // Clear first
  for (size_t i = 1; i < s.words.size(); i++) {
    s.words[i].lemma.assign("_");
    s.words[i].upostag.clear();
    s.words[i].xpostag.clear();
    s.words[i].feats.clear();
  }

  // Fill information from the tagger models
  for (auto&& tagger : taggers) {
    if (!tagger.tagger) return error.assign("No tagger defined for the UDPipe model!"), false;

    tagger.tagger->tag(c->forms_string_pieces, c->lemmas);

    for (size_t i = 0; i < c->lemmas.size(); i++)
      fill_word_analysis(c->lemmas[i], tagger.raw, tagger.upostag, tagger.lemma, tagger.xpostag, tagger.feats, s.words[i+1]);
  }

  // For raw tagger models, fill MorphoGuesser=Yes where appropriate
  if (taggers.size() == 1 && taggers[0].raw && taggers[0].tagger->get_morpho()) {
    const auto* morpho = taggers[0].tagger->get_morpho();
    for (size_t i = 0; i < c->forms_string_pieces.size(); i++) {
      if (morpho->analyze(c->forms_string_pieces[i], morphodita::morpho::GUESSER, c->lemmas) == morphodita::morpho::GUESSER)
        s.words[i + 1].misc.append(s.words[i + 1].misc.empty() ? "" : "|").append("MorphoGuesser=Yes");
    }
  }

  tagger_caches.push(c);
  return true;
}

bool model_morphodita_parsito::parse(sentence& s, const string& options, string& error) const {
  return parse(s, options, error, nullptr);
}

bool model_morphodita_parsito::parse(sentence& s, const string& options, string& error, double* cost) const {
  error.clear();

  if (!parser) return error.assign("No parser defined for the UDPipe model!"), false;
  if (s.empty()) return true;

  parser_cache* c = parser_caches.pop();
  if (!c) c = new parser_cache();

  int beam_search = 5;
  if (!named_values::parse(options, c->options, error))
    return false;
  if (c->options.count("beam_search"))
    if (!parse_int(c->options["beam_search"], "beam_search", beam_search, error))
      return false;

  c->tree.clear();
  for (size_t i = 1; i < s.words.size(); i++) {
    c->tree.add_node(string());
    normalize_form(s.words[i].form, c->tree.nodes.back().form);
    normalize_lemma(s.words[i].lemma, c->tree.nodes.back().lemma);
    c->tree.nodes.back().upostag.assign(s.words[i].upostag);
    c->tree.nodes.back().xpostag.assign(s.words[i].xpostag);
    c->tree.nodes.back().feats.assign(s.words[i].feats);
    c->tree.nodes.back().deps.assign(s.words[i].deps);
    c->tree.nodes.back().misc.assign(s.words[i].misc);
  }

  parser->parse(c->tree, beam_search, cost);
  for (size_t i = 1; i < s.words.size(); i++)
    s.set_head(i, c->tree.nodes[i].head, c->tree.nodes[i].deprel);

  parser_caches.push(c);
  return true;
}

model* model_morphodita_parsito::load(istream& is) {
  char version;
  if (!is.get(version)) return nullptr;
  if (!(version >= 1 && version <= VERSION_LATEST)) return nullptr;

  // Because UDPipe 1.0 does not check the model version,
  // a specific sentinel was added since version 2 so that
  // loading of such model fail on UDPipe 1.0
  if (version >= 2) {
    char sentinel;
    if (!is.get(sentinel) || sentinel != 0x7F) return nullptr;
    if (!is.get(sentinel) || sentinel != 0x7F) return nullptr;
  }

  unique_ptr m(new model_morphodita_parsito((unsigned char)version));
  if (!m) return nullptr;

  char tokenizer;
  if (!is.get(tokenizer)) return nullptr;
  m->tokenizer_factory.reset(tokenizer ? morphodita::tokenizer_factory::load(is) : nullptr);
  if (tokenizer && !m->tokenizer_factory) return nullptr;
  m->splitter.reset(tokenizer ? multiword_splitter::load(is) : nullptr);
  if (tokenizer && !m->splitter) return nullptr;

  m->taggers.clear();
  char taggers; if (!is.get(taggers)) return nullptr;
  for (char i = 0; i < taggers; i++) {
    char lemma; if (!is.get(lemma)) return nullptr;
    char xpostag; if (!is.get(xpostag)) return nullptr;
    char feats; if (!is.get(feats)) return nullptr;
    int model_type = is.peek();
    bool raw = !(model_type == morphodita::tagger_ids::CONLLU2 ||
                 model_type == morphodita::tagger_ids::CONLLU2_3 ||
                 model_type == morphodita::tagger_ids::CONLLU3);
    morphodita::tagger* tagger = morphodita::tagger::load(is);
    if (!tagger) return nullptr;
    m->taggers.emplace_back(raw, i == 0, int(lemma), bool(xpostag), bool(feats), tagger);
  }

  char parser;
  if (!is.get(parser)) return nullptr;
  m->parser.reset(parser ? parsito::parser::load(is) : nullptr);
  if (parser && !m->parser) return nullptr;

  return m.release();
}

model_morphodita_parsito::model_morphodita_parsito(unsigned version) : version(version) {}

bool model_morphodita_parsito::joint_with_parsing_tokenizer::read_block(istream& is, string& block) const {
  block.clear();

  for (string line; getline(is, line); ) {
    block.append(line);
    block.push_back('\n');
  }

  if (is.eof() && !block.empty()) is.clear(istream::eofbit);
  return bool(is);
}

void model_morphodita_parsito::joint_with_parsing_tokenizer::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  sentence_id = 1;
  set_text("");
  sentences.clear();
  sentences_index = 0;
}

void model_morphodita_parsito::joint_with_parsing_tokenizer::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text.str = text_copy.c_str();
  }
  this->text = text;
}

bool model_morphodita_parsito::joint_with_parsing_tokenizer::next_sentence(sentence& s, string& error) {
  error.clear();

  if (text.len) {
    sentences.clear();
    sentences_index = 0;

    tokenizer->set_text(text, false);

    sentence input;
    vector paragraph;
    while (tokenizer->next_sentence(input, error)) {
      if (input.get_new_par() && !paragraph.empty()) {
        if (!parse_paragraph(paragraph, error)) return false;
        for (auto&& sentence : paragraph)
          sentences.push_back(sentence);
        paragraph.clear();
      }
      paragraph.push_back(input);
    }
    if (!error.empty()) return false;

    if (!paragraph.empty()) {
      if (!parse_paragraph(paragraph, error)) return false;
      for (auto&& sentence : paragraph)
        sentences.push_back(sentence);
    }

    text.len = 0;
  }

  if (sentences_index < sentences.size()) {
    s = sentences[sentences_index++];
    return true;
  }

  return false;
}

bool model_morphodita_parsito::joint_with_parsing_tokenizer::parse_paragraph(vector& paragraph, string& error) {
  sentence all_words;
  vector sentence_boundary(1, true);
  vector token_boundary(1, true);

  for (auto&& s : paragraph) {
    unsigned offset = all_words.words.size() - 1;
    for (unsigned i = 1; i < s.words.size(); i++) {
      all_words.words.push_back(s.words[i]);
      all_words.words.back().id += offset;
      sentence_boundary.push_back(i+1 == s.words.size());
      token_boundary.push_back(true);
    }

    for (auto&& mwt : s.multiword_tokens) {
      all_words.multiword_tokens.push_back(mwt);
      all_words.multiword_tokens.back().id_first += offset;
      all_words.multiword_tokens.back().id_last += offset;
      for (int i = all_words.multiword_tokens.back().id_first; i < all_words.multiword_tokens.back().id_last; i++)
        token_boundary[i] = false;
    }
  }

  vector best_logprob(all_words.words.size(), -numeric_limits::infinity()); best_logprob[0] = 0.;
  vector best_length(all_words.words.size(), 0);
  sentence s;

  for (unsigned start = 1; start < all_words.words.size(); start++) {
    if (!token_boundary[start - 1]) continue;
    s.clear();
    for (unsigned end = start + 1; end <= all_words.words.size() && (end - start) <= unsigned(max_sentence_len); end++) {
      s.words.push_back(all_words.words[end - 1]);
      s.words.back().id -= start - 1;
      if (!token_boundary[end - 1]) continue;

      for (unsigned i = 1; i < s.words.size(); i++) {
        s.words[i].head = -1;
        s.words[i].children.clear();
      }

      double cost;
      if (!model.parse(s, DEFAULT, error, &cost)) return false;
      cost += sentence_logprob + change_boundary_logprob * (2 - int(sentence_boundary[start - 1]) - int(sentence_boundary[end - 1]));
      if (best_logprob[start - 1] + cost > best_logprob[end - 1]) {
        best_logprob[end - 1] = best_logprob[start - 1] + cost;
        best_length[end - 1] = end - start;
      }
    }
  }

  vector sentence_lengths;
  for (unsigned end = all_words.words.size(); end > 1; end -= best_length[end - 1])
    sentence_lengths.push_back(best_length[end - 1]);

  paragraph.clear();

  sentence_lengths.push_back(1);
  reverse(sentence_lengths.begin(), sentence_lengths.end());
  for (unsigned i = 1; i < sentence_lengths.size(); i++) {
    sentence_lengths[i] += sentence_lengths[i - 1];

    paragraph.emplace_back();
    while (!all_words.multiword_tokens.empty() && unsigned(all_words.multiword_tokens.front().id_first) < sentence_lengths[i]) {
      paragraph.back().multiword_tokens.push_back(all_words.multiword_tokens.front());
      paragraph.back().multiword_tokens.back().id_first -= sentence_lengths[i-1] - 1;
      paragraph.back().multiword_tokens.back().id_last -= sentence_lengths[i-1] - 1;
      all_words.multiword_tokens.erase(all_words.multiword_tokens.begin());
    }

    for (unsigned word = sentence_lengths[i - 1]; word < sentence_lengths[i]; word++) {
      paragraph.back().words.push_back(all_words.words[word]);
      paragraph.back().words.back().id -= sentence_lengths[i-1] - 1;
      paragraph.back().words.back().head = -1;
      paragraph.back().words.back().children.clear();
    }
  }

  if (!paragraph.empty()) {
    if (new_document) {
      paragraph.front().set_new_doc(true, document_id);
      new_document = false;
    }

    paragraph.front().set_new_par(true);
  }

  return true;
}

void model_morphodita_parsito::fill_word_analysis(const morphodita::tagged_lemma& analysis, bool raw, bool upostag, int lemma, bool xpostag, bool feats, word& word) const {
  // Handle raw MorphoDiTa models.
  if (raw) {
    if (lemma) word.lemma.assign(analysis.lemma);
    if (xpostag) word.xpostag.assign(analysis.tag);
    return;
  }

  // Lemma
  if (lemma == 1) {
    word.lemma.assign(analysis.lemma);
  } else if (lemma == 2) {
    word.lemma.assign(analysis.lemma);

    // Lemma matching ~replacement~normalized_form is changed to replacement.
    if (analysis.lemma[0] == '~') {
      auto end = analysis.lemma.find('~', 1);
      if (end != string::npos) {
        normalize_form(word.form, word.lemma);
        if (analysis.lemma.compare(end + 1, string::npos, word.lemma) == 0)
          word.lemma.assign(analysis.lemma, 1, end - 1);
        else
          word.lemma.assign(analysis.lemma);
      }
    }
  }
  if (version == 2) {
    // Replace '\001' back to spaces
    for (auto && chr : word.lemma)
      if (chr == '\001')
        chr = ' ';
  } else if (version >= 3) {
    // Replace '0xC2 0xA0' back to spaces
    for (size_t i = 0; i + 1 < word.lemma.size(); i++)
      if (word.lemma[i] == char(0xC2) && word.lemma[i+1] == char(0xA0))
        word.lemma.replace(i, 2, 1, ' ');
  }

  if (!upostag && !xpostag && !feats) return;

  // UPOSTag
  char separator = analysis.tag[0];
  size_t start = min(size_t(1), analysis.tag.size()), end = min(analysis.tag.find(separator, 1), analysis.tag.size());
  if (upostag) word.upostag.assign(analysis.tag, start, end - start);

  if (!xpostag && !feats) return;

  // XPOSTag
  start = min(end + 1, analysis.tag.size());
  end = min(analysis.tag.find(separator, start), analysis.tag.size());
  if (xpostag) word.xpostag.assign(analysis.tag, start, end - start);

  if (!feats) return;

  // Features
  start = min(end + 1, analysis.tag.size());
  word.feats.assign(analysis.tag, start, analysis.tag.size() - start);
}

const string& model_morphodita_parsito::normalize_form(string_piece form, string& output) const {
  using unilib::utf8;

  // No normalization on version 1
  if (version <= 1) return output.assign(form.str, form.len);

  // If requested, replace space by \001 in version 2 and by   (\u00a0) since version 3

  // Arabic normalization since version 2, implementation resulted from
  // discussion with Otakar Smrz and Nasrin Taghizadeh.
  // 1. Remove https://codepoints.net/U+0640 without any reasonable doubt :)
  // 2. Remove https://codepoints.net/U+0652
  // 3. Remove https://codepoints.net/U+0670
  // 4. Remove everything from https://codepoints.net/U+0653 to
  //    https://codepoints.net/U+0657 though they are probably very rare in date
  // 5. Remove everything from https://codepoints.net/U+064B to
  //    https://codepoints.net/U+0650
  // 6. Remove https://codepoints.net/U+0651
  // 7. Replace https://codepoints.net/U+0671 with https://codepoints.net/U+0627
  // 8. Replace https://codepoints.net/U+0622 with https://codepoints.net/U+0627
  // 9. Replace https://codepoints.net/U+0623 with https://codepoints.net/U+0627
  // 10. Replace https://codepoints.net/U+0625 with https://codepoints.net/U+0627
  // 11. Replace https://codepoints.net/U+0624 with https://codepoints.net/U+0648
  // 12. Replace https://codepoints.net/U+0626 with https://codepoints.net/U+064A
  // One might also consider replacing some Farsi characters that might be typed
  // unintentionally (by Iranians writing Arabic language texts):
  // 13. Replace https://codepoints.net/U+06CC with https://codepoints.net/U+064A
  // 14. Replace https://codepoints.net/U+06A9 with https://codepoints.net/U+0643
  // 15. Replace https://codepoints.net/U+06AA with https://codepoints.net/U+0643
  //
  // Not implemented:
  // There is additional challenge with data coming from Egypt (such as printed
  // or online newspapers), where the word-final https://codepoints.net/U+064A
  // may be switched for https://codepoints.net/U+0649 and visa versa. Also, the
  // word-final https://codepoints.net/U+0647 could actually represent https://
  // codepoints.net/U+0629. You can experiment with the following replacements,
  // but I would rather apply them only after classifying the whole document as
  // following such convention:
  // 1. Replace https://codepoints.net/U+0629 with https://codepoints.net/U+0647
  //    (frequent femine ending markers would appear like a third-person
  //    masculine pronoun clitic instead)
  // 2. Replace https://codepoints.net/U+0649 with https://codepoints.net/U+064A
  //    (some "weak" words would become even more ambiguous or appear as if
  //    with a first-person pronoun clitic)

  output.clear();
  for (auto&& chr : utf8::decoder(form.str, form.len)) {
    // Arabic normalization
    if (chr == 0x640 || (chr >= 0x64B && chr <= 0x657) || chr == 0x670) {}
    else if (chr == 0x622) utf8::append(output, 0x627);
    else if (chr == 0x623) utf8::append(output, 0x627);
    else if (chr == 0x624) utf8::append(output, 0x648);
    else if (chr == 0x625) utf8::append(output, 0x627);
    else if (chr == 0x626) utf8::append(output, 0x64A);
    else if (chr == 0x671) utf8::append(output, 0x627);
    else if (chr == 0x6A9) utf8::append(output, 0x643);
    else if (chr == 0x6AA) utf8::append(output, 0x643);
    else if (chr == 0x6CC) utf8::append(output, 0x64A);
    // Space normalization
    else if (chr == ' ' && version == 2) utf8::append(output, 0x01);
    else if (chr == ' ' && version >= 3) utf8::append(output, 0xA0);
    // Default
    else utf8::append(output, chr);
  }

  // Make sure we do not remove everything
  if (output.empty() && form.len)
    utf8::append(output, utf8::first(form.str, form.len));

  return output;
}

const string& model_morphodita_parsito::normalize_lemma(string_piece lemma, string& output) const {
  using unilib::utf8;

  // No normalization on version 1 and 2
  if (version <= 2) return output.assign(lemma.str, lemma.len);

  // Normalize spaces by   since version 3
  output.clear();
  for (size_t i = 0; i < lemma.len; i++) {
    // Space normalization
    if (lemma.str[i] == ' ') utf8::append(output, 0xA0);
    // Default
    else output.push_back(lemma.str[i]);
  }

  return output;
}

/////////
// File: model/pipeline.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class pipeline {
 public:
  pipeline(const model* m, const string& input, const string& tagger, const string& parser, const string& output);

  void set_model(const model* m);
  void set_input(const string& input);
  void set_tagger(const string& tagger);
  void set_parser(const string& parser);
  void set_output(const string& output);

  void set_immediate(bool immediate);
  void set_document_id(const string& document_id);

  bool process(istream& is, ostream& os, string& error) const;

  static const string DEFAULT;
  static const string NONE;

 private:
  const model* m;
  string input, tokenizer, tagger, parser, output;
  string document_id;
  bool immediate;
};

/////////
// File: sentence/output_format.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class output_format {
 public:
  virtual ~output_format() {}

  virtual void write_sentence(const sentence& s, ostream& os) = 0;
  virtual void finish_document(ostream& /*os*/) {}

  // Static factory methods
  static output_format* new_output_format(const string& name);
  static output_format* new_conllu_output_format(const string& options = string());
  static output_format* new_epe_output_format(const string& options = string());
  static output_format* new_matxin_output_format(const string& options = string());
  static output_format* new_horizontal_output_format(const string& options = string());
  static output_format* new_plaintext_output_format(const string& options = string());
  static output_format* new_vertical_output_format(const string& options = string());

  static const string CONLLU_V1;
  static const string CONLLU_V2;
  static const string HORIZONTAL_PARAGRAPHS;
  static const string PLAINTEXT_NORMALIZED_SPACES;
  static const string VERTICAL_PARAGRAPHS;
};

/////////
// File: utils/getwhole.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Read whole content until EOF. All encountered \n are stored.
inline istream& getwhole(istream& is, string& whole);

//
// Definitions
//

istream& getwhole(istream& is, string& whole) {
  whole.clear();

  for (string line; getline(is, line); )
    whole.append(line).push_back('\n');

  if (is.eof() && !whole.empty()) is.clear(istream::eofbit);
  return is;
}

} // namespace utils

/////////
// File: model/pipeline.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string pipeline::DEFAULT;
const string pipeline::NONE = "none";

pipeline::pipeline(const model* m, const string& input, const string& tagger, const string& parser, const string& output) : immediate(false) {
  set_model(m);
  set_input(input);
  set_tagger(tagger);
  set_parser(parser);
  set_output(output);
}

void pipeline::set_model(const model* m) {
  this->m = m;
}

void pipeline::set_input(const string& input) {
  tokenizer.clear();

  if (input.empty()) {
    this->input = "conllu";
  } else if (input == "tokenize" || input == "tokenizer") {
    this->input = "tokenizer";
  } else if (input.compare(0, 10, "tokenizer=") == 0) {
    this->input = "tokenizer";
    tokenizer.assign(input, 10, string::npos);
  } else {
    this->input = input;
  }
}

void pipeline::set_tagger(const string& tagger) {
  this->tagger = tagger;
}

void pipeline::set_parser(const string& parser) {
  this->parser = parser;
}

void pipeline::set_output(const string& output) {
  this->output = output.empty() ? "conllu" : output;
}

void pipeline::set_immediate(bool immediate) {
  this->immediate = immediate;
}

void pipeline::set_document_id(const string& document_id) {
  this->document_id = document_id;
}

bool pipeline::process(istream& is, ostream& os, string& error) const {
  error.clear();

  sentence s;

  unique_ptr reader;
  if (input == "tokenizer") {
    reader.reset(m->new_tokenizer(tokenizer));
    if (!reader) return error.assign("The model does not have a tokenizer!"), false;
  } else {
    reader.reset(input_format::new_input_format(input));
    if (!reader) return error.assign("The requested input format '").append(input).append("' does not exist!"), false;
  }
  reader->reset_document(document_id);

  unique_ptr writer(output_format::new_output_format(output));
  if (!writer) return error.assign("The requested output format '").append(output).append("' does not exist!"), false;

  string block;
  while (immediate ? reader->read_block(is, block) : bool(getwhole(is, block))) {
    reader->set_text(block);
    while (reader->next_sentence(s, error)) {
      if (tagger != NONE)
        if (!m->tag(s, tagger, error))
          return false;

      if (parser != NONE)
        if (!m->parse(s, parser, error))
          return false;

      writer->write_sentence(s, os);
    }
    if (!error.empty()) return false;
  }
  writer->finish_document(os);

  return true;
}

/////////
// File: morphodita/tagset_converter/tagset_converter.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tagset_converter {
 public:
  virtual ~tagset_converter() {}

  // Convert a tag-lemma pair to a different tag set.
  virtual void convert(tagged_lemma& tagged_lemma) const = 0;
  // Convert a result of analysis to a different tag set. Apart from calling
  // convert, any repeated entry is removed.
  virtual void convert_analyzed(vector& tagged_lemmas) const = 0;
  // Convert a result of generation to a different tag set. Apart from calling
  // convert, any repeated entry is removed.
  virtual void convert_generated(vector& forms) const = 0;

  // Static factory methods
  static tagset_converter* new_identity_converter();

  static tagset_converter* new_pdt_to_conll2009_converter();
  static tagset_converter* new_strip_lemma_comment_converter(const morpho& dictionary);
  static tagset_converter* new_strip_lemma_id_converter(const morpho& dictionary);
};

// Helper method for creating tagset_converter from instance name.
tagset_converter* new_tagset_converter(const string& name, const morpho& dictionary);

// Helper methods making sure remapped results are unique.
void tagset_converter_unique_analyzed(vector& tagged_lemmas);
void tagset_converter_unique_generated(vector& forms);

} // namespace morphodita

/////////
// File: morphodita/derivator/derivation_formatter.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class derivation_formatter {
 public:
  virtual ~derivation_formatter() {}

  // Perform the required derivation and store it directly in the lemma.
  virtual void format_derivation(string& lemma) const;

  // Perform the required derivation and store it directly in the tagged_lemma.
  // If a tagset_converter is given, it is also applied.
  virtual void format_tagged_lemma(tagged_lemma& lemma, const tagset_converter* converter = nullptr) const = 0;

  // Perform the required derivation on a list of tagged_lemmas.
  // If a tagset_converter is given, it is also applied.
  // Either way, only unique entries are returned.
  virtual void format_tagged_lemmas(vector& lemmas, const tagset_converter* converter = nullptr) const;

  // Static factory methods.
  static derivation_formatter* new_none_derivation_formatter();
  static derivation_formatter* new_root_derivation_formatter(const derivator* derinet);
  static derivation_formatter* new_path_derivation_formatter(const derivator* derinet);
  static derivation_formatter* new_tree_derivation_formatter(const derivator* derinet);
  // String version of static factory method.
  static derivation_formatter* new_derivation_formatter(string_piece name, const derivator* derinet);
};

} // namespace morphodita

/////////
// File: morphodita/derivator/derivation_formatter.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void derivation_formatter::format_derivation(string& lemma) const {
  tagged_lemma result;
  result.lemma.swap(lemma);
  format_tagged_lemma(result);
  lemma.swap(result.lemma);
}

void derivation_formatter::format_tagged_lemmas(vector& lemmas, const tagset_converter* converter) const {
  for (auto&& lemma : lemmas)
    format_tagged_lemma(lemma, converter);

  if (lemmas.size() > 1)
    tagset_converter_unique_analyzed(lemmas);
}

class none_derivation_formatter : public derivation_formatter {
  virtual void format_derivation(string& /*lemma*/) const override {}

  virtual void format_tagged_lemma(tagged_lemma& lemma, const tagset_converter* converter) const override {
    if (converter) converter->convert(lemma);
  }

  virtual void format_tagged_lemmas(vector& lemmas, const tagset_converter* converter) const override {
    if (converter) converter->convert_analyzed(lemmas);
  }
};

derivation_formatter* derivation_formatter::new_none_derivation_formatter() {
  return new none_derivation_formatter();
}

class root_derivation_formatter : public derivation_formatter {
 public:
  root_derivation_formatter(const derivator* derinet) : derinet(derinet) {}

  virtual void format_tagged_lemma(tagged_lemma& lemma, const tagset_converter* converter) const override {
    for (derivated_lemma parent; derinet->parent(lemma.lemma, parent); )
      lemma.lemma.assign(parent.lemma);
    if (converter) converter->convert(lemma);
  }

 private:
  const derivator* derinet;
};

derivation_formatter* derivation_formatter::new_root_derivation_formatter(const derivator* derinet) {
  return derinet ? new root_derivation_formatter(derinet) : nullptr;
}

class path_derivation_formatter : public derivation_formatter {
 public:
  path_derivation_formatter(const derivator* derinet) : derinet(derinet) {}

  virtual void format_tagged_lemma(tagged_lemma& lemma, const tagset_converter* converter) const override {
    tagged_lemma current(lemma);
    if (converter) converter->convert(lemma);
    for (derivated_lemma parent; derinet->parent(current.lemma, parent); current.lemma.swap(parent.lemma)) {
      tagged_lemma parrent_lemma(parent.lemma, current.tag);
      if (converter) converter->convert(parrent_lemma);
      lemma.lemma.append(" ").append(parrent_lemma.lemma);
    }
  }

 private:
  const derivator* derinet;
};

derivation_formatter* derivation_formatter::new_path_derivation_formatter(const derivator* derinet) {
  return derinet ? new path_derivation_formatter(derinet) : nullptr;
}

class tree_derivation_formatter : public derivation_formatter {
 public:
  tree_derivation_formatter(const derivator* derinet) : derinet(derinet) {}

  virtual void format_tagged_lemma(tagged_lemma& lemma, const tagset_converter* converter) const override {
    string root(lemma.lemma), tag(lemma.tag);
    if (converter) converter->convert(lemma);
    for (derivated_lemma parent; derinet->parent(root, parent); root.swap(parent.lemma)) {}
    format_tree(root, tag, lemma, converter);
  }

  void format_tree(const string& root, const string& tag, tagged_lemma& tree, const tagset_converter* converter) const {
    vector children;

    if (converter) {
      tagged_lemma current(root, tag);
      converter->convert(current);
      tree.lemma.append(" ").append(current.lemma);
    } else {
      tree.lemma.append(" ").append(root);
    }

    if (derinet->children(root, children))
      for (auto&& child : children)
        format_tree(child.lemma, tag, tree, converter);
    tree.lemma.push_back(' ');
  }

 private:
  const derivator* derinet;
};

derivation_formatter* derivation_formatter::new_tree_derivation_formatter(const derivator* derinet) {
  return derinet ? new tree_derivation_formatter(derinet) : nullptr;
}

derivation_formatter* derivation_formatter::new_derivation_formatter(string_piece name, const derivator* derinet) {
  if (name == "none") return new_none_derivation_formatter();
  if (name == "root") return new_root_derivation_formatter(derinet);
  if (name == "path") return new_path_derivation_formatter(derinet);
  if (name == "tree") return new_tree_derivation_formatter(derinet);
  return nullptr;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/small_stringops.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
inline bool small_memeq(const void* a, const void* b, size_t len);
inline void small_memcpy(void* dest, const void* src, size_t len);

// Definitions
bool small_memeq(const void* a_void, const void* b_void, size_t len) {
  const char* a = (const char*)a_void;
  const char* b = (const char*)b_void;

  while (len--)
    if (*a++ != *b++)
      return false;
  return true;
}

void small_memcpy(void* dest_void, const void* src_void, size_t len) {
  char* dest = (char*)dest_void;
  const char* src = (const char*)src_void;

  while (len--)
    *dest++ = *src++;
}

} // namespace morphodita

/////////
// File: trainer/training_failure.h
/////////

// This file is part of UDPipe .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

class training_error : public runtime_error {
 public:
  training_error();

  static ostringstream message_collector;
};

#define training_failure(message) throw (training_error::message_collector << message, training_error())

} // namespace utils

/////////
// File: utils/binary_encoder.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

class binary_encoder {
 public:
  inline binary_encoder();

  inline void add_1B(unsigned val);
  inline void add_2B(unsigned val);
  inline void add_4B(unsigned val);
  inline void add_float(double val);
  inline void add_double(double val);
  inline void add_str(string_piece str);
  inline void add_data(string_piece data);
  template  inline void add_data(const vector& data);
  template  inline void add_data(const T* data, size_t elements);

  vector data;
};

//
// Definitions
//

binary_encoder::binary_encoder() {
  data.reserve(16);
}

void binary_encoder::add_1B(unsigned val) {
  if (uint8_t(val) != val) training_failure("Should encode value " << val << " in one byte!");
  data.push_back(val);
}

void binary_encoder::add_2B(unsigned val) {
  if (uint16_t(val) != val) training_failure("Should encode value " << val << " in two bytes!");
  data.insert(data.end(), (unsigned char*) &val, ((unsigned char*) &val) + sizeof(uint16_t));
}

void binary_encoder::add_4B(unsigned val) {
  if (uint32_t(val) != val) training_failure("Should encode value " << val << " in four bytes!");
  data.insert(data.end(), (unsigned char*) &val, ((unsigned char*) &val) + sizeof(uint32_t));
}

void binary_encoder::add_float(double val) {
  data.insert(data.end(), (unsigned char*) &val, ((unsigned char*) &val) + sizeof(float));
}

void binary_encoder::add_double(double val) {
  data.insert(data.end(), (unsigned char*) &val, ((unsigned char*) &val) + sizeof(double));
}

void binary_encoder::add_str(string_piece str) {
  add_1B(str.len < 255 ? str.len : 255);
  if (!(str.len < 255)) add_4B(str.len);
  add_data(str);
}

void binary_encoder::add_data(string_piece data) {
  this->data.insert(this->data.end(), (const unsigned char*) data.str, (const unsigned char*) (data.str + data.len));
}

template
void binary_encoder::add_data(const vector& data) {
  this->data.insert(this->data.end(), (const unsigned char*) data.data(), (const unsigned char*) (data.data() + data.size()));
}

template
void binary_encoder::add_data(const T* data, size_t elements) {
  this->data.insert(this->data.end(), (const unsigned char*) data, (const unsigned char*) (data + elements));
}

} // namespace utils

/////////
// File: utils/pointer_decoder.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

class pointer_decoder {
 public:
  inline pointer_decoder(const unsigned char*& data);
  inline unsigned next_1B();
  inline unsigned next_2B();
  inline unsigned next_4B();
  inline void next_str(string& str);
  template  inline const T* next(unsigned elements);

 private:
  const unsigned char*& data;
};

//
// Definitions
//

pointer_decoder::pointer_decoder(const unsigned char*& data) : data(data) {}

unsigned pointer_decoder::next_1B() {
  return *data++;
}

unsigned pointer_decoder::next_2B() {
  uint16_t result;
  memcpy(&result, data, sizeof(uint16_t));
  data += sizeof(uint16_t);
  return result;
}

unsigned pointer_decoder::next_4B() {
  uint32_t result;
  memcpy(&result, data, sizeof(uint32_t));
  data += sizeof(uint32_t);
  return result;
}

void pointer_decoder::next_str(string& str) {
  unsigned len = next_1B();
  if (len == 255) len = next_4B();
  str.assign(next(len), len);
}

template  const T* pointer_decoder::next(unsigned elements) {
  const T* result = (const T*) data;
  data += sizeof(T) * elements;
  return result;
}

} // namespace utils

/////////
// File: utils/unaligned_access.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2017 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

template
inline T unaligned_load(const P* ptr);

template
inline T unaligned_load_inc(const P*& ptr);

template
inline void unaligned_store(P* ptr, T value);

template
inline void unaligned_store_inc(P*& ptr, T value);

template
T* unaligned_lower_bound(T* first, size_t size, T val);

template
T* unaligned_upper_bound(T* first, size_t size, T val);

//
// Definitions
//

template
inline T unaligned_load(const P* ptr) {
  T value;
  memcpy(&value, ptr, sizeof(T));
  return value;
}

template
inline T unaligned_load_inc(const P*& ptr) {
  T value;
  memcpy(&value, ptr, sizeof(T));
  ((const char*&)ptr) += sizeof(T);
  return value;
}

template
inline void unaligned_store(P* ptr, T value) {
  memcpy(ptr, &value, sizeof(T));
}

template
inline void unaligned_store_inc(P*& ptr, T value) {
  memcpy(ptr, &value, sizeof(T));
  ((char*&)ptr) += sizeof(T);
}

template
T* unaligned_lower_bound(T* first, size_t size, T val) {
  while (size) {
    size_t step = size >> 1;
    if (unaligned_load(first + step) < val) {
      first += step + 1;
      size -= step + 1;
    } else {
      size = step;
    }
  }
  return first;
}

template
T* unaligned_upper_bound(T* first, size_t size, T val) {
  while (size) {
    size_t step = size >> 1;
    if (!(val < unaligned_load(first + step))) {
      first += step + 1;
      size -= step + 1;
    } else {
      size = step;
    }
  }
  return first;
}

} // namespace utils

/////////
// File: morphodita/morpho/persistent_unordered_map.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
class persistent_unordered_map {
 public:
  // Accessing function
  template
  inline const unsigned char* at(const char* str, int len, EntrySize entry_size) const;

  template
  inline const T* at_typed(const char* str, int len) const;

  template
  inline void iter(const char* str, int len, EntryProcess entry_process) const;

  template
  inline void iter_all(EntryProcess entry_process) const;

  // Two helper functions accessing some internals
  inline int max_length() const;
  inline const unsigned char* data_start(int len) const;

  // Creation functions
  persistent_unordered_map() {}
  template
  persistent_unordered_map(const unordered_map& map, double load_factor, EntryEncode entry_encode);
  template
  persistent_unordered_map(const unordered_map& map, double load_factor, bool add_prefixes, bool add_suffixes, EntryEncode entry_encode);

  // Manual creation functions
  inline void resize(unsigned elems);
  inline void add(const char* str, int str_len, int data_len);
  inline void done_adding();
  inline unsigned char* fill(const char* str, int str_len, int data_len);
  inline void done_filling();

  // Serialization
  inline void load(binary_decoder& data);
  inline void save(binary_encoder& enc);

 private:
  struct fnv_hash;
  vector hashes;

  template
  void construct(const map& map, double load_factor, EntryEncode entry_encode);
};

// Definitions
struct persistent_unordered_map::fnv_hash {
  fnv_hash(unsigned num) {
    mask = 1;
    while (mask < num)
      mask <<= 1;
    hash.resize(mask + 1);
    mask--;
  }
  fnv_hash(binary_decoder& data) {
    uint32_t size = data.next_4B();
    mask = size - 2;
    hash.resize(size);
    memcpy(hash.data(), data.next(size), size * sizeof(uint32_t));

    size = data.next_4B();
    this->data.resize(size);
    if (size) memcpy(this->data.data(), data.next(size), size);
  }

  inline uint32_t index(const char* data, int len) const {
    if (len <= 0) return 0;
    if (len == 1) return unaligned_load(data);
    if (len == 2) return unaligned_load(data);

    uint32_t hash = 2166136261U;
    while (len--)
      hash = (hash ^ unsigned((signed char)*data++)) * 16777619U;
    return hash & mask;
  }

  inline void save(binary_encoder& enc);

  unsigned mask;
  vector hash;
  vector data;
};

template
const unsigned char* persistent_unordered_map::at(const char* str, int len, EntrySize entry_size) const {
  if (unsigned(len) >= hashes.size()) return nullptr;

  unsigned index = hashes[len].index(str, len);
  const unsigned char* data = hashes[len].data.data() + hashes[len].hash[index];
  const unsigned char* end = hashes[len].data.data() + hashes[len].hash[index+1];

  if (len <= 2)
    return data != end ? data + len : nullptr;

  while (data < end) {
    if (small_memeq(str, data, len)) return data + len;
    data += len;
    pointer_decoder decoder(data);
    entry_size(decoder);
  }

  return nullptr;
}

template
const T* persistent_unordered_map::at_typed(const char* str, int len) const {
  if (unsigned(len) >= hashes.size()) return nullptr;

  unsigned index = hashes[len].index(str, len);
  const unsigned char* data = hashes[len].data.data() + hashes[len].hash[index];
  const unsigned char* end = hashes[len].data.data() + hashes[len].hash[index+1];

  if (len <= 2)
    return data != end ? (const T*)(data + len) : nullptr;

  while (data < end) {
    if (small_memeq(str, data, len)) return (const T*)(data + len);
    data += len + sizeof(T);
  }

  return nullptr;
}

template
void persistent_unordered_map::iter(const char* str, int len, EntryProcess entry_process) const {
  if (unsigned(len) >= hashes.size()) return;

  unsigned index = hashes[len].index(str, len);
  const unsigned char* data = hashes[len].data.data() + hashes[len].hash[index];
  const unsigned char* end = hashes[len].data.data() + hashes[len].hash[index+1];

  while (data < end) {
    auto start = (const char*) data;
    data += len;
    pointer_decoder decoder(data);
    entry_process(start, decoder);
  }
}

template
void persistent_unordered_map::iter_all(EntryProcess entry_process) const {
  for (unsigned len = 0; len < hashes.size(); len++) {
    const unsigned char* data = hashes[len].data.data();
    const unsigned char* end = data + hashes[len].data.size();

    while (data < end) {
      auto start = (const char*) data;
      data += len;
      pointer_decoder decoder(data);
      entry_process(start, len, decoder);
    }
  }
}

int persistent_unordered_map::max_length() const {
  return hashes.size();
}

const unsigned char* persistent_unordered_map::data_start(int len) const {
  return unsigned(len) < hashes.size() ? hashes[len].data.data() : nullptr;
}

void persistent_unordered_map::resize(unsigned elems) {
  if (hashes.size() == 0) hashes.emplace_back(1);
  else if (hashes.size() == 1) hashes.emplace_back(1<<8);
  else if (hashes.size() == 2) hashes.emplace_back(1<<16);
  else hashes.emplace_back(elems);
}

void persistent_unordered_map::add(const char* str, int str_len, int data_len) {
  if (unsigned(str_len) < hashes.size())
    hashes[str_len].hash[hashes[str_len].index(str, str_len)] += str_len + data_len;
}

void persistent_unordered_map::done_adding() {
  for (auto&& hash : hashes) {
    int total = 0;
    for (auto&& len : hash.hash) total += len, len = total - len;
    hash.data.resize(total);
  }
}

unsigned char* persistent_unordered_map::fill(const char* str, int str_len, int data_len) {
  if (unsigned(str_len) < hashes.size()) {
    unsigned index = hashes[str_len].index(str, str_len);
    unsigned offset = hashes[str_len].hash[index];
    small_memcpy(hashes[str_len].data.data() + offset, str, str_len);
    hashes[str_len].hash[index] += str_len + data_len;
    return hashes[str_len].data.data() + offset + str_len;
  }
  return nullptr;
}

void persistent_unordered_map::done_filling() {
  for (auto&& hash : hashes)
    for (int i = hash.hash.size() - 1; i >= 0; i--)
      hash.hash[i] = i > 0 ? hash.hash[i-1] : 0;
}

void persistent_unordered_map::load(binary_decoder& data) {
  unsigned sizes = data.next_1B();

  hashes.clear();
  for (unsigned i = 0; i < sizes; i++)
    hashes.emplace_back(data);
}

} // namespace morphodita

/////////
// File: morphodita/derivator/derivator_dictionary.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class derivator_dictionary : public derivator {
 public:
  virtual bool parent(string_piece lemma, derivated_lemma& parent) const override;
  virtual bool children(string_piece lemma, vector& children) const override;

  bool load(istream& is);

 private:
  friend class morpho;
  const morpho* dictionary;
  persistent_unordered_map derinet;
};

} // namespace morphodita

/////////
// File: utils/compressor.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

class binary_decoder;
class binary_encoder;

class compressor {
 public:
  static bool load(istream& is, binary_decoder& data);
  static bool save(ostream& os, const binary_encoder& enc);
};

} // namespace utils

/////////
// File: morphodita/derivator/derivator_dictionary.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool derivator_dictionary::parent(string_piece lemma, derivated_lemma& parent) const {
  if (dictionary) lemma.len = dictionary->lemma_id_len(lemma);

  auto lemma_data = derinet.at(lemma.str, lemma.len, [](pointer_decoder& data) {
    data.next(data.next_1B());
    data.next_4B();
    data.next(data.next_2B());
  });
  if (lemma_data) {
    auto parent_encoded = *(uint32_t*)(lemma_data + 1 + *lemma_data);
    if (parent_encoded) {
      unsigned parent_len = parent_encoded & 0xFF;
      auto parent_data = derinet.data_start(parent_len) + (parent_encoded >> 8);
      parent.lemma.assign((const char*) parent_data, parent_len);
      if (parent_data[parent_len])
        parent.lemma.append((const char*) parent_data + parent_len + 1, parent_data[parent_len]);
      return true;
    }
  }
  parent.lemma.clear();
  return false;
}

bool derivator_dictionary::children(string_piece lemma, vector& children) const {
  if (dictionary) lemma.len = dictionary->lemma_id_len(lemma);

  auto lemma_data = derinet.at(lemma.str, lemma.len, [](pointer_decoder& data) {
    data.next(data.next_1B());
    data.next_4B();
    data.next(data.next_2B());
  });
  if (lemma_data) {
    auto children_len = *(uint16_t*)(lemma_data + 1 + *lemma_data + 4);
    auto children_encoded = (uint32_t*)(lemma_data + 1 + *lemma_data + 4 + 2);
    if (children_len) {
      children.resize(children_len);
      for (unsigned i = 0; i < children_len; i++) {
        unsigned child_len = children_encoded[i] & 0xFF;
        auto child_data = derinet.data_start(child_len) + (children_encoded[i] >> 8);
        children[i].lemma.assign((const char*) child_data, child_len);
        if (child_data[child_len])
          children[i].lemma.append((const char*) child_data + child_len + 1, child_data[child_len]);
      }
      return true;
    }
  }
  children.clear();
  return false;
}

bool derivator_dictionary::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    for (int i = data.next_1B(); i > 0; i--)
      derinet.resize(data.next_4B());

    unsigned data_position = data.tell();
    vector lemma, parent;
    for (int pass = 1; pass <= 3; pass++) {
      if (pass > 1) data.seek(data_position);

      lemma.clear();
      for (int i = data.next_4B(); i > 0; i--) {
        lemma.resize(lemma.size() - data.next_1B());
        for (int i = data.next_1B(); i > 0; i--)
          lemma.push_back(data.next_1B());

        unsigned char lemma_comment_len = data.next_1B();
        const char* lemma_comment = lemma_comment_len ? data.next(lemma_comment_len) : nullptr;

        unsigned children = data.next_2B();

        if (pass == 3) parent.clear();
        enum { REMOVE_START = 1, REMOVE_END = 2, ADD_START = 4, ADD_END = 8 };
        int operations = data.next_1B();
        if (operations) {
          int remove_start = operations & REMOVE_START ? data.next_1B() : 0;
          int remove_end = operations & REMOVE_END ? data.next_1B() : 0;
          if (operations & ADD_START) {
            int add_start = data.next_1B();
            const char* str = data.next(add_start);
            if (pass == 3) parent.assign(str, str + add_start);
          }
          if (pass == 3) parent.insert(parent.end(), lemma.begin() + remove_start, lemma.end() - remove_end);
          if (operations & ADD_END) {
            int add_end = data.next_1B();
            const char* str = data.next(add_end);
            if (pass == 3) parent.insert(parent.end(), str, str + add_end);
          }
        }

        if (pass == 1) {
          derinet.add(lemma.data(), lemma.size(), 1 + lemma_comment_len + 4 + 2 + 4 * children);
        } else if (pass == 2) {
          unsigned char* lemma_data = derinet.fill(lemma.data(), lemma.size(), 1 + lemma_comment_len + 4 + 2 + 4 * children);
          *lemma_data++ = lemma_comment_len;
          while (lemma_comment_len--) *lemma_data++ = *lemma_comment++;
          unaligned_store_inc(lemma_data, 0);
          unaligned_store_inc(lemma_data, children);
          if (children) unaligned_store(((uint32_t*)lemma_data) + children - 1, 0);
        } else if (pass == 3 && !parent.empty()) {
          auto lemma_data = derinet.at(lemma.data(), lemma.size(), [](pointer_decoder& data) {
            data.next(data.next_1B());
            data.next_4B();
            data.next(data.next_2B());
          });
          auto parent_data = derinet.at(parent.data(), parent.size(), [](pointer_decoder& data) {
            data.next(data.next_1B());
            data.next_4B();
            data.next(data.next_2B());
          });
          assert(lemma_data && parent_data);

          unsigned parent_offset = parent_data - parent.size() - derinet.data_start(parent.size());
          assert(parent.size() < (1<<8) && parent_offset < (1<<24));
          unaligned_store((void *)(lemma_data + 1 + *lemma_data), (parent_offset << 8) | parent.size());

          unsigned lemma_offset = lemma_data - lemma.size() - derinet.data_start(lemma.size());
          assert(lemma.size() < (1<<8) && lemma_offset < (1<<24));
          auto children_len = unaligned_load(parent_data + 1 + *parent_data + 4);
          auto children = (uint32_t*)(parent_data + 1 + *parent_data + 4 + 2);
          auto child_index = unaligned_load(children + children_len - 1);
          unaligned_store(children + child_index, (lemma_offset << 8) | lemma.size());
          if (child_index+1 < children_len)
            unaligned_store(children + children_len - 1, unaligned_load(children + children_len - 1) + 1);
        }
      }

      if (pass == 1)
        derinet.done_adding();
      if (pass == 2)
        derinet.done_filling();
    }
  } catch (binary_decoder_error&) {
    return false;
  }
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/casing_variants.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

inline void generate_casing_variants(string_piece form, string& form_uclc, string& form_lc) {
  using namespace unilib;

  // Detect uppercase+titlecase characters.
  bool first_Lut = false; // first character is uppercase or titlecase
  bool rest_has_Lut = false; // any character but first is uppercase or titlecase
  {
    string_piece form_tmp = form;
    first_Lut = unicode::category(utf8::decode(form_tmp.str, form_tmp.len)) & unicode::Lut;
    while (form_tmp.len && !rest_has_Lut)
      rest_has_Lut = unicode::category(utf8::decode(form_tmp.str, form_tmp.len)) & unicode::Lut;
  }

  // Generate all casing variants if needed (they are different than given form).
  // We only replace letters with their lowercase variants.
  // - form_uclc: first uppercase, rest lowercase
  // - form_lc: all lowercase

  if (first_Lut && !rest_has_Lut) { // common case allowing fast execution
    form_lc.reserve(form.len);
    string_piece form_tmp = form;
    utf8::append(form_lc, unicode::lowercase(utf8::decode(form_tmp.str, form_tmp.len)));
    form_lc.append(form_tmp.str, form_tmp.len);
  } else if (!first_Lut && rest_has_Lut) {
    form_lc.reserve(form.len);
    utf8::map(unicode::lowercase, form.str, form.len, form_lc);
  } else if (first_Lut && rest_has_Lut) {
    form_lc.reserve(form.len);
    form_uclc.reserve(form.len);
    string_piece form_tmp = form;
    char32_t first = utf8::decode(form_tmp.str, form_tmp.len);
    utf8::append(form_lc, unicode::lowercase(first));
    utf8::append(form_uclc, first);
    while (form_tmp.len) {
      char32_t lowercase = unicode::lowercase(utf8::decode(form_tmp.str, form_tmp.len));
      utf8::append(form_lc, lowercase);
      utf8::append(form_uclc, lowercase);
    }
  }
}

} // namespace morphodita

/////////
// File: morphodita/morpho/czech_lemma_addinfo.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
struct czech_lemma_addinfo {
  inline static int raw_lemma_len(string_piece lemma);
  inline static int lemma_id_len(string_piece lemma);
  inline static string format(const unsigned char* addinfo, int addinfo_len);
  inline static bool generatable(const unsigned char* addinfo, int addinfo_len);

  inline int parse(string_piece lemma, bool die_on_failure = false);
  inline bool match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len);

  vector data;
};

// Definitions
int czech_lemma_addinfo::raw_lemma_len(string_piece lemma) {
  // Lemma ends by a '-[0-9]', '`' or '_' on non-first position.
  for (unsigned len = 1; len < lemma.len; len++)
    if (lemma.str[len] == '`' || lemma.str[len] == '_' ||
        (lemma.str[len] == '-' && len+1 < lemma.len && lemma.str[len+1] >= '0' && lemma.str[len+1] <= '9'))
      return len;
  return lemma.len;
}

int czech_lemma_addinfo::lemma_id_len(string_piece lemma) {
  // Lemma ends by a '-[0-9]', '`' or '_' on non-first position.
  for (unsigned len = 1; len < lemma.len; len++) {
    if (lemma.str[len] == '`' || lemma.str[len] == '_')
      return len;
    if (lemma.str[len] == '-' && len+1 < lemma.len && lemma.str[len+1] >= '0' && lemma.str[len+1] <= '9') {
      len += 2;
      while (len < lemma.len && lemma.str[len] >= '0' && lemma.str[len] <= '9') len++;
      return len;
    }
  }
  return lemma.len;
}

string czech_lemma_addinfo::format(const unsigned char* addinfo, int addinfo_len) {
  string res;

  if (addinfo_len) {
    res.reserve(addinfo_len + 4);
    if (addinfo[0] != 255) {
      char num[5];
      snprintf(num, sizeof(num), "-%u", addinfo[0]);
      res += num;
    }
    for (int i = 1; i < addinfo_len; i++)
      res += addinfo[i];
  }

  return res;
}

bool czech_lemma_addinfo::generatable(const unsigned char* addinfo, int addinfo_len) {
  for (int i = 1; i + 2 < addinfo_len; i++)
    if (addinfo[i] == '_' && addinfo[i+1] == ',' && addinfo[i+2] == 'x')
      return false;

  return true;
}

int czech_lemma_addinfo::parse(string_piece lemma, bool die_on_failure) {
  data.clear();

  const char* lemma_info = lemma.str + raw_lemma_len(lemma);
  if (lemma_info < lemma.str + lemma.len) {
    int lemma_num = 255;
    const char* lemma_additional_info = lemma_info;

    if (*lemma_info == '-') {
      lemma_num = 0;
      for (lemma_additional_info = lemma_info + 1;
           lemma_additional_info < lemma.str + lemma.len && (*lemma_additional_info >= '0' && *lemma_additional_info <= '9');
           lemma_additional_info++)
        lemma_num = 10 * lemma_num + (*lemma_additional_info - '0');

      if (lemma_additional_info == lemma_info + 1 || (lemma_additional_info < lemma.str + lemma.len && *lemma_additional_info != '`' && *lemma_additional_info != '_') || lemma_num >= 255) {
        if (die_on_failure)
          training_failure("Lemma number " << lemma_num << " in lemma " << lemma << " out of range!");
        else
          lemma_num = 255;
      }
    }
    data.emplace_back(lemma_num);
    while (lemma_additional_info < lemma.str + lemma.len)
      data.push_back(*(unsigned char*)lemma_additional_info++);

    if (data.size() > 255) {
      if (die_on_failure)
        training_failure("Too long lemma info " << lemma_info << " in lemma " << lemma << '!');
      else
        data.resize(255);
    }
  }

  return lemma_info - lemma.str;
}

bool czech_lemma_addinfo::match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len) {
  if (data.empty()) return true;
  if (data[0] != 255 && (!other_addinfo_len || other_addinfo[0] != data[0])) return false;
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/tag_filter.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
class tag_filter {
 public:
  tag_filter(const char* filter = nullptr);

  inline bool matches(const char* tag) const;

 private:
  struct char_filter {
    char_filter(int pos, bool negate, int chars_offset, int chars_len)
        : pos(pos), negate(negate), chars_offset(chars_offset), chars_len(chars_len) {}

    int pos;
    bool negate;
    int chars_offset, chars_len;
  };

  string wildcard;
  std::vector filters;
};

// Definitions
inline bool tag_filter::matches(const char* tag) const {
  if (filters.empty()) return true;

  int tag_pos = 0;
  for (auto&& filter : filters) {
    // Skip until next filter position. If the tag ends prematurely, accept.
    while (tag_pos < filter.pos)
      if (!tag[tag_pos++])
        return true;
    if (!tag[tag_pos])
      return true;

    // We assume filter.chars_len >= 1.
    bool matched = (wildcard[filter.chars_offset] == tag[tag_pos]) ^ filter.negate;
    for (int i = 1; i < filter.chars_len && ((!matched) ^ filter.negate); i++)
      matched = (wildcard[filter.chars_offset + i] == tag[tag_pos]) ^ filter.negate;
    if (!matched) return false;
  }
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_dictionary.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template
class morpho_dictionary {
 public:
  void load(binary_decoder& data);
  void analyze(string_piece form, vector& lemmas) const;
  bool generate(string_piece lemma, const tag_filter& filter, vector& lemmas_forms) const;
 private:
  persistent_unordered_map lemmas, roots, suffixes;

  vector tags;
  vector>>> classes;
};

// Definitions
template
void morpho_dictionary::load(binary_decoder& data) {
  // Prepare lemmas and roots hashes
  for (int i = data.next_1B(); i > 0; i--)
    lemmas.resize(data.next_4B());
  for (int i = data.next_1B(); i > 0; i--)
    roots.resize(data.next_4B());

  // Perform two pass over the lemmas and roots data, filling the hashes.

  vector lemma(max(lemmas.max_length(), roots.max_length()));
  vector root(max(lemmas.max_length(), roots.max_length()));
  unsigned data_position = data.tell();
  for (int pass = 1; pass <= 2; pass++) {
    if (pass > 1) data.seek(data_position);

    int lemma_len = 0;
    int root_len = 0;

    for (int i = data.next_4B(); i > 0; i--) {
      lemma_len -= data.next_1B();
      for (int i = data.next_1B(); i > 0; i--)
        lemma[lemma_len++] = data.next_1B();
      unsigned char lemma_info_len = data.next_1B();
      const char* lemma_info = lemma_info_len ? data.next(lemma_info_len) : nullptr;
      unsigned lemma_roots = data.next_1B();

      unsigned char* lemma_data /* to keep compiler happy */ = nullptr;
      unsigned lemma_offset /* to keep compiler happy */ = 0;

      if (pass == 1) {
        lemmas.add(lemma.data(), lemma_len, 1 + lemma_info_len + 1 + lemma_roots * (sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint16_t)));
      } else /*if (pass == 2)*/ {
        lemma_data = lemmas.fill(lemma.data(), lemma_len, 1 + lemma_info_len + 1 + lemma_roots * (sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint16_t)));
        lemma_offset = lemma_data - lemma_len - lemmas.data_start(lemma_len);

        *lemma_data++ = lemma_info_len;
        if (lemma_info_len) small_memcpy(lemma_data, lemma_info, lemma_info_len), lemma_data += lemma_info_len;
        *lemma_data++ = lemma_roots;
      }

      small_memcpy(root.data(), lemma.data(), lemma_len); root_len = lemma_len;
      for (unsigned i = 0; i < lemma_roots; i++) {
        enum { REMOVE_START = 1, REMOVE_END = 2, ADD_START = 4, ADD_END = 8 };
        int operations = data.next_1B();
        if (operations & REMOVE_START) { int from = data.next_1B(), to = 0; while (from < root_len) root[to++] = root[from++]; root_len = to; }
        if (operations & REMOVE_END) root_len -= data.next_1B();
        if (operations & ADD_START) {
          int from = root_len, to = from + data.next_1B(); while (from > 0) root[--to] = root[--from]; root_len += to;
          for (int i = 0; i < to; i++) root[i] = data.next_1B();
        }
        if (operations & ADD_END)
          for (int len = data.next_1B(); len > 0; len--)
            root[root_len++] = data.next_1B();
        uint16_t clas = data.next_2B();

        if (pass == 1) { // for each root
          roots.add(root.data(), root_len, sizeof(uint16_t) + sizeof(uint32_t) + sizeof(uint8_t));
        } else /*if (pass == 2)*/ {
          unsigned char* root_data = roots.fill(root.data(), root_len, sizeof(uint16_t) + sizeof(uint32_t) + sizeof(uint8_t));
          unsigned root_offset = root_data - root_len - roots.data_start(root_len);

          unaligned_store_inc(root_data, clas);
          unaligned_store_inc(root_data, lemma_offset);
          unaligned_store_inc(root_data, lemma_len);
          assert(uint8_t(lemma_len) == lemma_len);

          unaligned_store_inc(lemma_data, root_offset);
          unaligned_store_inc(lemma_data, root_len);
          unaligned_store_inc(lemma_data, clas);
          assert(uint8_t(root_len) == root_len);
        }
      }
    }

    if (pass == 1) { // after the whole pass
      lemmas.done_adding();
      roots.done_adding();
    } else /*if (pass == 2)*/ {
      lemmas.done_filling();
      roots.done_filling();
    }
  }

  // Load tags
  tags.resize(data.next_2B());
  for (auto&& tag : tags) {
    tag.resize(data.next_1B());
    for (unsigned i = 0; i < tag.size(); i++)
      tag[i] = data.next_1B();
  }

  // Load suffixes
  suffixes.load(data);

  // Fill classes from suffixes
  suffixes.iter_all([this](const char* suffix, int len, pointer_decoder& data) mutable {
    unsigned classes_len = data.next_2B();
    const uint16_t* classes_ptr = data.next(classes_len);
    const uint16_t* indices_ptr = data.next(classes_len + 1);
    uint32_t tags_len = unaligned_load(indices_ptr);
    for (unsigned i = 0; i < classes_len; i++)
      tags_len += uint16_t(unaligned_load(indices_ptr + i + 1) - unaligned_load(indices_ptr + i));
    const uint16_t* tags_ptr = data.next(tags_len);

    string suffix_str(suffix, len);
    uint32_t index = unaligned_load(indices_ptr), prev_index = 0;
    for (unsigned i = 0; i < classes_len; i++) {
      auto classes_ptr_i = unaligned_load(classes_ptr + i);
      if (classes_ptr_i >= classes.size()) classes.resize(classes_ptr_i + 1);
      prev_index = index;
      index += uint16_t(unaligned_load(indices_ptr + i + 1) - unaligned_load(indices_ptr + i));
      classes[classes_ptr_i].emplace_back(suffix_str, vector());
      for (const uint16_t* ptr = tags_ptr + prev_index; ptr < tags_ptr + index; ptr++)
        classes[classes_ptr_i].back().second.emplace_back(unaligned_load(ptr));
    }
  });
}

template
void morpho_dictionary::analyze(string_piece form, vector& lemmas) const {
  int max_suffix_len = suffixes.max_length();

  uint16_t* suff_stack[16]; vector suff_heap;
  uint16_t** suff = max_suffix_len <= 16 ? suff_stack : (suff_heap.resize(max_suffix_len), suff_heap.data());
  int suff_len = 0;
  for (int i = form.len; i >= 0 && suff_len < max_suffix_len; i--, suff_len++) {
    suff[suff_len] = (uint16_t*) suffixes.at(form.str + i, suff_len, [](pointer_decoder& data) {
      data.next(2 * data.next_2B());
      data.next(data.next_2B());
    });
    if (!suff[suff_len]) break;
  }

  for (int root_len = int(form.len) - --suff_len; suff_len >= 0 && root_len < int(roots.max_length()); suff_len--, root_len++)
    if (unaligned_load(suff[suff_len])) {
      unsigned suff_classes = unaligned_load(suff[suff_len]);
      uint16_t* suff_data = suff[suff_len] + 1;

      roots.iter(form.str, root_len, [&](const char* root, pointer_decoder& root_data) {
        uint16_t root_class = root_data.next_2B();
        unsigned lemma_offset = root_data.next_4B();
        unsigned lemma_len = root_data.next_1B();

        if (small_memeq(form.str, root, root_len)) {
          uint16_t* suffix_class_ptr = unaligned_lower_bound(suff_data, suff_classes, root_class);
          if (suffix_class_ptr < suff_data + suff_classes && unaligned_load(suffix_class_ptr) == root_class) {
            const unsigned char* lemma_data = this->lemmas.data_start(lemma_len) + lemma_offset;
            string lemma((const char*)lemma_data, lemma_len);
            if (lemma_data[lemma_len]) lemma += LemmaAddinfo::format(lemma_data + lemma_len + 1, lemma_data[lemma_len]);

            uint16_t* suff_tag_indices = suff_data + suff_classes;
            uint16_t* suff_tags = suff_tag_indices + suff_classes + 1;
            for (unsigned i = unaligned_load(suff_tag_indices + (suffix_class_ptr - suff_data));
                 i < unaligned_load(suff_tag_indices + (suffix_class_ptr - suff_data) + 1); i++)
              lemmas.emplace_back(lemma, tags[unaligned_load(suff_tags + i)]);
          }
        }
      });
    }
}

template
bool morpho_dictionary::generate(string_piece lemma, const tag_filter& filter, vector& lemmas_forms) const {
  LemmaAddinfo addinfo;
  int raw_lemma_len = addinfo.parse(lemma);
  bool matched_lemma = false;

  lemmas.iter(lemma.str, raw_lemma_len, [&](const char* lemma_str, pointer_decoder& data) {
    unsigned lemma_info_len = data.next_1B();
    const auto* lemma_info = data.next(lemma_info_len);
    unsigned lemma_roots_len = data.next_1B();
    auto* lemma_roots_ptr = data.next(lemma_roots_len * (sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint16_t)));

    if (small_memeq(lemma.str, lemma_str, raw_lemma_len) && addinfo.match_lemma_id(lemma_info, lemma_info_len) && LemmaAddinfo::generatable(lemma_info, lemma_info_len)) {
      matched_lemma = true;

      vector* forms = nullptr;
      pointer_decoder lemma_roots(lemma_roots_ptr);
      for (unsigned i = 0; i < lemma_roots_len; i++) {
        unsigned root_offset = lemma_roots.next_4B();
        unsigned root_len = lemma_roots.next_1B();
        unsigned clas = lemma_roots.next_2B();

        const unsigned char* root_data = roots.data_start(root_len) + root_offset;
        for (auto&& suffix : classes[clas]) {
          string root_with_suffix;
          for (auto&& tag : suffix.second)
            if (filter.matches(tags[tag].c_str())) {
              if (!forms) {
                lemmas_forms.emplace_back(string(lemma.str, raw_lemma_len) + LemmaAddinfo::format(lemma_info, lemma_info_len));
                forms = &lemmas_forms.back().forms;
              }

              if (root_with_suffix.empty() && root_len + suffix.first.size()) {
                root_with_suffix.reserve(root_len + suffix.first.size());
                root_with_suffix.assign((const char*)root_data, root_len);
                root_with_suffix.append(suffix.first);
              }

              forms->emplace_back(root_with_suffix, tags[tag]);
            }
        }
      }
    }
  });

  return matched_lemma;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_prefix_guesser.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template
class morpho_prefix_guesser {
 public:
  morpho_prefix_guesser(const MorphoDictionary& dictionary) : dictionary(dictionary) {}

  void load(binary_decoder& data);
  void analyze(string_piece form, vector& lemmas);
  bool generate(string_piece lemma, const tag_filter& filter, vector& lemmas_forms);

 private:
  const MorphoDictionary& dictionary;
  vector tag_filters;
  persistent_unordered_map prefixes_initial, prefixes_middle;
};

// Definitions
template
void morpho_prefix_guesser::load(binary_decoder& data) {
  // Load and construct tag filters
  for (unsigned tag_filters_len = data.next_1B(); tag_filters_len; tag_filters_len--) {
    unsigned tag_filter_len = data.next_1B();
    string tag_filter(data.next(tag_filter_len), tag_filter_len);

    tag_filters.emplace_back(tag_filter.c_str());
  }

  // Load prefixes
  prefixes_initial.load(data);
  prefixes_middle.load(data);
}

// Analyze can return non-unique lemma-tag pairs.
template
void morpho_prefix_guesser::analyze(string_piece form, vector& lemmas) {
  if (!form.len) return;

  vector form_tmp;
  vector middle_masks;
  middle_masks.reserve(form.len);

  for (unsigned initial = 0; initial < form.len; initial++) {
    // Match the initial prefix.
    unsigned initial_mask = (1<
    if (initial) {
      auto found = prefixes_initial.at_typed(form.str, initial);
      if (!found) break;
      initial_mask = unaligned_load(found);
    }

    // If we have found an initial prefix (including the empty one), match middle prefixes.
    if (initial_mask) {
      middle_masks.resize(initial);
      middle_masks.emplace_back(initial_mask);
      for (unsigned middle = initial; middle < middle_masks.size(); middle++) {
        if (!middle_masks[middle]) continue;
        // Try matching middle prefixes from current index.
        for (unsigned i = middle + 1; i < form.len; i++) {
          auto found = prefixes_middle.at_typed(form.str + middle, i - middle);
          if (!found) break;
          if (unaligned_load(found)) {
            if (i + 1 > middle_masks.size()) middle_masks.resize(i + 1);
            middle_masks[i] |= middle_masks[middle] & unaligned_load(found);
          }
        }

        // Try matching word forms if at least one middle prefix was found.
        if (middle > initial && middle < form.len ) {
          if (initial) {
            if (form_tmp.empty()) form_tmp.assign(form.str, form.str + form.len);
            small_memcpy(form_tmp.data() + middle - initial, form.str, initial);
          }
          unsigned lemmas_ori_size = lemmas.size();
          dictionary.analyze(string_piece((initial ? form_tmp.data() : form.str) + middle - initial, form.len - middle + initial), lemmas);
          unsigned lemmas_new_size = lemmas_ori_size;
          for (unsigned i = lemmas_ori_size; i < lemmas.size(); i++) {
            for (unsigned filter = 0; filter < tag_filters.size(); filter++)
              if ((middle_masks[middle] & (1<
                if (i == lemmas_new_size) {
                  lemmas[lemmas_new_size].lemma.insert(0, form.str + initial, middle - initial);
                } else {
                  lemmas[lemmas_new_size].lemma.reserve(lemmas[i].lemma.size() + middle - initial);
                  lemmas[lemmas_new_size].lemma.assign(form.str + initial, middle - initial);
                  lemmas[lemmas_new_size].lemma.append(lemmas[i].lemma);
                  lemmas[lemmas_new_size].tag = lemmas[i].tag;
                }
                lemmas_new_size++;
                break;
              }
          }
          if (lemmas_new_size < lemmas.size()) lemmas.erase(lemmas.begin() + lemmas_new_size, lemmas.end());
        }
      }
    }
  }
}

template
bool morpho_prefix_guesser::generate(string_piece /*lemma*/, const tag_filter& /*filter*/, vector& /*lemmas_forms*/) {
  // Not implemented yet. Is it actually needed?
  return false;
}
} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_statistical_guesser {
 public:
  void load(binary_decoder& data);
  typedef vector used_rules;
  void analyze(string_piece form, vector& lemmas, used_rules* used);

 private:
  vector tags;
  unsigned default_tag;
  persistent_unordered_map rules;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/unicode_tokenizer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class unicode_tokenizer : public tokenizer {
 public:
  enum { URL_EMAIL_LATEST = 2 };
  unicode_tokenizer(unsigned url_email_tokenizer);

  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(vector* forms, vector* tokens) override;

  virtual bool next_sentence(vector& tokens) = 0;

 protected:
  struct char_info {
    char32_t chr;
    unilib::unicode::category_t cat;
    const char* str;

    char_info(char32_t chr, const char* str) : chr(chr), cat(unilib::unicode::category(chr)), str(str) {}
  };
  vector chars;
  size_t current;

  bool tokenize_url_email(vector& tokens);
  bool emergency_sentence_split(const vector& tokens);
  bool is_eos(const vector& tokens, char32_t eos_chr, const unordered_set* abbreviations);

 private:
  unsigned url_email_tokenizer;
  string text_buffer;
  vector tokens_buffer;
  string eos_buffer;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/ragel_tokenizer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class ragel_tokenizer : public unicode_tokenizer {
 public:
  ragel_tokenizer(unsigned url_email_tokenizer);

 protected:
  static inline uint8_t ragel_char(const char_info& chr);

 private:
  static void initialize_ragel_map();
  static vector ragel_map;
  static atomic_flag ragel_map_flag;
  static void ragel_map_add(char32_t chr, uint8_t mapping);

  friend class unicode_tokenizer;
  static bool ragel_url_email(unsigned version, const vector& chars, size_t& current_char, vector& tokens);
};

uint8_t ragel_tokenizer::ragel_char(const char_info& chr) {
  return chr.chr < ragel_map.size() && ragel_map[chr.chr] != 128 ? ragel_map[chr.chr] : 128 + (uint32_t(chr.cat) * uint32_t(0x077CB531U) >> 27);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/czech_tokenizer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class czech_tokenizer : public ragel_tokenizer {
 public:
  enum tokenizer_language { CZECH = 0, SLOVAK = 1 };
  enum { LATEST = 2 };
  czech_tokenizer(tokenizer_language language, unsigned version, const morpho* m = nullptr);

  virtual bool next_sentence(vector& tokens) override;

 private:
  const morpho* m;
  const unordered_set* abbreviations;
  vector lemmas;

  void merge_hyphenated(vector& tokens);

  static const unordered_set abbreviations_czech;
  static const unordered_set abbreviations_slovak;
};

} // namespace morphodita

/////////
// File: morphodita/morpho/czech_morpho.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class czech_morpho : public morpho {
 public:
  using morpho_language = czech_tokenizer::tokenizer_language;

  czech_morpho(morpho_language language, unsigned version) : language(language), version(version) {}

  virtual int analyze(string_piece form, morpho::guesser_mode guesser, vector& lemmas) const override;
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector& forms) const override;
  virtual int raw_lemma_len(string_piece lemma) const override;
  virtual int lemma_id_len(string_piece lemma) const override;
  virtual int raw_form_len(string_piece form) const override;
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);
 private:
  inline void analyze_special(string_piece form, vector& lemmas) const;

  morpho_language language;
  unsigned version;
  morpho_dictionary dictionary;
  unique_ptr> prefix_guesser;
  unique_ptr statistical_guesser;

  string unknown_tag = "X@-------------";
  string number_tag = "C=-------------";
  string punctuation_tag = "Z:-------------";
};

} // namespace morphodita

/////////
// File: morphodita/morpho/czech_morpho.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool czech_morpho::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    // Load tag length
    unsigned tag_length = data.next_1B();
    if (tag_length < unknown_tag.size()) unknown_tag.erase(tag_length);
    if (tag_length < number_tag.size()) number_tag.erase(tag_length);
    if (tag_length < punctuation_tag.size()) punctuation_tag.erase(tag_length);

    // Load dictionary
    dictionary.load(data);

    // Optionally prefix guesser if present
    prefix_guesser.reset();
    if (data.next_1B()) {
      prefix_guesser.reset(new morpho_prefix_guesser(dictionary));
      prefix_guesser->load(data);
    }

    // Optionally statistical guesser if present
    statistical_guesser.reset();
    if (data.next_1B()) {
      statistical_guesser.reset(new morpho_statistical_guesser());
      statistical_guesser->load(data);
    }
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

int czech_morpho::analyze(string_piece form, guesser_mode guesser, vector& lemmas) const {
  lemmas.clear();

  if (form.len) {
    // Generate all casing variants if needed (they are different than given form).
    string form_uclc; // first uppercase, rest lowercase
    string form_lc;   // all lowercase
    generate_casing_variants(form, form_uclc, form_lc);

    // Start by analysing using the dictionary and all casing variants.
    dictionary.analyze(form, lemmas);
    if (!form_uclc.empty()) dictionary.analyze(form_uclc, lemmas);
    if (!form_lc.empty()) dictionary.analyze(form_lc, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // Then call analyze_special to handle numbers and punctuation.
    analyze_special(form, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // For the prefix guesser, use only form_lc.
    if (guesser == GUESSER && prefix_guesser)
      prefix_guesser->analyze(form_lc.empty() ? form : form_lc, lemmas);
    bool prefix_guesser_guesses = !lemmas.empty();

    // For the statistical guesser, use all casing variants.
    if (guesser == GUESSER && statistical_guesser) {
      if (form_uclc.empty() && form_lc.empty())
        statistical_guesser->analyze(form, lemmas, nullptr);
      else {
        morpho_statistical_guesser::used_rules used_rules; used_rules.reserve(3);
        statistical_guesser->analyze(form, lemmas, &used_rules);
        if (!form_uclc.empty()) statistical_guesser->analyze(form_uclc, lemmas, &used_rules);
        if (!form_lc.empty()) statistical_guesser->analyze(form_lc, lemmas, &used_rules);
      }
    }

    // Make sure results are unique lemma-tag pairs. Statistical guesser produces
    // unique lemma-tag pairs, but prefix guesser does not.
    if (prefix_guesser_guesses) {
      sort(lemmas.begin(), lemmas.end(), [](const tagged_lemma& a, const tagged_lemma& b) {
        int lemma_compare = a.lemma.compare(b.lemma);
        return lemma_compare < 0 || (lemma_compare == 0 && a.tag < b.tag);
      });
      auto lemmas_end = unique(lemmas.begin(), lemmas.end(), [](const tagged_lemma& a, const tagged_lemma& b) {
        return a.lemma == b.lemma && a.tag == b.tag;
      });
      if (lemmas_end != lemmas.end()) lemmas.erase(lemmas_end, lemmas.end());
    }

    if (!lemmas.empty()) return GUESSER;
  }

  lemmas.emplace_back(string(form.str, form.len), unknown_tag);
  return -1;
}

int czech_morpho::generate(string_piece lemma, const char* tag_wildcard, morpho::guesser_mode guesser, vector& forms) const {
  forms.clear();

  tag_filter filter(tag_wildcard);

  if (lemma.len) {
    if (dictionary.generate(lemma, filter, forms))
      return NO_GUESSER;

    if (guesser == GUESSER && prefix_guesser)
      if (prefix_guesser->generate(lemma, filter, forms))
        return GUESSER;
  }

  return -1;
}

int czech_morpho::raw_lemma_len(string_piece lemma) const {
  return czech_lemma_addinfo::raw_lemma_len(lemma);
}

int czech_morpho::lemma_id_len(string_piece lemma) const {
  return czech_lemma_addinfo::lemma_id_len(lemma);
}

int czech_morpho::raw_form_len(string_piece form) const {
  return form.len;
}

tokenizer* czech_morpho::new_tokenizer() const {
  return new czech_tokenizer(language, version, this);
}

// What characters are considered punctuation except for the ones in unicode Punctuation category.
static bool punctuation_additional[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1/*$*/,
  0,0,0,0,0,0,1/*+*/,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1/*<*/,1/*=*/,1/*>*/,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,1/*^*/,0,1/*`*/,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1/*|*/,0,1/*~*/,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1/*caron*/};

// What characters of unicode Punctuation category are not considered punctuation.
static bool punctuation_exceptions[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
  0,0,0,0,0,0,0,0,0,1/*paragraph*/};

void czech_morpho::analyze_special(string_piece form, vector& lemmas) const {
  using namespace unilib;

  // Analyzer for numbers and punctuation.
  // Number is anything matching [+-]? is_Pn* ([.,] is_Pn*)? ([Ee] [+-]? is_Pn+)? for at least one is_Pn* nonempty.
  // Punctuation is any form beginning with either unicode punctuation or punctuation_exceptions character.
  // Beware that numbers takes precedence, so - is punctuation, -3 is number, -. is punctuation, -.3 is number.
  if (!form.len) return;

  string_piece form_ori = form;
  char32_t first = utf8::decode(form.str, form.len);

  // Try matching a number.
  char32_t codepoint = first;
  bool any_digit = false;
  if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(form.str, form.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(form.str, form.len);
  if ((codepoint == '.' && form.len) || codepoint == ',') codepoint = utf8::decode(form.str, form.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(form.str, form.len);
  if (any_digit && (codepoint == 'e' || codepoint == 'E')) {
    codepoint = utf8::decode(form.str, form.len);
    if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(form.str, form.len);
    any_digit = false;
    while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(form.str, form.len);
  }

  if (any_digit && !form.len && (!codepoint || codepoint == '.')) {
    lemmas.emplace_back(string(form_ori.str, form_ori.len), number_tag);
  } else if ((first < sizeof(punctuation_additional) && punctuation_additional[first]) ||
             ((unicode::category(first) & unicode::P) && (first >= sizeof(punctuation_exceptions) || !punctuation_exceptions[first])))
    lemmas.emplace_back(string(form_ori.str, form_ori.len), punctuation_tag);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/english_lemma_addinfo.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
struct english_lemma_addinfo {
  inline static int raw_lemma_len(string_piece lemma);
  inline static int lemma_id_len(string_piece lemma);
  inline static string format(const unsigned char* addinfo, int addinfo_len);
  inline static bool generatable(const unsigned char* addinfo, int addinfo_len);

  inline int parse(string_piece lemma, bool die_on_failure = false);
  inline bool match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len);

  vector data;
};

// Definitions
int english_lemma_addinfo::raw_lemma_len(string_piece lemma) {
  // Lemma ends either by
  // - '^' on non-first position followed by nothing or [A-Za-z][-A-Za-z]*
  // - '+' on non-first position followed by nothing
  for (unsigned len = 1; len < lemma.len; len++) {
    if (len + 1 == lemma.len && (lemma.str[len] == '^' || lemma.str[len] == '+'))
      return len;
    if (len + 1 < lemma.len && lemma.str[len] == '^') {
      bool ok = true;
      for (unsigned i = len + 1; ok && i < lemma.len; i++)
        ok &= (lemma.str[i] >= 'A' && lemma.str[i] <= 'Z') ||
            (lemma.str[i] >= 'a' && lemma.str[i] <= 'z') ||
            (i > len + 1 && lemma.str[i] == '-');
      if (ok) return len;
    }
  }
  return lemma.len;
}

int english_lemma_addinfo::lemma_id_len(string_piece lemma) {
  // No lemma comments.
  return lemma.len;
}

string english_lemma_addinfo::format(const unsigned char* addinfo, int addinfo_len) {
  return string((const char*) addinfo, addinfo_len);
}

bool english_lemma_addinfo::generatable(const unsigned char* /*addinfo*/, int /*addinfo_len*/) {
  return true;
}

int english_lemma_addinfo::parse(string_piece lemma, bool /*die_on_failure*/) {
  data.clear();

  size_t len = raw_lemma_len(lemma);
  for (size_t i = len; i < lemma.len; i++)
    data.push_back(lemma.str[i]);

  return len;
}

bool english_lemma_addinfo::match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len) {
  if (data.empty()) return true;
  if (data.size() == 1 && data[0] == '^') return other_addinfo_len > 0 && other_addinfo[0] == '^';
  if (data.size() == 1 && data[0] == '+') return other_addinfo_len == 0;
  return data.size() == size_t(other_addinfo_len) && small_memeq(data.data(), other_addinfo, other_addinfo_len);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/english_morpho_guesser.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class english_morpho_guesser {
 public:
  void load(binary_decoder& data);
  void analyze(string_piece form, string_piece form_lc, vector& lemmas) const;
  bool analyze_proper_names(string_piece form, string_piece form_lc, vector& lemmas) const;

 private:
  inline void add(const string& tag, const string& form, vector& lemmas) const;
  inline void add(const string& tag, const string& tag2, const string& form, vector& lemmas) const;
  inline void add(const string& tag, const string& form, unsigned negation_len, vector& lemmas) const;
  inline void add(const string& tag, const string& tag2, const string& form, unsigned negation_len, vector& lemmas) const;
  void add_NNS(const string& form, unsigned negation_len, vector& lemmas) const;
  void add_NNPS(const string& form, vector& lemmas) const;
  void add_VBG(const string& form, vector& lemmas) const;
  void add_VBD_VBN(const string& form, vector& lemmas) const;
  void add_VBZ(const string& form, vector& lemmas) const;
  void add_JJR_RBR(const string& form, unsigned negation_len, vector& lemmas) const;
  void add_JJS_RBS(const string& form, unsigned negation_len, vector& lemmas) const;

  enum { NEGATION_LEN = 0, TO_FOLLOW = 1, TOTAL = 2 };
  vector exceptions_tags;
  persistent_unordered_map exceptions;
  persistent_unordered_map negations;
  string CD = "CD", FW = "FW", JJ = "JJ", JJR = "JJR", JJS = "JJS",
         NN = "NN", NNP = "NNP", NNPS = "NNPS", NNS = "NNS", RB = "RB",
         RBR = "RBR", RBS = "RBS", SYM = "SYM", VB = "VB", VBD = "VBD",
         VBG = "VBG", VBN = "VBN", VBP = "VBP", VBZ = "VBZ";
};

} // namespace morphodita

/////////
// File: morphodita/morpho/english_morpho.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class english_morpho : public morpho {
 public:
  english_morpho(unsigned version) : version(version) {}

  virtual int analyze(string_piece form, morpho::guesser_mode guesser, vector& lemmas) const override;
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector& forms) const override;
  virtual int raw_lemma_len(string_piece lemma) const override;
  virtual int lemma_id_len(string_piece lemma) const override;
  virtual int raw_form_len(string_piece form) const override;
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);
 private:
  inline void analyze_special(string_piece form, vector& lemmas) const;

  unsigned version;
  morpho_dictionary dictionary;
  english_morpho_guesser morpho_guesser;

  string unknown_tag = "UNK";
  string number_tag = "CD", nnp_tag = "NNP", ls_tag = "LS";
  string open_quotation_tag = "``", close_quotation_tag = "''";
  string open_parenthesis_tag = "(", close_parenthesis_tag = ")";
  string comma_tag = ",", dot_tag = ".", punctuation_tag = ":", hash_tag = "#", dollar_tag = "$";
  string sym_tag = "SYM", jj_tag = "JJ", nn_tag = "NN", nns_tag = "NNS", cc_tag = "CC", pos_tag = "POS", in_tag = "IN";
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/english_tokenizer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class english_tokenizer : public ragel_tokenizer {
 public:
  enum { LATEST = 2 };
  english_tokenizer(unsigned version);

  virtual bool next_sentence(vector& tokens) override;

 private:
  void split_token(vector& tokens);

  static const unordered_set abbreviations;
};

} // namespace morphodita

/////////
// File: morphodita/morpho/english_morpho.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool english_morpho::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    dictionary.load(data);
    morpho_guesser.load(data);
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

int english_morpho::analyze(string_piece form, guesser_mode guesser, vector& lemmas) const {
  lemmas.clear();

  if (form.len) {
    // Generate all casing variants if needed (they are different than given form).
    string form_uclc; // first uppercase, rest lowercase
    string form_lc;   // all lowercase
    generate_casing_variants(form, form_uclc, form_lc);

    // Start by analysing using the dictionary and all casing variants.
    dictionary.analyze(form, lemmas);
    if (!form_uclc.empty()) dictionary.analyze(form_uclc, lemmas);
    if (!form_lc.empty()) dictionary.analyze(form_lc, lemmas);
    if (!lemmas.empty())
      return guesser == NO_GUESSER || !morpho_guesser.analyze_proper_names(form, form_lc.empty() ? form : form_lc, lemmas) ? NO_GUESSER : GUESSER;

    // Then call analyze_special to handle numbers, punctuation and symbols.
    analyze_special(form, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // Use English guesser on form_lc if allowed.
    if (guesser == GUESSER)
      morpho_guesser.analyze(form, form_lc.empty() ? form : form_lc, lemmas);
    if (!lemmas.empty()) return GUESSER;
  }

  lemmas.emplace_back(string(form.str, form.len), unknown_tag);
  return -1;
}

int english_morpho::generate(string_piece lemma, const char* tag_wildcard, morpho::guesser_mode /*guesser*/, vector& forms) const {
  forms.clear();

  tag_filter filter(tag_wildcard);

  if (lemma.len) {
    if (dictionary.generate(lemma, filter, forms))
      return NO_GUESSER;
  }

  return -1;
}

int english_morpho::raw_lemma_len(string_piece lemma) const {
  return english_lemma_addinfo::raw_lemma_len(lemma);
}

int english_morpho::lemma_id_len(string_piece lemma) const {
  return english_lemma_addinfo::lemma_id_len(lemma);
}

int english_morpho::raw_form_len(string_piece form) const {
  return form.len;
}

tokenizer* english_morpho::new_tokenizer() const {
  return new english_tokenizer(version <= 2 ? 1 : 2);
}

void english_morpho::analyze_special(string_piece form, vector& lemmas) const {
  using namespace unilib;

  // Analyzer for numbers and punctuation.
  if (!form.len) return;

  // One-letter punctuation exceptions.
  if (form.len == 1)
    switch(*form.str) {
      case '.':
      case '!':
      case '?': lemmas.emplace_back(string(form.str, form.len), dot_tag); return;
      case ',': lemmas.emplace_back(string(form.str, form.len), comma_tag); return;
      case '#': lemmas.emplace_back(string(form.str, form.len), hash_tag); return;
      case '$': lemmas.emplace_back(string(form.str, form.len), dollar_tag); return;
      case '[': lemmas.emplace_back(string(form.str, form.len), sym_tag); return;
      case ']': lemmas.emplace_back(string(form.str, form.len), sym_tag); return;
      case '%': lemmas.emplace_back(string(form.str, form.len), jj_tag);
                lemmas.emplace_back(string(form.str, form.len), nn_tag); return;
      case '&': lemmas.emplace_back(string(form.str, form.len), cc_tag);
                lemmas.emplace_back(string(form.str, form.len), sym_tag); return;
      case '*': lemmas.emplace_back(string(form.str, form.len), sym_tag);
                lemmas.emplace_back(string(form.str, form.len), nn_tag); return;
      case '@': lemmas.emplace_back(string(form.str, form.len), sym_tag);
                lemmas.emplace_back(string(form.str, form.len), in_tag); return;
      case '\'': lemmas.emplace_back(string(form.str, form.len), close_quotation_tag);
                 lemmas.emplace_back(string(form.str, form.len), pos_tag); return;
    }

  // Try matching a number: [+-]? is_Pn* (, is_Pn{3})? (. is_Pn*)? (s | [Ee] [+-]? is_Pn+)? with at least one digit
  string_piece number = form;
  char32_t codepoint = utf8::decode(number.str, number.len);
  bool any_digit = false;
  if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(number.str, number.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  while (codepoint == ',') {
    string_piece group = number;
    if (unicode::category(utf8::decode(group.str, group.len) & ~unicode::N)) break;
    if (unicode::category(utf8::decode(group.str, group.len) & ~unicode::N)) break;
    if (unicode::category(utf8::decode(group.str, group.len) & ~unicode::N)) break;
    any_digit = true;
    number = group;
    codepoint = utf8::decode(number.str, number.len);
  }
  if (codepoint == '.' && number.len) {
    codepoint = utf8::decode(number.str, number.len);
    while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  }
  if (version >= 2 && any_digit && codepoint == 's' && !number.len) {
    lemmas.emplace_back(string(form.str, form.len), number_tag);
    lemmas.emplace_back(string(form.str, form.len - 1), nns_tag);
    return;
  }
  if (any_digit && (codepoint == 'e' || codepoint == 'E')) {
    codepoint = utf8::decode(number.str, number.len);
    if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(number.str, number.len);
    any_digit = false;
    while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  }
  if (any_digit && !number.len && (!codepoint || codepoint == '.')) {
    lemmas.emplace_back(string(form.str, form.len), number_tag);
    lemmas.emplace_back(string(form.str, form.len), nnp_tag);
    if (form.len == 1 + (codepoint == '.') && *form.str >= '1' && *form.str <= '9')
      lemmas.emplace_back(string(form.str, form.len), ls_tag);
    return;
  }

  // Open quotation, end quotation, open parentheses, end parentheses, symbol, or other
  string_piece punctuation = form;
  bool open_quotation = true, close_quotation = true, open_parenthesis = true, close_parenthesis = true, any_punctuation = true, symbol = true;
  while ((symbol || any_punctuation) && punctuation.len) {
    codepoint = utf8::decode(punctuation.str, punctuation.len);
    if (open_quotation) open_quotation = codepoint == '`' || unicode::category(codepoint) & unicode::Pi;
    if (close_quotation) close_quotation = codepoint == '\'' || codepoint == '"' || unicode::category(codepoint) & unicode::Pf;
    if (open_parenthesis) open_parenthesis = unicode::category(codepoint) & unicode::Ps;
    if (close_parenthesis) close_parenthesis = unicode::category(codepoint) & unicode::Pe;
    if (any_punctuation) any_punctuation = unicode::category(codepoint) & unicode::P;
    if (symbol) symbol = codepoint == '*' || unicode::category(codepoint) & unicode::S;
  }
  if (!punctuation.len && open_quotation) { lemmas.emplace_back(string(form.str, form.len), open_quotation_tag); return; }
  if (!punctuation.len && close_quotation) { lemmas.emplace_back(string(form.str, form.len), close_quotation_tag); return; }
  if (!punctuation.len && open_parenthesis) { lemmas.emplace_back(string(form.str, form.len), open_parenthesis_tag); return; }
  if (!punctuation.len && close_parenthesis) { lemmas.emplace_back(string(form.str, form.len), close_parenthesis_tag); return; }
  if (!punctuation.len && symbol) { lemmas.emplace_back(string(form.str, form.len), sym_tag); return; }
  if (!punctuation.len && any_punctuation) { lemmas.emplace_back(string(form.str, form.len), punctuation_tag); return; }
}

} // namespace morphodita

/////////
// File: morphodita/morpho/english_morpho_guesser.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// This code is a reimplementation of morphologic analyzer Morphium
// by Johanka Spoustova (Treex::Tool::EnglishMorpho::Analysis Perl module)
// and reimplementation of morphologic lemmatizer by Martin Popel
// (Treex::Tool::EnglishMorpho::Lemmatizer Perl module). The latter is based
// on morpha:
//   Minnen, G., J. Carroll and D. Pearce (2001). Applied morphological
//   processing of English, Natural Language Engineering, 7(3). 207-223.
// Morpha has been released under LGPL as a part of RASP system
//   http://ilexir.co.uk/applications/rasp/.

namespace morphodita {

void english_morpho_guesser::load(binary_decoder& data) {
  unsigned tags = data.next_2B();
  exceptions_tags.clear();
  exceptions_tags.reserve(tags);
  while (tags--) {
    unsigned len = data.next_1B();
    exceptions_tags.emplace_back(string(data.next(len), len));
  }

  exceptions.load(data);
  negations.load(data);
}

static const char _tag_guesser_actions[] = {
	0, 1, 0, 1, 1, 1, 2, 1,
	3, 1, 4, 1, 5, 1, 6, 1,
	7, 2, 2, 6, 2, 2, 7, 2,
	4, 6, 2, 4, 7, 2, 5, 6,
	2, 5, 7, 2, 6, 7, 3, 2,
	6, 7, 3, 4, 6, 7, 3, 5,
	6, 7
};

static const unsigned char _tag_guesser_key_offsets[] = {
	0, 19, 26, 34, 42, 50, 58, 66,
	74, 82, 90, 100, 108, 116, 124, 132,
	145, 153, 161, 168, 179, 195, 212, 220,
	228, 236
};

static const char _tag_guesser_trans_keys[] = {
	45, 46, 99, 100, 103, 105, 109, 110,
	114, 115, 116, 118, 120, 48, 57, 65,
	90, 97, 122, 45, 48, 57, 65, 90,
	97, 122, 45, 114, 48, 57, 65, 90,
	97, 122, 45, 111, 48, 57, 65, 90,
	97, 122, 45, 109, 48, 57, 65, 90,
	97, 122, 45, 101, 48, 57, 65, 90,
	97, 122, 45, 115, 48, 57, 65, 90,
	97, 122, 45, 101, 48, 57, 65, 90,
	97, 122, 45, 108, 48, 57, 65, 90,
	97, 122, 45, 115, 48, 57, 65, 90,
	97, 122, 45, 97, 101, 111, 48, 57,
	65, 90, 98, 122, 45, 101, 48, 57,
	65, 90, 97, 122, 45, 108, 48, 57,
	65, 90, 97, 122, 45, 109, 48, 57,
	65, 90, 97, 122, 45, 105, 48, 57,
	65, 90, 97, 122, 45, 97, 101, 105,
	111, 117, 121, 48, 57, 65, 90, 98,
	122, 45, 115, 48, 57, 65, 90, 97,
	122, 45, 101, 48, 57, 65, 90, 97,
	122, 45, 48, 57, 65, 90, 97, 122,
	45, 101, 114, 115, 116, 48, 57, 65,
	90, 97, 122, 45, 46, 105, 109, 118,
	120, 48, 57, 65, 90, 97, 98, 99,
	100, 101, 122, 45, 46, 101, 105, 109,
	118, 120, 48, 57, 65, 90, 97, 98,
	99, 100, 102, 122, 45, 110, 48, 57,
	65, 90, 97, 122, 45, 105, 48, 57,
	65, 90, 97, 122, 45, 101, 48, 57,
	65, 90, 97, 122, 45, 115, 48, 57,
	65, 90, 97, 122, 0
};

static const char _tag_guesser_single_lengths[] = {
	13, 1, 2, 2, 2, 2, 2, 2,
	2, 2, 4, 2, 2, 2, 2, 7,
	2, 2, 1, 5, 6, 7, 2, 2,
	2, 2
};

static const char _tag_guesser_range_lengths[] = {
	3, 3, 3, 3, 3, 3, 3, 3,
	3, 3, 3, 3, 3, 3, 3, 3,
	3, 3, 3, 3, 5, 5, 3, 3,
	3, 3
};

static const unsigned char _tag_guesser_index_offsets[] = {
	0, 17, 22, 28, 34, 40, 46, 52,
	58, 64, 70, 78, 84, 90, 96, 102,
	113, 119, 125, 130, 139, 151, 164, 170,
	176, 182
};

static const char _tag_guesser_indicies[] = {
	1, 2, 5, 6, 7, 5, 5, 8,
	9, 10, 11, 5, 5, 3, 4, 4,
	0, 13, 14, 15, 15, 12, 13, 16,
	14, 15, 15, 12, 13, 17, 14, 15,
	15, 12, 13, 18, 14, 15, 15, 12,
	13, 18, 14, 15, 15, 12, 13, 19,
	14, 15, 15, 12, 13, 20, 14, 15,
	15, 12, 13, 18, 14, 15, 15, 12,
	13, 21, 14, 15, 15, 12, 13, 22,
	23, 24, 14, 15, 15, 12, 13, 25,
	14, 15, 15, 12, 13, 23, 14, 15,
	15, 12, 13, 23, 14, 15, 15, 12,
	13, 26, 14, 15, 15, 12, 28, 15,
	15, 15, 15, 15, 15, 29, 26, 26,
	27, 31, 4, 32, 33, 33, 30, 13,
	23, 14, 15, 15, 12, 13, 14, 15,
	15, 12, 13, 34, 35, 36, 37, 14,
	15, 15, 12, 13, 38, 39, 39, 39,
	39, 14, 15, 15, 39, 15, 12, 13,
	38, 40, 39, 39, 39, 39, 14, 15,
	15, 39, 15, 12, 13, 41, 14, 15,
	15, 12, 13, 42, 14, 15, 15, 12,
	13, 18, 14, 15, 15, 12, 13, 43,
	14, 15, 15, 12, 0
};

static const char _tag_guesser_trans_targs[] = {
	18, 19, 20, 18, 18, 20, 21, 22,
	23, 24, 16, 25, 18, 19, 18, 1,
	3, 4, 18, 7, 8, 10, 11, 18,
	13, 12, 18, 18, 19, 18, 18, 19,
	18, 18, 2, 5, 6, 9, 20, 20,
	18, 14, 15, 17
};

static const char _tag_guesser_trans_actions[] = {
	29, 46, 29, 32, 11, 11, 11, 11,
	11, 11, 0, 11, 13, 35, 15, 0,
	0, 0, 1, 0, 0, 0, 0, 3,
	0, 0, 5, 17, 38, 20, 23, 42,
	26, 9, 0, 0, 0, 0, 13, 0,
	7, 0, 0, 0
};

static const char _tag_guesser_eof_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 15, 15, 0, 0,
	0, 0
};

static const int tag_guesser_start = 0;

void english_morpho_guesser::analyze(string_piece form, string_piece form_lc, vector& lemmas) const {
  // Try exceptions list
  auto* exception = exceptions.at(form_lc.str, form_lc.len, [](pointer_decoder& data){
    for (unsigned len = data.next_1B(); len; len--) {
      data.next(data.next_1B());
      data.next(data.next_1B());
    }
  });

  if (exception) {
    // Found in exceptions list
    pointer_decoder data(exception);
    for (unsigned len = data.next_1B(); len; len--) {
      unsigned lemma_len = data.next_1B();
      string lemma(data.next(lemma_len), lemma_len);
      for (unsigned tags = data.next_1B(); tags; tags--)
        lemmas.emplace_back(lemma, exceptions_tags[data.next_2B()]);
    }
  } else {
    // Try stripping negative prefix and use rule guesser
    string lemma_lc(form_lc.str, form_lc.len);
    // Try finding negative prefix
    unsigned negation_len = 0;
    for (unsigned prefix = 1; prefix <= form_lc.len; prefix++) {
      auto found = negations.at(form_lc.str, prefix, [](pointer_decoder& data){ data.next(TOTAL); });
      if (!found) break;
      if (found[NEGATION_LEN]) {
        if (form_lc.len - prefix >= found[TO_FOLLOW]) negation_len = found[NEGATION_LEN];
      }
    }

    // Add default tags
    add(FW, lemma_lc, lemmas);
    add(JJ, lemma_lc, negation_len, lemmas);
    add(RB, lemma_lc, negation_len, lemmas);
    add(NN, lemma_lc, negation_len, lemmas);
    add_NNS(lemma_lc, negation_len, lemmas);

    // Add specialized tags
    const char* p = form_lc.str; int cs;
    bool added_JJR_RBR = false, added_JJS_RBS = false, added_SYM = false, added_CD = false;

	{
	cs = tag_guesser_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form_lc.str + form_lc.len)) )
		goto _test_eof;
_resume:
	_keys = _tag_guesser_trans_keys + _tag_guesser_key_offsets[cs];
	_trans = _tag_guesser_index_offsets[cs];

	_klen = _tag_guesser_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form_lc.str[form_lc.len - 1 - (p - form_lc.str)]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form_lc.str[form_lc.len - 1 - (p - form_lc.str)]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _tag_guesser_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form_lc.str[form_lc.len - 1 - (p - form_lc.str)]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form_lc.str[form_lc.len - 1 - (p - form_lc.str)]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _tag_guesser_indicies[_trans];
	cs = _tag_guesser_trans_targs[_trans];

	if ( _tag_guesser_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _tag_guesser_actions + _tag_guesser_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (!added_JJR_RBR) added_JJR_RBR = true, add_JJR_RBR(lemma_lc, negation_len, lemmas); }
	break;
	case 1:
	{ if (!added_JJS_RBS) added_JJS_RBS = true, add_JJS_RBS(lemma_lc, negation_len, lemmas); }
	break;
	case 2:
	{ add_VBG(lemma_lc, lemmas); }
	break;
	case 3:
	{ add_VBD_VBN(lemma_lc, lemmas); }
	break;
	case 4:
	{ add_VBZ(lemma_lc, lemmas); }
	break;
	case 5:
	{ add(VB, lemma_lc, lemmas); add(VBP, lemma_lc, lemmas); }
	break;
	case 6:
	{ if (!added_SYM) added_SYM = true, add(SYM, lemma_lc, lemmas); }
	break;
	case 7:
	{ if (!added_CD) added_CD = true, add(CD, lemma_lc, lemmas); }
	break;
		}
	}

_again:
	if ( ++p != ( (form_lc.str + form_lc.len)) )
		goto _resume;
	_test_eof: {}
	if ( p == ( (form_lc.str + form_lc.len)) )
	{
	const char *__acts = _tag_guesser_actions + _tag_guesser_eof_actions[cs];
	unsigned int __nacts = (unsigned int) *__acts++;
	while ( __nacts-- > 0 ) {
		switch ( *__acts++ ) {
	case 7:
	{ if (!added_CD) added_CD = true, add(CD, lemma_lc, lemmas); }
	break;
		}
	}
	}

	}

  }

  // Add proper names
  analyze_proper_names(form, form_lc, lemmas);
}

bool english_morpho_guesser::analyze_proper_names(string_piece form, string_piece form_lc, vector& lemmas) const {
  // NNP if form_lc != form or form.str[0] =~ /[0-9']/, NNPS if form_lc != form
  bool is_NNP = form.str != form_lc.str || (form.len && (*form.str == '\'' || (*form.str >= '0' && *form.str <= '9')));
  bool is_NNPS = form.str != form_lc.str;
  if (!is_NNP && !is_NNPS) return false;

  bool was_NNP = false, was_NNPS = false;
  for (auto&& lemma : lemmas) {
    was_NNP |= lemma.tag == NNP;
    was_NNPS |= lemma.tag == NNPS;
  }
  if (!((is_NNP && !was_NNP) || (is_NNPS && !was_NNPS))) return false;

  string lemma(form.str, form.len);
  if (is_NNP && !was_NNP) add(NNP, lemma, lemmas);
  if (is_NNPS && !was_NNPS) add_NNPS(lemma, lemmas);
  return true;
}

inline void english_morpho_guesser::add(const string& tag, const string& form, vector& lemmas) const {
  lemmas.emplace_back(form, tag);
}

inline void english_morpho_guesser::add(const string& tag, const string& tag2, const string& form, vector& lemmas) const {
  add(tag, form, lemmas);
  add(tag2, form, lemmas);
}

inline void english_morpho_guesser::add(const string& tag, const string& form, unsigned negation_len, vector& lemmas) const {
  lemmas.emplace_back(negation_len ? form.substr(negation_len) + "^" + form.substr(0, negation_len) : form, tag);
}

inline void english_morpho_guesser::add(const string& tag, const string& tag2, const string& form, unsigned negation_len, vector& lemmas) const {
  add(tag, form, negation_len, lemmas);
  add(tag2, form, negation_len, lemmas);
}

// Common definitions (written backwards)
#define REM(str, len) (str.substr(0, str.size() - len))
#define REM_ADD(str, len, add) (str.substr(0, str.size() - len).append(add))

static const char _NNS_actions[] = {
	0, 1, 0, 1, 1, 1, 2, 1,
	3, 1, 4, 1, 5, 1, 6, 1,
	7, 1, 8, 1, 9, 1, 10, 1,
	11, 1, 12, 1, 13
};

static const char _NNS_key_offsets[] = {
	0, 0, 2, 3, 4, 5, 7, 17,
	17, 29, 30, 35, 35, 36, 37, 37,
	37, 44, 45, 53, 63, 72
};

static const char _NNS_trans_keys[] = {
	110, 115, 101, 109, 101, 99, 115, 98,
	100, 102, 104, 106, 110, 112, 116, 118,
	122, 104, 122, 98, 100, 102, 103, 106,
	110, 112, 116, 118, 120, 111, 97, 101,
	105, 111, 117, 105, 119, 104, 105, 111,
	115, 118, 120, 122, 115, 97, 101, 105,
	110, 111, 114, 115, 117, 98, 100, 102,
	104, 106, 110, 112, 116, 118, 122, 97,
	101, 105, 111, 117, 121, 122, 98, 120,
	0
};

static const char _NNS_single_lengths[] = {
	0, 2, 1, 1, 1, 2, 0, 0,
	2, 1, 5, 0, 1, 1, 0, 0,
	7, 1, 8, 0, 7, 0
};

static const char _NNS_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 5, 0,
	5, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 5, 1, 0
};

static const char _NNS_index_offsets[] = {
	0, 0, 3, 5, 7, 9, 12, 18,
	19, 27, 29, 35, 36, 38, 40, 41,
	42, 50, 52, 61, 67, 76
};

static const char _NNS_indicies[] = {
	0, 2, 1, 3, 1, 4, 1, 6,
	5, 7, 7, 1, 8, 8, 8, 8,
	8, 1, 9, 11, 10, 10, 10, 10,
	10, 10, 1, 12, 1, 13, 13, 13,
	13, 13, 1, 14, 15, 1, 16, 1,
	17, 1, 18, 19, 20, 21, 22, 7,
	23, 1, 24, 1, 25, 25, 25, 26,
	25, 27, 28, 29, 1, 30, 30, 30,
	30, 30, 1, 31, 31, 31, 31, 31,
	31, 33, 32, 1, 17, 0
};

static const char _NNS_trans_targs[] = {
	2, 0, 4, 3, 15, 15, 16, 15,
	7, 15, 15, 17, 15, 11, 15, 13,
	15, 15, 5, 6, 8, 18, 12, 20,
	15, 15, 9, 10, 15, 19, 15, 15,
	14, 21
};

static const char _NNS_trans_actions[] = {
	0, 0, 0, 0, 1, 27, 27, 21,
	0, 23, 25, 25, 19, 0, 17, 0,
	5, 11, 0, 0, 0, 21, 0, 21,
	3, 9, 0, 0, 15, 9, 7, 13,
	0, 15
};

static const int NNS_start = 1;

void english_morpho_guesser::add_NNS(const string& form, unsigned negation_len, vector& lemmas) const {
  const char* p = form.c_str() + negation_len; int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = NNS_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _NNS_trans_keys + _NNS_key_offsets[cs];
	_trans = _NNS_index_offsets[cs];

	_klen = _NNS_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _NNS_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _NNS_indicies[_trans];
	cs = _NNS_trans_targs[_trans];

	if ( _NNS_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _NNS_actions + _NNS_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 2, append = "an";    }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 1, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 3, append = "fe";    }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 1, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 1, append = nullptr; }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 2, append = nullptr; }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 1, append = nullptr; }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 1, append = nullptr; }
	break;
	case 10:
	{ if (best > 'k') best = 'k', remove = 2, append = nullptr; }
	break;
	case 11:
	{ if (best > 'l') best = 'l', remove = 3, append = "y";     }
	break;
	case 12:
	{ if (best > 'm') best = 'm', remove = 2, append = nullptr; }
	break;
	case 13:
	{ if (best > 'n') best = 'n', remove = 1, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(NNS, form.substr(0, form.size() - remove).append(append ? append : ""), negation_len, lemmas);
}

static const char _NNPS_actions[] = {
	0, 1, 1, 1, 2, 1, 4, 1,
	5, 1, 6, 1, 7, 1, 8, 1,
	9, 1, 10, 1, 11, 1, 12, 1,
	14, 1, 15, 1, 16, 2, 0, 1,
	2, 3, 4, 2, 13, 14
};

static const unsigned char _NNPS_key_offsets[] = {
	0, 0, 4, 6, 8, 10, 12, 16,
	36, 36, 60, 62, 72, 72, 74, 76,
	78, 78, 98, 98, 100, 102, 104, 104,
	118, 120, 136, 156, 174, 174
};

static const char _NNPS_trans_keys[] = {
	78, 83, 110, 115, 69, 101, 77, 109,
	77, 109, 69, 101, 67, 83, 99, 115,
	66, 68, 70, 72, 74, 78, 80, 84,
	86, 90, 98, 100, 102, 104, 106, 110,
	112, 116, 118, 122, 72, 90, 104, 122,
	66, 68, 70, 71, 74, 78, 80, 84,
	86, 88, 98, 100, 102, 103, 106, 110,
	112, 116, 118, 120, 79, 111, 65, 69,
	73, 79, 85, 97, 101, 105, 111, 117,
	73, 105, 87, 119, 87, 119, 66, 68,
	70, 72, 74, 78, 80, 84, 86, 90,
	98, 100, 102, 104, 106, 110, 112, 116,
	118, 122, 73, 105, 69, 101, 69, 101,
	72, 73, 79, 83, 86, 88, 90, 104,
	105, 111, 115, 118, 120, 122, 83, 115,
	65, 69, 73, 78, 79, 82, 83, 85,
	97, 101, 105, 110, 111, 114, 115, 117,
	66, 68, 70, 72, 74, 78, 80, 84,
	86, 90, 98, 100, 102, 104, 106, 110,
	112, 116, 118, 122, 65, 69, 73, 79,
	85, 89, 90, 97, 101, 105, 111, 117,
	121, 122, 66, 88, 98, 120, 72, 73,
	79, 83, 86, 88, 90, 104, 105, 111,
	115, 118, 120, 122, 0
};

static const char _NNPS_single_lengths[] = {
	0, 4, 2, 2, 2, 2, 4, 0,
	0, 4, 2, 10, 0, 2, 2, 2,
	0, 0, 0, 2, 2, 2, 0, 14,
	2, 16, 0, 14, 0, 14
};

static const char _NNPS_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 10,
	0, 10, 0, 0, 0, 0, 0, 0,
	0, 10, 0, 0, 0, 0, 0, 0,
	0, 0, 10, 2, 0, 0
};

static const unsigned char _NNPS_index_offsets[] = {
	0, 0, 5, 8, 11, 14, 17, 22,
	33, 34, 49, 52, 63, 64, 67, 70,
	73, 74, 85, 86, 89, 92, 95, 96,
	111, 114, 131, 142, 159, 160
};

static const char _NNPS_indicies[] = {
	0, 2, 3, 4, 1, 5, 6, 1,
	7, 8, 1, 8, 8, 1, 10, 11,
	9, 12, 12, 12, 12, 1, 13, 13,
	13, 13, 13, 13, 13, 13, 13, 13,
	1, 14, 16, 15, 16, 15, 15, 15,
	15, 15, 15, 15, 15, 15, 15, 15,
	1, 17, 17, 1, 18, 18, 18, 18,
	18, 18, 18, 18, 18, 18, 1, 19,
	20, 21, 1, 22, 23, 1, 23, 23,
	1, 24, 25, 25, 25, 25, 25, 25,
	25, 25, 25, 25, 1, 26, 21, 21,
	1, 6, 6, 1, 11, 11, 9, 1,
	27, 28, 29, 30, 31, 12, 32, 27,
	33, 29, 30, 34, 12, 32, 1, 35,
	35, 1, 36, 36, 36, 37, 36, 38,
	39, 40, 36, 36, 36, 37, 36, 38,
	39, 40, 1, 41, 41, 41, 41, 41,
	41, 41, 41, 41, 41, 1, 42, 42,
	42, 42, 42, 42, 44, 42, 42, 42,
	42, 42, 42, 44, 43, 43, 1, 24,
	27, 33, 29, 30, 34, 12, 32, 27,
	33, 29, 30, 34, 12, 32, 1, 0
};

static const char _NNPS_trans_targs[] = {
	2, 0, 5, 20, 21, 3, 4, 22,
	22, 22, 23, 29, 22, 8, 22, 22,
	24, 22, 12, 22, 14, 15, 22, 22,
	22, 18, 22, 6, 7, 9, 25, 13,
	27, 17, 19, 22, 22, 10, 11, 22,
	26, 22, 22, 16, 28
};

static const char _NNPS_trans_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 29,
	1, 27, 27, 27, 21, 0, 35, 25,
	25, 19, 0, 17, 0, 0, 32, 5,
	11, 0, 23, 0, 0, 0, 21, 0,
	21, 0, 0, 3, 9, 0, 0, 15,
	9, 7, 13, 0, 15
};

static const int NNPS_start = 1;

void english_morpho_guesser::add_NNPS(const string& form, vector& lemmas) const {
  const char* p = form.c_str(); int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = NNPS_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _NNPS_trans_keys + _NNPS_key_offsets[cs];
	_trans = _NNPS_index_offsets[cs];

	_klen = _NNPS_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _NNPS_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _NNPS_indicies[_trans];
	cs = _NNPS_trans_targs[_trans];

	if ( _NNPS_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _NNPS_actions + _NNPS_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 2, append = "AN";    }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 2, append = "an";    }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 1, append = nullptr; }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 3, append = "FE";    }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 3, append = "fe";    }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 1, append = nullptr; }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 2, append = nullptr; }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 1, append = nullptr; }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 2, append = nullptr; }
	break;
	case 10:
	{ if (best > 'k') best = 'k', remove = 1, append = nullptr; }
	break;
	case 11:
	{ if (best > 'l') best = 'l', remove = 1, append = nullptr; }
	break;
	case 12:
	{ if (best > 'm') best = 'm', remove = 2, append = nullptr; }
	break;
	case 13:
	{ if (best > 'n') best = 'n', remove = 3, append = "Y";     }
	break;
	case 14:
	{ if (best > 'o') best = 'o', remove = 3, append = "y";     }
	break;
	case 15:
	{ if (best > 'p') best = 'p', remove = 2, append = nullptr; }
	break;
	case 16:
	{ if (best > 'q') best = 'q', remove = 1, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(NNPS, form.substr(0, form.size() - remove).append(append ? append : ""), lemmas);
}

static const char _VBG_actions[] = {
	0, 1, 1, 1, 2, 1, 4, 1,
	5, 1, 6, 1, 7, 1, 9, 1,
	10, 1, 11, 1, 12, 1, 13, 1,
	14, 1, 15, 1, 16, 1, 17, 2,
	0, 12, 2, 3, 4, 2, 5, 9,
	2, 5, 10, 2, 8, 9, 2, 9,
	10, 2, 11, 12, 3, 0, 2, 12,
	3, 2, 11, 12
};

static const short _VBG_key_offsets[] = {
	0, 0, 1, 2, 3, 9, 14, 24,
	29, 34, 44, 46, 47, 48, 49, 50,
	51, 52, 59, 66, 68, 70, 71, 72,
	73, 74, 75, 76, 81, 89, 90, 91,
	92, 93, 94, 96, 97, 98, 99, 100,
	101, 102, 127, 127, 136, 137, 142, 153,
	162, 171, 181, 186, 191, 197, 207, 207,
	216, 228, 229, 240, 240, 249, 258, 267,
	276, 285, 290, 302, 313, 318, 324, 334,
	344, 355, 362, 373, 382, 391, 391, 402,
	413, 415, 416, 417, 417, 418, 426, 437,
	442, 448, 458, 468, 479, 486, 497, 504,
	510, 519, 528, 537, 543
};

static const char _VBG_trans_keys[] = {
	103, 110, 105, 97, 101, 105, 111, 117,
	121, 97, 101, 105, 111, 117, 98, 100,
	102, 104, 106, 110, 112, 116, 118, 122,
	97, 101, 105, 111, 117, 97, 101, 105,
	111, 117, 98, 100, 102, 104, 106, 110,
	112, 116, 118, 122, 98, 114, 105, 114,
	112, 105, 109, 101, 97, 101, 105, 111,
	117, 98, 122, 97, 101, 105, 111, 117,
	98, 122, 97, 122, 98, 114, 105, 114,
	112, 105, 109, 101, 97, 101, 105, 111,
	117, 97, 101, 105, 110, 111, 115, 117,
	120, 105, 112, 105, 109, 101, 98, 114,
	105, 114, 112, 105, 109, 101, 98, 99,
	100, 102, 103, 104, 106, 107, 108, 109,
	110, 111, 112, 113, 114, 115, 116, 117,
	118, 119, 120, 121, 122, 97, 105, 97,
	98, 101, 105, 111, 117, 122, 99, 120,
	113, 97, 101, 105, 111, 117, 98, 99,
	100, 105, 111, 117, 122, 97, 101, 102,
	120, 97, 100, 101, 105, 111, 117, 122,
	98, 120, 97, 101, 102, 105, 111, 117,
	122, 98, 120, 97, 101, 103, 105, 110,
	111, 117, 122, 98, 120, 97, 101, 105,
	111, 117, 101, 110, 111, 115, 120, 101,
	110, 111, 112, 115, 120, 97, 101, 104,
	105, 111, 116, 117, 122, 98, 120, 97,
	101, 105, 106, 111, 117, 122, 98, 120,
	98, 99, 100, 105, 107, 111, 117, 122,
	97, 101, 102, 120, 105, 97, 101, 105,
	108, 111, 114, 117, 119, 122, 98, 120,
	97, 101, 105, 109, 111, 117, 122, 98,
	120, 97, 101, 105, 110, 111, 117, 122,
	98, 120, 97, 101, 105, 111, 112, 117,
	122, 98, 120, 97, 101, 105, 111, 113,
	117, 122, 98, 120, 97, 101, 105, 111,
	114, 117, 122, 98, 120, 97, 101, 105,
	111, 117, 98, 99, 100, 105, 108, 111,
	116, 117, 97, 101, 102, 122, 101, 110,
	111, 115, 120, 98, 104, 106, 116, 118,
	122, 101, 110, 111, 115, 120, 101, 110,
	111, 112, 115, 120, 101, 105, 110, 111,
	115, 120, 98, 116, 118, 122, 101, 105,
	110, 111, 115, 120, 98, 116, 118, 122,
	101, 110, 111, 115, 120, 98, 104, 106,
	116, 118, 122, 98, 101, 110, 111, 114,
	115, 120, 101, 110, 111, 115, 120, 98,
	104, 106, 116, 118, 122, 97, 101, 105,
	111, 115, 117, 122, 98, 120, 97, 101,
	105, 111, 116, 117, 122, 98, 120, 122,
	98, 100, 102, 104, 106, 110, 112, 116,
	118, 120, 122, 98, 100, 102, 104, 106,
	110, 112, 116, 118, 120, 98, 114, 112,
	114, 113, 97, 101, 105, 108, 111, 117,
	98, 122, 101, 110, 111, 115, 120, 98,
	104, 106, 116, 118, 122, 101, 110, 111,
	115, 120, 101, 110, 111, 112, 115, 120,
	101, 105, 110, 111, 115, 120, 98, 116,
	118, 122, 101, 105, 110, 111, 115, 120,
	98, 116, 118, 122, 101, 110, 111, 115,
	120, 98, 104, 106, 116, 118, 122, 98,
	101, 110, 111, 114, 115, 120, 101, 110,
	111, 115, 120, 98, 104, 106, 116, 118,
	122, 97, 101, 105, 111, 117, 98, 122,
	97, 101, 105, 111, 117, 121, 97, 101,
	105, 111, 117, 118, 122, 98, 120, 97,
	101, 105, 111, 117, 119, 122, 98, 120,
	97, 101, 105, 111, 117, 120, 122, 98,
	119, 97, 101, 105, 111, 117, 121, 97,
	101, 105, 111, 117, 121, 122, 98, 120,
	0
};

static const char _VBG_single_lengths[] = {
	0, 1, 1, 1, 6, 5, 0, 5,
	5, 0, 2, 1, 1, 1, 1, 1,
	1, 5, 5, 0, 2, 1, 1, 1,
	1, 1, 1, 5, 8, 1, 1, 1,
	1, 1, 2, 1, 1, 1, 1, 1,
	1, 23, 0, 7, 1, 5, 7, 7,
	7, 8, 5, 5, 6, 8, 0, 7,
	8, 1, 9, 0, 7, 7, 7, 7,
	7, 5, 8, 5, 5, 6, 6, 6,
	5, 7, 5, 7, 7, 0, 1, 1,
	2, 1, 1, 0, 1, 6, 5, 5,
	6, 6, 6, 5, 7, 5, 5, 6,
	7, 7, 7, 6, 7
};

static const char _VBG_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 5, 0,
	0, 5, 0, 0, 0, 0, 0, 0,
	0, 1, 1, 1, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 1, 0, 1, 0, 0, 2, 1,
	1, 1, 0, 0, 0, 1, 0, 1,
	2, 0, 1, 0, 1, 1, 1, 1,
	1, 0, 2, 3, 0, 0, 2, 2,
	3, 0, 3, 1, 1, 0, 5, 5,
	0, 0, 0, 0, 0, 1, 3, 0,
	0, 2, 2, 3, 0, 3, 1, 0,
	1, 1, 1, 0, 1
};

static const short _VBG_index_offsets[] = {
	0, 0, 2, 4, 6, 13, 19, 25,
	31, 37, 43, 46, 48, 50, 52, 54,
	56, 58, 65, 72, 74, 77, 79, 81,
	83, 85, 87, 89, 95, 104, 106, 108,
	110, 112, 114, 117, 119, 121, 123, 125,
	127, 129, 154, 155, 164, 166, 172, 182,
	191, 200, 210, 216, 222, 229, 239, 240,
	249, 260, 262, 273, 274, 283, 292, 301,
	310, 319, 325, 336, 345, 351, 358, 367,
	376, 385, 393, 402, 411, 420, 421, 428,
	435, 438, 440, 442, 443, 445, 453, 462,
	468, 475, 484, 493, 502, 510, 519, 526,
	533, 542, 551, 560, 567
};

static const unsigned char _VBG_indicies[] = {
	0, 1, 2, 1, 3, 1, 4, 4,
	4, 4, 4, 4, 1, 5, 5, 5,
	5, 6, 1, 7, 7, 7, 7, 7,
	1, 8, 8, 8, 8, 9, 1, 5,
	5, 5, 5, 10, 1, 11, 11, 11,
	11, 11, 1, 11, 12, 1, 11, 1,
	13, 1, 11, 1, 14, 1, 11, 1,
	11, 1, 5, 5, 5, 5, 6, 15,
	1, 5, 5, 5, 5, 6, 16, 1,
	4, 1, 17, 18, 1, 17, 1, 19,
	1, 17, 1, 20, 1, 17, 1, 17,
	1, 21, 22, 21, 23, 24, 1, 25,
	26, 25, 27, 28, 29, 25, 30, 1,
	31, 1, 31, 1, 32, 1, 31, 1,
	31, 1, 33, 34, 1, 33, 1, 35,
	1, 33, 1, 36, 1, 33, 1, 33,
	1, 38, 39, 40, 41, 42, 43, 44,
	45, 46, 47, 48, 49, 50, 51, 52,
	53, 54, 55, 56, 57, 58, 59, 60,
	37, 1, 1, 61, 62, 61, 61, 61,
	61, 63, 63, 1, 64, 1, 65, 65,
	65, 65, 65, 1, 67, 68, 67, 66,
	66, 66, 67, 66, 67, 1, 69, 62,
	69, 69, 69, 69, 63, 63, 1, 61,
	61, 62, 61, 61, 61, 63, 63, 1,
	66, 66, 68, 66, 70, 66, 66, 67,
	67, 1, 71, 71, 71, 71, 71, 1,
	72, 73, 74, 75, 76, 1, 72, 73,
	74, 11, 75, 76, 1, 61, 61, 62,
	61, 61, 77, 61, 63, 63, 1, 78,
	61, 61, 61, 62, 61, 61, 63, 63,
	1, 63, 79, 63, 61, 62, 61, 61,
	63, 61, 63, 1, 7, 1, 61, 61,
	61, 68, 61, 80, 61, 80, 67, 67,
	1, 5, 61, 61, 61, 62, 61, 61,
	63, 63, 1, 81, 81, 82, 62, 81,
	81, 63, 63, 1, 81, 81, 81, 81,
	62, 81, 63, 63, 1, 61, 61, 61,
	61, 62, 61, 63, 63, 1, 61, 83,
	61, 84, 62, 61, 63, 63, 1, 5,
	5, 5, 5, 6, 1, 85, 86, 85,
	5, 86, 5, 86, 6, 5, 85, 1,
	87, 88, 89, 90, 91, 85, 85, 85,
	1, 87, 92, 89, 93, 94, 1, 87,
	92, 89, 17, 93, 94, 1, 87, 17,
	88, 89, 90, 91, 85, 85, 1, 87,
	20, 88, 89, 90, 91, 85, 85, 1,
	95, 88, 89, 90, 91, 85, 85, 85,
	1, 17, 87, 92, 89, 18, 93, 94,
	1, 87, 97, 89, 98, 99, 96, 96,
	96, 1, 66, 66, 66, 66, 100, 66,
	67, 67, 1, 101, 102, 103, 61, 62,
	61, 63, 63, 1, 104, 106, 106, 106,
	106, 106, 106, 105, 107, 107, 107, 107,
	107, 107, 1, 31, 108, 1, 31, 1,
	109, 1, 105, 110, 104, 5, 5, 5,
	112, 5, 6, 111, 1, 113, 114, 115,
	116, 117, 111, 111, 111, 1, 113, 118,
	115, 119, 120, 1, 113, 118, 115, 33,
	119, 120, 1, 113, 33, 114, 115, 116,
	117, 111, 111, 1, 113, 36, 114, 115,
	116, 117, 111, 111, 1, 121, 114, 115,
	116, 117, 111, 111, 111, 1, 33, 113,
	118, 115, 34, 119, 120, 1, 113, 123,
	115, 124, 125, 122, 122, 122, 1, 5,
	5, 5, 5, 6, 111, 1, 4, 4,
	4, 4, 4, 4, 1, 66, 66, 66,
	66, 66, 68, 67, 67, 1, 81, 81,
	81, 81, 81, 62, 63, 63, 1, 81,
	81, 81, 81, 81, 62, 63, 63, 1,
	126, 126, 126, 126, 126, 4, 1, 127,
	127, 127, 127, 127, 129, 130, 128, 1,
	0
};

static const char _VBG_trans_targs[] = {
	2, 0, 3, 41, 42, 42, 44, 42,
	42, 44, 44, 51, 52, 13, 15, 42,
	42, 68, 69, 23, 25, 77, 78, 83,
	84, 42, 80, 29, 82, 31, 33, 42,
	32, 87, 88, 37, 39, 4, 43, 46,
	47, 48, 49, 53, 55, 56, 58, 60,
	61, 19, 62, 63, 64, 75, 76, 95,
	96, 97, 98, 99, 100, 5, 45, 42,
	42, 6, 7, 42, 45, 8, 50, 9,
	10, 11, 12, 14, 16, 54, 42, 57,
	59, 17, 18, 65, 66, 67, 74, 20,
	70, 22, 71, 72, 21, 24, 26, 73,
	67, 70, 71, 72, 45, 27, 85, 94,
	42, 42, 79, 28, 81, 30, 42, 86,
	93, 34, 89, 36, 90, 91, 35, 38,
	40, 92, 86, 89, 90, 91, 65, 65,
	42, 42, 45
};

static const char _VBG_trans_actions[] = {
	0, 0, 0, 29, 23, 15, 15, 3,
	46, 46, 40, 0, 0, 0, 0, 5,
	34, 0, 0, 0, 0, 15, 15, 15,
	15, 11, 11, 0, 11, 0, 0, 9,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 21,
	0, 0, 0, 23, 0, 0, 19, 19,
	7, 0, 0, 49, 49, 0, 49, 0,
	0, 0, 0, 0, 0, 19, 17, 19,
	49, 0, 0, 27, 27, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	25, 25, 25, 25, 56, 0, 9, 9,
	13, 43, 43, 0, 9, 0, 37, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 7, 7, 7, 7, 23, 1,
	31, 1, 52
};

static const char _VBG_eof_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 3, 0, 0, 3, 3,
	3, 3, 0, 3, 3, 3, 0, 3,
	3, 0, 3, 0, 3, 3, 3, 3,
	3, 0, 0, 25, 25, 25, 25, 25,
	25, 25, 25, 3, 3, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 7, 7,
	7, 7, 7, 7, 7, 7, 0, 0,
	3, 3, 3, 0, 3
};

static const int VBG_start = 1;

void english_morpho_guesser::add_VBG(const string& form, vector& lemmas) const {
  const char* p = form.c_str(); int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = VBG_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _VBG_trans_keys + _VBG_key_offsets[cs];
	_trans = _VBG_index_offsets[cs];

	_klen = _VBG_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _VBG_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _VBG_indicies[_trans];
	cs = _VBG_trans_targs[_trans];

	if ( _VBG_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _VBG_actions + _VBG_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 3, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 3, append = "e";     }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 3, append = nullptr; }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 3, append = "e";     }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 3, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 3, append = "e";     }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 3, append = nullptr; }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 3, append = "e";     }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 3, append = nullptr; }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 3, append = "e";     }
	break;
	case 10:
	{ if (best > 'k') best = 'k', remove = 3, append = nullptr; }
	break;
	case 11:
	{ if (best > 'l') best = 'l', remove = 3, append = "e";     }
	break;
	case 12:
	{ if (best > 'm') best = 'm', remove = 3, append = nullptr; }
	break;
	case 13:
	{ if (best > 'n') best = 'n', remove = 3, append = "e";     }
	break;
	case 14:
	{ if (best > 'o') best = 'o', remove = 3, append = nullptr; }
	break;
	case 15:
	{ if (best > 'p') best = 'p', remove = 3, append = "e";     }
	break;
	case 16:
	{ if (best > 'q') best = 'q', remove = 3, append = nullptr; }
	break;
	case 17:
	{ if (best > 'r') best = 'r', remove = 3, append = "e";     }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	if ( p == ( (form.c_str() + form.size())) )
	{
	const char *__acts = _VBG_actions + _VBG_eof_actions[cs];
	unsigned int __nacts = (unsigned int) *__acts++;
	while ( __nacts-- > 0 ) {
		switch ( *__acts++ ) {
	case 2:
	{ if (best > 'c') best = 'c', remove = 3, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 3, append = "e";     }
	break;
	case 15:
	{ if (best > 'p') best = 'p', remove = 3, append = "e";     }
	break;
		}
	}
	}

	_out: {}
	}

  add(VBG, form.substr(0, form.size() - remove).append(append ? append : ""), lemmas);
}

static const char _VBD_VBN_actions[] = {
	0, 1, 0, 1, 2, 1, 3, 1,
	4, 1, 5, 1, 6, 1, 7, 1,
	8, 1, 9, 1, 10, 1, 11, 1,
	13, 1, 14, 1, 15, 1, 16, 1,
	17, 2, 1, 16, 2, 4, 5, 2,
	8, 16, 2, 9, 13, 2, 9, 14,
	2, 12, 13, 2, 13, 14, 2, 15,
	16, 3, 1, 3, 16, 3, 3, 15,
	16
};

static const short _VBD_VBN_key_offsets[] = {
	0, 0, 2, 3, 9, 14, 24, 29,
	34, 44, 46, 47, 48, 49, 50, 51,
	52, 60, 67, 74, 76, 77, 78, 79,
	80, 81, 82, 87, 95, 96, 97, 98,
	99, 100, 102, 103, 104, 105, 106, 107,
	108, 114, 115, 140, 140, 149, 150, 155,
	166, 175, 184, 194, 199, 204, 210, 220,
	220, 229, 241, 242, 253, 253, 262, 271,
	280, 289, 298, 303, 316, 327, 332, 338,
	348, 358, 369, 376, 387, 396, 405, 405,
	416, 427, 429, 430, 431, 431, 432, 440,
	451, 456, 462, 472, 482, 493, 500, 511,
	518, 524, 533, 542, 551
};

static const char _VBD_VBN_trans_keys[] = {
	100, 110, 101, 97, 101, 105, 111, 117,
	121, 97, 101, 105, 111, 117, 98, 100,
	102, 104, 106, 110, 112, 116, 118, 122,
	97, 101, 105, 111, 117, 97, 101, 105,
	111, 117, 98, 100, 102, 104, 106, 110,
	112, 116, 118, 122, 98, 114, 105, 114,
	112, 105, 109, 101, 97, 101, 105, 111,
	117, 121, 98, 122, 97, 101, 105, 111,
	117, 98, 122, 97, 101, 105, 111, 117,
	98, 122, 98, 114, 105, 114, 112, 105,
	109, 101, 97, 101, 105, 111, 117, 97,
	101, 105, 110, 111, 115, 117, 120, 105,
	112, 105, 109, 101, 98, 114, 105, 114,
	112, 105, 109, 101, 97, 101, 105, 111,
	117, 121, 101, 98, 99, 100, 102, 103,
	104, 105, 106, 107, 108, 109, 110, 112,
	113, 114, 115, 116, 117, 118, 119, 120,
	121, 122, 97, 111, 97, 98, 101, 105,
	111, 117, 122, 99, 120, 113, 97, 101,
	105, 111, 117, 98, 99, 100, 105, 111,
	117, 122, 97, 101, 102, 120, 97, 100,
	101, 105, 111, 117, 122, 98, 120, 97,
	101, 102, 105, 111, 117, 122, 98, 120,
	97, 101, 103, 105, 110, 111, 117, 122,
	98, 120, 97, 101, 105, 111, 117, 101,
	110, 111, 115, 120, 101, 110, 111, 112,
	115, 120, 97, 101, 104, 105, 111, 116,
	117, 122, 98, 120, 97, 101, 105, 106,
	111, 117, 122, 98, 120, 98, 99, 100,
	105, 107, 111, 117, 122, 97, 101, 102,
	120, 105, 97, 101, 105, 108, 111, 114,
	117, 119, 122, 98, 120, 97, 101, 105,
	109, 111, 117, 122, 98, 120, 97, 101,
	105, 110, 111, 117, 122, 98, 120, 97,
	101, 105, 111, 112, 117, 122, 98, 120,
	97, 101, 105, 111, 113, 117, 122, 98,
	120, 97, 101, 105, 111, 114, 117, 122,
	98, 120, 97, 101, 105, 111, 117, 98,
	99, 100, 105, 108, 110, 111, 116, 117,
	97, 101, 102, 122, 101, 110, 111, 115,
	120, 98, 104, 106, 116, 118, 122, 101,
	110, 111, 115, 120, 101, 110, 111, 112,
	115, 120, 101, 105, 110, 111, 115, 120,
	98, 116, 118, 122, 101, 105, 110, 111,
	115, 120, 98, 116, 118, 122, 101, 110,
	111, 115, 120, 98, 104, 106, 116, 118,
	122, 98, 101, 110, 111, 114, 115, 120,
	101, 110, 111, 115, 120, 98, 104, 106,
	116, 118, 122, 97, 101, 105, 111, 115,
	117, 122, 98, 120, 97, 101, 105, 111,
	116, 117, 122, 98, 120, 122, 98, 100,
	102, 104, 106, 110, 112, 116, 118, 120,
	122, 98, 100, 102, 104, 106, 110, 112,
	116, 118, 120, 98, 114, 112, 114, 113,
	97, 101, 105, 108, 111, 117, 98, 122,
	101, 110, 111, 115, 120, 98, 104, 106,
	116, 118, 122, 101, 110, 111, 115, 120,
	101, 110, 111, 112, 115, 120, 101, 105,
	110, 111, 115, 120, 98, 116, 118, 122,
	101, 105, 110, 111, 115, 120, 98, 116,
	118, 122, 101, 110, 111, 115, 120, 98,
	104, 106, 116, 118, 122, 98, 101, 110,
	111, 114, 115, 120, 101, 110, 111, 115,
	120, 98, 104, 106, 116, 118, 122, 97,
	101, 105, 111, 117, 98, 122, 97, 101,
	105, 111, 117, 121, 97, 101, 105, 111,
	117, 118, 122, 98, 120, 97, 101, 105,
	111, 117, 119, 122, 98, 120, 97, 101,
	105, 111, 117, 120, 122, 98, 119, 97,
	101, 105, 111, 117, 121, 122, 98, 120,
	0
};

static const char _VBD_VBN_single_lengths[] = {
	0, 2, 1, 6, 5, 0, 5, 5,
	0, 2, 1, 1, 1, 1, 1, 1,
	6, 5, 5, 2, 1, 1, 1, 1,
	1, 1, 5, 8, 1, 1, 1, 1,
	1, 2, 1, 1, 1, 1, 1, 1,
	6, 1, 23, 0, 7, 1, 5, 7,
	7, 7, 8, 5, 5, 6, 8, 0,
	7, 8, 1, 9, 0, 7, 7, 7,
	7, 7, 5, 9, 5, 5, 6, 6,
	6, 5, 7, 5, 7, 7, 0, 1,
	1, 2, 1, 1, 0, 1, 6, 5,
	5, 6, 6, 6, 5, 7, 5, 5,
	6, 7, 7, 7, 7
};

static const char _VBD_VBN_range_lengths[] = {
	0, 0, 0, 0, 0, 5, 0, 0,
	5, 0, 0, 0, 0, 0, 0, 0,
	1, 1, 1, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 1, 0, 1, 0, 0, 2,
	1, 1, 1, 0, 0, 0, 1, 0,
	1, 2, 0, 1, 0, 1, 1, 1,
	1, 1, 0, 2, 3, 0, 0, 2,
	2, 3, 0, 3, 1, 1, 0, 5,
	5, 0, 0, 0, 0, 0, 1, 3,
	0, 0, 2, 2, 3, 0, 3, 1,
	0, 1, 1, 1, 1
};

static const short _VBD_VBN_index_offsets[] = {
	0, 0, 3, 5, 12, 18, 24, 30,
	36, 42, 45, 47, 49, 51, 53, 55,
	57, 65, 72, 79, 82, 84, 86, 88,
	90, 92, 94, 100, 109, 111, 113, 115,
	117, 119, 122, 124, 126, 128, 130, 132,
	134, 141, 143, 168, 169, 178, 180, 186,
	196, 205, 214, 224, 230, 236, 243, 253,
	254, 263, 274, 276, 287, 288, 297, 306,
	315, 324, 333, 339, 351, 360, 366, 373,
	382, 391, 400, 408, 417, 426, 435, 436,
	443, 450, 453, 455, 457, 458, 460, 468,
	477, 483, 490, 499, 508, 517, 525, 534,
	541, 548, 557, 566, 575
};

static const unsigned char _VBD_VBN_indicies[] = {
	0, 2, 1, 3, 1, 4, 4, 4,
	4, 4, 4, 1, 5, 5, 5, 5,
	6, 1, 7, 7, 7, 7, 7, 1,
	8, 8, 8, 8, 9, 1, 5, 5,
	5, 5, 10, 1, 11, 11, 11, 11,
	11, 1, 11, 12, 1, 11, 1, 13,
	1, 11, 1, 14, 1, 11, 1, 11,
	1, 4, 4, 4, 4, 4, 16, 15,
	1, 5, 5, 5, 5, 6, 17, 1,
	5, 5, 5, 5, 6, 18, 1, 19,
	20, 1, 19, 1, 21, 1, 19, 1,
	22, 1, 19, 1, 19, 1, 23, 24,
	23, 25, 26, 1, 27, 28, 27, 29,
	30, 31, 27, 32, 1, 33, 1, 33,
	1, 34, 1, 33, 1, 33, 1, 35,
	36, 1, 35, 1, 37, 1, 35, 1,
	38, 1, 35, 1, 35, 1, 39, 39,
	39, 39, 39, 4, 1, 40, 1, 42,
	43, 44, 45, 46, 47, 48, 49, 50,
	51, 52, 53, 54, 55, 56, 57, 58,
	59, 60, 61, 62, 63, 64, 41, 1,
	1, 65, 66, 65, 65, 65, 65, 4,
	4, 1, 67, 1, 68, 68, 68, 68,
	68, 1, 70, 71, 70, 69, 69, 69,
	70, 69, 70, 1, 72, 66, 72, 72,
	72, 72, 4, 4, 1, 65, 65, 66,
	65, 65, 65, 4, 4, 1, 69, 69,
	71, 69, 73, 69, 69, 70, 70, 1,
	74, 74, 74, 74, 74, 1, 75, 76,
	77, 78, 79, 1, 75, 76, 77, 11,
	78, 79, 1, 65, 65, 66, 65, 65,
	80, 65, 4, 4, 1, 81, 65, 65,
	65, 66, 65, 65, 4, 4, 1, 4,
	82, 4, 65, 66, 65, 65, 4, 65,
	4, 1, 7, 1, 65, 65, 65, 71,
	65, 83, 65, 83, 70, 70, 1, 5,
	65, 65, 65, 66, 65, 65, 4, 4,
	1, 84, 84, 85, 66, 84, 84, 4,
	4, 1, 84, 84, 84, 84, 66, 84,
	4, 4, 1, 65, 65, 65, 65, 66,
	65, 4, 4, 1, 65, 86, 65, 87,
	66, 65, 4, 4, 1, 5, 5, 5,
	5, 6, 1, 88, 89, 88, 5, 89,
	89, 5, 89, 6, 5, 88, 1, 90,
	91, 92, 93, 94, 88, 88, 88, 1,
	90, 95, 92, 96, 97, 1, 90, 95,
	92, 19, 96, 97, 1, 90, 19, 91,
	92, 93, 94, 88, 88, 1, 90, 22,
	91, 92, 93, 94, 88, 88, 1, 98,
	91, 92, 93, 94, 88, 88, 88, 1,
	19, 90, 95, 92, 20, 96, 97, 1,
	90, 100, 92, 101, 102, 99, 99, 99,
	1, 69, 69, 69, 69, 103, 69, 70,
	70, 1, 104, 105, 106, 65, 66, 65,
	4, 4, 1, 107, 109, 109, 109, 109,
	109, 109, 108, 110, 110, 110, 110, 110,
	110, 1, 33, 111, 1, 33, 1, 112,
	1, 108, 113, 107, 5, 5, 5, 115,
	5, 6, 114, 1, 116, 117, 118, 119,
	120, 114, 114, 114, 1, 116, 121, 118,
	122, 123, 1, 116, 121, 118, 35, 122,
	123, 1, 116, 35, 117, 118, 119, 120,
	114, 114, 1, 116, 38, 117, 118, 119,
	120, 114, 114, 1, 124, 117, 118, 119,
	120, 114, 114, 114, 1, 35, 116, 121,
	118, 36, 122, 123, 1, 116, 126, 118,
	127, 128, 125, 125, 125, 1, 5, 5,
	5, 5, 6, 114, 1, 4, 4, 4,
	4, 4, 4, 1, 69, 69, 69, 69,
	69, 71, 70, 70, 1, 84, 84, 84,
	84, 84, 66, 4, 4, 1, 84, 84,
	84, 84, 84, 66, 4, 4, 1, 129,
	129, 129, 129, 129, 131, 132, 130, 1,
	0
};

static const char _VBD_VBN_trans_targs[] = {
	2, 0, 41, 42, 43, 43, 45, 43,
	43, 45, 45, 52, 53, 12, 14, 43,
	43, 43, 43, 69, 70, 22, 24, 78,
	79, 84, 85, 43, 81, 28, 83, 30,
	32, 43, 31, 88, 89, 36, 38, 66,
	43, 3, 44, 47, 48, 49, 50, 54,
	16, 56, 57, 59, 61, 62, 63, 64,
	65, 76, 77, 96, 97, 98, 99, 40,
	100, 4, 46, 43, 5, 6, 43, 46,
	7, 51, 8, 9, 10, 11, 13, 15,
	55, 43, 58, 60, 17, 18, 66, 67,
	68, 75, 19, 71, 21, 72, 73, 20,
	23, 25, 74, 68, 71, 72, 73, 46,
	26, 86, 95, 43, 43, 80, 27, 82,
	29, 43, 87, 94, 33, 90, 35, 91,
	92, 34, 37, 39, 93, 87, 90, 91,
	92, 66, 43, 43, 46
};

static const char _VBD_VBN_trans_actions[] = {
	0, 0, 0, 31, 29, 25, 25, 5,
	51, 51, 45, 0, 0, 0, 0, 15,
	39, 9, 36, 0, 0, 0, 0, 25,
	25, 25, 25, 21, 21, 0, 21, 0,
	0, 19, 0, 0, 0, 0, 0, 29,
	1, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 27, 0, 0, 0, 0,
	0, 0, 29, 17, 0, 0, 54, 54,
	0, 54, 0, 0, 0, 0, 0, 0,
	29, 27, 29, 54, 0, 0, 13, 13,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 7, 7, 7, 7, 61,
	0, 19, 19, 23, 48, 48, 0, 19,
	0, 42, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 17, 17, 17,
	17, 3, 33, 3, 57
};

static const char _VBD_VBN_eof_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 5, 0, 0, 5,
	5, 5, 5, 0, 5, 5, 5, 0,
	5, 5, 0, 5, 0, 5, 5, 5,
	5, 5, 0, 0, 11, 11, 11, 11,
	11, 11, 11, 11, 5, 5, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 17,
	17, 17, 17, 17, 17, 17, 17, 0,
	0, 5, 5, 5, 5
};

static const int VBD_VBN_start = 1;

void english_morpho_guesser::add_VBD_VBN(const string& form, vector& lemmas) const {
  const char* p = form.c_str(); int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = VBD_VBN_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _VBD_VBN_trans_keys + _VBD_VBN_key_offsets[cs];
	_trans = _VBD_VBN_index_offsets[cs];

	_klen = _VBD_VBN_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _VBD_VBN_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _VBD_VBN_indicies[_trans];
	cs = _VBD_VBN_trans_targs[_trans];

	if ( _VBD_VBN_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _VBD_VBN_actions + _VBD_VBN_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 1, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 2, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 1, append = nullptr; }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 1, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 2, append = nullptr; }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 3, append = "y";     }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 1, append = nullptr; }
	break;
	case 10:
	{ if (best > 'k') best = 'k', remove = 2, append = nullptr; }
	break;
	case 11:
	{ if (best > 'l') best = 'l', remove = 1, append = nullptr; }
	break;
	case 12:
	{ if (best > 'm') best = 'm', remove = 2, append = nullptr; }
	break;
	case 13:
	{ if (best > 'n') best = 'n', remove = 1, append = nullptr; }
	break;
	case 14:
	{ if (best > 'o') best = 'o', remove = 2, append = nullptr; }
	break;
	case 15:
	{ if (best > 'p') best = 'p', remove = 1, append = nullptr; }
	break;
	case 16:
	{ if (best > 'q') best = 'q', remove = 2, append = nullptr; }
	break;
	case 17:
	{ if (best > 'r') best = 'r', remove = 1, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	if ( p == ( (form.c_str() + form.size())) )
	{
	const char *__acts = _VBD_VBN_actions + _VBD_VBN_eof_actions[cs];
	unsigned int __nacts = (unsigned int) *__acts++;
	while ( __nacts-- > 0 ) {
		switch ( *__acts++ ) {
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 1, append = nullptr; }
	break;
	case 9:
	{ if (best > 'j') best = 'j', remove = 1, append = nullptr; }
	break;
		}
	}
	}

	_out: {}
	}

  add(VBD, VBN, form.substr(0, form.size() - remove).append(append ? append : ""), lemmas);
}

static const char _VBZ_actions[] = {
	0, 1, 0, 1, 1, 1, 2, 1,
	3, 1, 4, 1, 5, 1, 6, 1,
	7, 1, 8
};

static const char _VBZ_key_offsets[] = {
	0, 0, 1, 2, 4, 14, 14, 25,
	26, 31, 31, 31, 31, 37, 45, 54
};

static const char _VBZ_trans_keys[] = {
	115, 101, 99, 115, 98, 100, 102, 104,
	106, 110, 112, 116, 118, 122, 122, 98,
	100, 102, 104, 106, 110, 112, 116, 118,
	120, 111, 97, 101, 105, 111, 117, 104,
	105, 111, 115, 120, 122, 97, 101, 105,
	110, 111, 114, 115, 117, 97, 101, 105,
	111, 117, 121, 122, 98, 120, 0
};

static const char _VBZ_single_lengths[] = {
	0, 1, 1, 2, 0, 0, 1, 1,
	5, 0, 0, 0, 6, 8, 7, 0
};

static const char _VBZ_range_lengths[] = {
	0, 0, 0, 0, 5, 0, 5, 0,
	0, 0, 0, 0, 0, 0, 1, 0
};

static const char _VBZ_index_offsets[] = {
	0, 0, 2, 4, 7, 13, 14, 21,
	23, 29, 30, 31, 32, 39, 48, 57
};

static const char _VBZ_indicies[] = {
	0, 1, 3, 2, 4, 4, 1, 5,
	5, 5, 5, 5, 1, 6, 7, 7,
	7, 7, 7, 7, 1, 8, 1, 9,
	9, 9, 9, 9, 1, 8, 10, 1,
	11, 12, 13, 14, 4, 15, 1, 16,
	16, 16, 17, 16, 18, 19, 16, 1,
	20, 20, 20, 20, 20, 20, 22, 21,
	1, 10, 0
};

static const char _VBZ_trans_targs[] = {
	2, 0, 11, 12, 11, 5, 11, 11,
	11, 9, 11, 3, 4, 6, 13, 14,
	11, 7, 8, 11, 11, 10, 15
};

static const char _VBZ_trans_actions[] = {
	0, 0, 17, 17, 11, 0, 13, 15,
	9, 0, 3, 0, 0, 0, 11, 11,
	1, 0, 0, 7, 5, 0, 7
};

static const int VBZ_start = 1;

void english_morpho_guesser::add_VBZ(const string& form, vector& lemmas) const {
  const char* p = form.c_str(); int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = VBZ_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _VBZ_trans_keys + _VBZ_key_offsets[cs];
	_trans = _VBZ_index_offsets[cs];

	_klen = _VBZ_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _VBZ_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str())]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str())]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _VBZ_indicies[_trans];
	cs = _VBZ_trans_targs[_trans];

	if ( _VBZ_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _VBZ_actions + _VBZ_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 1, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 2, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 1, append = nullptr; }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 1, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
	case 6:
	{ if (best > 'g') best = 'g', remove = 3, append = "y";     }
	break;
	case 7:
	{ if (best > 'h') best = 'h', remove = 2, append = nullptr; }
	break;
	case 8:
	{ if (best > 'i') best = 'i', remove = 1, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(VBZ, form.substr(0, form.size() - remove).append(append ? append : ""), lemmas);
}

static const char _JJR_RBR_actions[] = {
	0, 1, 0, 1, 1, 1, 3, 1,
	4, 1, 5, 2, 1, 4, 2, 2,
	5, 2, 4, 5
};

static const unsigned char _JJR_RBR_key_offsets[] = {
	0, 0, 1, 2, 26, 26, 32, 37,
	50, 56, 62, 73, 79, 85, 91, 102,
	103, 109, 115, 117, 123, 129, 135, 146,
	152, 163, 169, 175, 181
};

static const char _JJR_RBR_trans_keys[] = {
	114, 101, 98, 99, 100, 101, 102, 103,
	104, 105, 106, 107, 108, 109, 110, 112,
	113, 114, 115, 116, 117, 118, 119, 120,
	121, 122, 97, 98, 101, 105, 111, 117,
	97, 101, 105, 111, 117, 98, 99, 100,
	105, 111, 117, 122, 97, 101, 102, 109,
	112, 120, 97, 100, 101, 105, 111, 117,
	97, 101, 102, 105, 111, 117, 97, 101,
	103, 105, 111, 117, 122, 98, 109, 112,
	120, 97, 101, 104, 105, 111, 117, 97,
	101, 105, 106, 111, 117, 97, 101, 105,
	107, 111, 117, 97, 101, 105, 108, 111,
	117, 122, 98, 109, 112, 120, 101, 97,
	101, 105, 109, 111, 117, 97, 101, 105,
	110, 111, 117, 97, 122, 97, 101, 105,
	111, 112, 117, 97, 101, 105, 111, 113,
	117, 97, 101, 105, 111, 114, 117, 97,
	101, 105, 111, 115, 117, 122, 98, 109,
	112, 120, 97, 101, 105, 111, 116, 117,
	97, 101, 105, 111, 117, 118, 122, 98,
	109, 112, 120, 97, 101, 105, 111, 117,
	119, 97, 101, 105, 111, 117, 120, 97,
	101, 105, 111, 117, 121, 97, 101, 105,
	111, 117, 122, 0
};

static const char _JJR_RBR_single_lengths[] = {
	0, 1, 1, 24, 0, 6, 5, 7,
	6, 6, 7, 6, 6, 6, 7, 1,
	6, 6, 0, 6, 6, 6, 7, 6,
	7, 6, 6, 6, 6
};

static const char _JJR_RBR_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 3,
	0, 0, 2, 0, 0, 0, 2, 0,
	0, 0, 1, 0, 0, 0, 2, 0,
	2, 0, 0, 0, 0
};

static const unsigned char _JJR_RBR_index_offsets[] = {
	0, 0, 2, 4, 29, 30, 37, 43,
	54, 61, 68, 78, 85, 92, 99, 109,
	111, 118, 125, 127, 134, 141, 148, 158,
	165, 175, 182, 189, 196
};

static const char _JJR_RBR_indicies[] = {
	0, 1, 2, 1, 4, 5, 6, 7,
	8, 9, 10, 11, 12, 13, 14, 15,
	16, 17, 18, 19, 20, 21, 7, 22,
	23, 24, 25, 26, 3, 1, 27, 28,
	27, 27, 27, 27, 1, 29, 29, 29,
	29, 29, 1, 30, 31, 30, 27, 27,
	27, 30, 27, 30, 30, 1, 27, 28,
	27, 27, 27, 27, 1, 27, 27, 28,
	27, 27, 27, 1, 27, 27, 31, 27,
	27, 27, 30, 30, 30, 1, 27, 27,
	28, 27, 27, 27, 1, 27, 27, 27,
	28, 27, 27, 1, 27, 27, 27, 28,
	27, 27, 1, 27, 27, 27, 32, 27,
	27, 30, 30, 30, 1, 1, 33, 27,
	27, 27, 28, 27, 27, 1, 34, 34,
	34, 28, 34, 34, 1, 29, 1, 34,
	34, 34, 34, 28, 34, 1, 27, 27,
	27, 27, 28, 27, 1, 27, 27, 27,
	27, 28, 27, 1, 27, 27, 27, 27,
	31, 27, 30, 30, 30, 1, 27, 27,
	27, 27, 28, 27, 1, 27, 27, 27,
	27, 27, 31, 30, 30, 30, 1, 34,
	34, 34, 34, 34, 28, 1, 34, 34,
	34, 34, 34, 28, 1, 27, 27, 27,
	27, 27, 28, 1, 27, 27, 27, 27,
	27, 28, 1, 0
};

static const char _JJR_RBR_trans_targs[] = {
	2, 0, 3, 4, 5, 7, 8, 4,
	9, 10, 11, 4, 12, 13, 14, 16,
	17, 19, 20, 21, 22, 23, 24, 25,
	26, 27, 28, 6, 4, 4, 4, 4,
	15, 4, 18
};

static const char _JJR_RBR_trans_actions[] = {
	0, 0, 0, 9, 9, 9, 9, 17,
	9, 9, 9, 14, 9, 9, 9, 9,
	9, 9, 9, 9, 9, 9, 9, 9,
	9, 9, 9, 7, 3, 5, 7, 11,
	11, 1, 7
};

static const int JJR_RBR_start = 1;

void english_morpho_guesser::add_JJR_RBR(const string& form, unsigned negation_len, vector& lemmas) const {
  const char* p = form.c_str() + negation_len; int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = JJR_RBR_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _JJR_RBR_trans_keys + _JJR_RBR_key_offsets[cs];
	_trans = _JJR_RBR_index_offsets[cs];

	_klen = _JJR_RBR_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _JJR_RBR_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _JJR_RBR_indicies[_trans];
	cs = _JJR_RBR_trans_targs[_trans];

	if ( _JJR_RBR_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _JJR_RBR_actions + _JJR_RBR_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 2, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 3, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 3, append = "y";     }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 2, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 1, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 2, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(JJR, RBR, form.substr(0, form.size() - remove).append(append ? append : ""), negation_len, lemmas);
}

static const char _JJS_RBS_actions[] = {
	0, 1, 1, 1, 2, 1, 4, 1,
	5, 2, 0, 5, 2, 1, 4, 2,
	3, 5
};

static const unsigned char _JJS_RBS_key_offsets[] = {
	0, 0, 1, 2, 3, 25, 25, 25,
	31, 44, 50, 56, 67, 73, 79, 85,
	96, 102, 108, 114, 120, 126, 137, 143,
	154, 160, 166, 172, 178, 178, 183, 183,
	183, 184
};

static const char _JJS_RBS_trans_keys[] = {
	116, 115, 101, 98, 99, 100, 102, 103,
	104, 105, 106, 107, 108, 109, 110, 112,
	113, 114, 115, 116, 118, 119, 120, 121,
	122, 97, 98, 101, 105, 111, 117, 98,
	99, 100, 105, 111, 117, 122, 97, 101,
	102, 109, 112, 120, 97, 100, 101, 105,
	111, 117, 97, 101, 102, 105, 111, 117,
	97, 101, 103, 105, 111, 117, 122, 98,
	109, 112, 120, 97, 101, 104, 105, 111,
	117, 97, 101, 105, 106, 111, 117, 97,
	101, 105, 107, 111, 117, 97, 101, 105,
	108, 111, 117, 122, 98, 109, 112, 120,
	97, 101, 105, 109, 111, 117, 97, 101,
	105, 110, 111, 117, 97, 101, 105, 111,
	112, 117, 97, 101, 105, 111, 113, 117,
	97, 101, 105, 111, 114, 117, 97, 101,
	105, 111, 115, 117, 122, 98, 109, 112,
	120, 97, 101, 105, 111, 116, 117, 97,
	101, 105, 111, 117, 118, 122, 98, 109,
	112, 120, 97, 101, 105, 111, 117, 119,
	97, 101, 105, 111, 117, 120, 97, 101,
	105, 111, 117, 121, 97, 101, 105, 111,
	117, 122, 97, 101, 105, 111, 117, 101,
	97, 122, 0
};

static const char _JJS_RBS_single_lengths[] = {
	0, 1, 1, 1, 22, 0, 0, 6,
	7, 6, 6, 7, 6, 6, 6, 7,
	6, 6, 6, 6, 6, 7, 6, 7,
	6, 6, 6, 6, 0, 5, 0, 0,
	1, 0
};

static const char _JJS_RBS_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	3, 0, 0, 2, 0, 0, 0, 2,
	0, 0, 0, 0, 0, 2, 0, 2,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 1
};

static const unsigned char _JJS_RBS_index_offsets[] = {
	0, 0, 2, 4, 6, 29, 30, 31,
	38, 49, 56, 63, 73, 80, 87, 94,
	104, 111, 118, 125, 132, 139, 149, 156,
	166, 173, 180, 187, 194, 195, 201, 202,
	203, 205
};

static const char _JJS_RBS_indicies[] = {
	0, 1, 2, 1, 3, 1, 5, 6,
	7, 8, 9, 10, 11, 12, 13, 14,
	15, 16, 17, 18, 19, 20, 21, 22,
	23, 24, 25, 26, 4, 27, 28, 29,
	30, 29, 29, 29, 29, 27, 31, 32,
	31, 29, 29, 29, 31, 29, 31, 31,
	27, 29, 30, 29, 29, 29, 29, 27,
	29, 29, 30, 29, 29, 29, 27, 29,
	29, 32, 29, 29, 29, 31, 31, 31,
	27, 29, 29, 30, 29, 29, 29, 27,
	29, 29, 29, 30, 29, 29, 27, 29,
	29, 29, 30, 29, 29, 27, 29, 29,
	29, 33, 29, 29, 31, 31, 31, 27,
	29, 29, 29, 30, 29, 29, 27, 34,
	34, 34, 30, 34, 34, 27, 34, 34,
	34, 34, 30, 34, 27, 29, 29, 29,
	29, 30, 29, 27, 29, 29, 29, 29,
	30, 29, 27, 29, 29, 29, 29, 32,
	29, 31, 31, 31, 27, 29, 29, 29,
	29, 30, 29, 27, 29, 29, 29, 29,
	29, 32, 31, 31, 31, 27, 34, 34,
	34, 34, 34, 30, 27, 34, 34, 34,
	34, 34, 30, 27, 29, 29, 29, 29,
	29, 30, 27, 29, 29, 29, 29, 29,
	30, 27, 1, 35, 35, 35, 35, 35,
	28, 28, 27, 28, 36, 35, 28, 0
};

static const char _JJS_RBS_trans_targs[] = {
	2, 0, 3, 4, 5, 7, 8, 9,
	10, 11, 12, 31, 13, 14, 15, 16,
	17, 18, 19, 20, 21, 22, 23, 24,
	25, 26, 27, 6, 28, 29, 30, 30,
	30, 32, 33, 28, 28
};

static const char _JJS_RBS_trans_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 3, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 7, 5, 1, 5,
	12, 12, 5, 15, 9
};

static const int JJS_RBS_start = 1;

void english_morpho_guesser::add_JJS_RBS(const string& form, unsigned negation_len, vector& lemmas) const {
  const char* p = form.c_str() + negation_len; int cs;
  char best = 'z'; unsigned remove = 0; const char* append = nullptr;

	{
	cs = JJS_RBS_start;
	}

	{
	int _klen;
	unsigned int _trans;
	const char *_acts;
	unsigned int _nacts;
	const char *_keys;

	if ( p == ( (form.c_str() + form.size())) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _JJS_RBS_trans_keys + _JJS_RBS_key_offsets[cs];
	_trans = _JJS_RBS_index_offsets[cs];

	_klen = _JJS_RBS_single_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < *_mid )
				_upper = _mid - 1;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _JJS_RBS_range_lengths[cs];
	if ( _klen > 0 ) {
		const char *_lower = _keys;
		const char *_mid;
		const char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( form[form.size() - 1 - (p - form.c_str() - negation_len)]) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _JJS_RBS_indicies[_trans];
	cs = _JJS_RBS_trans_targs[_trans];

	if ( _JJS_RBS_trans_actions[_trans] == 0 )
		goto _again;

	_acts = _JJS_RBS_actions + _JJS_RBS_trans_actions[_trans];
	_nacts = (unsigned int) *_acts++;
	while ( _nacts-- > 0 )
	{
		switch ( *_acts++ )
		{
	case 0:
	{ if (best > 'a') best = 'a', remove = 3, append = nullptr; }
	break;
	case 1:
	{ if (best > 'b') best = 'b', remove = 4, append = nullptr; }
	break;
	case 2:
	{ if (best > 'c') best = 'c', remove = 4, append = "y";     }
	break;
	case 3:
	{ if (best > 'd') best = 'd', remove = 3, append = nullptr; }
	break;
	case 4:
	{ if (best > 'e') best = 'e', remove = 2, append = nullptr; }
	break;
	case 5:
	{ if (best > 'f') best = 'f', remove = 3, append = nullptr; }
	break;
		}
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++p != ( (form.c_str() + form.size())) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  add(JJS, RBS, form.substr(0, form.size() - remove).append(append ? append : ""), negation_len, lemmas);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/external_morpho.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class external_morpho : public morpho {
 public:
  external_morpho(unsigned version) : version(version) {}

  virtual int analyze(string_piece form, morpho::guesser_mode guesser, vector& lemmas) const override;
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector& forms) const override;
  virtual int raw_lemma_len(string_piece lemma) const override;
  virtual int lemma_id_len(string_piece lemma) const override;
  virtual int raw_form_len(string_piece form) const override;
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);

 private:
  unsigned version;

  string unknown_tag;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_tokenizer : public ragel_tokenizer {
 public:
  enum { LATEST = 2 };
  generic_tokenizer(unsigned version);

  virtual bool next_sentence(vector& tokens) override;
};

} // namespace morphodita

/////////
// File: morphodita/morpho/external_morpho.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool external_morpho::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    // Load unknown_tag
    unsigned length = data.next_1B();
    unknown_tag.assign(data.next(length), length);
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

int external_morpho::analyze(string_piece form, guesser_mode /*guesser*/, vector& lemmas) const {
  lemmas.clear();

  if (form.len) {
    // Start by skipping the first form
    string_piece lemmatags = form;
    while (lemmatags.len && *lemmatags.str != ' ') lemmatags.len--, lemmatags.str++;
    if (lemmatags.len) lemmatags.len--, lemmatags.str++;

    // Split lemmatags using ' ' into lemma-tag pairs.
    while (lemmatags.len) {
      auto lemma_start = lemmatags.str;
      while (lemmatags.len && *lemmatags.str != ' ') lemmatags.len--, lemmatags.str++;
      if (!lemmatags.len) break;
      auto lemma_len = lemmatags.str - lemma_start;
      lemmatags.len--, lemmatags.str++;

      auto tag_start = lemmatags.str;
      while (lemmatags.len && *lemmatags.str != ' ') lemmatags.len--, lemmatags.str++;
      auto tag_len = lemmatags.str - tag_start;
      if (lemmatags.len) lemmatags.len--, lemmatags.str++;

      lemmas.emplace_back(string(lemma_start, lemma_len), string(tag_start, tag_len));
    }

    if (!lemmas.empty()) return NO_GUESSER;
  }

  lemmas.emplace_back(string(form.str, form.len), unknown_tag);
  return -1;
}

int external_morpho::generate(string_piece lemma, const char* tag_wildcard, morpho::guesser_mode /*guesser*/, vector& forms) const {
  forms.clear();

  tag_filter filter(tag_wildcard);

  if (lemma.len) {
    // Start by locating the lemma
    string_piece formtags = lemma;
    while (formtags.len && *formtags.str != ' ') formtags.len--, formtags.str++;
    string_piece real_lemma(lemma.str, lemma.len - formtags.len);
    if (formtags.len) formtags.len--, formtags.str++;

    // Split formtags using ' ' into form-tag pairs.
    bool any_result = false;
    while (formtags.len) {
      auto form_start = formtags.str;
      while (formtags.len && *formtags.str != ' ') formtags.len--, formtags.str++;
      if (!formtags.len) break;
      auto form_len = formtags.str - form_start;
      formtags.len--, formtags.str++;

      auto tag_start = formtags.str;
      while (formtags.len && *formtags.str != ' ') formtags.len--, formtags.str++;
      auto tag_len = formtags.str - tag_start;
      if (formtags.len) formtags.len--, formtags.str++;

      any_result = true;
      string tag(tag_start, tag_len);
      if (filter.matches(tag.c_str())) {
        if (forms.empty()) forms.emplace_back(string(real_lemma.str, real_lemma.len));
        forms.back().forms.emplace_back(string(form_start, form_len), tag);
      }
    }

    if (any_result) return NO_GUESSER;
  }

  return -1;
}

int external_morpho::raw_lemma_len(string_piece lemma) const {
  unsigned lemma_len = 0;
  while (lemma_len < lemma.len && lemma.str[lemma_len] != ' ') lemma_len++;
  return lemma_len;
}

int external_morpho::lemma_id_len(string_piece lemma) const {
  unsigned lemma_len = 0;
  while (lemma_len < lemma.len && lemma.str[lemma_len] != ' ') lemma_len++;
  return lemma_len;
}

int external_morpho::raw_form_len(string_piece form) const {
  unsigned form_len = 0;
  while (form_len < form.len && form.str[form_len] != ' ') form_len++;
  return form_len;
}

tokenizer* external_morpho::new_tokenizer() const {
  return new generic_tokenizer(version);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_lemma_addinfo.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
struct generic_lemma_addinfo {
  inline static int raw_lemma_len(string_piece lemma);
  inline static int lemma_id_len(string_piece lemma);
  inline static string format(const unsigned char* addinfo, int addinfo_len);
  inline static bool generatable(const unsigned char* addinfo, int addinfo_len);

  inline int parse(string_piece lemma, bool die_on_failure = false);
  inline bool match_lemma_id(const unsigned char* other_addinfo, int other_addinfo_len);

  vector data;
};

// Definitions
int generic_lemma_addinfo::raw_lemma_len(string_piece lemma) {
  return lemma.len;
}

int generic_lemma_addinfo::lemma_id_len(string_piece lemma) {
  return lemma.len;
}

string generic_lemma_addinfo::format(const unsigned char* /*addinfo*/, int /*addinfo_len*/) {
  return string();
}

bool generic_lemma_addinfo::generatable(const unsigned char* /*addinfo*/, int /*addinfo_len*/) {
  return true;
}

int generic_lemma_addinfo::parse(string_piece lemma, bool /*die_on_failure*/) {
  return lemma.len;
}

bool generic_lemma_addinfo::match_lemma_id(const unsigned char* /*other_addinfo*/, int /*other_addinfo_len*/) {
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_morpho.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_morpho : public morpho {
 public:
  generic_morpho(unsigned version) : version(version) {}

  virtual int analyze(string_piece form, morpho::guesser_mode guesser, vector& lemmas) const override;
  virtual int generate(string_piece lemma, const char* tag_wildcard, guesser_mode guesser, vector& forms) const override;
  virtual int raw_lemma_len(string_piece lemma) const override;
  virtual int lemma_id_len(string_piece lemma) const override;
  virtual int raw_form_len(string_piece form) const override;
  virtual tokenizer* new_tokenizer() const override;

  bool load(istream& is);
 private:
  inline void analyze_special(string_piece form, vector& lemmas) const;

  unsigned version;
  morpho_dictionary dictionary;
  unique_ptr statistical_guesser;

  string unknown_tag, number_tag, punctuation_tag, symbol_tag;
};

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_morpho.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool generic_morpho::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    // Load tags
    unsigned length = data.next_1B();
    unknown_tag.assign(data.next(length), length);
    length = data.next_1B();
    number_tag.assign(data.next(length), length);
    length = data.next_1B();
    punctuation_tag.assign(data.next(length), length);
    length = data.next_1B();
    symbol_tag.assign(data.next(length), length);

    // Load dictionary
    dictionary.load(data);

    // Optionally statistical guesser if present
    statistical_guesser.reset();
    if (data.next_1B()) {
      statistical_guesser.reset(new morpho_statistical_guesser());
      statistical_guesser->load(data);
    }
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

int generic_morpho::analyze(string_piece form, guesser_mode guesser, vector& lemmas) const {
  lemmas.clear();

  if (form.len) {
    // Generate all casing variants if needed (they are different than given form).
    string form_uclc; // first uppercase, rest lowercase
    string form_lc;   // all lowercase
    generate_casing_variants(form, form_uclc, form_lc);

    // Start by analysing using the dictionary and all casing variants.
    dictionary.analyze(form, lemmas);
    if (!form_uclc.empty()) dictionary.analyze(form_uclc, lemmas);
    if (!form_lc.empty()) dictionary.analyze(form_lc, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // Then call analyze_special to handle numbers, punctuation and symbols.
    analyze_special(form, lemmas);
    if (!lemmas.empty()) return NO_GUESSER;

    // For the statistical guesser, use all casing variants.
    if (guesser == GUESSER && statistical_guesser) {
      if (form_uclc.empty() && form_lc.empty())
        statistical_guesser->analyze(form, lemmas, nullptr);
      else {
        morpho_statistical_guesser::used_rules used_rules; used_rules.reserve(3);
        statistical_guesser->analyze(form, lemmas, &used_rules);
        if (!form_uclc.empty()) statistical_guesser->analyze(form_uclc, lemmas, &used_rules);
        if (!form_lc.empty()) statistical_guesser->analyze(form_lc, lemmas, &used_rules);
      }
    }
    if (!lemmas.empty()) return GUESSER;
  }

  lemmas.emplace_back(string(form.str, form.len), unknown_tag);
  return -1;
}

int generic_morpho::generate(string_piece lemma, const char* tag_wildcard, morpho::guesser_mode /*guesser*/, vector& forms) const {
  forms.clear();

  tag_filter filter(tag_wildcard);

  if (lemma.len) {
    if (dictionary.generate(lemma, filter, forms))
      return NO_GUESSER;
  }

  return -1;
}

int generic_morpho::raw_lemma_len(string_piece lemma) const {
  return generic_lemma_addinfo::raw_lemma_len(lemma);
}

int generic_morpho::lemma_id_len(string_piece lemma) const {
  return generic_lemma_addinfo::lemma_id_len(lemma);
}

int generic_morpho::raw_form_len(string_piece form) const {
  return form.len;
}

tokenizer* generic_morpho::new_tokenizer() const {
  return new generic_tokenizer(version);
}

void generic_morpho::analyze_special(string_piece form, vector& lemmas) const {
  using namespace unilib;

  // Analyzer for numbers, punctuation and symbols.
  // Number is anything matching [+-]? is_Pn* ([.,] is_Pn*)? ([Ee] [+-]? is_Pn+)? for at least one is_Pn* nonempty.
  // Punctuation is any form beginning with either unicode punctuation or punctuation_exceptions character.
  // Beware that numbers takes precedence, so - is punctuation, -3 is number, -. is punctuation, -.3 is number.
  if (!form.len) return;

  string_piece number = form;
  char32_t first = utf8::decode(number.str, number.len);

  // Try matching a number.
  char32_t codepoint = first;
  bool any_digit = false;
  if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(number.str, number.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  if ((codepoint == '.' && number.len) || codepoint == ',') codepoint = utf8::decode(number.str, number.len);
  while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  if (any_digit && (codepoint == 'e' || codepoint == 'E')) {
    codepoint = utf8::decode(number.str, number.len);
    if (codepoint == '+' || codepoint == '-') codepoint = utf8::decode(number.str, number.len);
    any_digit = false;
    while (unicode::category(codepoint) & unicode::N) any_digit = true, codepoint = utf8::decode(number.str, number.len);
  }

  if (any_digit && !number.len && (!codepoint || codepoint == '.')) {
    lemmas.emplace_back(string(form.str, form.len), number_tag);
    return;
  }

  // Try matching punctuation or symbol.
  bool punctuation = true, symbol = true;
  string_piece form_ori = form;
  while (form.len) {
    codepoint = utf8::decode(form.str, form.len);
    punctuation = punctuation && unicode::category(codepoint) & unicode::P;
    symbol = symbol && unicode::category(codepoint) & unicode::S;
  }
  if (punctuation)
    lemmas.emplace_back(string(form_ori.str, form_ori.len), punctuation_tag);
  else if (symbol)
    lemmas.emplace_back(string(form_ori.str, form_ori.len), symbol_tag);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_morpho_encoder.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_morpho_encoder {
 public:
  struct tags {
    string unknown_tag, number_tag, punctuation_tag, symbol_tag;
  };
  static void encode(istream& in_dictionary, int max_suffix_len, const tags& tags, istream& in_statistical_guesser, ostream& out_morpho);
};

} // namespace morphodita

/////////
// File: morphodita/morpho/persistent_unordered_map_encoder.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

template
persistent_unordered_map::persistent_unordered_map(const unordered_map& map, double load_factor, EntryEncode entry_encode) {
  construct(std::map(map.begin(), map.end()), load_factor, entry_encode);
}

template
persistent_unordered_map::persistent_unordered_map(const unordered_map& map, double load_factor, bool add_prefixes, bool add_suffixes, EntryEncode entry_encode) {
  // Copy data, possibly including prefixes and suffixes
  std::map enlarged_map(map.begin(), map.end());

  for (auto&& entry : map) {
    const string& key = entry.first;

    if (!key.empty() && add_prefixes)
      for (unsigned i = key.size() - 1; i; i--)
        enlarged_map[key.substr(0, i)];

    if (!key.empty() && add_suffixes)
      for (unsigned i = 1; i < key.size(); i++)
        enlarged_map[key.substr(i)];
  }

  construct(enlarged_map, load_factor, entry_encode);
}

// We could (and used to) use unordered_map as input parameter.
// Nevertheless, as order is unspecified, the resulting persistent_unordered_map
// has different collision chains when generated on 32-bit and 64-bit machines.
// To guarantee uniform binary representation, we use map instead.
template
void persistent_unordered_map::construct(const map& map, double load_factor, EntryEncode entry_encode) {
  // 1) Count number of elements for each size
  vector sizes;
  for (auto&& elem : map) {
    unsigned len = elem.first.size();
    if (len >= sizes.size()) sizes.resize(len + 1);
    sizes[len]++;
  }
  for (auto&& size : sizes)
    resize(unsigned(load_factor * size));

  // 2) Add sizes of element data
  for (auto&& elem : map) {
    binary_encoder enc;
    entry_encode(enc, elem.second);
    add(elem.first.c_str(), elem.first.size(), enc.data.size());
  }
  done_adding();

  // 3) Fill in element data
  for (auto&& elem : map) {
    binary_encoder enc;
    entry_encode(enc, elem.second);
    small_memcpy(fill(elem.first.c_str(), elem.first.size(), enc.data.size()), enc.data.data(), enc.data.size());
  }
  done_filling();
}

void persistent_unordered_map::save(binary_encoder& enc) {
  enc.add_1B(hashes.size());

  for (auto&& hash : hashes)
    hash.save(enc);
}

void persistent_unordered_map::fnv_hash::save(binary_encoder& enc) {
  enc.add_4B(hash.size());
  enc.add_data(hash);

  enc.add_4B(data.size());
  enc.add_data(data);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/raw_morpho_dictionary_reader.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class raw_morpho_dictionary_reader {
 public:
  raw_morpho_dictionary_reader(istream& in) : in(in) {}
  bool next_lemma(string& lemma, vector>& tagged_forms);
 private:
  istream& in;
  string line;
  vector tokens;
  unordered_set seen_lemmas;
};

} // namespace morphodita

/////////
// File: utils/new_unique_ptr.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

template
unique_ptr new_unique_ptr(Args&&... args) {
  return unique_ptr(new T(std::forward(args)...));
}

} // namespace utils

/////////
// File: morphodita/morpho/morpho_dictionary_encoder.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template
class morpho_dictionary_encoder {
 public:
  static void encode(istream& is, int max_suffix_len, binary_encoder& enc);
};

// Definitions
template
class dictionary {
 public:
  void load(istream& is, int max_suffix_len);
  void encode(binary_encoder& enc);

 private:
  class trie {
   public:
    trie() : depth(0) {}

    void add(const char* str) {
      if (!*str) return;

      for (auto&& child : children)
        if (child.first == *str) {
          child.second->add(str + 1);
          depth = max(depth, 1 + child.second->depth);
          return;
        }
      children.emplace_back(*str, new_unique_ptr());
      children.back().second->add(str + 1);
      depth = max(depth, 1 + children.back().second->depth);
    }

    string find_candidate_prefix(int max_suffix_len) {
      string current, best;
      int best_length = 0;
      find_candidate_prefix(max_suffix_len, current, best, best_length, 0);
      return best;
    }
    void find_candidate_prefix(int max_suffix_len, string& current, string& best, int& best_length, int length) {
      if (depth < max_suffix_len && length > best_length) {
        best = current;
        best_length = length;
      }
      for (auto&& child : children) {
        current.push_back(child.first);
        child.second->find_candidate_prefix(max_suffix_len, current, best, best_length, children.size() == 1 ? length + 1 : 1);
        current.resize(current.size() - 1);
      }
    }

    vector>> children;
    int depth;
  };

  class histogram {
   public:
    void add(const string& str) {
      if (str.size() >= lengths.size()) lengths.resize(str.size() + 1);
      lengths[str.size()].insert(str);
    }

    void encode(binary_encoder& enc) {
      enc.add_1B(lengths.size());
      for (auto&& set : lengths)
        enc.add_4B(set.size());
    }

    vector> lengths;
  };

  struct lemma_info {
    lemma_info(string lemma) {
      this->lemma = lemma.substr(0, addinfo.parse(lemma, true));
    }

    string lemma;
    LemmaAddinfo addinfo;
    struct lemma_form_info {
      lemma_form_info(string form, int clas) : form(form), clas(clas) {}

      string form;
      int clas;

      bool operator<(const lemma_form_info& other) const { return form < other.form || (form == other.form && clas < other.clas); }
    };
    vector forms;

    bool operator<(const lemma_info& other) const { return lemma < other.lemma || (lemma == other.lemma && addinfo.data < other.addinfo.data); }
  };

  unordered_map classes;
  unordered_map>> suffixes;

  vector tags;
  unordered_map tags_map;

  histogram lemmas_hist, forms_hist;

  vector lemmas;
};

template
void morpho_dictionary_encoder::encode(istream& is, int max_suffix_len, binary_encoder& enc) {
  dictionary dict;

  // Load the dictionary and create classes
  dict.load(is, max_suffix_len);

  // Encode the dictionary
  dict.encode(enc);
}

template
void dictionary::load(istream& is, int max_suffix_len) {
  // Load lemmas and create classes
  raw_morpho_dictionary_reader raw(is);
  string lemma;
  vector> forms;
  while(raw.next_lemma(lemma, forms)) {
    // Make sure forms are unique
    sort(forms.begin(), forms.end());
    auto forms_end = unique(forms.begin(), forms.end());
    if (forms_end != forms.end()) {
//      cerr << "Warning: repeated form-tag in lemma " << lemma << '.' << endl;
      forms.erase(forms_end, forms.end());
    }

    // Create lemma_info
    lemmas.emplace_back(lemma);
    auto& lemma_info = lemmas.back();
    lemmas_hist.add(lemma_info.lemma);

    // Create classes
    while (!forms.empty()) {
      trie t;
      for (auto&& form : forms)
        t.add(form.first.c_str());

      // Find prefix of forms in class being added.
      string prefix = t.find_candidate_prefix(max_suffix_len);

      // Find forms of the class being added.
      auto start = forms.begin();
      while (start != forms.end() && start->first.compare(0, prefix.size(), prefix) != 0) start++;
      if (start == forms.end()) training_failure("Internal error when generating classes, cannot find prefix '" << prefix << "'!");
      auto end = start;
      while (end != forms.end() && end->first.compare(0, prefix.size(), prefix) == 0) end++;

      // Find common prefix of class forms -- may be larger than prefix.
      int common_prefix = prefix.size();
      while (common_prefix < int(start->first.size()) && start->first[common_prefix] == (end-1)->first[common_prefix]) common_prefix++;

      string clas;
      for (auto form = start; form != end; form++) {
        if (!clas.empty()) clas.push_back('\t');
        clas.append(form->first, common_prefix, string::npos);
        clas.push_back('\t');
        clas.append(form->second);
      }

      auto class_it = classes.emplace(clas, int(classes.size()));
      int class_id = class_it.first->second;
      if (class_it.second) {
        // New class, add it, together with its tags.
        for (auto form = start; form != end; form++) {
          int tag = tags_map.emplace(form->second, int(tags.size())).first->second;
          if (tag >= int(tags.size())) tags.emplace_back(form->second);
          suffixes[form->first.substr(common_prefix)][class_id].emplace_back(tag);
        }
      }

      // Move forms in the class being added to lemma and remove them from unprocessed forms.
      lemma_info.forms.emplace_back(start->first.substr(0, common_prefix), class_id);
      forms_hist.add(lemma_info.forms.back().form);
      forms.erase(start, end);
    }
    stable_sort(lemma_info.forms.begin(), lemma_info.forms.end());
  }
  stable_sort(lemmas.begin(), lemmas.end());
}

template
void dictionary::encode(binary_encoder& enc) {
  // Encode lemmas and forms
  lemmas_hist.encode(enc);
  forms_hist.encode(enc);

  string prev = "";
  enc.add_4B(lemmas.size());
  for (auto&& lemma : lemmas) {
    int cpl = 0;
    while (prev[cpl] && prev[cpl] == lemma.lemma[cpl]) cpl++;

    enc.add_1B(prev.length() - cpl);
    enc.add_1B(lemma.lemma.size() - cpl);
    enc.add_data(lemma.lemma.substr(cpl));
    enc.add_1B(lemma.addinfo.data.size());
    enc.add_data(lemma.addinfo.data);
    enc.add_1B(lemma.forms.size());

    string prev_form = lemma.lemma;
    for (auto&& lemma_form : lemma.forms) {
      unsigned best_prev_from = 0, best_form_from = 0, best_len = 0;
      for (unsigned prev_from = 0; prev_from < prev_form.size(); prev_from++)
        for (unsigned form_from = 0; form_from < lemma_form.form.size(); form_from++) {
          unsigned len = 0;
          while (prev_from + len < prev_form.size() && form_from + len < lemma_form.form.size() && prev_form[prev_from+len] == lemma_form.form[form_from+len]) len++;
          if (len > best_len) best_prev_from = prev_from, best_form_from = form_from, best_len = len;
        }

      enum { REMOVE_START = 1, REMOVE_END = 2, ADD_START = 4, ADD_END = 8 };
      enc.add_1B(REMOVE_START * (best_prev_from>0) + REMOVE_END * (best_prev_from+best_len
             ADD_START * (best_form_from>0) + ADD_END * (best_form_from+best_len
      if (best_prev_from > 0) enc.add_1B(best_prev_from);
      if (best_prev_from + best_len < prev_form.size()) enc.add_1B(prev_form.size() - best_prev_from - best_len);
      if (best_form_from > 0) {
        enc.add_1B(best_form_from);
        enc.add_data(lemma_form.form.substr(0, best_form_from));
      }
      if (best_form_from + best_len < lemma_form.form.size()) {
        enc.add_1B(lemma_form.form.size() - best_form_from - best_len);
        enc.add_data(lemma_form.form.substr(best_form_from + best_len));
      }
      enc.add_2B(lemma_form.clas);

      prev_form = lemma_form.form;
    }

    prev = lemma.lemma;
  }

  // Encode tags
  enc.add_2B(tags.size());
  for (auto&& tag : tags) {
    enc.add_1B(tag.size());
    enc.add_data(tag);
  }

  // Encode classes
  persistent_unordered_map(suffixes, 5, false, true, [](binary_encoder& enc, const map>& suffix) {
    enc.add_2B(suffix.size());
    for (auto&& clas : suffix)
      enc.add_2B(clas.first);
    uint32_t tags = 0, prev_tags = 0;
    for (auto&& clas : suffix) {
      enc.add_2B(tags - prev_tags < (1<<16) ? uint16_t(tags) : tags);
      prev_tags = tags;
      tags += clas.second.size();
    }
    enc.add_2B(tags - prev_tags < (1<<16) ? uint16_t(tags) : tags);
    for (auto&& clas : suffix)
      for (auto&& tag : clas.second)
        enc.add_2B(tag);
  }).save(enc);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_prefix_guesser_encoder.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_prefix_guesser_encoder {
 public:
  static void encode(istream& is, binary_encoder& enc);
};

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser_encoder.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_statistical_guesser_encoder {
 public:
  static void encode(istream& is, binary_encoder& enc);
};

} // namespace morphodita

/////////
// File: morphodita/morpho/generic_morpho_encoder.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void generic_morpho_encoder::encode(istream& in_dictionary, int max_suffix_len, const tags& tags, istream& in_statistical_guesser, ostream& out_morpho) {
  binary_encoder enc;

  enc.add_1B(tags.unknown_tag.size());
  enc.add_data(tags.unknown_tag);
  enc.add_1B(tags.number_tag.size());
  enc.add_data(tags.number_tag);
  enc.add_1B(tags.punctuation_tag.size());
  enc.add_data(tags.punctuation_tag);
  enc.add_1B(tags.symbol_tag.size());
  enc.add_data(tags.symbol_tag);

//  cerr << "Encoding dictionary." << endl;
  morpho_dictionary_encoder::encode(in_dictionary, max_suffix_len, enc);

  // Load and encode statistical guesser if requested
  enc.add_1B(bool(in_statistical_guesser));
  if (in_statistical_guesser) {
//    cerr << "Encoding statistical guesser." << endl;
    morpho_statistical_guesser_encoder::encode(in_statistical_guesser, enc);
  }

  // done, save the dictionary
//  cerr << "Compressing dictionary." << endl;
  if (!compressor::save(out_morpho, enc)) training_failure("Cannot compress and write dictionary to file!");
//  cerr << "Dictionary saved." << endl;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_ids.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_ids {
 public:
  enum morpho_id {
    CZECH = 0,
    ENGLISH_V1 = 1,
    GENERIC = 2,
    EXTERNAL = 3,
    ENGLISH_V2 = 4,
    ENGLISH_V3 = 5, ENGLISH = ENGLISH_V3,
    SLOVAK_PDT = 6,
    DERIVATOR_DICTIONARY = 7,
  };

  static bool parse(const string& str, morpho_id& id) {
    if (str == "czech") return id = CZECH, true;
    if (str == "english") return id = ENGLISH, true;
    if (str == "external") return id = EXTERNAL, true;
    if (str == "generic") return id = GENERIC, true;
    if (str == "slovak_pdt") return id = SLOVAK_PDT, true;
    return false;
  }
};

typedef morpho_ids::morpho_id morpho_id;

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

morpho* morpho::load(istream& is) {
  morpho_id id = morpho_id(is.get());
  switch (id) {
    case morpho_ids::CZECH:
      {
        auto res = new_unique_ptr(czech_morpho::morpho_language::CZECH, 1);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::ENGLISH_V1:
    case morpho_ids::ENGLISH_V2:
    case morpho_ids::ENGLISH_V3:
      {
        auto res = new_unique_ptr(id == morpho_ids::ENGLISH_V1 ? 1 :
                                                  id == morpho_ids::ENGLISH_V2 ? 2 :
                                                  3);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::EXTERNAL:
      {
        auto res = new_unique_ptr(1);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::GENERIC:
      {
        auto res = new_unique_ptr(1);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::SLOVAK_PDT:
      {
        auto res = new_unique_ptr(czech_morpho::morpho_language::SLOVAK, 3);
        if (res->load(is)) return res.release();
        break;
      }
    case morpho_ids::DERIVATOR_DICTIONARY:
      {
        auto derinet = new_unique_ptr();
        if (!derinet->load(is)) return nullptr;

        unique_ptr dictionary(load(is));
        if (!dictionary) return nullptr;
        derinet->dictionary = dictionary.get();
        dictionary->derinet.reset(derinet.release());
        return dictionary.release();
      }
  }

  return nullptr;
}

morpho* morpho::load(const char* fname) {
  ifstream f(path_from_utf8(fname).c_str(), ifstream::binary);
  if (!f) return nullptr;

  return load(f);
}

const derivator* morpho::get_derivator() const {
  return derinet.get();
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void morpho_statistical_guesser::load(binary_decoder& data) {
  // Load tags and default tag
  tags.resize(data.next_2B());
  for (auto&& tag : tags) {
    tag.resize(data.next_1B());
    for (unsigned i = 0; i < tag.size(); i++)
      tag[i] = data.next_1B();
  }
  default_tag = data.next_2B();

  // Load rules
  rules.load(data);
}

// Helper method for analyze.
static bool contains(morpho_statistical_guesser::used_rules* used, const string& rule) {
  if (!used) return false;

  for (auto&& used_rule : *used)
    if (used_rule == rule)
      return true;

  return false;
}

// Produces unique lemma-tag pairs.
void morpho_statistical_guesser::analyze(string_piece form, vector& lemmas, morpho_statistical_guesser::used_rules* used) {
  unsigned lemmas_initial_size = lemmas.size();

  // We have rules in format "suffix prefix" in rules.
  // Find the matching rule with longest suffix and of those with longest prefix.
  string rule_label; rule_label.reserve(12);
  unsigned suffix_len = 0;
  for (; suffix_len < form.len; suffix_len++) {
    rule_label.push_back(form.str[form.len - (suffix_len + 1)]);
    if (!rules.at(rule_label.c_str(), rule_label.size(), [](pointer_decoder& data){ data.next(data.next_2B()); }))
      break;
  }

  for (suffix_len++; suffix_len--; ) {
    rule_label.resize(suffix_len);
    rule_label.push_back(' ');

    const unsigned char* rule = nullptr;
    unsigned rule_prefix_len = 0;
    for (unsigned prefix_len = 0; prefix_len + suffix_len <= form.len; prefix_len++) {
      if (prefix_len) rule_label.push_back(form.str[prefix_len - 1]);
      const unsigned char* found = rules.at(rule_label.c_str(), rule_label.size(), [](pointer_decoder& data){ data.next(data.next_2B()); });
      if (!found) break;
      if (*(found += sizeof(uint16_t))) {
        rule = found;
        rule_prefix_len = prefix_len;
      }
    }

    if (rule) {
      rule_label.resize(suffix_len + 1 + rule_prefix_len);
      if (rule_label.size() > 1 && !contains(used, rule_label)) { // ignore rule ' '
        if (used) used->push_back(rule_label);
        for (int rules_len = *rule++; rules_len; rules_len--) {
          unsigned pref_del_len = *rule++; const char* pref_del = (const char*)rule; rule += pref_del_len;
          unsigned pref_add_len = *rule++; const char* pref_add = (const char*)rule; rule += pref_add_len;
          unsigned suff_del_len = *rule++; const char* suff_del = (const char*)rule; rule += suff_del_len;
          unsigned suff_add_len = *rule++; const char* suff_add = (const char*)rule; rule += suff_add_len;
          unsigned tags_len = *rule++; const uint16_t* tags = (const uint16_t*)rule; rule += tags_len * sizeof(uint16_t);

          if (pref_del_len + suff_del_len > form.len ||
              (pref_del_len && !small_memeq(pref_del, form.str, pref_del_len)) ||
              (suff_del_len && !small_memeq(suff_del, form.str + form.len - suff_del_len, suff_del_len)) ||
              (form.len + pref_add_len - pref_del_len + suff_add_len - suff_del_len == 0))
            continue;

          string lemma;
          lemma.reserve(form.len + pref_add_len - pref_del_len + suff_add_len - suff_del_len);
          if (pref_add_len) lemma.append(pref_add, pref_add_len);
          if (pref_del_len + suff_del_len < form.len) lemma.append(form.str + pref_del_len, form.len - pref_del_len - suff_del_len);
          if (suff_add_len) lemma.append(suff_add, suff_add_len);
          while (tags_len--)
            lemmas.emplace_back(lemma, this->tags[unaligned_load_inc(tags)]);
        }
      }
      break;
    }
  }

  // If nothing was found, use default tag.
  if (lemmas.size() == lemmas_initial_size)
    if (!contains(used, string())) {
      if (used) used->push_back(string());
      lemmas.emplace_back(string(form.str, form.len), tags[default_tag]);
    }
}

} // namespace morphodita

/////////
// File: utils/split.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Split given text on the separator character.
inline void split(const string& text, char sep, vector& tokens);
inline void split(string_piece text, char sep, vector& tokens);

//
// Definitions
//

void split(const string& text, char sep, vector& tokens) {
  tokens.clear();
  if (text.empty()) return;

  string::size_type index = 0;
  for (string::size_type next; (next = text.find(sep, index)) != string::npos; index = next + 1)
    tokens.emplace_back(text, index, next - index);

  tokens.emplace_back(text, index);
}

void split(string_piece text, char sep, vector& tokens) {
  tokens.clear();
  if (!text.len) return;

  const char* str = text.str;
  for (const char* next; (next = (const char*) memchr(str, sep, text.str + text.len - str)); str = next + 1)
    tokens.emplace_back(str, next - str);

  tokens.emplace_back(str, text.str + text.len - str);
}

} // namespace utils

/////////
// File: morphodita/morpho/morpho_statistical_guesser_encoder.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void morpho_statistical_guesser_encoder::encode(istream& is, binary_encoder& enc) {
  unordered_map, vector>>> statistical_guesser;
  vector tags;
  unordered_map tags_map;

  // Load statistical guesser
  string line;
  vector tokens;
  if (!getline(is, line)) training_failure("Missing first line with default tag in statistical guesser file");
  int statistical_guesser_default = tags_map.emplace(line.data(), int(tags.size())).first->second;
  if (unsigned(statistical_guesser_default) >= tags.size()) tags.emplace_back(line.data());

  while (getline(is, line)) {
    split(line, '\t', tokens);
    if (tokens.size() < 3 || (tokens.size() % 2) != 1) training_failure("Cannot parse line " << line << " in statistical guesser file!");

    vector affixes;
    split(tokens[0], ' ', affixes);
    if (affixes.size() != 2) training_failure("Cannot parse prefix_suffix '" << tokens[0] << "' in statistical guesser file!");
    reverse(affixes[1].begin(), affixes[1].end());

    auto& rules = statistical_guesser[affixes[1] + ' ' + affixes[0]];
    for (unsigned i = 1; i < tokens.size(); i+= 2) {
      vector replacements;
      split(tokens[i], ' ', replacements);
      if (replacements.size() != 4) training_failure("Cannot parse replacement rule '" << tokens[i] << "' in statistical guesser file!");

      vector rule_tags;
      split(tokens[i+1], ' ', rule_tags);
      vector decoded_tags;
      for (auto&& rule_tag : rule_tags) {
        int tag = tags_map.emplace(rule_tag, int(tags.size())).first->second;
        if (unsigned(tag) >= tags.size()) tags.emplace_back(rule_tag);
        decoded_tags.emplace_back(tag);
      }

      rules.emplace_back(replacements, decoded_tags);
    }
  }

  // Encode statistical guesser
  enc.add_2B(tags.size());
  for (auto&& tag : tags) {
    enc.add_1B(tag.size());
    enc.add_data(tag);
  }
  enc.add_2B(statistical_guesser_default);

  persistent_unordered_map(statistical_guesser, 5, true, false, [](binary_encoder& enc, vector, vector>> rules) {
    binary_encoder e;
    e.add_1B(rules.size());
    for (auto&& rule : rules) {
      if (rule.first.size() != 4) training_failure("Replacement rule not of size 4 in statistical guesser!");
      for (auto&& affix : rule.first) {
        e.add_1B(affix.size());
        e.add_data(affix);
      }
      e.add_1B(rule.second.size());
      for (auto&& tag : rule.second)
        e.add_2B(tag);
    }
    enc.add_2B(e.data.size());
    enc.add_data(e.data);
  }).save(enc);
}

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser_trainer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class morpho_statistical_guesser_trainer {
 public:
  static void train(istream& is, unsigned suffix_len, unsigned rules_per_suffix, unsigned max_prefixes, unsigned min_prefix_count, ostream& os);

 private:
  struct instance {
    string form, lemma, tag;
    string lemma_rule, form_prefix;

    instance(const string& form, const string& lemma, const string& tag);
  };

  enum casing { CASE_LC, CASE_UCLC, CASE_UC, CASE_OTHER };
  static casing get_casing(const string& word, bool allow_nonletters);
  static void set_casing(const string& original, casing c, string& word);
  static bool suffix(const string& word, unsigned& length);
};

} // namespace morphodita

/////////
// File: morphodita/morpho/morpho_statistical_guesser_trainer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void morpho_statistical_guesser_trainer::train(istream& is, unsigned suffix_len, unsigned rules_per_suffix, unsigned max_prefixes, unsigned min_prefix_count, ostream& os) {
  vector data;

  // Load training data
  string form;
  vector tokens;
  for (string line; getline(is, line);) {
    if (line.empty()) continue;

    split(line, '\t', tokens);
    if (tokens.size() != 3) training_failure("The guesser training line '" << line << "' does not contain three columns!");
    if (tokens[0].empty() || tokens[1].empty() || tokens[2].empty()) training_failure("The guesser training line '" << line << "' contains an empty column!");

    // Normalize case
    casing form_case = get_casing(tokens[0], false);
    casing lemma_case = get_casing(tokens[1], true);
    if ((lemma_case == CASE_LC && (form_case == CASE_UCLC || form_case == CASE_UC)) ||
        (lemma_case == CASE_UCLC && form_case == CASE_UC)) {
      set_casing(tokens[0], lemma_case, form);
    } else {
      form.swap(tokens[0]);
    }

    data.emplace_back(form, tokens[1], tokens[2]);
  }

  // Generate at most max_prefixes prefixes with min_prefix_count
  unordered_map> prefixes_with_forms;
  for (auto&& instance : data)
    if (!instance.form_prefix.empty())
      prefixes_with_forms[instance.form_prefix].insert(instance.form);

  vector> prefixes_with_counts;
  for (auto&& prefix : prefixes_with_forms)
    if (prefix.second.size() >= min_prefix_count)
      prefixes_with_counts.emplace_back(unsigned(prefix.second.size()), prefix.first);

  if (prefixes_with_counts.size() > max_prefixes) {
    sort(prefixes_with_counts.begin(), prefixes_with_counts.end(), greater>());
    prefixes_with_counts.resize(max_prefixes);
  }

  unordered_set prefixes;
  prefixes.emplace();
  for (auto&& prefix : prefixes_with_counts)
    prefixes.insert(prefix.second);

  // Generate the guesser rules
  unordered_map> tags;
  unordered_map>> rules;
  unordered_set suffixes;
  string prefix_suffix, tag_lemma_rule;
  for (auto&& instance : data) {
    // Add tag
    tags[instance.tag].insert(instance.form);

    // Find longest matching prefix
    unsigned prefix_length = 0;
    for (auto&& prefix : prefixes)
      if (prefix.size() > prefix_length && instance.form.compare(0, prefix.size(), prefix) == 0)
        prefix_length = prefix.size();

    tag_lemma_rule.assign(instance.lemma_rule).append("\t").append(instance.tag);

    // Add prefix + all suffixes of length 1..suffix_len to rules
    for (unsigned length = 0, utf8_length = 0; length < suffix_len && suffix(instance.form, utf8_length); length++) {
      prefix_suffix.assign(instance.form, 0, prefix_length).append(" ").append(instance.form, instance.form.size() - utf8_length, utf8_length);
      rules[prefix_suffix][tag_lemma_rule].insert(instance.form);
      suffixes.emplace(instance.form, instance.form.size() - utf8_length, utf8_length);
    }
  }

  // Start generating the guesser description by writing the most "frequent" tag
  string most_frequent_tag; unsigned most_frequent_tag_count = 0;
  for (auto&& tag : tags)
    if (tag.second.size() > most_frequent_tag_count)
      most_frequent_tag.assign(tag.first), most_frequent_tag_count = tag.second.size();

  os << most_frequent_tag << endl;

  // For every prefix-suffix, write at most rules_per_suffix most "frequent" rules
  string rule_key, output;
  unordered_set rules_set;
  vector> rules_counts;
  for (auto&& suffix : suffixes) {
    for (auto&& prefix : prefixes) {
      rules_counts.clear();
      rules_set.clear();

      // Gather at most rules_per_suffix rules
      for (int prefix_len = int(prefix.size()); prefix_len >= 0; prefix_len -= prefix.empty() ? 1 : prefix.size()) {
        for (int suffix_len = int(suffix.size()); rules_counts.size() < rules_per_suffix && suffix_len > 0; suffix_len--) {
          rule_key.assign(prefix, 0, prefix_len).append(" ").append(suffix, suffix.size() - suffix_len, suffix_len);
          if (!rules.count(rule_key)) continue;

          unsigned rules_counts_original = rules_counts.size();
          for (auto&& entry : rules[rule_key])
            if (!rules_set.count(entry.first)) {
              rules_counts.emplace_back(unsigned(entry.second.size()), entry.first);
              rules_set.insert(entry.first);
            }

          sort(rules_counts.begin() + rules_counts_original, rules_counts.end(), greater>());

          if (rules_counts.size() >= rules_per_suffix) {
            rules_counts.resize(rules_per_suffix);
            break;
          }
        }
        // Stop if there are no rules for given prefix
        if (rules_set.empty()) break;
      }
      if (!rules_set.empty()) {
        // Write the chosen rules
        output.assign(prefix).append(" ").append(suffix);
        for (unsigned i = 0; i < rules_counts.size(); i++) {
          unsigned tab = rules_counts[i].second.find('\t');

          output.append("\t").append(rules_counts[i].second, 0, tab).append("\t").append(rules_counts[i].second, tab + 1, string::npos);

          // Join rules with same lemma_rule
          for (unsigned start = i; i+1 < rules_counts.size() && rules_counts[i+1].second.compare(0, tab + 1, rules_counts[start].second, 0, tab + 1) == 0; i++)
            output.append(" ").append(rules_counts[i+1].second, tab + 1, string::npos);
        }
        os << output << endl;
      }
    }
  }
}

morpho_statistical_guesser_trainer::instance::instance(const string& form, const string& lemma, const string& tag)
  : form(form), lemma(lemma), tag(tag)
{
  using namespace unilib;

  unsigned length_best = 0;
  int form_best = 0, lemma_best = 0;
  for (int offset = -int(lemma.size() - 1); offset < int(form.size()) - 1; offset++) {
    unsigned form_offset = max(0, offset);
    unsigned lemma_offset = max(0, -offset);
    for (unsigned length = 0; form_offset < form.size() && lemma_offset < lemma.size(); form_offset++, lemma_offset++)
      if (form[form_offset] == lemma[lemma_offset]) {
        if (++length > length_best && utf8::valid(form.c_str() + form_offset + 1 - length, length))
          length_best = length, form_best = form_offset + 1 - length, lemma_best = lemma_offset + 1 - length;
      } else {
        length = 0;
      }
  }

  form_prefix.assign(form, 0, lemma_best == 0 ? form_best : 0);
  lemma_rule.assign(form, 0, form_best).append(" ").append(lemma, 0, lemma_best).append(" ")
      .append(form, form_best + length_best, string::npos).append(" ").append(lemma, lemma_best + length_best, string::npos);
}

morpho_statistical_guesser_trainer::casing morpho_statistical_guesser_trainer::get_casing(const string& word, bool allow_nonletters) {
  using namespace unilib;

  casing c = CASE_OTHER;
  int index = 0;
  for (auto&& chr : utf8::decoder(word)) {
    auto cat = unicode::category(chr);

    // Return OTHER for non-letters
    if (allow_nonletters && index >= 2 && cat & ~unicode::L) continue;
    if (cat & ~unicode::L) return CASE_OTHER;

    if (index == 0) {
      c = cat & unicode::Ll ? CASE_LC : CASE_UC;
    } else if (c == CASE_UC && index == 1) {
      c = cat & unicode::Ll ? CASE_UCLC : CASE_UC;
    } else if (c == CASE_UC) {
      if (cat & ~unicode::Lut) return CASE_OTHER;
    } else /*CASE_LC or CASE_UCLC*/ {
      if (cat & ~unicode::Ll) return CASE_OTHER;
    }
    index++;
  }
  return c;
}

void morpho_statistical_guesser_trainer::set_casing(const string& original, casing c, string& word) {
  using namespace unilib;

  word.clear();
  bool first = true;
  for (auto&& chr : utf8::decoder(original)) {
    utf8::append(word, (c == CASE_UC || (c == CASE_UCLC && first)) ? unicode::uppercase(chr) : unicode::lowercase(chr));
    first = false;
  }
}

bool morpho_statistical_guesser_trainer::suffix(const string& word, unsigned& length) {
  using namespace unilib;

  unsigned additional = 1;
  while (additional + length <= word.size() && !utf8::valid(word.c_str() + word.size() - length - additional, additional))
    additional++;

  if (additional + length > word.size()) return false;

  length += additional;
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/raw_morpho_dictionary_reader.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool raw_morpho_dictionary_reader::next_lemma(string& lemma, vector>& tagged_forms) {
  if (line.empty()) {
    if (!getline(in, line))
      return false;
    split(line, '\t', tokens);
    if (tokens.size() != 3) training_failure("Line " << line << " does not have three columns!");
  }

  lemma = tokens[0];
  if (seen_lemmas.count(lemma))
    training_failure("Raw morphological dictionary contains lemma '" << lemma << "' multiple times - all forms of one lemma must be in continuous region!");
  seen_lemmas.insert(lemma);

  tagged_forms.clear();
  tagged_forms.emplace_back(tokens[2], tokens[1]);
  while (getline(in, line)) {
    split(line, '\t', tokens);
    if (tokens.size() != 3) training_failure("Line " << line << " does not have three columns!");

    if (lemma != tokens[0]) break;
    tagged_forms.emplace_back(tokens[2], tokens[1]);
  }

  return true;
}

} // namespace morphodita

/////////
// File: morphodita/morpho/tag_filter.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tag_filter::tag_filter(const char* filter) {
  if (!filter) return;

  wildcard.assign(filter);
  filter = wildcard.c_str();

  for (int tag_pos = 0, filter_pos = 0; filter[filter_pos]; tag_pos++, filter_pos++) {
    if (filter[filter_pos] == '?') continue;
    if (filter[filter_pos] == '[') {
      filter_pos++;

      bool negate = false;
      if (filter[filter_pos] == '^') negate = true, filter_pos++;

      int chars_start = filter_pos;
      for (bool first = true; filter[filter_pos] && (first || filter[filter_pos] != ']'); first = false)
        filter_pos++;

      filters.emplace_back(tag_pos, negate, chars_start, filter_pos - chars_start);
      if (!filter[filter_pos]) break;
    } else {
      filters.emplace_back(tag_pos, false, filter_pos, 1);
    }
  }
}

} // namespace morphodita

/////////
// File: morphodita/tagger/elementary_features.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
enum elementary_feature_type { PER_FORM, PER_TAG, DYNAMIC };
enum elementary_feature_range { ONLY_CURRENT, ANY_OFFSET };

typedef uint32_t elementary_feature_value;
enum :elementary_feature_value { elementary_feature_unknown = 0, elementary_feature_empty = 1 };

struct elementary_feature_description {
  string name;
  elementary_feature_type type;
  elementary_feature_range range;
  int index;
  int map_index;
};

template
class elementary_features {
 public:
  bool load(istream& is);
  bool save(ostream& out);

  vector maps;
};

class persistent_elementary_feature_map : public persistent_unordered_map {
 public:
  persistent_elementary_feature_map() : persistent_unordered_map() {}
  persistent_elementary_feature_map(const persistent_unordered_map&& map) : persistent_unordered_map(map) {}

  elementary_feature_value value(const char* feature, int len) const {
    auto* it = at_typed(feature, len);
    return it ? unaligned_load(it) : elementary_feature_unknown;
  }
};

// Definitions
template
inline bool elementary_features::load(istream& is) {
  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    maps.resize(data.next_1B());
    for (auto&& map : maps)
      map.load(data);
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

} // namespace morphodita

/////////
// File: morphodita/tagger/vli.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template
class vli {
 public:
  static int max_length();
  static void encode(T value, char*& where);
  static T decode(const char*& from);
};

// Definitions
template <>
inline int vli::max_length() {
  return 5;
}

template <>
inline void vli::encode(uint32_t value, char*& where) {
  if (value < 0x80) *where++ = value;
  else if (value < 0x4000) *where++ = (value >> 7) | 0x80u, *where++ = value & 0x7Fu;
  else if (value < 0x200000) *where++ = (value >> 14) | 0x80u, *where++ = ((value >> 7) & 0x7Fu) | 0x80u, *where++ = value & 0x7Fu;
  else if (value < 0x10000000) *where++ = (value >> 21) | 0x80u, *where++ = ((value >> 14) & 0x7Fu) | 0x80u, *where++ = ((value >> 7) & 0x7Fu) | 0x80u, *where++ = value & 0x7Fu;
  else *where++ = (value >> 28) | 0x80u, *where++ = ((value >> 21) & 0x7Fu) | 0x80u, *where++ = ((value >> 14) & 0x7Fu) | 0x80u, *where++ = ((value >> 7) & 0x7Fu) | 0x80u, *where++ = value & 0x7Fu;
}

template <>
inline uint32_t vli::decode(const char*& from) {
  uint32_t value = 0;
  while (((unsigned char)(*from)) & 0x80u) value = (value << 7) | (((unsigned char)(*from++)) ^ 0x80u);
  value = (value << 7) | ((unsigned char)(*from++));
  return value;
}

} // namespace morphodita

/////////
// File: morphodita/tagger/feature_sequences.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
typedef int32_t feature_sequence_score;
typedef int64_t feature_sequences_score;

struct feature_sequence_element {
  elementary_feature_type type;
  int elementary_index;
  int sequence_index;

  feature_sequence_element() {}
  feature_sequence_element(elementary_feature_type type, int elementary_index, int sequence_index) : type(type), elementary_index(elementary_index), sequence_index(sequence_index) {}
};

struct feature_sequence {
  vector elements;
  int dependant_range = 1;
};

template
class feature_sequences {
 public:
  typedef typename ElementaryFeatures::per_form_features per_form_features;
  typedef typename ElementaryFeatures::per_tag_features per_tag_features;
  typedef typename ElementaryFeatures::dynamic_features dynamic_features;

  void parse(int window_size, istream& is);
  bool load(istream& is);
  bool save(ostream& os);

  struct cache;

  inline void initialize_sentence(const vector& forms, const vector>& analyses, cache& c) const;
  inline void compute_dynamic_features(int form_index, int tag_index, const dynamic_features* prev_dynamic, dynamic_features& dynamic, cache& c) const;
  inline feature_sequences_score score(int form_index, int tags_window[], int tags_unchanged, dynamic_features& dynamic, cache& c) const;
  void feature_keys(int form_index, int tags_window[], int tags_unchanged, dynamic_features& dynamic, vector& keys, cache& c) const;

  ElementaryFeatures elementary;
  vector scores;
  vector sequences;
};

class persistent_feature_sequence_map : public persistent_unordered_map {
 public:
  persistent_feature_sequence_map() : persistent_unordered_map() {}
  persistent_feature_sequence_map(const persistent_unordered_map&& map) : persistent_unordered_map(map) {}

  feature_sequence_score score(const char* feature, int len) const {
    auto* it = at_typed(feature, len);
    return it ? unaligned_load(it) : 0;
  }
};

template  using persistent_feature_sequences = feature_sequences;

// Definitions
template
inline bool feature_sequences::load(istream& is) {
  if (!elementary.load(is)) return false;

  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    sequences.resize(data.next_1B());
    for (auto&& sequence : sequences) {
      sequence.dependant_range = data.next_4B();
      sequence.elements.resize(data.next_1B());
      for (auto&& element : sequence.elements) {
        element.type = elementary_feature_type(data.next_4B());
        element.elementary_index = data.next_4B();
        element.sequence_index = data.next_4B();
      }
    }

    scores.resize(data.next_1B());
    for (auto&& score : scores)
      score.load(data);
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

template
struct feature_sequences::cache {
  const vector* forms;
  const vector>* analyses;
  vector elementary_per_form;
  vector> elementary_per_tag;

  struct cache_element {
    vector key;
    int key_size;
    feature_sequence_score score;

    cache_element(int elements) : key(vli::max_length() * elements), key_size(0), score(0) {}
  };
  vector caches;
  vector window;
  vector key;
  feature_sequences_score score;

  cache(const feature_sequences& self) : score(0) {
    caches.reserve(self.sequences.size());
    int max_sequence_elements = 0, max_window_size = 1;
    for (auto&& sequence : self.sequences) {
      caches.emplace_back(int(sequence.elements.size()));
      if (int(sequence.elements.size()) > max_sequence_elements) max_sequence_elements = sequence.elements.size();
      for (auto&& element : sequence.elements)
        if (element.type == PER_TAG && 1 - element.sequence_index > max_window_size)
          max_window_size = 1 - element.sequence_index;
    }
    key.resize(max_sequence_elements * vli::max_length());
    window.resize(max_window_size);
  }
};

template
void feature_sequences::initialize_sentence(const vector& forms, const vector>& analyses, cache& c) const {
  // Store forms and forms_size
  c.forms = &forms;
  c.analyses = &analyses;

  // Enlarge elementary features vectors if needed
  if (forms.size() > c.elementary_per_form.size()) c.elementary_per_form.resize(forms.size() * 2);
  if (forms.size() > c.elementary_per_tag.size()) c.elementary_per_tag.resize(forms.size() * 2);
  for (unsigned i = 0; i < forms.size(); i++)
    if (analyses[i].size() > c.elementary_per_tag[i].size())
      c.elementary_per_tag[i].resize(analyses[i].size() * 2);

  // Compute elementary features
  elementary.compute_features(forms, analyses, c.elementary_per_form, c.elementary_per_tag);

  // Clear score cache, because scores may have been modified
  c.score = 0;
  for (auto&& cache : c.caches)
    cache.key_size = cache.score = 0;
}

template
void feature_sequences::compute_dynamic_features(int form_index, int tag_index, const dynamic_features* prev_dynamic, dynamic_features& dynamic, cache& c) const {
  elementary.compute_dynamic_features((*c.analyses)[form_index][tag_index], c.elementary_per_form[form_index], c.elementary_per_tag[form_index][tag_index], form_index > 0 ? prev_dynamic : nullptr, dynamic);
}

template
feature_sequences_score feature_sequences::score(int form_index, int tags_window[], int tags_unchanged, dynamic_features& dynamic, cache& c) const {
  // Start by creating a window of per_tag_features*
  for (int i = 0; i < int(c.window.size()) && form_index - i >= 0; i++)
    c.window[i] = &c.elementary_per_tag[form_index - i][tags_window[i]];

  // Compute the score
  feature_sequences_score result = c.score;
  for (unsigned i = 0; i < sequences.size(); i++) {
    if (tags_unchanged >= sequences[i].dependant_range)
      break;

    char* key = c.key.data();
    for (unsigned j = 0; j < sequences[i].elements.size(); j++) {
      auto& element = sequences[i].elements[j];
      elementary_feature_value value;

      switch (element.type) {
        case PER_FORM:
          value = form_index + element.sequence_index < 0 || unsigned(form_index + element.sequence_index) >= c.forms->size() ? elementary_feature_empty : c.elementary_per_form[form_index + element.sequence_index].values[element.elementary_index];
          break;
        case PER_TAG:
          value = form_index + element.sequence_index < 0 ? elementary_feature_empty : c.window[-element.sequence_index]->values[element.elementary_index];
          break;
        case DYNAMIC:
        default:
          value = dynamic.values[element.elementary_index];
      }

      if (value == elementary_feature_unknown) {
        key = c.key.data();
        break;
      }
      vli::encode(value, key);
    }

    result -= c.caches[i].score;
    int key_size = key - c.key.data();
    if (!key_size) {
      c.caches[i].score = 0;
      c.caches[i].key_size = 0;
    } else if (key_size != c.caches[i].key_size || !small_memeq(c.key.data(), c.caches[i].key.data(), key_size)) {
      c.caches[i].score = scores[i].score(c.key.data(), key_size);
      c.caches[i].key_size = key_size;
      small_memcpy(c.caches[i].key.data(), c.key.data(), key_size);
    }
    result += c.caches[i].score;
  }

  c.score = result;
  return result;
}

template
void feature_sequences::feature_keys(int form_index, int tags_window[], int tags_unchanged, dynamic_features& dynamic, vector& keys, cache& c) const {
  score(form_index, tags_window, tags_unchanged, dynamic, c);

  keys.resize(c.caches.size());
  for (unsigned i = 0; i < c.caches.size(); i++)
    keys[i].assign(c.caches[i].key.data(), c.caches[i].key_size);
}

} // namespace morphodita

/////////
// File: morphodita/tagger/viterbi.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template
class viterbi {
 public:
  viterbi(const FeatureSequences& features, int decoding_order, int window_size)
      : features(features), decoding_order(decoding_order), window_size(window_size) {}

  struct cache;
  void tag(const vector& forms, const vector>& analyses, cache& c, vector& tags) const;

 private:
  struct node;

  const FeatureSequences& features;
  int decoding_order, window_size;
};

// Definitions
template
struct viterbi::cache {
  vector nodes;
  typename FeatureSequences::cache features_cache;

  cache(const viterbi& self) : features_cache(self.features) {}
};

template
struct viterbi::node {
  int tag;
  int prev;
  feature_sequences_score score;
  typename FeatureSequences::dynamic_features dynamic;
};

template
void viterbi::tag(const vector& forms, const vector>& analyses, cache& c, vector& tags) const {
  if (!forms.size()) return;

  // Count number of nodes and allocate
  unsigned nodes = 0;
  for (unsigned i = 0, states = 1; i < forms.size(); i++) {
    if (analyses[i].empty()) return;
    states = (i+1 >= unsigned(decoding_order) ? states / analyses[i-decoding_order+1].size() : states) * analyses[i].size();
    nodes += states;
  }
  if (nodes > c.nodes.size()) c.nodes.resize(nodes);

  // Init feature sequences
  features.initialize_sentence(forms, analyses, c.features_cache);

  int window_stack[16]; vector window_heap;
  int* window = window_size <= 16 ? window_stack : (window_heap.resize(window_size), window_heap.data());
  typename FeatureSequences::dynamic_features dynamic;
  feature_sequences_score score;

  // Compute all nodes score
  int nodes_prev = -1, nodes_now = 0;
  for (unsigned i = 0; i < forms.size(); i++) {
    int nodes_next = nodes_now;

    for (int j = 0; j < window_size; j++) window[j] = -1;
    for (int tag = 0; tag < int(analyses[i].size()); tag++)
      for (int prev = nodes_prev; prev < nodes_now; prev++) {
        // Compute predecessors and number of unchanges
        int same_tags = window[0] == tag;
        window[0] = tag;
        for (int p = prev, n = 1; p >= 0 && n < window_size; p = c.nodes[p].prev, n++) {
          same_tags += same_tags == n && window[n] == c.nodes[p].tag;
          window[n] = c.nodes[p].tag;
        }

        // Compute dynamic elementary features and score
        features.compute_dynamic_features(i, tag, prev >= 0 ? &c.nodes[prev].dynamic : nullptr, dynamic, c.features_cache);
        score = (nodes_prev + 1 == nodes_now && analyses[i].size() == 1 ? 0 : features.score(i, window, same_tags, dynamic, c.features_cache)) +
            (prev >= 0 ? c.nodes[prev].score : 0);

        // Update existing node or create a new one
        if (same_tags >= decoding_order-1) {
          if (score <= c.nodes[nodes_next-1].score) continue;
          nodes_next--;
        }
        c.nodes[nodes_next].tag = tag;
        c.nodes[nodes_next].prev = prev;
        c.nodes[nodes_next].score = score;
        c.nodes[nodes_next++].dynamic = dynamic;
      }

    nodes_prev = nodes_now;
    nodes_now = nodes_next;
  }

  // Choose the best ending node
  int best = nodes_prev;
  for (int node = nodes_prev + 1; node < nodes_now; node++)
    if (c.nodes[node].score > c.nodes[best].score)
      best = node;

  for (int i = forms.size() - 1; i >= 0; i--, best = c.nodes[best].prev)
    tags[i] = c.nodes[best].tag;
}

} // namespace morphodita

/////////
// File: morphodita/tagger/conllu_elementary_features.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template
class conllu_elementary_features : public elementary_features {
 public:
  conllu_elementary_features();

  enum features_per_form { FORM, FOLLOWING_VERB_TAG, FOLLOWING_VERB_FORM, NUM, CAP, DASH, PREFIX1, PREFIX2, PREFIX3, PREFIX4, PREFIX5, PREFIX6, PREFIX7, PREFIX8, PREFIX9, SUFFIX1, SUFFIX2, SUFFIX3, SUFFIX4, SUFFIX5, SUFFIX6, SUFFIX7, SUFFIX8, SUFFIX9, PER_FORM_TOTAL };
  enum features_per_tag { TAG, TAG_UPOS, TAG_CASE, TAG_GENDER, TAG_NUMBER, TAG_NEGATIVE, TAG_PERSON, LEMMA, PER_TAG_TOTAL };
  enum features_dynamic { PREVIOUS_VERB_TAG, PREVIOUS_VERB_FORM, PREVIOUS_OR_CURRENT_VERB_TAG, PREVIOUS_OR_CURRENT_VERB_FORM, DYNAMIC_TOTAL };
  enum features_map { MAP_NONE = -1, MAP_FORM, MAP_PREFIX1, MAP_PREFIX2, MAP_PREFIX3, MAP_PREFIX4, MAP_PREFIX5, MAP_PREFIX6, MAP_PREFIX7, MAP_PREFIX8, MAP_PREFIX9, MAP_SUFFIX1, MAP_SUFFIX2, MAP_SUFFIX3, MAP_SUFFIX4, MAP_SUFFIX5, MAP_SUFFIX6, MAP_SUFFIX7, MAP_SUFFIX8, MAP_SUFFIX9, MAP_TAG, MAP_TAG_UPOS, MAP_TAG_CASE, MAP_TAG_GENDER, MAP_TAG_NUMBER, MAP_TAG_NEGATIVE, MAP_TAG_PERSON, MAP_LEMMA, MAP_TOTAL } ;

  struct per_form_features { elementary_feature_value values[PER_FORM_TOTAL]; };
  struct per_tag_features { elementary_feature_value values[PER_TAG_TOTAL]; };
  struct dynamic_features { elementary_feature_value values[DYNAMIC_TOTAL]; };

  static vector descriptions;

  void compute_features(const vector& forms, const vector>& analyses, vector& per_form, vector>& per_tag) const;
  inline void compute_dynamic_features(const tagged_lemma& tag, const per_form_features& per_form, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const;

  using elementary_features::maps;
};

typedef conllu_elementary_features persistent_conllu_elementary_features;

// Definitions
template
conllu_elementary_features::conllu_elementary_features() {
  maps.resize(MAP_TOTAL);
}

template
vector conllu_elementary_features::descriptions = {
  {"Form", PER_FORM, ANY_OFFSET, FORM, MAP_FORM},
  {"FollowingVerbTag", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_TAG, MAP_TAG},
  {"FollowingVerbForm", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_FORM, MAP_FORM},
  {"Num", PER_FORM, ONLY_CURRENT, NUM, MAP_NONE},
  {"Cap", PER_FORM, ONLY_CURRENT, CAP, MAP_NONE},
  {"Dash", PER_FORM, ONLY_CURRENT, DASH, MAP_NONE},
  {"Prefix1", PER_FORM, ONLY_CURRENT, PREFIX1, MAP_PREFIX1},
  {"Prefix2", PER_FORM, ONLY_CURRENT, PREFIX2, MAP_PREFIX2},
  {"Prefix3", PER_FORM, ONLY_CURRENT, PREFIX3, MAP_PREFIX3},
  {"Prefix4", PER_FORM, ONLY_CURRENT, PREFIX4, MAP_PREFIX4},
  {"Prefix5", PER_FORM, ONLY_CURRENT, PREFIX5, MAP_PREFIX5},
  {"Prefix6", PER_FORM, ONLY_CURRENT, PREFIX6, MAP_PREFIX6},
  {"Prefix7", PER_FORM, ONLY_CURRENT, PREFIX7, MAP_PREFIX7},
  {"Prefix8", PER_FORM, ONLY_CURRENT, PREFIX8, MAP_PREFIX8},
  {"Prefix9", PER_FORM, ONLY_CURRENT, PREFIX9, MAP_PREFIX9},
  {"Suffix1", PER_FORM, ONLY_CURRENT, SUFFIX1, MAP_SUFFIX1},
  {"Suffix2", PER_FORM, ONLY_CURRENT, SUFFIX2, MAP_SUFFIX2},
  {"Suffix3", PER_FORM, ONLY_CURRENT, SUFFIX3, MAP_SUFFIX3},
  {"Suffix4", PER_FORM, ONLY_CURRENT, SUFFIX4, MAP_SUFFIX4},
  {"Suffix5", PER_FORM, ONLY_CURRENT, SUFFIX5, MAP_SUFFIX5},
  {"Suffix6", PER_FORM, ONLY_CURRENT, SUFFIX6, MAP_SUFFIX6},
  {"Suffix7", PER_FORM, ONLY_CURRENT, SUFFIX7, MAP_SUFFIX7},
  {"Suffix8", PER_FORM, ONLY_CURRENT, SUFFIX8, MAP_SUFFIX8},
  {"Suffix9", PER_FORM, ONLY_CURRENT, SUFFIX9, MAP_SUFFIX9},

  {"Tag", PER_TAG, ANY_OFFSET, TAG, MAP_TAG},
  {"TagUPos", PER_TAG, ANY_OFFSET, TAG_UPOS, MAP_TAG_UPOS},
  {"TagCase", PER_TAG, ANY_OFFSET, TAG_CASE, MAP_TAG_CASE},
  {"TagGender", PER_TAG, ANY_OFFSET, TAG_GENDER, MAP_TAG_GENDER},
  {"TagNumber", PER_TAG, ANY_OFFSET, TAG_NUMBER, MAP_TAG_NUMBER},
  {"TagNegative", PER_TAG, ANY_OFFSET, TAG_NEGATIVE, MAP_TAG_NEGATIVE},
  {"TagPerson", PER_TAG, ANY_OFFSET, TAG_PERSON, MAP_TAG_PERSON},
  {"Lemma", PER_TAG, ANY_OFFSET, LEMMA, MAP_LEMMA},

  {"PreviousVerbTag", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_TAG, MAP_TAG},
  {"PreviousVerbForm", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_FORM, MAP_FORM},
};

template
void conllu_elementary_features::compute_features(const vector& forms, const vector>& analyses, vector& per_form, vector>& per_tag) const {
  using namespace unilib;

  // We process the sentence in reverse order, so that we can compute FollowingVerbTag and FollowingVerbLemma directly.
  elementary_feature_value following_verb_tag = elementary_feature_empty, following_verb_form = elementary_feature_empty;
  for (unsigned i = forms.size(); i--;) {
    int verb_candidate = -1;

    // Per_tag features and verb_candidate
    for (unsigned j = 0; j < analyses[i].size(); j++) {
      const string& tag = analyses[i][j].tag;
      const string& lemma = analyses[i][j].lemma;

      // Tag consists of three parts separated by tag[0] character
      // - first is TAG_UPOS,
      // - second is TAG_LPOS,
      // - then there is any number of | separated named fields in format Name=Value
      per_tag[i][j].values[TAG] = maps[MAP_TAG].value(tag.c_str(), tag.size());
      per_tag[i][j].values[TAG_UPOS] = per_tag[i][j].values[TAG_CASE] = per_tag[i][j].values[TAG_GENDER] = elementary_feature_empty;
      per_tag[i][j].values[TAG_NUMBER] = per_tag[i][j].values[TAG_NEGATIVE] = per_tag[i][j].values[TAG_PERSON] = elementary_feature_empty;
      per_tag[i][j].values[LEMMA] = j && analyses[i][j-1].lemma == lemma ? per_tag[i][j-1].values[LEMMA] :
          maps[MAP_LEMMA].value(lemma.c_str(), lemma.size());

      char separator = tag[0];
      size_t index = tag.find(separator, 1);
      if (index == string::npos) index = tag.size();
      per_tag[i][j].values[TAG_UPOS] = maps[MAP_TAG_UPOS].value(tag.c_str() + (index ? 1 : 0), index - (index ? 1 : 0));

      if (index < tag.size()) index++;
      if (index < tag.size()) index = tag.find(separator, index);
      if (index < tag.size()) index++;
      for (size_t length; index < tag.size(); index += length + 1) {
        length = tag.find('|', index);
        length = (length == string::npos ? tag.size() : length) - index;

        for (size_t equal_sign = 0; equal_sign + 1 < length; equal_sign++)
          if (tag[index + equal_sign] == '=') {
            int value = -1, map;
            switch (equal_sign) {
              case 4:
                if (tag.compare(index, equal_sign, "Case") == 0) value = TAG_CASE, map = MAP_TAG_CASE;
                break;
              case 6:
                if (tag.compare(index, equal_sign, "Gender") == 0) value = TAG_GENDER, map = MAP_TAG_GENDER;
                if (tag.compare(index, equal_sign, "Number") == 0) value = TAG_NUMBER, map = MAP_TAG_NUMBER;
                if (tag.compare(index, equal_sign, "Person") == 0) value = TAG_PERSON, map = MAP_TAG_PERSON;
                break;
              case 8:
                if (tag.compare(index, equal_sign, "Negative") == 0) value = TAG_NEGATIVE, map = MAP_TAG_NEGATIVE;
                break;
            }

            if (value >= 0)
              per_tag[i][j].values[value] = maps[map].value(tag.c_str() + index + equal_sign + 1, length - equal_sign - 1);
            break;
          }
      }

      if (tag.size() >= 2 && tag[1] == 'V') {
        int tag_compare;
        verb_candidate = verb_candidate < 0 || (tag_compare = tag.compare(analyses[i][verb_candidate].tag), tag_compare < 0) || (tag_compare == 0 && lemma < analyses[i][verb_candidate].lemma) ? j : verb_candidate;
      }
    }

    // Per_form features
    per_form[i].values[FORM] = maps[MAP_FORM].value(forms[i].str, forms[i].len);
    per_form[i].values[FOLLOWING_VERB_TAG] = following_verb_tag;
    per_form[i].values[FOLLOWING_VERB_FORM] = following_verb_form;

    // Update following_verb_{tag,lemma} _after_ filling FOLLOWING_VERB_{TAG,LEMMA}.
    if (verb_candidate >= 0) {
      following_verb_tag = per_tag[i][verb_candidate].values[TAG];
      following_verb_form = per_form[i].values[FORM];
    }

    // Ortographic per_form features if needed
    if (analyses[i].size() == 1) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_unknown;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = elementary_feature_unknown;
      per_form[i].values[PREFIX4] = per_form[i].values[PREFIX5] = per_form[i].values[PREFIX6] = elementary_feature_unknown;
      per_form[i].values[PREFIX7] = per_form[i].values[PREFIX8] = per_form[i].values[PREFIX9] = elementary_feature_unknown;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = elementary_feature_unknown;
      per_form[i].values[SUFFIX4] = per_form[i].values[SUFFIX5] = per_form[i].values[SUFFIX6] = elementary_feature_unknown;
      per_form[i].values[SUFFIX7] = per_form[i].values[SUFFIX8] = per_form[i].values[SUFFIX9] = elementary_feature_unknown;
    } else if (forms[i].len <= 0) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_empty + 1;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = elementary_feature_empty;
      per_form[i].values[PREFIX4] = per_form[i].values[PREFIX5] = per_form[i].values[PREFIX6] = elementary_feature_empty;
      per_form[i].values[PREFIX7] = per_form[i].values[PREFIX8] = per_form[i].values[PREFIX9] = elementary_feature_empty;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = elementary_feature_empty;
      per_form[i].values[SUFFIX4] = per_form[i].values[SUFFIX5] = per_form[i].values[SUFFIX6] = elementary_feature_empty;
      per_form[i].values[SUFFIX7] = per_form[i].values[SUFFIX8] = per_form[i].values[SUFFIX9] = elementary_feature_empty;
    } else {
      string_piece form = forms[i];
      const char* form_start = form.str;

      bool num = false, cap = false, dash = false;
      size_t indices[18] = {0, form.len, form.len, form.len, form.len, form.len, form.len, form.len, form.len, form.len, 0, 0, 0, 0, 0, 0, 0, 0}; // careful here regarding forms shorter than 9 characters
      int index = 0;
      while (form.len) {
        indices[(index++) % 18] = form.str - form_start;

        unicode::category_t cat = unicode::category(utf8::decode(form.str, form.len));
        num = num || cat & unicode::N;
        cap = cap || cat & unicode::Lut;
        dash = dash || cat & unicode::Pd;

        if (index == 10 || (!form.len && index < 10)) {
          per_form[i].values[PREFIX1] = maps[MAP_PREFIX1].value(form_start, indices[1]);
          per_form[i].values[PREFIX2] = maps[MAP_PREFIX2].value(form_start, indices[2]);
          per_form[i].values[PREFIX3] = maps[MAP_PREFIX3].value(form_start, indices[3]);
          per_form[i].values[PREFIX4] = maps[MAP_PREFIX4].value(form_start, indices[4]);
          per_form[i].values[PREFIX5] = maps[MAP_PREFIX5].value(form_start, indices[5]);
          per_form[i].values[PREFIX6] = maps[MAP_PREFIX6].value(form_start, indices[6]);
          per_form[i].values[PREFIX7] = maps[MAP_PREFIX7].value(form_start, indices[7]);
          per_form[i].values[PREFIX8] = maps[MAP_PREFIX8].value(form_start, indices[8]);
          per_form[i].values[PREFIX9] = maps[MAP_PREFIX9].value(form_start, indices[9]);
        }
      }
      per_form[i].values[SUFFIX1] = maps[MAP_SUFFIX1].value(form_start + indices[(index+18-1) % 18], form.str - form_start - indices[(index+18-1) % 18]);
      per_form[i].values[SUFFIX2] = maps[MAP_SUFFIX2].value(form_start + indices[(index+18-2) % 18], form.str - form_start - indices[(index+18-2) % 18]);
      per_form[i].values[SUFFIX3] = maps[MAP_SUFFIX3].value(form_start + indices[(index+18-3) % 18], form.str - form_start - indices[(index+18-3) % 18]);
      per_form[i].values[SUFFIX4] = maps[MAP_SUFFIX4].value(form_start + indices[(index+18-4) % 18], form.str - form_start - indices[(index+18-4) % 18]);
      per_form[i].values[SUFFIX5] = maps[MAP_SUFFIX5].value(form_start + indices[(index+18-5) % 18], form.str - form_start - indices[(index+18-5) % 18]);
      per_form[i].values[SUFFIX6] = maps[MAP_SUFFIX6].value(form_start + indices[(index+18-6) % 18], form.str - form_start - indices[(index+18-6) % 18]);
      per_form[i].values[SUFFIX7] = maps[MAP_SUFFIX7].value(form_start + indices[(index+18-7) % 18], form.str - form_start - indices[(index+18-7) % 18]);
      per_form[i].values[SUFFIX8] = maps[MAP_SUFFIX8].value(form_start + indices[(index+18-8) % 18], form.str - form_start - indices[(index+18-8) % 18]);
      per_form[i].values[SUFFIX9] = maps[MAP_SUFFIX9].value(form_start + indices[(index+18-9) % 18], form.str - form_start - indices[(index+18-9) % 18]);
      per_form[i].values[NUM] = elementary_feature_empty + 1 + num;
      per_form[i].values[CAP] = elementary_feature_empty + 1 + cap;
      per_form[i].values[DASH] = elementary_feature_empty + 1 + dash;
    }
  }
}

template
void conllu_elementary_features::compute_dynamic_features(const tagged_lemma& tag, const per_form_features& per_form, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const {
  if (prev_dynamic) {
    dynamic.values[PREVIOUS_VERB_TAG] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_TAG];
    dynamic.values[PREVIOUS_VERB_FORM] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_FORM];
  } else {
    dynamic.values[PREVIOUS_VERB_TAG] = elementary_feature_empty;
    dynamic.values[PREVIOUS_VERB_FORM] = elementary_feature_empty;
  }

  if (tag.tag.size() >= 2 && tag.tag[1] == 'V') {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = per_tag.values[TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_FORM] = per_form.values[FORM];
  } else {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = dynamic.values[PREVIOUS_VERB_TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_FORM] = dynamic.values[PREVIOUS_VERB_FORM];
  }
}

} // namespace morphodita

/////////
// File: morphodita/tagger/czech_elementary_features.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template
class czech_elementary_features : public elementary_features {
 public:
  czech_elementary_features();

  enum features_per_form { FORM, FOLLOWING_VERB_TAG, FOLLOWING_VERB_LEMMA, NUM, CAP, DASH, PREFIX1, PREFIX2, PREFIX3, PREFIX4, SUFFIX1, SUFFIX2, SUFFIX3, SUFFIX4, PER_FORM_TOTAL };
  enum features_per_tag { TAG, TAG3, TAG5, TAG25, LEMMA, PER_TAG_TOTAL };
  enum features_dynamic { PREVIOUS_VERB_TAG, PREVIOUS_VERB_LEMMA, PREVIOUS_OR_CURRENT_VERB_TAG, PREVIOUS_OR_CURRENT_VERB_LEMMA, DYNAMIC_TOTAL };
  enum features_map { MAP_NONE = -1, MAP_FORM, MAP_LEMMA, MAP_PREFIX1, MAP_PREFIX2, MAP_PREFIX3, MAP_PREFIX4, MAP_SUFFIX1, MAP_SUFFIX2, MAP_SUFFIX3, MAP_SUFFIX4, MAP_TAG, MAP_TAG3, MAP_TAG5, MAP_TAG25, MAP_TOTAL } ;

  struct per_form_features { elementary_feature_value values[PER_FORM_TOTAL]; };
  struct per_tag_features { elementary_feature_value values[PER_TAG_TOTAL]; };
  struct dynamic_features { elementary_feature_value values[DYNAMIC_TOTAL]; };

  static vector descriptions;

  void compute_features(const vector& forms, const vector>& analyses, vector& per_form, vector>& per_tag) const;
  inline void compute_dynamic_features(const tagged_lemma& tag, const per_form_features& per_form, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const;

  using elementary_features::maps;
};

typedef czech_elementary_features persistent_czech_elementary_features;

// Definitions
template
czech_elementary_features::czech_elementary_features() {
  maps.resize(MAP_TOTAL);
}

template
vector czech_elementary_features::descriptions = {
  {"Form", PER_FORM, ANY_OFFSET, FORM, MAP_FORM},
  {"FollowingVerbTag", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_TAG, MAP_TAG},
  {"FollowingVerbLemma", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_LEMMA, MAP_LEMMA },
  {"Num", PER_FORM, ONLY_CURRENT, NUM, MAP_NONE},
  {"Cap", PER_FORM, ONLY_CURRENT, CAP, MAP_NONE},
  {"Dash", PER_FORM, ONLY_CURRENT, DASH, MAP_NONE},
  {"Prefix1", PER_FORM, ONLY_CURRENT, PREFIX1, MAP_PREFIX1},
  {"Prefix2", PER_FORM, ONLY_CURRENT, PREFIX2, MAP_PREFIX2},
  {"Prefix3", PER_FORM, ONLY_CURRENT, PREFIX3, MAP_PREFIX3},
  {"Prefix4", PER_FORM, ONLY_CURRENT, PREFIX4, MAP_PREFIX4},
  {"Suffix1", PER_FORM, ONLY_CURRENT, SUFFIX1, MAP_SUFFIX1},
  {"Suffix2", PER_FORM, ONLY_CURRENT, SUFFIX2, MAP_SUFFIX2},
  {"Suffix3", PER_FORM, ONLY_CURRENT, SUFFIX3, MAP_SUFFIX3},
  {"Suffix4", PER_FORM, ONLY_CURRENT, SUFFIX4, MAP_SUFFIX4},

  {"Tag", PER_TAG, ANY_OFFSET, TAG, MAP_TAG},
  {"Tag3", PER_TAG, ANY_OFFSET, TAG3, MAP_TAG3},
  {"Tag5", PER_TAG, ANY_OFFSET, TAG5, MAP_TAG5},
  {"Tag25", PER_TAG, ANY_OFFSET, TAG25, MAP_TAG25},
  {"Lemma", PER_TAG, ANY_OFFSET, LEMMA, MAP_LEMMA},

  {"PreviousVerbTag", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_TAG, MAP_TAG},
  {"PreviousVerbLemma", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_LEMMA, MAP_LEMMA}
};

template
void czech_elementary_features::compute_features(const vector& forms, const vector>& analyses, vector& per_form, vector>& per_tag) const {
  using namespace unilib;

  // We process the sentence in reverse order, so that we can compute FollowingVerbTag and FollowingVerbLemma directly.
  elementary_feature_value following_verb_tag = elementary_feature_empty, following_verb_lemma = elementary_feature_empty;
  for (unsigned i = forms.size(); i--;) {
    int verb_candidate = -1;

    // Per_tag features and verb_candidate
    for (unsigned j = 0; j < analyses[i].size(); j++) {
      char tag25[2];
      per_tag[i][j].values[TAG] = maps[MAP_TAG].value(analyses[i][j].tag.c_str(), analyses[i][j].tag.size());
      per_tag[i][j].values[TAG3] = analyses[i][j].tag.size() >= 3 ? maps[MAP_TAG3].value(analyses[i][j].tag.c_str() + 2, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG5] = analyses[i][j].tag.size() >= 5 ? maps[MAP_TAG5].value(analyses[i][j].tag.c_str() + 4, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG25] = analyses[i][j].tag.size() >= 5 ? maps[MAP_TAG25].value((tag25[0] = analyses[i][j].tag[1], tag25[1] = analyses[i][j].tag[4], tag25), 2) : elementary_feature_empty;
      per_tag[i][j].values[LEMMA] = j && analyses[i][j-1].lemma == analyses[i][j].lemma ? per_tag[i][j-1].values[LEMMA] :
          maps[MAP_LEMMA].value(analyses[i][j].lemma.c_str(), analyses[i][j].lemma.size());

      if (analyses[i][j].tag[0] == 'V') {
        int tag_compare;
        verb_candidate = verb_candidate < 0 || (tag_compare = analyses[i][j].tag.compare(analyses[i][verb_candidate].tag), tag_compare < 0) || (tag_compare == 0 && analyses[i][j].lemma < analyses[i][verb_candidate].lemma) ? j : verb_candidate;
      }
    }

    // Per_form features
    per_form[i].values[FORM] = maps[MAP_FORM].value(forms[i].str, forms[i].len);
    per_form[i].values[FOLLOWING_VERB_TAG] = following_verb_tag;
    per_form[i].values[FOLLOWING_VERB_LEMMA] = following_verb_lemma;

    // Update following_verb_{tag,lemma} _after_ filling FOLLOWING_VERB_{TAG,LEMMA}.
    if (verb_candidate >= 0) {
      following_verb_tag = per_tag[i][verb_candidate].values[TAG];
      following_verb_lemma = per_tag[i][verb_candidate].values[LEMMA];
    }

    // Ortographic per_form features if needed
    if (analyses[i].size() == 1) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_unknown;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = per_form[i].values[PREFIX4] = elementary_feature_unknown;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = per_form[i].values[SUFFIX4] = elementary_feature_unknown;
    } else if (forms[i].len <= 0) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_empty + 1;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = per_form[i].values[PREFIX4] = elementary_feature_empty;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = per_form[i].values[SUFFIX4] = elementary_feature_empty;
    } else {
      string_piece form = forms[i];
      const char* form_start = form.str;

      bool num = false, cap = false, dash = false;
      size_t indices[8] = {0, form.len, form.len, form.len, form.len, 0, 0, 0}; // careful here regarding forms shorter than 4 characters
      int index = 0;
      while (form.len) {
        indices[(index++)&7] = form.str - form_start;

        unicode::category_t cat = unicode::category(utf8::decode(form.str, form.len));
        num = num || cat & unicode::N;
        cap = cap || cat & unicode::Lut;
        dash = dash || cat & unicode::Pd;

        if (index == 5 || (!form.len && index < 5)) {
          per_form[i].values[PREFIX1] = maps[MAP_PREFIX1].value(form_start, indices[1]);
          per_form[i].values[PREFIX2] = maps[MAP_PREFIX2].value(form_start, indices[2]);
          per_form[i].values[PREFIX3] = maps[MAP_PREFIX3].value(form_start, indices[3]);
          per_form[i].values[PREFIX4] = maps[MAP_PREFIX4].value(form_start, indices[4]);
        }
      }
      per_form[i].values[SUFFIX1] = maps[MAP_SUFFIX1].value(form_start + indices[(index-1)&7], form.str - form_start - indices[(index-1)&7]);
      per_form[i].values[SUFFIX2] = maps[MAP_SUFFIX2].value(form_start + indices[(index-2)&7], form.str - form_start - indices[(index-2)&7]);
      per_form[i].values[SUFFIX3] = maps[MAP_SUFFIX3].value(form_start + indices[(index-3)&7], form.str - form_start - indices[(index-3)&7]);
      per_form[i].values[SUFFIX4] = maps[MAP_SUFFIX4].value(form_start + indices[(index-4)&7], form.str - form_start - indices[(index-4)&7]);
      per_form[i].values[NUM] = elementary_feature_empty + 1 + num;
      per_form[i].values[CAP] = elementary_feature_empty + 1 + cap;
      per_form[i].values[DASH] = elementary_feature_empty + 1 + dash;
    }
  }
}

template
void czech_elementary_features::compute_dynamic_features(const tagged_lemma& tag, const per_form_features& /*per_form*/, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const {
  if (prev_dynamic) {
    dynamic.values[PREVIOUS_VERB_TAG] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_TAG];
    dynamic.values[PREVIOUS_VERB_LEMMA] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_LEMMA];
  } else {
    dynamic.values[PREVIOUS_VERB_TAG] = elementary_feature_empty;
    dynamic.values[PREVIOUS_VERB_LEMMA] = elementary_feature_empty;
  }

  if (tag.tag[0] == 'V') {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = per_tag.values[TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_LEMMA] = per_tag.values[LEMMA];
  } else {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = dynamic.values[PREVIOUS_VERB_TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_LEMMA] = dynamic.values[PREVIOUS_VERB_LEMMA];
  }
}

} // namespace morphodita

/////////
// File: morphodita/tagger/generic_elementary_features.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template
class generic_elementary_features : public elementary_features {
 public:
  generic_elementary_features();

  enum features_per_form { FORM, FOLLOWING_VERB_TAG, FOLLOWING_VERB_LEMMA, NUM, CAP, DASH, PREFIX1, PREFIX2, PREFIX3, PREFIX4, PREFIX5, PREFIX6, PREFIX7, PREFIX8, PREFIX9, SUFFIX1, SUFFIX2, SUFFIX3, SUFFIX4, SUFFIX5, SUFFIX6, SUFFIX7, SUFFIX8, SUFFIX9, PER_FORM_TOTAL };
  enum features_per_tag { TAG, TAG1, TAG2, TAG3, TAG4, TAG5, LEMMA, PER_TAG_TOTAL };
  enum features_dynamic { PREVIOUS_VERB_TAG, PREVIOUS_VERB_LEMMA, PREVIOUS_OR_CURRENT_VERB_TAG, PREVIOUS_OR_CURRENT_VERB_LEMMA, DYNAMIC_TOTAL };
  enum features_map { MAP_NONE = -1, MAP_FORM, MAP_PREFIX1, MAP_PREFIX2, MAP_PREFIX3, MAP_PREFIX4, MAP_PREFIX5, MAP_PREFIX6, MAP_PREFIX7, MAP_PREFIX8, MAP_PREFIX9, MAP_SUFFIX1, MAP_SUFFIX2, MAP_SUFFIX3, MAP_SUFFIX4, MAP_SUFFIX5, MAP_SUFFIX6, MAP_SUFFIX7, MAP_SUFFIX8, MAP_SUFFIX9, MAP_TAG, MAP_TAG1, MAP_TAG2, MAP_TAG3, MAP_TAG4, MAP_TAG5, MAP_LEMMA, MAP_TOTAL } ;

  struct per_form_features { elementary_feature_value values[PER_FORM_TOTAL]; };
  struct per_tag_features { elementary_feature_value values[PER_TAG_TOTAL]; };
  struct dynamic_features { elementary_feature_value values[DYNAMIC_TOTAL]; };

  static vector descriptions;

  void compute_features(const vector& forms, const vector>& analyses, vector& per_form, vector>& per_tag) const;
  inline void compute_dynamic_features(const tagged_lemma& tag, const per_form_features& per_form, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const;

  using elementary_features::maps;
};

typedef generic_elementary_features persistent_generic_elementary_features;

// Definitions
template
generic_elementary_features::generic_elementary_features() {
  maps.resize(MAP_TOTAL);
}

template
vector generic_elementary_features::descriptions = {
  {"Form", PER_FORM, ANY_OFFSET, FORM, MAP_FORM},
  {"FollowingVerbTag", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_TAG, MAP_TAG},
  {"FollowingVerbLemma", PER_FORM, ANY_OFFSET, FOLLOWING_VERB_LEMMA, MAP_LEMMA },
  {"Num", PER_FORM, ONLY_CURRENT, NUM, MAP_NONE},
  {"Cap", PER_FORM, ONLY_CURRENT, CAP, MAP_NONE},
  {"Dash", PER_FORM, ONLY_CURRENT, DASH, MAP_NONE},
  {"Prefix1", PER_FORM, ONLY_CURRENT, PREFIX1, MAP_PREFIX1},
  {"Prefix2", PER_FORM, ONLY_CURRENT, PREFIX2, MAP_PREFIX2},
  {"Prefix3", PER_FORM, ONLY_CURRENT, PREFIX3, MAP_PREFIX3},
  {"Prefix4", PER_FORM, ONLY_CURRENT, PREFIX4, MAP_PREFIX4},
  {"Prefix5", PER_FORM, ONLY_CURRENT, PREFIX5, MAP_PREFIX5},
  {"Prefix6", PER_FORM, ONLY_CURRENT, PREFIX6, MAP_PREFIX6},
  {"Prefix7", PER_FORM, ONLY_CURRENT, PREFIX7, MAP_PREFIX7},
  {"Prefix8", PER_FORM, ONLY_CURRENT, PREFIX8, MAP_PREFIX8},
  {"Prefix9", PER_FORM, ONLY_CURRENT, PREFIX9, MAP_PREFIX9},
  {"Suffix1", PER_FORM, ONLY_CURRENT, SUFFIX1, MAP_SUFFIX1},
  {"Suffix2", PER_FORM, ONLY_CURRENT, SUFFIX2, MAP_SUFFIX2},
  {"Suffix3", PER_FORM, ONLY_CURRENT, SUFFIX3, MAP_SUFFIX3},
  {"Suffix4", PER_FORM, ONLY_CURRENT, SUFFIX4, MAP_SUFFIX4},
  {"Suffix5", PER_FORM, ONLY_CURRENT, SUFFIX5, MAP_SUFFIX5},
  {"Suffix6", PER_FORM, ONLY_CURRENT, SUFFIX6, MAP_SUFFIX6},
  {"Suffix7", PER_FORM, ONLY_CURRENT, SUFFIX7, MAP_SUFFIX7},
  {"Suffix8", PER_FORM, ONLY_CURRENT, SUFFIX8, MAP_SUFFIX8},
  {"Suffix9", PER_FORM, ONLY_CURRENT, SUFFIX9, MAP_SUFFIX9},

  {"Tag", PER_TAG, ANY_OFFSET, TAG, MAP_TAG},
  {"Tag1", PER_TAG, ANY_OFFSET, TAG1, MAP_TAG1},
  {"Tag2", PER_TAG, ANY_OFFSET, TAG2, MAP_TAG2},
  {"Tag3", PER_TAG, ANY_OFFSET, TAG3, MAP_TAG3},
  {"Tag4", PER_TAG, ANY_OFFSET, TAG4, MAP_TAG4},
  {"Tag5", PER_TAG, ANY_OFFSET, TAG5, MAP_TAG5},
  {"Lemma", PER_TAG, ANY_OFFSET, LEMMA, MAP_LEMMA},

  {"PreviousVerbTag", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_TAG, MAP_TAG},
  {"PreviousVerbLemma", DYNAMIC, ANY_OFFSET, PREVIOUS_VERB_LEMMA, MAP_LEMMA}
};

template
void generic_elementary_features::compute_features(const vector& forms, const vector>& analyses, vector& per_form, vector>& per_tag) const {
  using namespace unilib;

  // We process the sentence in reverse order, so that we can compute FollowingVerbTag and FollowingVerbLemma directly.
  elementary_feature_value following_verb_tag = elementary_feature_empty, following_verb_lemma = elementary_feature_empty;
  for (unsigned i = forms.size(); i--;) {
    int verb_candidate = -1;

    // Per_tag features and verb_candidate
    for (unsigned j = 0; j < analyses[i].size(); j++) {
      per_tag[i][j].values[TAG] = maps[MAP_TAG].value(analyses[i][j].tag.c_str(), analyses[i][j].tag.size());
      per_tag[i][j].values[TAG1] = analyses[i][j].tag.size() >= 1 ? maps[MAP_TAG1].value(analyses[i][j].tag.c_str() + 0, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG2] = analyses[i][j].tag.size() >= 2 ? maps[MAP_TAG2].value(analyses[i][j].tag.c_str() + 1, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG3] = analyses[i][j].tag.size() >= 3 ? maps[MAP_TAG3].value(analyses[i][j].tag.c_str() + 2, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG4] = analyses[i][j].tag.size() >= 4 ? maps[MAP_TAG4].value(analyses[i][j].tag.c_str() + 3, 1) : elementary_feature_empty;
      per_tag[i][j].values[TAG5] = analyses[i][j].tag.size() >= 5 ? maps[MAP_TAG5].value(analyses[i][j].tag.c_str() + 4, 1) : elementary_feature_empty;
      per_tag[i][j].values[LEMMA] = j && analyses[i][j-1].lemma == analyses[i][j].lemma ? per_tag[i][j-1].values[LEMMA] :
          maps[MAP_LEMMA].value(analyses[i][j].lemma.c_str(), analyses[i][j].lemma.size());

      if (analyses[i][j].tag[0] == 'V') {
        int tag_compare;
        verb_candidate = verb_candidate < 0 || (tag_compare = analyses[i][j].tag.compare(analyses[i][verb_candidate].tag), tag_compare < 0) || (tag_compare == 0 && analyses[i][j].lemma < analyses[i][verb_candidate].lemma) ? j : verb_candidate;
      }
    }

    // Per_form features
    per_form[i].values[FORM] = maps[MAP_FORM].value(forms[i].str, forms[i].len);
    per_form[i].values[FOLLOWING_VERB_TAG] = following_verb_tag;
    per_form[i].values[FOLLOWING_VERB_LEMMA] = following_verb_lemma;

    // Update following_verb_{tag,lemma} _after_ filling FOLLOWING_VERB_{TAG,LEMMA}.
    if (verb_candidate >= 0) {
      following_verb_tag = per_tag[i][verb_candidate].values[TAG];
      following_verb_lemma = per_tag[i][verb_candidate].values[LEMMA];
    }

    // Ortographic per_form features if needed
    if (analyses[i].size() == 1) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_unknown;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = elementary_feature_unknown;
      per_form[i].values[PREFIX4] = per_form[i].values[PREFIX5] = per_form[i].values[PREFIX6] = elementary_feature_unknown;
      per_form[i].values[PREFIX7] = per_form[i].values[PREFIX8] = per_form[i].values[PREFIX9] = elementary_feature_unknown;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = elementary_feature_unknown;
      per_form[i].values[SUFFIX4] = per_form[i].values[SUFFIX5] = per_form[i].values[SUFFIX6] = elementary_feature_unknown;
      per_form[i].values[SUFFIX7] = per_form[i].values[SUFFIX8] = per_form[i].values[SUFFIX9] = elementary_feature_unknown;
    } else if (forms[i].len <= 0) {
      per_form[i].values[NUM] = per_form[i].values[CAP] = per_form[i].values[DASH] = elementary_feature_empty + 1;
      per_form[i].values[PREFIX1] = per_form[i].values[PREFIX2] = per_form[i].values[PREFIX3] = elementary_feature_empty;
      per_form[i].values[PREFIX4] = per_form[i].values[PREFIX5] = per_form[i].values[PREFIX6] = elementary_feature_empty;
      per_form[i].values[PREFIX7] = per_form[i].values[PREFIX8] = per_form[i].values[PREFIX9] = elementary_feature_empty;
      per_form[i].values[SUFFIX1] = per_form[i].values[SUFFIX2] = per_form[i].values[SUFFIX3] = elementary_feature_empty;
      per_form[i].values[SUFFIX4] = per_form[i].values[SUFFIX5] = per_form[i].values[SUFFIX6] = elementary_feature_empty;
      per_form[i].values[SUFFIX7] = per_form[i].values[SUFFIX8] = per_form[i].values[SUFFIX9] = elementary_feature_empty;
    } else {
      string_piece form = forms[i];
      const char* form_start = form.str;

      bool num = false, cap = false, dash = false;
      size_t indices[18] = {0, form.len, form.len, form.len, form.len, form.len, form.len, form.len, form.len, form.len, 0, 0, 0, 0, 0, 0, 0, 0}; // careful here regarding forms shorter than 9 characters
      int index = 0;
      while (form.len) {
        indices[(index++) % 18] = form.str - form_start;

        unicode::category_t cat = unicode::category(utf8::decode(form.str, form.len));
        num = num || cat & unicode::N;
        cap = cap || cat & unicode::Lut;
        dash = dash || cat & unicode::Pd;

        if (index == 10 || (!form.len && index < 10)) {
          per_form[i].values[PREFIX1] = maps[MAP_PREFIX1].value(form_start, indices[1]);
          per_form[i].values[PREFIX2] = maps[MAP_PREFIX2].value(form_start, indices[2]);
          per_form[i].values[PREFIX3] = maps[MAP_PREFIX3].value(form_start, indices[3]);
          per_form[i].values[PREFIX4] = maps[MAP_PREFIX4].value(form_start, indices[4]);
          per_form[i].values[PREFIX5] = maps[MAP_PREFIX5].value(form_start, indices[5]);
          per_form[i].values[PREFIX6] = maps[MAP_PREFIX6].value(form_start, indices[6]);
          per_form[i].values[PREFIX7] = maps[MAP_PREFIX7].value(form_start, indices[7]);
          per_form[i].values[PREFIX8] = maps[MAP_PREFIX8].value(form_start, indices[8]);
          per_form[i].values[PREFIX9] = maps[MAP_PREFIX9].value(form_start, indices[9]);
        }
      }
      per_form[i].values[SUFFIX1] = maps[MAP_SUFFIX1].value(form_start + indices[(index+18-1) % 18], form.str - form_start - indices[(index+18-1) % 18]);
      per_form[i].values[SUFFIX2] = maps[MAP_SUFFIX2].value(form_start + indices[(index+18-2) % 18], form.str - form_start - indices[(index+18-2) % 18]);
      per_form[i].values[SUFFIX3] = maps[MAP_SUFFIX3].value(form_start + indices[(index+18-3) % 18], form.str - form_start - indices[(index+18-3) % 18]);
      per_form[i].values[SUFFIX4] = maps[MAP_SUFFIX4].value(form_start + indices[(index+18-4) % 18], form.str - form_start - indices[(index+18-4) % 18]);
      per_form[i].values[SUFFIX5] = maps[MAP_SUFFIX5].value(form_start + indices[(index+18-5) % 18], form.str - form_start - indices[(index+18-5) % 18]);
      per_form[i].values[SUFFIX6] = maps[MAP_SUFFIX6].value(form_start + indices[(index+18-6) % 18], form.str - form_start - indices[(index+18-6) % 18]);
      per_form[i].values[SUFFIX7] = maps[MAP_SUFFIX7].value(form_start + indices[(index+18-7) % 18], form.str - form_start - indices[(index+18-7) % 18]);
      per_form[i].values[SUFFIX8] = maps[MAP_SUFFIX8].value(form_start + indices[(index+18-8) % 18], form.str - form_start - indices[(index+18-8) % 18]);
      per_form[i].values[SUFFIX9] = maps[MAP_SUFFIX9].value(form_start + indices[(index+18-9) % 18], form.str - form_start - indices[(index+18-9) % 18]);
      per_form[i].values[NUM] = elementary_feature_empty + 1 + num;
      per_form[i].values[CAP] = elementary_feature_empty + 1 + cap;
      per_form[i].values[DASH] = elementary_feature_empty + 1 + dash;
    }
  }
}

template
void generic_elementary_features::compute_dynamic_features(const tagged_lemma& tag, const per_form_features& /*per_form*/, const per_tag_features& per_tag, const dynamic_features* prev_dynamic, dynamic_features& dynamic) const {
  if (prev_dynamic) {
    dynamic.values[PREVIOUS_VERB_TAG] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_TAG];
    dynamic.values[PREVIOUS_VERB_LEMMA] = prev_dynamic->values[PREVIOUS_OR_CURRENT_VERB_LEMMA];
  } else {
    dynamic.values[PREVIOUS_VERB_TAG] = elementary_feature_empty;
    dynamic.values[PREVIOUS_VERB_LEMMA] = elementary_feature_empty;
  }

  if (tag.tag[0] == 'V') {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = per_tag.values[TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_LEMMA] = per_tag.values[LEMMA];
  } else {
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_TAG] = dynamic.values[PREVIOUS_VERB_TAG];
    dynamic.values[PREVIOUS_OR_CURRENT_VERB_LEMMA] = dynamic.values[PREVIOUS_VERB_LEMMA];
  }
}

} // namespace morphodita

/////////
// File: morphodita/tagger/perceptron_tagger.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
template
class perceptron_tagger : public tagger {
 public:
  perceptron_tagger(int decoding_order, int window_size);

  bool load(istream& is);
  virtual const morpho* get_morpho() const override;
  virtual void tag(const vector& forms, vector& tags, morpho::guesser_mode guesser = morpho::guesser_mode(-1)) const override;
  virtual void tag_analyzed(const vector& forms, const vector>& analyses, vector& tags) const override;

 private:
  int decoding_order, window_size;

  unique_ptr dict;
  bool use_guesser;
  FeatureSequences features;
  typedef viterbi viterbi_decoder;
  viterbi_decoder decoder;
  struct cache {
    vector forms;
    vector> analyses;
    vector tags;
    typename viterbi_decoder::cache decoder_cache;

    cache(const perceptron_tagger& self) : decoder_cache(self.decoder) {}
  };

  mutable threadsafe_stack caches;
};

// Definitions

template
perceptron_tagger::perceptron_tagger(int decoding_order, int window_size)
  : decoding_order(decoding_order), window_size(window_size), decoder(features, decoding_order, window_size) {}

template
bool perceptron_tagger::load(istream& is) {
  if (dict.reset(morpho::load(is)), !dict) return false;
  use_guesser = is.get();
  if (!features.load(is)) return false;
  return true;
}

template
const morpho* perceptron_tagger::get_morpho() const {
  return dict.get();
}

template
void perceptron_tagger::tag(const vector& forms, vector& tags, morpho::guesser_mode guesser) const {
  tags.clear();
  if (!dict) return;

  cache* c = caches.pop();
  if (!c) c = new cache(*this);

  c->forms.resize(forms.size());
  if (c->analyses.size() < forms.size()) c->analyses.resize(forms.size());
  for (unsigned i = 0; i < forms.size(); i++) {
    c->forms[i] = forms[i];
    c->forms[i].len = dict->raw_form_len(forms[i]);
    dict->analyze(forms[i], guesser >= 0 ? guesser : use_guesser ? morpho::GUESSER : morpho::NO_GUESSER, c->analyses[i]);
  }

  if (c->tags.size() < forms.size()) c->tags.resize(forms.size() * 2);
  decoder.tag(c->forms, c->analyses, c->decoder_cache, c->tags);

  for (unsigned i = 0; i < forms.size(); i++)
    tags.emplace_back(c->analyses[i][c->tags[i]]);

  caches.push(c);
}

template
void perceptron_tagger::tag_analyzed(const vector& forms, const vector>& analyses, vector& tags) const {
  tags.clear();

  cache* c = caches.pop();
  if (!c) c = new cache(*this);

  tags.resize(forms.size());
  decoder.tag(forms, analyses, c->decoder_cache, tags);

  caches.push(c);
}

} // namespace morphodita

/////////
// File: morphodita/tagger/tagger.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tagger* tagger::load(istream& is) {
  tagger_id id = tagger_id(is.get());
  switch (id) {
    case tagger_ids::CZECH2:
    case tagger_ids::CZECH2_3:
    case tagger_ids::CZECH3:
      {
        auto res = new_unique_ptr>>(tagger_ids::decoding_order(id), tagger_ids::window_size(id));
        if (res->load(is)) return res.release();
        break;
      }
    case tagger_ids::GENERIC2:
    case tagger_ids::GENERIC2_3:
    case tagger_ids::GENERIC3:
    case tagger_ids::GENERIC4:
      {
        auto res = new_unique_ptr>>(tagger_ids::decoding_order(id), tagger_ids::window_size(id));
        if (res->load(is)) return res.release();
        break;
      }
    case tagger_ids::CONLLU2:
    case tagger_ids::CONLLU2_3:
    case tagger_ids::CONLLU3:
      {
        auto res = new_unique_ptr>>(tagger_ids::decoding_order(id), tagger_ids::window_size(id));
        if (res->load(is)) return res.release();
        break;
      }
  }

  return nullptr;
}

tagger* tagger::load(const char* fname) {
  ifstream f(path_from_utf8(fname).c_str(), ifstream::binary);
  if (!f) return nullptr;

  return load(f);
}

tokenizer* tagger::new_tokenizer() const {
  auto morpho = get_morpho();
  return morpho ? morpho->new_tokenizer() : nullptr;
}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/identity_tagset_converter.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class identity_tagset_converter : public tagset_converter {
 public:
  virtual void convert(tagged_lemma& tagged_lemma) const override;
  virtual void convert_analyzed(vector& tagged_lemmas) const override;
  virtual void convert_generated(vector& forms) const override;
};

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/identity_tagset_converter.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void identity_tagset_converter::convert(tagged_lemma& /*tagged_lemma*/) const {}

void identity_tagset_converter::convert_analyzed(vector& /*tagged_lemmas*/) const {}

void identity_tagset_converter::convert_generated(vector& /*forms*/) const {}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/pdt_to_conll2009_tagset_converter.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class pdt_to_conll2009_tagset_converter : public tagset_converter {
 public:
  virtual void convert(tagged_lemma& tagged_lemma) const override;
  virtual void convert_analyzed(vector& tagged_lemmas) const override;
  virtual void convert_generated(vector& forms) const override;

 private:
  inline void convert_tag(const string& lemma, string& tag) const;
  inline bool convert_lemma(string& lemma) const;
};

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/pdt_to_conll2009_tagset_converter.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

static const char* names[15] = {"POS", "SubPOS", "Gen", "Num", "Cas", "PGe", "PNu", "Per", "Ten", "Gra", "Neg", "Voi", "", "", "Var"};

inline void pdt_to_conll2009_tagset_converter::convert_tag(const string& lemma, string& tag) const {
  char pdt_tag[16];
  strncpy(pdt_tag, tag.c_str(), 15);

  // Clear the tag
  tag.clear();

  // Fill FEAT of filled tag characters
  for (int i = 0; i < 15 && pdt_tag[i]; i++)
    if (pdt_tag[i] != '-') {
      if (!tag.empty()) tag.push_back('|');
      tag.append(names[i]);
      tag.push_back('=');
      tag.push_back(pdt_tag[i]);
    }

  // Try adding Sem FEAT
  for (unsigned i = 0; i + 2 < lemma.size(); i++)
    if (lemma[i] == '_' && lemma[i + 1] == ';') {
      if (!tag.empty()) tag.push_back('|');
      tag.append("Sem=");
      tag.push_back(lemma[i + 2]);
      break;
    }
}

inline bool pdt_to_conll2009_tagset_converter::convert_lemma(string& lemma) const {
  unsigned raw_lemma = czech_lemma_addinfo::raw_lemma_len(lemma);
  return raw_lemma < lemma.size() ? (lemma.resize(raw_lemma), true) : false;
}

void pdt_to_conll2009_tagset_converter::convert(tagged_lemma& tagged_lemma) const {
  convert_tag(tagged_lemma.lemma, tagged_lemma.tag);
  convert_lemma(tagged_lemma.lemma);
}

void pdt_to_conll2009_tagset_converter::convert_analyzed(vector& tagged_lemmas) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma : tagged_lemmas) {
    convert_tag(tagged_lemma.lemma, tagged_lemma.tag);
    lemma_changed |= convert_lemma(tagged_lemma.lemma);
  }

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || tagged_lemmas.size() < 2) return;

  tagset_converter_unique_analyzed(tagged_lemmas);
}

void pdt_to_conll2009_tagset_converter::convert_generated(vector& forms) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma_forms : forms) {
    for (auto&& tagged_form : tagged_lemma_forms.forms)
      convert_tag(tagged_lemma_forms.lemma, tagged_form.tag);
    lemma_changed |= convert_lemma(tagged_lemma_forms.lemma);
  }

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || forms.size() < 2) return;

  tagset_converter_unique_generated(forms);
}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/strip_lemma_comment_tagset_converter.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class strip_lemma_comment_tagset_converter : public tagset_converter {
 public:
  strip_lemma_comment_tagset_converter(const morpho& dictionary) : dictionary(dictionary) {}

  virtual void convert(tagged_lemma& tagged_lemma) const override;
  virtual void convert_analyzed(vector& tagged_lemmas) const override;
  virtual void convert_generated(vector& forms) const override;

 private:
  inline bool convert_lemma(string& lemma) const;
  const morpho& dictionary;
};

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/strip_lemma_comment_tagset_converter.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

inline bool strip_lemma_comment_tagset_converter::convert_lemma(string& lemma) const {
  unsigned lemma_id_len = dictionary.lemma_id_len(lemma);
  return lemma_id_len < lemma.size() ? (lemma.resize(lemma_id_len), true) : false;
}

void strip_lemma_comment_tagset_converter::convert(tagged_lemma& tagged_lemma) const {
  convert_lemma(tagged_lemma.lemma);
}

void strip_lemma_comment_tagset_converter::convert_analyzed(vector& tagged_lemmas) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma : tagged_lemmas)
    lemma_changed |= convert_lemma(tagged_lemma.lemma);

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || tagged_lemmas.size() < 2) return;

  tagset_converter_unique_analyzed(tagged_lemmas);
}

void strip_lemma_comment_tagset_converter::convert_generated(vector& forms) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma_forms : forms)
    lemma_changed |= convert_lemma(tagged_lemma_forms.lemma);

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || forms.size() < 2) return;

  tagset_converter_unique_generated(forms);
}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/strip_lemma_id_tagset_converter.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class strip_lemma_id_tagset_converter : public tagset_converter {
 public:
  strip_lemma_id_tagset_converter(const morpho& dictionary) : dictionary(dictionary) {}

  virtual void convert(tagged_lemma& tagged_lemma) const override;
  virtual void convert_analyzed(vector& tagged_lemmas) const override;
  virtual void convert_generated(vector& forms) const override;

 private:
  inline bool convert_lemma(string& lemma) const;
  const morpho& dictionary;
};

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/strip_lemma_id_tagset_converter.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

inline bool strip_lemma_id_tagset_converter::convert_lemma(string& lemma) const {
  unsigned raw_lemma_len = dictionary.raw_lemma_len(lemma);
  return raw_lemma_len < lemma.size() ? (lemma.resize(raw_lemma_len), true) : false;
}

void strip_lemma_id_tagset_converter::convert(tagged_lemma& tagged_lemma) const {
  convert_lemma(tagged_lemma.lemma);
}

void strip_lemma_id_tagset_converter::convert_analyzed(vector& tagged_lemmas) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma : tagged_lemmas)
    lemma_changed |= convert_lemma(tagged_lemma.lemma);

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || tagged_lemmas.size() < 2) return;

  tagset_converter_unique_analyzed(tagged_lemmas);
}

void strip_lemma_id_tagset_converter::convert_generated(vector& forms) const {
  bool lemma_changed = false;

  for (auto&& tagged_lemma_forms : forms)
    lemma_changed |= convert_lemma(tagged_lemma_forms.lemma);

  // If no lemma was changed or there is 1 analysis, no duplicates could be created.
  if (!lemma_changed || forms.size() < 2) return;

  tagset_converter_unique_generated(forms);
}

} // namespace morphodita

/////////
// File: morphodita/tagset_converter/tagset_converter.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tagset_converter* tagset_converter::new_identity_converter() {
  return new identity_tagset_converter();
}

tagset_converter* tagset_converter::new_pdt_to_conll2009_converter() {
  return new pdt_to_conll2009_tagset_converter();
}

tagset_converter* tagset_converter::new_strip_lemma_comment_converter(const morpho& dictionary) {
  return new strip_lemma_comment_tagset_converter(dictionary);
}

tagset_converter* tagset_converter::new_strip_lemma_id_converter(const morpho& dictionary) {
  return new strip_lemma_id_tagset_converter(dictionary);
}

tagset_converter* new_tagset_converter(const string& name, const morpho& dictionary) {
  if (name == "pdt_to_conll2009") return tagset_converter::new_pdt_to_conll2009_converter();
  if (name == "strip_lemma_comment") return tagset_converter::new_strip_lemma_comment_converter(dictionary);
  if (name == "strip_lemma_id") return tagset_converter::new_strip_lemma_id_converter(dictionary);
  return nullptr;
}

void tagset_converter_unique_analyzed(vector& tagged_lemmas) {
  // Remove possible lemma-tag pair duplicates
  struct tagged_lemma_comparator {
    inline static bool eq(const tagged_lemma& a, const tagged_lemma& b) { return a.lemma == b.lemma && a.tag == b.tag; }
    inline static bool lt(const tagged_lemma& a, const tagged_lemma& b) { int lemma_compare = a.lemma.compare(b.lemma); return lemma_compare < 0 || (lemma_compare == 0 && a.tag < b.tag); }
  };

  sort(tagged_lemmas.begin(), tagged_lemmas.end(), tagged_lemma_comparator::lt);
  tagged_lemmas.resize(unique(tagged_lemmas.begin(), tagged_lemmas.end(), tagged_lemma_comparator::eq) - tagged_lemmas.begin());
}

void tagset_converter_unique_generated(vector& forms) {
  // Regroup and if needed remove duplicate form-tag pairs for each lemma
  for (unsigned i = 0; i < forms.size(); i++) {
    bool any_merged = false;
    for (unsigned j = forms.size() - 1; j > i; j--)
      if (forms[j].lemma == forms[i].lemma) {
        // Same lemma was found. Merge form-tag pairs
        for (auto&& tagged_form : forms[j].forms)
          forms[i].forms.emplace_back(move(tagged_form));

        // Remove lemma j by moving it to end and deleting
        if (j < forms.size() - 1) {
          forms[j].lemma.swap(forms[forms.size() - 1].lemma);
          forms[j].forms.swap(forms[forms.size() - 1].forms);
        }
        forms.pop_back();
        any_merged = true;
      }

    if (any_merged && forms[i].forms.size() > 1) {
      // Remove duplicate form-tag pairs
      struct tagged_form_comparator {
        inline static bool eq(const tagged_form& a, const tagged_form& b) { return a.tag == b.tag && a.form == b.form; }
        inline static bool lt(const tagged_form& a, const tagged_form& b) { int tag_compare = a.tag.compare(b.tag); return tag_compare < 0 || (tag_compare == 0 && a.form < b.form); }
      };

      sort(forms[i].forms.begin(), forms[i].forms.end(), tagged_form_comparator::lt);
      forms[i].forms.resize(unique(forms[i].forms.begin(), forms[i].forms.end(), tagged_form_comparator::eq) - forms[i].forms.begin());
    }
  }
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/czech_tokenizer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

static const char _czech_tokenizer_cond_offsets[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	2, 2, 2, 2, 2, 2, 2, 2,
	2, 2, 2, 2, 2, 2, 2
};

static const char _czech_tokenizer_cond_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 2,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const short _czech_tokenizer_cond_keys[] = {
	43u, 43u, 45u, 45u, 0
};

static const char _czech_tokenizer_cond_spaces[] = {
	1, 0, 0
};

static const unsigned char _czech_tokenizer_key_offsets[] = {
	0, 0, 17, 29, 43, 46, 51, 54,
	89, 94, 98, 101, 105, 110, 111, 116,
	117, 122, 136, 143, 148, 151, 163
};

static const short _czech_tokenizer_trans_keys[] = {
	13u, 32u, 34u, 40u, 91u, 96u, 123u, 129u,
	133u, 135u, 147u, 150u, 162u, 9u, 10u, 65u,
	90u, 34u, 40u, 91u, 96u, 123u, 129u, 133u,
	135u, 150u, 162u, 65u, 90u, 13u, 32u, 34u,
	39u, 41u, 59u, 93u, 125u, 139u, 141u, 147u,
	161u, 9u, 10u, 159u, 48u, 57u, 43u, 45u,
	159u, 48u, 57u, 159u, 48u, 57u, 9u, 10u,
	13u, 32u, 33u, 44u, 46u, 47u, 63u, 129u,
	131u, 135u, 142u, 147u, 157u, 159u, 160u, 301u,
	557u, 811u, 1067u, 0u, 42u, 48u, 57u, 58u,
	64u, 65u, 90u, 91u, 96u, 97u, 122u, 123u,
	255u, 9u, 10u, 13u, 32u, 147u, 9u, 13u,
	32u, 147u, 9u, 32u, 147u, 9u, 10u, 32u,
	147u, 9u, 10u, 13u, 32u, 147u, 13u, 9u,
	10u, 13u, 32u, 147u, 10u, 9u, 10u, 13u,
	32u, 147u, 13u, 32u, 34u, 39u, 41u, 59u,
	93u, 125u, 139u, 141u, 147u, 161u, 9u, 10u,
	44u, 46u, 69u, 101u, 159u, 48u, 57u, 69u,
	101u, 159u, 48u, 57u, 159u, 48u, 57u, 129u,
	131u, 135u, 151u, 155u, 157u, 65u, 90u, 97u,
	122u, 142u, 143u, 159u, 48u, 57u, 0
};

static const char _czech_tokenizer_single_lengths[] = {
	0, 13, 10, 12, 1, 3, 1, 21,
	5, 4, 3, 4, 5, 1, 5, 1,
	5, 12, 5, 3, 1, 6, 1
};

static const char _czech_tokenizer_range_lengths[] = {
	0, 2, 1, 1, 1, 1, 1, 7,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 1, 1, 1, 1, 3, 1
};

static const unsigned char _czech_tokenizer_index_offsets[] = {
	0, 0, 16, 28, 42, 45, 50, 53,
	82, 88, 93, 97, 102, 108, 110, 116,
	118, 124, 138, 145, 150, 153, 163
};

static const char _czech_tokenizer_indicies[] = {
	1, 1, 2, 2, 2, 2, 2, 3,
	2, 3, 1, 2, 2, 1, 3, 0,
	2, 2, 2, 2, 2, 3, 2, 3,
	2, 2, 3, 0, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 0, 6, 6, 0, 7, 7, 8,
	8, 0, 8, 8, 0, 10, 11, 12,
	10, 13, 9, 13, 9, 13, 16, 16,
	16, 16, 10, 16, 15, 13, 9, 17,
	9, 17, 9, 15, 9, 16, 9, 16,
	9, 14, 10, 19, 20, 10, 10, 18,
	10, 21, 10, 10, 18, 10, 10, 10,
	18, 10, 21, 10, 10, 18, 10, 22,
	23, 10, 10, 18, 25, 24, 10, 22,
	26, 10, 10, 18, 25, 24, 10, 23,
	26, 10, 10, 18, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 27, 28, 28, 29, 29, 15, 15,
	27, 29, 29, 6, 6, 27, 8, 8,
	27, 16, 16, 16, 16, 16, 16, 16,
	16, 16, 27, 15, 15, 27, 0
};

static const char _czech_tokenizer_trans_targs[] = {
	7, 1, 2, 7, 1, 3, 19, 6,
	20, 7, 8, 12, 16, 17, 0, 18,
	21, 22, 7, 9, 11, 10, 13, 14,
	7, 7, 15, 7, 4, 5
};

static const char _czech_tokenizer_trans_actions[] = {
	1, 0, 0, 2, 3, 0, 4, 0,
	0, 7, 0, 0, 0, 4, 0, 4,
	0, 0, 8, 0, 0, 0, 0, 0,
	9, 10, 0, 11, 0, 0
};

static const char _czech_tokenizer_to_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 5,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const char _czech_tokenizer_from_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 6,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const unsigned char _czech_tokenizer_eof_trans[] = {
	0, 1, 1, 1, 1, 1, 1, 0,
	19, 19, 19, 19, 19, 25, 19, 25,
	19, 28, 28, 28, 28, 28, 28
};

static const int czech_tokenizer_start = 7;

// The list of lower cased words that when preceding eos do not end sentence.
// Note: because of VS, we cannot list the abbreviations directly in UTF-8,
// because the compilation of utf-8 encoded sources fail on some locales
// (e.g., Japanese).
// perl -CS -ple 'use Encode;s/([^[:ascii:]])/join("", map {sprintf "\\%o", ord($_)} split(m@@, encode("utf-8", $1)))/ge'
// perl -CS -ple 'use Encode;s/\\([0-7]{3})\\([0-7]{3})/decode("utf-8", chr(oct($1)).chr(oct($2)))/ge'
const unordered_set czech_tokenizer::abbreviations_czech = {
  // Titles
  "prof", "csc", "drsc", "doc", "phd", "ph", "d",
  "judr", "mddr", "mudr", "mvdr", "paeddr", "paedr", "phdr", "rndr", "rsdr", "dr",
  "ing", "arch", "mgr", "bc", "mag", "mba", "bca", "mga",
  "gen", "plk", "pplk", "npor", "por", "ppor", "kpt", "mjr", "sgt", "pls", "p", "s",
  "p", "p\303\255", "fa", "fy", "mr", "mrs", "ms", "miss", "tr", "sv",
  // Geographic names
  "angl", "fr", "\304\215es", "ces", "\304\215s", "cs", "slov", "n\304\233m", "nem", "it", "pol", "ma\304\217", "mad", "rus",
  "sev", "v\303\275ch", "vych", "ji\305\276", "jiz", "z\303\241p", "zap",
  // Common abbrevs
  "adr", "\304\215", "c", "eg", "ev", "g", "hod", "j", "kr", "m", "max", "min", "mj", "nap\305\231", "napr",
  "okr", "pop\305\231", "popr", "pozn", "r", "\305\231", "red", "rep", "resp", "srov", "st", "st\305\231", "str",
  "sv", "tel", "tj", "tzv", "\303\272", "u", "uh", "ul", "um", "zl", "zn",
};

const unordered_set czech_tokenizer::abbreviations_slovak = {
  // Titles
  "prof", "csc", "drsc", "doc", "phd", "ph", "d",
  "judr", "mddr", "mudr", "mvdr", "paeddr", "paedr", "phdr", "rndr", "rsdr", "dr",
  "ing", "arch", "mgr", "bc", "mag", "mba", "bca", "mga",
  "gen", "plk", "pplk", "npor", "por", "ppor", "kpt", "mjr", "sgt", "pls", "p", "s",
  "p", "p\303\255", "fa", "fy", "mr", "mrs", "ms", "miss", "tr", "sv",
  // Geographic names
  "angl", "fr", "\304\215es", "ces", "\304\215s", "cs", "slov", "nem", "it", "po\304\276", "pol", "ma\304\217", "mad",
  "rus", "sev", "v\303\275ch", "vych", "ju\305\276", "juz", "z\303\241p", "zap",
  // Common abbrevs
  "adr", "\304\215", "c", "eg", "ev", "g", "hod", "j", "kr", "m", "max", "min", "mj", "napr",
  "okr", "popr", "pozn", "r", "red", "rep", "resp", "srov", "st", "str",
  "sv", "tel", "tj", "tzv", "\303\272", "u", "uh", "ul", "um", "zl", "zn",
};

czech_tokenizer::czech_tokenizer(tokenizer_language language, unsigned version, const morpho* m)
  : ragel_tokenizer(version <= 1 ? 1 : 2), m(m) {
  switch (language) {
    case CZECH:
      abbreviations = &abbreviations_czech;
      break;
    case SLOVAK:
      abbreviations = &abbreviations_slovak;
      break;
  }
}

void czech_tokenizer::merge_hyphenated(vector& tokens) {
  using namespace unilib;

  if (!m) return;
  if (tokens.empty() || chars[tokens.back().start].cat & ~unicode::L) return;

  unsigned matched_hyphens = 0;
  for (unsigned hyphens = 1; hyphens <= 2; hyphens++) {
    // Are the tokens a sequence of 'hyphens' hyphenated tokens?
    if (tokens.size() < 2*hyphens + 1) break;
    unsigned first_hyphen = tokens.size() - 2*hyphens;
    if (tokens[first_hyphen].length != 1 || chars[tokens[first_hyphen].start].cat & ~unicode::P ||
        tokens[first_hyphen].start + tokens[first_hyphen].length != tokens[first_hyphen + 1].start ||
        tokens[first_hyphen-1].start + tokens[first_hyphen-1].length != tokens[first_hyphen].start ||
        chars[tokens[first_hyphen-1].start].cat & ~unicode::L)
      break;

    if (m->analyze(string_piece(chars[tokens[first_hyphen-1].start].str, chars[tokens.back().start + tokens.back().length].str - chars[tokens[first_hyphen-1].start].str), morpho::NO_GUESSER, lemmas) >= 0)
      matched_hyphens = hyphens;
  }

  if (matched_hyphens) {
    unsigned first = tokens.size() - 2*matched_hyphens - 1;
    tokens[first].length = tokens.back().start + tokens.back().length - tokens[first].start;
    tokens.resize(first + 1);
  }
}

bool czech_tokenizer::next_sentence(vector& tokens) {
  using namespace unilib;

  int cs, act;
  size_t ts, te;
  size_t whitespace = 0; // Suppress "may be uninitialized" warning

  while (tokenize_url_email(tokens))
    if (emergency_sentence_split(tokens))
      return true;

	{
	cs = czech_tokenizer_start;
	ts = 0;
	te = 0;
	act = 0;
	}

	{
	int _klen;
	const short *_keys;
	int _trans;
	short _widec;

	if ( ( current) == ( (chars.size() - 1)) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	switch ( _czech_tokenizer_from_state_actions[cs] ) {
	case 6:
	{ts = ( current);}
	break;
	}

	_widec = ( ragel_char(chars[current]));
	_klen = _czech_tokenizer_cond_lengths[cs];
	_keys = _czech_tokenizer_cond_keys + (_czech_tokenizer_cond_offsets[cs]*2);
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				switch ( _czech_tokenizer_cond_spaces[_czech_tokenizer_cond_offsets[cs] + ((_mid - _keys)>>1)] ) {
	case 0: {
		_widec = (short)(256u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || (chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N | unicode::Pd))  ) _widec += 256;
		break;
	}
	case 1: {
		_widec = (short)(768u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || ((chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N)) && chars[current-1].chr != '+')  ) _widec += 256;
		break;
	}
				}
				break;
			}
		}
	}

	_keys = _czech_tokenizer_trans_keys + _czech_tokenizer_key_offsets[cs];
	_trans = _czech_tokenizer_index_offsets[cs];

	_klen = _czech_tokenizer_single_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( _widec < *_mid )
				_upper = _mid - 1;
			else if ( _widec > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _czech_tokenizer_range_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _czech_tokenizer_indicies[_trans];
_eof_trans:
	cs = _czech_tokenizer_trans_targs[_trans];

	if ( _czech_tokenizer_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _czech_tokenizer_trans_actions[_trans] ) {
	case 3:
	{ whitespace = current; }
	break;
	case 4:
	{te = ( current)+1;}
	break;
	case 7:
	{te = ( current)+1;{ tokens.emplace_back(ts, te - ts);
          merge_hyphenated(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 2:
	{te = ( current)+1;{
          bool eos = is_eos(tokens, chars[ts].chr, abbreviations);
          for (current = ts; current < whitespace; current++)
            tokens.emplace_back(current, 1);
          {( current) = (( whitespace))-1;}
          if (eos) {( current)++; goto _out; }
        }}
	break;
	case 10:
	{te = ( current)+1;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 11:
	{te = ( current);( current)--;{ tokens.emplace_back(ts, te - ts);
          merge_hyphenated(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 8:
	{te = ( current);( current)--;{
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 9:
	{te = ( current);( current)--;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 1:
	{{( current) = ((te))-1;}{ tokens.emplace_back(ts, te - ts);
          merge_hyphenated(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	}

_again:
	switch ( _czech_tokenizer_to_state_actions[cs] ) {
	case 5:
	{ts = 0;}
	break;
	}

	if ( cs == 0 )
		goto _out;
	if ( ++( current) != ( (chars.size() - 1)) )
		goto _resume;
	_test_eof: {}
	if ( ( current) == ( (chars.size() - 1)) )
	{
	if ( _czech_tokenizer_eof_trans[cs] > 0 ) {
		_trans = _czech_tokenizer_eof_trans[cs] - 1;
		goto _eof_trans;
	}
	}

	_out: {}
	}

  (void)act; // Suppress unused variable warning

  return !tokens.empty();
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/czech_tokenizer_factory.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2019 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class czech_tokenizer_factory : public tokenizer_factory {
 public:
  // Construct a new tokenizer instance.
  virtual tokenizer* new_tokenizer(const morpho* m) const override;

  bool load(istream& is);
 private:
  czech_tokenizer::tokenizer_language language;
  unsigned version;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/czech_tokenizer_factory.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2019 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tokenizer* czech_tokenizer_factory::new_tokenizer(const morpho* m) const {
  return new czech_tokenizer(language, version, m);
}

bool czech_tokenizer_factory::load(istream& is) {
  language = czech_tokenizer::tokenizer_language(is.get());
  version = is.get();

  return bool(is) && (language == czech_tokenizer::CZECH || language == czech_tokenizer::SLOVAK);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/czech_tokenizer_factory_encoder.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2019 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class czech_tokenizer_factory_encoder {
 public:
  static void encode(czech_tokenizer::tokenizer_language language, unsigned version, ostream& os);
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/czech_tokenizer_factory_encoder.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2019 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void czech_tokenizer_factory_encoder::encode(czech_tokenizer::tokenizer_language language, unsigned version, ostream& os) {
  os.put(language);
  os.put(version);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/english_tokenizer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// The list of lowercased words that when preceding eos do not end sentence.
const unordered_set english_tokenizer::abbreviations = {
  // Titles
  "adj", "adm", "adv", "assoc", "asst", "bart", "bldg", "brig", "bros", "capt",
  "cmdr", "col", "comdr", "con", "corp", "cpl", "d", "dr", "dr", "drs", "ens",
  "gen", "gov", "hon", "hosp", "hr", "insp", "lt", "mm", "mr", "mrs", "ms",
  "maj", "messrs", "mlle", "mme", "mr", "mrs", "ms", "msgr", "op", "ord",
  "pfc", "ph", "phd", "prof", "pvt", "rep", "reps", "res", "rev", "rt", "sen",
  "sens", "sfc", "sgt", "sr", "st", "supt", "surg", "univ",
  // Common abbrevs
  "addr", "approx", "apr", "aug", "calif", "co", "corp", "dec", "def", "e",
  "e.g", "eg", "feb", "fla", "ft", "gen", "gov", "hrs", "i.", "i.e", "ie",
  "inc", "jan", "jr", "ltd", "mar", "max", "min", "mph", "mt", "n", "nov",
  "oct", "ont", "pa", "pres", "rep", "rev", "s", "sec", "sen", "sep", "sept",
  "sgt", "sr", "tel", "un", "univ", "v", "va", "vs", "w", "yrs",
};

static const char _english_tokenizer_split_token_key_offsets[] = {
	0, 0, 16, 20, 22, 26, 28, 30,
	32, 34, 36, 44, 46, 50, 52, 54,
	56, 58, 60, 62, 64, 66, 68, 72,
	74, 76, 78, 80, 82, 82
};

static const unsigned char _english_tokenizer_split_token_trans_keys[] = {
	65u, 68u, 69u, 76u, 77u, 78u, 83u, 84u,
	97u, 100u, 101u, 108u, 109u, 110u, 115u, 116u,
	78u, 84u, 110u, 116u, 78u, 110u, 65u, 79u,
	97u, 111u, 87u, 119u, 71u, 103u, 84u, 116u,
	79u, 111u, 39u, 161u, 77u, 82u, 86u, 89u,
	109u, 114u, 118u, 121u, 77u, 109u, 69u, 73u,
	101u, 105u, 76u, 108u, 39u, 161u, 68u, 100u,
	76u, 108u, 39u, 161u, 69u, 101u, 82u, 114u,
	79u, 111u, 77u, 109u, 39u, 79u, 111u, 161u,
	78u, 110u, 78u, 110u, 78u, 110u, 65u, 97u,
	67u, 99u, 0
};

static const char _english_tokenizer_split_token_single_lengths[] = {
	0, 16, 4, 2, 4, 2, 2, 2,
	2, 2, 8, 2, 4, 2, 2, 2,
	2, 2, 2, 2, 2, 2, 4, 2,
	2, 2, 2, 2, 0, 0
};

static const char _english_tokenizer_split_token_range_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0
};

static const unsigned char _english_tokenizer_split_token_index_offsets[] = {
	0, 0, 17, 22, 25, 30, 33, 36,
	39, 42, 45, 54, 57, 62, 65, 68,
	71, 74, 77, 80, 83, 86, 89, 94,
	97, 100, 103, 106, 109, 110
};

static const char _english_tokenizer_split_token_indicies[] = {
	0, 2, 3, 4, 2, 5, 2, 6,
	0, 2, 3, 4, 2, 5, 2, 6,
	1, 7, 8, 7, 8, 1, 9, 9,
	1, 10, 11, 10, 11, 1, 12, 12,
	1, 12, 12, 1, 13, 13, 1, 11,
	11, 1, 14, 14, 1, 15, 2, 2,
	16, 15, 2, 2, 16, 1, 17, 17,
	1, 18, 11, 18, 11, 1, 12, 12,
	1, 19, 19, 1, 12, 12, 1, 2,
	2, 1, 20, 20, 1, 21, 21, 1,
	22, 22, 1, 23, 23, 1, 12, 12,
	1, 24, 25, 25, 24, 1, 14, 14,
	1, 26, 26, 1, 27, 27, 1, 28,
	28, 1, 12, 12, 1, 1, 1, 0
};

static const char _english_tokenizer_split_token_trans_targs[] = {
	2, 0, 9, 10, 16, 17, 22, 3,
	7, 4, 5, 6, 28, 8, 29, 11,
	14, 12, 13, 15, 18, 19, 20, 21,
	23, 24, 25, 26, 27
};

static const char _english_tokenizer_split_token_trans_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 1,
	1, 0, 0, 0, 0, 0, 2, 1,
	1, 0, 0, 0, 1, 0, 0, 0,
	0, 0, 1, 0, 0
};

static const char _english_tokenizer_split_token_eof_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 3, 0
};

static const int english_tokenizer_split_token_start = 1;

void english_tokenizer::split_token(vector& tokens) {
  if (tokens.empty() || chars[tokens.back().start].cat & ~unilib::unicode::L) return;

  size_t index = tokens.back().start, end = index + tokens.back().length;
  int cs;
  size_t split_mark = 0, split_len = 0;

	{
	cs = english_tokenizer_split_token_start;
	}

	{
	int _klen;
	const unsigned char *_keys;
	int _trans;

	if ( ( index) == ( end) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_keys = _english_tokenizer_split_token_trans_keys + _english_tokenizer_split_token_key_offsets[cs];
	_trans = _english_tokenizer_split_token_index_offsets[cs];

	_klen = _english_tokenizer_split_token_single_lengths[cs];
	if ( _klen > 0 ) {
		const unsigned char *_lower = _keys;
		const unsigned char *_mid;
		const unsigned char *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( ( ragel_char(chars[tokens.back().start + end - index - 1])) < *_mid )
				_upper = _mid - 1;
			else if ( ( ragel_char(chars[tokens.back().start + end - index - 1])) > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _english_tokenizer_split_token_range_lengths[cs];
	if ( _klen > 0 ) {
		const unsigned char *_lower = _keys;
		const unsigned char *_mid;
		const unsigned char *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( ( ragel_char(chars[tokens.back().start + end - index - 1])) < _mid[0] )
				_upper = _mid - 2;
			else if ( ( ragel_char(chars[tokens.back().start + end - index - 1])) > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _english_tokenizer_split_token_indicies[_trans];
	cs = _english_tokenizer_split_token_trans_targs[_trans];

	if ( _english_tokenizer_split_token_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _english_tokenizer_split_token_trans_actions[_trans] ) {
	case 1:
	{ split_mark = index - tokens.back().start + 1; }
	break;
	case 2:
	{ split_mark = index - tokens.back().start + 1; }
	{ split_len = split_mark; {( index)++; goto _out; } }
	break;
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++( index) != ( end) )
		goto _resume;
	_test_eof: {}
	if ( ( index) == ( end) )
	{
	switch ( _english_tokenizer_split_token_eof_actions[cs] ) {
	case 3:
	{ split_len = split_mark; {( index)++; goto _out; } }
	break;
	}
	}

	_out: {}
	}

  if (split_len && split_len < end) {
    tokens.back().length -= split_len;
    tokens.emplace_back(end - split_len, split_len);
  }
}

static const char _english_tokenizer_cond_offsets[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 2, 2, 2, 2, 2,
	2, 2, 2, 2, 2, 2, 2, 2,
	2, 2, 2, 2, 2
};

static const char _english_tokenizer_cond_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 2, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0
};

static const short _english_tokenizer_cond_keys[] = {
	43u, 43u, 45u, 45u, 0
};

static const char _english_tokenizer_cond_spaces[] = {
	1, 0, 0
};

static const unsigned char _english_tokenizer_key_offsets[] = {
	0, 0, 17, 29, 43, 46, 49, 52,
	55, 60, 63, 98, 103, 107, 110, 114,
	119, 120, 125, 126, 131, 145, 152, 156,
	161, 164, 179, 192, 206
};

static const short _english_tokenizer_trans_keys[] = {
	13u, 32u, 34u, 40u, 91u, 96u, 123u, 129u,
	133u, 135u, 147u, 150u, 162u, 9u, 10u, 65u,
	90u, 34u, 40u, 91u, 96u, 123u, 129u, 133u,
	135u, 150u, 162u, 65u, 90u, 13u, 32u, 34u,
	39u, 41u, 59u, 93u, 125u, 139u, 141u, 147u,
	161u, 9u, 10u, 159u, 48u, 57u, 159u, 48u,
	57u, 159u, 48u, 57u, 159u, 48u, 57u, 43u,
	45u, 159u, 48u, 57u, 159u, 48u, 57u, 9u,
	10u, 13u, 32u, 33u, 44u, 46u, 47u, 63u,
	129u, 131u, 135u, 142u, 147u, 157u, 159u, 160u,
	301u, 557u, 811u, 1067u, 0u, 42u, 48u, 57u,
	58u, 64u, 65u, 90u, 91u, 96u, 97u, 122u,
	123u, 255u, 9u, 10u, 13u, 32u, 147u, 9u,
	13u, 32u, 147u, 9u, 32u, 147u, 9u, 10u,
	32u, 147u, 9u, 10u, 13u, 32u, 147u, 13u,
	9u, 10u, 13u, 32u, 147u, 10u, 9u, 10u,
	13u, 32u, 147u, 13u, 32u, 34u, 39u, 41u,
	59u, 93u, 125u, 139u, 141u, 147u, 161u, 9u,
	10u, 44u, 46u, 69u, 101u, 159u, 48u, 57u,
	44u, 46u, 69u, 101u, 69u, 101u, 159u, 48u,
	57u, 159u, 48u, 57u, 39u, 45u, 129u, 131u,
	135u, 151u, 155u, 157u, 161u, 65u, 90u, 97u,
	122u, 142u, 143u, 45u, 129u, 131u, 135u, 151u,
	155u, 157u, 65u, 90u, 97u, 122u, 142u, 143u,
	39u, 129u, 131u, 135u, 151u, 155u, 157u, 161u,
	65u, 90u, 97u, 122u, 142u, 143u, 159u, 48u,
	57u, 0
};

static const char _english_tokenizer_single_lengths[] = {
	0, 13, 10, 12, 1, 1, 1, 1,
	3, 1, 21, 5, 4, 3, 4, 5,
	1, 5, 1, 5, 12, 5, 4, 3,
	1, 9, 7, 8, 1
};

static const char _english_tokenizer_range_lengths[] = {
	0, 2, 1, 1, 1, 1, 1, 1,
	1, 1, 7, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 1, 1, 0, 1,
	1, 3, 3, 3, 1
};

static const unsigned char _english_tokenizer_index_offsets[] = {
	0, 0, 16, 28, 42, 45, 48, 51,
	54, 59, 62, 91, 97, 102, 106, 111,
	117, 119, 125, 127, 133, 147, 154, 159,
	164, 167, 180, 191, 203
};

static const char _english_tokenizer_indicies[] = {
	1, 1, 2, 2, 2, 2, 2, 3,
	2, 3, 1, 2, 2, 1, 3, 0,
	2, 2, 2, 2, 2, 3, 2, 3,
	2, 2, 3, 0, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 0, 6, 6, 0, 7, 7, 0,
	8, 8, 0, 9, 9, 0, 10, 10,
	11, 11, 0, 11, 11, 0, 13, 14,
	15, 13, 16, 12, 16, 12, 16, 19,
	19, 19, 19, 13, 19, 18, 16, 12,
	20, 12, 20, 12, 18, 12, 19, 12,
	19, 12, 17, 13, 22, 23, 13, 13,
	21, 13, 24, 13, 13, 21, 13, 13,
	13, 21, 13, 24, 13, 13, 21, 13,
	25, 26, 13, 13, 21, 28, 27, 13,
	25, 29, 13, 13, 21, 28, 27, 13,
	26, 29, 13, 13, 21, 4, 4, 5,
	5, 5, 5, 5, 5, 5, 5, 4,
	5, 4, 30, 31, 32, 33, 33, 18,
	18, 30, 31, 32, 33, 33, 30, 33,
	33, 9, 9, 30, 11, 11, 30, 34,
	35, 19, 19, 19, 19, 19, 19, 34,
	19, 19, 19, 30, 35, 19, 19, 19,
	19, 19, 19, 19, 19, 19, 30, 34,
	19, 19, 19, 19, 19, 19, 34, 19,
	19, 19, 30, 18, 18, 30, 0
};

static const char _english_tokenizer_trans_targs[] = {
	10, 1, 2, 10, 1, 3, 5, 6,
	22, 23, 9, 24, 10, 11, 15, 19,
	20, 0, 21, 25, 28, 10, 12, 14,
	13, 16, 17, 10, 10, 18, 10, 4,
	7, 8, 26, 27
};

static const char _english_tokenizer_trans_actions[] = {
	1, 0, 0, 2, 3, 0, 0, 0,
	4, 4, 0, 0, 7, 0, 0, 0,
	4, 0, 4, 0, 0, 8, 0, 0,
	0, 0, 0, 9, 10, 0, 11, 0,
	0, 0, 0, 0
};

static const char _english_tokenizer_to_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 5, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0
};

static const char _english_tokenizer_from_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 6, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0
};

static const unsigned char _english_tokenizer_eof_trans[] = {
	0, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 0, 22, 22, 22, 22, 22,
	28, 22, 28, 22, 31, 31, 31, 31,
	31, 31, 31, 31, 31
};

static const int english_tokenizer_start = 10;

english_tokenizer::english_tokenizer(unsigned version) : ragel_tokenizer(version <= 1 ? 1 : 2) {}

bool english_tokenizer::next_sentence(vector& tokens) {
  using namespace unilib;

  int cs, act;
  size_t ts, te;
  size_t whitespace = 0; // Suppress "may be uninitialized" warning

  while (tokenize_url_email(tokens))
    if (emergency_sentence_split(tokens))
      return true;

	{
	cs = english_tokenizer_start;
	ts = 0;
	te = 0;
	act = 0;
	}

	{
	int _klen;
	const short *_keys;
	int _trans;
	short _widec;

	if ( ( current) == ( (chars.size() - 1)) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	switch ( _english_tokenizer_from_state_actions[cs] ) {
	case 6:
	{ts = ( current);}
	break;
	}

	_widec = ( ragel_char(chars[current]));
	_klen = _english_tokenizer_cond_lengths[cs];
	_keys = _english_tokenizer_cond_keys + (_english_tokenizer_cond_offsets[cs]*2);
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				switch ( _english_tokenizer_cond_spaces[_english_tokenizer_cond_offsets[cs] + ((_mid - _keys)>>1)] ) {
	case 0: {
		_widec = (short)(256u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || (chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N | unicode::Pd))  ) _widec += 256;
		break;
	}
	case 1: {
		_widec = (short)(768u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || ((chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N)) && chars[current-1].chr != '+')  ) _widec += 256;
		break;
	}
				}
				break;
			}
		}
	}

	_keys = _english_tokenizer_trans_keys + _english_tokenizer_key_offsets[cs];
	_trans = _english_tokenizer_index_offsets[cs];

	_klen = _english_tokenizer_single_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( _widec < *_mid )
				_upper = _mid - 1;
			else if ( _widec > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _english_tokenizer_range_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _english_tokenizer_indicies[_trans];
_eof_trans:
	cs = _english_tokenizer_trans_targs[_trans];

	if ( _english_tokenizer_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _english_tokenizer_trans_actions[_trans] ) {
	case 3:
	{ whitespace = current; }
	break;
	case 4:
	{te = ( current)+1;}
	break;
	case 7:
	{te = ( current)+1;{ tokens.emplace_back(ts, te - ts);
          split_token(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 2:
	{te = ( current)+1;{
          bool eos = is_eos(tokens, chars[ts].chr, &abbreviations);
          for (current = ts; current < whitespace; current++)
            tokens.emplace_back(current, 1);
          {( current) = (( whitespace))-1;}
          if (eos) {( current)++; goto _out; }
        }}
	break;
	case 10:
	{te = ( current)+1;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 11:
	{te = ( current);( current)--;{ tokens.emplace_back(ts, te - ts);
          split_token(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 8:
	{te = ( current);( current)--;{
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 9:
	{te = ( current);( current)--;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 1:
	{{( current) = ((te))-1;}{ tokens.emplace_back(ts, te - ts);
          split_token(tokens);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	}

_again:
	switch ( _english_tokenizer_to_state_actions[cs] ) {
	case 5:
	{ts = 0;}
	break;
	}

	if ( cs == 0 )
		goto _out;
	if ( ++( current) != ( (chars.size() - 1)) )
		goto _resume;
	_test_eof: {}
	if ( ( current) == ( (chars.size() - 1)) )
	{
	if ( _english_tokenizer_eof_trans[cs] > 0 ) {
		_trans = _english_tokenizer_eof_trans[cs] - 1;
		goto _eof_trans;
	}
	}

	_out: {}
	}

  (void)act; // Suppress unused variable warning

  return !tokens.empty();
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

static const char _generic_tokenizer_cond_offsets[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	2, 2, 2, 2, 2, 2, 2, 2,
	2, 2, 2, 2, 2, 2, 2
};

static const char _generic_tokenizer_cond_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 2,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const short _generic_tokenizer_cond_keys[] = {
	43u, 43u, 45u, 45u, 0
};

static const char _generic_tokenizer_cond_spaces[] = {
	1, 0, 0
};

static const unsigned char _generic_tokenizer_key_offsets[] = {
	0, 0, 17, 29, 43, 46, 51, 54,
	89, 94, 98, 101, 105, 110, 111, 116,
	117, 122, 136, 142, 147, 150, 162
};

static const short _generic_tokenizer_trans_keys[] = {
	13u, 32u, 34u, 40u, 91u, 96u, 123u, 129u,
	133u, 135u, 147u, 150u, 162u, 9u, 10u, 65u,
	90u, 34u, 40u, 91u, 96u, 123u, 129u, 133u,
	135u, 150u, 162u, 65u, 90u, 13u, 32u, 34u,
	39u, 41u, 59u, 93u, 125u, 139u, 141u, 147u,
	161u, 9u, 10u, 159u, 48u, 57u, 43u, 45u,
	159u, 48u, 57u, 159u, 48u, 57u, 9u, 10u,
	13u, 32u, 33u, 44u, 46u, 47u, 63u, 129u,
	131u, 135u, 142u, 147u, 157u, 159u, 160u, 301u,
	557u, 811u, 1067u, 0u, 42u, 48u, 57u, 58u,
	64u, 65u, 90u, 91u, 96u, 97u, 122u, 123u,
	255u, 9u, 10u, 13u, 32u, 147u, 9u, 13u,
	32u, 147u, 9u, 32u, 147u, 9u, 10u, 32u,
	147u, 9u, 10u, 13u, 32u, 147u, 13u, 9u,
	10u, 13u, 32u, 147u, 10u, 9u, 10u, 13u,
	32u, 147u, 13u, 32u, 34u, 39u, 41u, 59u,
	93u, 125u, 139u, 141u, 147u, 161u, 9u, 10u,
	46u, 69u, 101u, 159u, 48u, 57u, 69u, 101u,
	159u, 48u, 57u, 159u, 48u, 57u, 129u, 131u,
	135u, 151u, 155u, 157u, 65u, 90u, 97u, 122u,
	142u, 143u, 159u, 48u, 57u, 0
};

static const char _generic_tokenizer_single_lengths[] = {
	0, 13, 10, 12, 1, 3, 1, 21,
	5, 4, 3, 4, 5, 1, 5, 1,
	5, 12, 4, 3, 1, 6, 1
};

static const char _generic_tokenizer_range_lengths[] = {
	0, 2, 1, 1, 1, 1, 1, 7,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 1, 1, 1, 1, 3, 1
};

static const unsigned char _generic_tokenizer_index_offsets[] = {
	0, 0, 16, 28, 42, 45, 50, 53,
	82, 88, 93, 97, 102, 108, 110, 116,
	118, 124, 138, 144, 149, 152, 162
};

static const char _generic_tokenizer_indicies[] = {
	1, 1, 2, 2, 2, 2, 2, 3,
	2, 3, 1, 2, 2, 1, 3, 0,
	2, 2, 2, 2, 2, 3, 2, 3,
	2, 2, 3, 0, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 0, 6, 6, 0, 7, 7, 8,
	8, 0, 8, 8, 0, 10, 11, 12,
	10, 13, 9, 13, 9, 13, 16, 16,
	16, 16, 10, 16, 15, 13, 9, 17,
	9, 17, 9, 15, 9, 16, 9, 16,
	9, 14, 10, 19, 20, 10, 10, 18,
	10, 21, 10, 10, 18, 10, 10, 10,
	18, 10, 21, 10, 10, 18, 10, 22,
	23, 10, 10, 18, 25, 24, 10, 22,
	26, 10, 10, 18, 25, 24, 10, 23,
	26, 10, 10, 18, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 4, 5,
	4, 27, 28, 29, 29, 15, 15, 27,
	29, 29, 6, 6, 27, 8, 8, 27,
	16, 16, 16, 16, 16, 16, 16, 16,
	16, 27, 15, 15, 27, 0
};

static const char _generic_tokenizer_trans_targs[] = {
	7, 1, 2, 7, 1, 3, 19, 6,
	20, 7, 8, 12, 16, 17, 0, 18,
	21, 22, 7, 9, 11, 10, 13, 14,
	7, 7, 15, 7, 4, 5
};

static const char _generic_tokenizer_trans_actions[] = {
	1, 0, 0, 2, 3, 0, 4, 0,
	0, 7, 0, 0, 0, 4, 0, 4,
	0, 0, 8, 0, 0, 0, 0, 0,
	9, 10, 0, 11, 0, 0
};

static const char _generic_tokenizer_to_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 5,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const char _generic_tokenizer_from_state_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 6,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0
};

static const unsigned char _generic_tokenizer_eof_trans[] = {
	0, 1, 1, 1, 1, 1, 1, 0,
	19, 19, 19, 19, 19, 25, 19, 25,
	19, 28, 28, 28, 28, 28, 28
};

static const int generic_tokenizer_start = 7;

generic_tokenizer::generic_tokenizer(unsigned version) : ragel_tokenizer(version <= 1 ? 1 : 2) {}

bool generic_tokenizer::next_sentence(vector& tokens) {
  using namespace unilib;

  int cs, act;
  size_t ts, te;
  size_t whitespace = 0; // Suppress "may be uninitialized" warning

  while (tokenize_url_email(tokens))
    if (emergency_sentence_split(tokens))
      return true;

	{
	cs = generic_tokenizer_start;
	ts = 0;
	te = 0;
	act = 0;
	}

	{
	int _klen;
	const short *_keys;
	int _trans;
	short _widec;

	if ( ( current) == ( (chars.size() - 1)) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	switch ( _generic_tokenizer_from_state_actions[cs] ) {
	case 6:
	{ts = ( current);}
	break;
	}

	_widec = ( ragel_char(chars[current]));
	_klen = _generic_tokenizer_cond_lengths[cs];
	_keys = _generic_tokenizer_cond_keys + (_generic_tokenizer_cond_offsets[cs]*2);
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				switch ( _generic_tokenizer_cond_spaces[_generic_tokenizer_cond_offsets[cs] + ((_mid - _keys)>>1)] ) {
	case 0: {
		_widec = (short)(256u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || (chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N | unicode::Pd))  ) _widec += 256;
		break;
	}
	case 1: {
		_widec = (short)(768u + (( ragel_char(chars[current])) - 0u));
		if (
 !current || ((chars[current-1].cat & ~(unicode::L | unicode::M | unicode::N)) && chars[current-1].chr != '+')  ) _widec += 256;
		break;
	}
				}
				break;
			}
		}
	}

	_keys = _generic_tokenizer_trans_keys + _generic_tokenizer_key_offsets[cs];
	_trans = _generic_tokenizer_index_offsets[cs];

	_klen = _generic_tokenizer_single_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( _widec < *_mid )
				_upper = _mid - 1;
			else if ( _widec > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _generic_tokenizer_range_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _generic_tokenizer_indicies[_trans];
_eof_trans:
	cs = _generic_tokenizer_trans_targs[_trans];

	if ( _generic_tokenizer_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _generic_tokenizer_trans_actions[_trans] ) {
	case 3:
	{ whitespace = current; }
	break;
	case 4:
	{te = ( current)+1;}
	break;
	case 7:
	{te = ( current)+1;{ tokens.emplace_back(ts, te - ts);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 2:
	{te = ( current)+1;{
          bool eos = is_eos(tokens, chars[ts].chr, nullptr);
          for (current = ts; current < whitespace; current++)
            tokens.emplace_back(current, 1);
          {( current) = (( whitespace))-1;}
          if (eos) {( current)++; goto _out; }
        }}
	break;
	case 10:
	{te = ( current)+1;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 11:
	{te = ( current);( current)--;{ tokens.emplace_back(ts, te - ts);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 8:
	{te = ( current);( current)--;{
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 9:
	{te = ( current);( current)--;{
          if (!tokens.empty()) {( current)++; goto _out; }
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	case 1:
	{{( current) = ((te))-1;}{ tokens.emplace_back(ts, te - ts);
          current = te;
          do
            if (emergency_sentence_split(tokens)) { ( current)--; {( current)++; goto _out; } }
          while (tokenize_url_email(tokens));
          ( current)--;
        }}
	break;
	}

_again:
	switch ( _generic_tokenizer_to_state_actions[cs] ) {
	case 5:
	{ts = 0;}
	break;
	}

	if ( cs == 0 )
		goto _out;
	if ( ++( current) != ( (chars.size() - 1)) )
		goto _resume;
	_test_eof: {}
	if ( ( current) == ( (chars.size() - 1)) )
	{
	if ( _generic_tokenizer_eof_trans[cs] > 0 ) {
		_trans = _generic_tokenizer_eof_trans[cs] - 1;
		goto _eof_trans;
	}
	}

	_out: {}
	}

  (void)act; // Suppress unused variable warning

  return !tokens.empty();
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer_factory.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_tokenizer_factory : public tokenizer_factory {
 public:
  // Construct a new tokenizer instance.
  virtual tokenizer* new_tokenizer(const morpho* m) const override;

  bool load(istream& is);
 private:
  unsigned version;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer_factory.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tokenizer* generic_tokenizer_factory::new_tokenizer(const morpho* /*m*/) const {
  return new generic_tokenizer(version);
}

bool generic_tokenizer_factory::load(istream& is) {
  version = is.get();

  return bool(is);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer_factory_encoder.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class generic_tokenizer_factory_encoder {
 public:
  static void encode(unsigned version, ostream& os);
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/generic_tokenizer_factory_encoder.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

void generic_tokenizer_factory_encoder::encode(unsigned version, ostream& os) {
  os.put(version);
}

} // namespace morphodita

/////////
// File: unilib/uninorms.h
/////////

// This file is part of UniLib .
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.3.0
// Unicode version: 15.0.0

namespace unilib {

class uninorms {
 public:
  static void nfc(std::u32string& str);
  static void nfd(std::u32string& str);
  static void nfkc(std::u32string& str);
  static void nfkd(std::u32string& str);

 private:
  static void compose(std::u32string& str);
  static void decompose(std::u32string& str, bool kanonical);

  static const char32_t CHARS = 0x110000;

  struct Hangul {
    // Hangul decomposition and composition
    static const char32_t SBase = 0xAC00, LBase = 0x1100, VBase = 0x1161, TBase = 0x11A7;
    static const char32_t LCount = 19, VCount = 21, TCount = 28, NCount = VCount * TCount, SCount = LCount * NCount;
  };

  static const uint8_t ccc_index[CHARS >> 8];
  static const uint8_t ccc_block[][256];

  static const uint8_t composition_index[CHARS >> 8];
  static const uint16_t composition_block[][257];
  static const char32_t composition_data[];

  static const uint8_t decomposition_index[CHARS >> 8];
  static const uint16_t decomposition_block[][257];
  static const char32_t decomposition_data[];
};

} // namespace unilib

/////////
// File: morphodita/tokenizer/gru_tokenizer_network.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations

class gru_tokenizer_network {
 public:
  virtual ~gru_tokenizer_network() {}

  template  struct matrix {
    float w[R][C];
    float b[R];

    void clear();
    void load(binary_decoder& data);
  };

  enum { NO_SPLIT, END_OF_TOKEN, END_OF_SENTENCE, OUTCOMES };
  struct outcome_t {
    int outcome;
    float w[3];
    const float* embedding;
  };
  struct char_info {
    char32_t chr;
    unilib::unicode::category_t cat;

    char_info() {}
    char_info(char32_t chr, unilib::unicode::category_t cat) : chr(chr), cat(cat) {}
  };

  virtual void classify(const vector& chars, vector& outcomes) const = 0;

  static gru_tokenizer_network* load(binary_decoder& data);
};

template
class gru_tokenizer_network_implementation : public gru_tokenizer_network {
 public:
  virtual void classify(const vector& chars, vector& outcomes) const override;

  static gru_tokenizer_network_implementation* load(binary_decoder& data);

 protected:
  void cache_embeddings();

  struct cached_embedding {
    matrix<1, D> e;
    matrix<6, D> cache;
  };

  struct gru {
    matrix X, X_r, X_z;
    matrix H, H_r, H_z;

    void load(binary_decoder& data);
  };

  unordered_map embeddings;
  cached_embedding empty_embedding;
  gru gru_fwd, gru_bwd;
  matrix<3, D> projection_fwd, projection_bwd;
  unordered_map unknown_chars;
};

// Definitions

template
void gru_tokenizer_network::matrix::clear() {
  for (int i = 0; i < R; i++) fill_n(w[i], C, 0.f);
  fill_n(b, R, 0.f);
}

template
void gru_tokenizer_network::matrix::load(binary_decoder& data) {
  for (int i = 0; i < R; i++) memcpy(w[i], data.next(C), sizeof(float) * C);
  memcpy(b, data.next(R), sizeof(float) * R);
}

template
void gru_tokenizer_network_implementation::gru::load(binary_decoder& data) {
  X.load(data);
  X_r.load(data);
  X_z.load(data);
  H.load(data);
  H_r.load(data);
  H_z.load(data);
}

template
void gru_tokenizer_network_implementation::classify(const vector& chars, vector& outcomes) const {
  if (chars.empty()) return;

  // Resolve embeddings, possibly with unknown_chars or empty_embedding
  u32string decomposition;
  for (size_t i = 0; i < chars.size(); i++) {
    auto embedding = embeddings.find(chars[i].chr);

    // Try finding substitute character if not found, by using NFKD
    // and by replacing IDEOGRAPHIC FULL STOP/COMMA.
    if (embedding == embeddings.end()) {
      decomposition.assign(1, chars[i].chr);
      unilib::uninorms::nfkd(decomposition);
      if (decomposition[0] == 0x3001) decomposition[0] = char32_t(',');
      if (decomposition[0] == 0x3002) decomposition[0] = char32_t('.');
      if (decomposition[0] != chars[i].chr) embedding = embeddings.find(decomposition[0]);
    }

    if (embedding != embeddings.end()) {
      outcomes[i].embedding = embedding->second.cache.w[0];
    } else {
      auto unknown_char = unknown_chars.find(chars[i].cat);
      if (unknown_char != unknown_chars.end()) embedding = embeddings.find(unknown_char->second);
      outcomes[i].embedding = embedding != embeddings.end() ? embedding->second.cache.w[0] : empty_embedding.cache.w[0];
    }
  }

  // Clear outcome probabilities
  for (auto&& outcome : outcomes)
    for (int i = 0; i < 3; i++)
      outcome.w[i] = projection_fwd.b[i];

  // Perform forward & backward GRU
  matrix<1, D> state, update, reset, candidate;
  for (int dir = 0; dir < 2; dir++) {
    auto& gru = dir == 0 ? gru_fwd : gru_bwd;
    auto& projection = dir == 0 ? projection_fwd : projection_bwd;

    state.clear();
    for (size_t i = 0; i < outcomes.size(); i++) {
      auto& outcome = outcomes[dir == 0 ? i : outcomes.size() - 1 - i];
      auto* embedding_cache = outcome.embedding + (dir == 1) * 3 * D;

      for (int j = 0; j < D; j++) {
        update.w[0][j] = gru.X_z.b[j] + embedding_cache[2*D + j];
        reset.w[0][j] = gru.X_r.b[j] + embedding_cache[D + j];
        for (int k = 0; k < D; k++) {
          update.w[0][j] += state.w[0][k] * gru.H_z.w[j][k];
          reset.w[0][j] += state.w[0][k] * gru.H_r.w[j][k];
        }
        update.w[0][j] = 1.f / (1.f + exp(-update.w[0][j]));
        reset.w[0][j] = 1.f / (1.f + exp(-reset.w[0][j]));
        reset.w[0][j] *= state.w[0][j];
      }
      for (int j = 0; j < D; j++) {
        candidate.w[0][j] = gru.X.b[j] + embedding_cache[j];
        for (int k = 0; k < D; k++)
          candidate.w[0][j] += reset.w[0][k] * gru.H.w[j][k];
        candidate.w[0][j] = tanh(candidate.w[0][j]);
        state.w[0][j] = update.w[0][j] * state.w[0][j] + (1.f - update.w[0][j]) * candidate.w[0][j];
      }

      for (int j = 0; j < 3; j++)
        for (int k = 0; k < D; k++)
          outcome.w[j] += projection.w[j][k] * state.w[0][k];
    }
  }

  // Choose the outcome with the highest weight
  for (auto&& outcome : outcomes) {
    outcome.outcome = outcome.w[1] > outcome.w[0];
    if (outcome.w[2] > outcome.w[outcome.outcome]) outcome.outcome = 2;
  }
}

template
gru_tokenizer_network_implementation* gru_tokenizer_network_implementation::load(binary_decoder& data) {
  unique_ptr> network(new gru_tokenizer_network_implementation());

  for (unsigned chars = data.next_4B(); chars; chars--) {
    auto& embedding = network->embeddings[data.next_4B()];
    copy_n(data.next(D), D, embedding.e.w[0]);
  }
  fill_n(network->empty_embedding.e.w[0], D, 0.f);

  network->gru_fwd.load(data);
  network->gru_bwd.load(data);
  network->projection_fwd.load(data);
  network->projection_bwd.load(data);

  network->unknown_chars.clear();
  for (unsigned unknown_chars_len = data.next_1B(); unknown_chars_len; unknown_chars_len--) {
    unilib::unicode::category_t cat = data.next_4B();
    network->unknown_chars[cat] = data.next_4B();
  }

  network->cache_embeddings();

  return network.release();
}

template
void gru_tokenizer_network_implementation::cache_embeddings() {
  for (auto&& embedding : embeddings) {
    auto& e = embedding.second.e;
    auto& cache = embedding.second.cache;

    for (int i = 0; i < 6; i++) fill_n(cache.w[i], D, 0.f);
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[0][i] += e.w[0][j] * gru_fwd.X.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[1][i] += e.w[0][j] * gru_fwd.X_r.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[2][i] += e.w[0][j] * gru_fwd.X_z.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[3][i] += e.w[0][j] * gru_bwd.X.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[4][i] += e.w[0][j] * gru_bwd.X_r.w[i][j];
    for (int i = 0; i < D; i++) for (int j = 0; j < D; j++) cache.w[5][i] += e.w[0][j] * gru_bwd.X_z.w[i][j];
  }
  for (int i = 0; i < 6; i++) fill_n(empty_embedding.cache.w[i], D, 0.f);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class gru_tokenizer : public unicode_tokenizer {
 public:
  gru_tokenizer(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, const gru_tokenizer_network& network)
      : unicode_tokenizer(url_email_tokenizer), segment(segment), allow_spaces(allow_spaces), network_index(0), network_length(0), network(network) {}

  virtual bool next_sentence(vector& tokens) override;

 private:
  inline bool is_space(size_t index);
  int next_outcome();

  unsigned segment;
  bool allow_spaces;
  unsigned network_index, network_length;
  vector network_chars;
  vector network_outcomes;
  vector network_offsets;
  const gru_tokenizer_network& network;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool gru_tokenizer::is_space(size_t index) {
  return (chars[index].cat & unilib::unicode::Zs) || chars[index].chr == '\r' || chars[index].chr == '\n' || chars[index].chr == '\t';
}

bool gru_tokenizer::next_sentence(vector& tokens) {
  tokens.clear();

  // Reset tokenizer on new text
  if (current == 0) network_index = network_length = 0;

  // Tokenize until EOS
  for (bool eos = false; !eos && !emergency_sentence_split(tokens); ) {
    while (current < chars.size() - 1 && is_space(current))
      if (next_outcome() == gru_tokenizer_network::END_OF_SENTENCE && !tokens.empty())
        break;

    if (current >= chars.size() - 1) break;

    // We have a beginning of a token. Try if it is an URL.
    if (tokenize_url_email(tokens)) {
      while (network_index < network_length && network_offsets[network_index] < current)
        if (network_outcomes[network_index++].outcome == gru_tokenizer_network::END_OF_SENTENCE && !tokens.empty())
          eos = true;
      continue;
    }

    // Slurp current token
    size_t token_start = current;
    do {
      int outcome = next_outcome();
      eos = outcome == gru_tokenizer_network::END_OF_SENTENCE;
      if (outcome != gru_tokenizer_network::NO_SPLIT) break;
    } while (current < chars.size() - 1);
    tokens.emplace_back(token_start, current - token_start);
  }

  return !tokens.empty();
}

int gru_tokenizer::next_outcome() {
  if (network_index >= network_length) {
    // Compute required window
    network_index = 0;
    network_length = 0;
    network_chars.clear();
    network_outcomes.clear();
    network_offsets.clear();

    // Prepare data for the classification
    for (size_t offset = current;
         network_offsets.push_back(offset), offset < chars.size() - 1 && network_length < segment;
         network_length++, offset++) {
      if (is_space(offset)) {
        network_chars.emplace_back(' ', unilib::unicode::Zs);
        while (offset + 1 < chars.size() - 1 && is_space(offset + 1)) offset++;
      } else {
        network_chars.emplace_back(chars[offset].chr, chars[offset].cat);
      }
    }
    // Add a space to the end on the EOD
    if (network_length < segment && network_chars.back().chr != ' ')
      network_chars.emplace_back(' ', unilib::unicode::Zs);
    network_outcomes.resize(network_chars.size());

    // Perform the classification
    network.classify(network_chars, network_outcomes);

    // Add spacing token/sentence breaks
    for (size_t i = 0; i < network_length - 1; i++)
      if (is_space(network_offsets[i+1])) {
        // Detect EOS on the following space or \n\n or \r\n\r\n, or if there is end of text
        bool eos = network_outcomes[i+1].outcome == gru_tokenizer_network::END_OF_SENTENCE;
        if (i + 2 == network_length) eos = true;
        for (size_t j = network_offsets[i+1]; j + 1 < network_offsets[i+2] && !eos; j++)
          eos = (chars[j].chr == '\n' && chars[j+1].chr == '\n') ||
                (j + 3 < network_offsets[i+2] && chars[j].chr == '\r' && chars[j+1].chr == '\n' && chars[j+2].chr == '\r' && chars[j+3].chr == '\n');
        if (eos) network_outcomes[i].outcome = gru_tokenizer_network::END_OF_SENTENCE;

        if (network_outcomes[i].outcome == gru_tokenizer_network::NO_SPLIT)
          // Force EOT if not allowing spaces, and also detect EOT on the following space
          if (!allow_spaces || network_outcomes[i+1].outcome == gru_tokenizer_network::END_OF_TOKEN)
            network_outcomes[i].outcome = gru_tokenizer_network::END_OF_TOKEN;
      }

    // Adjust network_length to suitable break
    if (network_length == segment && network_length >= 10) {
      network_length -= 5;
      while (network_length > segment / 2)
        if (network_outcomes[--network_length].outcome != gru_tokenizer_network::NO_SPLIT)
          break;
    }
  }
  return current = network_offsets[network_index + 1], network_outcomes[network_index++].outcome;
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_factory.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class gru_tokenizer_factory : public tokenizer_factory {
 public:
  // Construct a new tokenizer instance.
  virtual tokenizer* new_tokenizer(const morpho* m) const override;

  bool load(istream& is);

 private:
  unsigned url_email_tokenizer;
  unsigned segment;
  bool allow_spaces;

  unique_ptr network;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_factory.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tokenizer* gru_tokenizer_factory::new_tokenizer(const morpho* /*m*/) const {
  return new gru_tokenizer(url_email_tokenizer, segment, allow_spaces, *network);
}

bool gru_tokenizer_factory::load(istream& is) {
  char version;
  if (!is.get(version)) return false;
  if (!(version >= 1 && version <= 2)) return false;

  binary_decoder data;
  if (!compressor::load(is, data)) return false;

  try {
    url_email_tokenizer = data.next_1B();
    segment = data.next_2B();
    allow_spaces = version >= 2 ? data.next_1B() : false /*false was default for version 1*/;

    network.reset(gru_tokenizer_network::load(data));
    if (!network) return false;
  } catch (binary_decoder_error&) {
    return false;
  }

  return data.is_end();
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_network.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

gru_tokenizer_network* gru_tokenizer_network::load(binary_decoder& data) {
  if (data.next_1B() != 1) return nullptr;
  switch (data.next_1B()) {
    case 16: return gru_tokenizer_network_implementation<16>::load(data);
    case 24: return gru_tokenizer_network_implementation<24>::load(data);
    case 64: return gru_tokenizer_network_implementation<64>::load(data);
  }
  return nullptr;
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_trainer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

struct tokenized_sentence {
  u32string sentence;
  vector tokens;
};

class gru_tokenizer_trainer {
 public:
  enum { URL_EMAIL_LATEST = unicode_tokenizer::URL_EMAIL_LATEST };

  static bool train(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, unsigned dimension, unsigned epochs,
                    unsigned batch_size, float learning_rate, float learning_rate_final, float dropout,
                    float initialization_range, bool early_stopping, const vector& data,
                    const vector& heldout, ostream& os, string& error);
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_network_trainer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

//
// Declarations
//

template
class gru_tokenizer_network_trainer : public gru_tokenizer_network_implementation {
 public:
  bool train(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, unsigned epochs, unsigned batch_size,
             float learning_rate, float learning_rate_final, float dropout, float initialization_range,
             bool early_stopping, const vector& data, const vector& heldout,
             binary_encoder& enc, string& error);

 private:
  template  using matrix = typename gru_tokenizer_network_implementation::template matrix;
  using typename gru_tokenizer_network_implementation::cached_embedding;
  using typename gru_tokenizer_network_implementation::gru;

  template  struct matrix_trainer {
    matrix& original;
    float w_g[R][C], b_g[R];
    float w_m[R][C], b_m[R];
    float w_v[R][C], b_v[R];

    matrix_trainer(matrix& original) : original(original), w_g(), b_g(), w_m(), b_m(), w_v(), b_v() {}
    void update_weights(float learning_rate);
  };
  struct gru_trainer {
    matrix_trainer X, X_r, X_z;
    matrix_trainer H, H_r, H_z;
    vector> states, updates, resets, resetstates, candidates, dropouts;

    gru_trainer(gru& g, unsigned segment)
        : X(g.X), X_r(g.X_r), X_z(g.X_z), H(g.H), H_r(g.H_r), H_z(g.H_z), states(segment + 1),
        updates(segment), resets(segment), resetstates(segment), candidates(segment), dropouts(segment) {}
    void update_weights(float learning_rate);
  };

  struct f1_info { double precision, recall, f1; };
  void evaluate(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, const vector& heldout,
                f1_info& tokens_f1, f1_info& sentences_f1);
  void evaluate_f1(const vector& system, const vector& gold, f1_info& f1);

  template  void random_matrix(matrix& m, mt19937& generator, float range, float bias);
  void random_gru(gru& g,  mt19937& generator, float range);

  template  void save_matrix(const matrix& m, binary_encoder& enc);
  void save_gru(const gru& g, binary_encoder& enc);
};

//
// Definitions
//

template
bool gru_tokenizer_network_trainer::train(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, unsigned epochs, unsigned batch_size,
                                             float learning_rate_initial, float learning_rate_final, float dropout,
                                             float initialization_range, bool early_stopping, const vector& data,
                                             const vector& heldout, binary_encoder& enc, string& error) {
  if (segment < 10) return error.assign("Segment size must be at least 10!"), false;

  unsigned characters = 0;
  for (auto&& sentence : data)
    characters += sentence.sentence.size();
  if (characters < segment) return error.assign("Not enought training data for the gru_tokenizer!"), false;

  mt19937 generator;

  float dropout_multiplier = 1.f / (1.f - dropout);
  bernoulli_distribution dropout_distribution(dropout);

  // Generate embeddings
  for (auto&& sentence : data)
    for (auto&& chr : sentence.sentence)
      if (!this->embeddings.count(chr)) {
        cached_embedding embedding;
        random_matrix(embedding.e, generator, initialization_range, 0.f);
        this->embeddings.emplace(chr, embedding);
      }
  this->empty_embedding.e.clear();

  // Initialize weights
  random_gru(this->gru_fwd, generator, initialization_range);
  random_gru(this->gru_bwd, generator, initialization_range);
  random_matrix(this->projection_fwd, generator, initialization_range, 0.f); this->projection_fwd.b[this->NO_SPLIT] = 1.f;
  random_matrix(this->projection_bwd, generator, initialization_range, 0.f); this->projection_bwd.b[this->NO_SPLIT] = 1.f;

  // Train the network
  unordered_map> embeddings;
  for (auto&& embedding : this->embeddings)
    embeddings.emplace(embedding.first, embedding.second.e);
  vector*> chosen_embeddings(segment);
  vector> embedding_dropouts(segment);
  gru_trainer gru_fwd(this->gru_fwd, segment), gru_bwd(this->gru_bwd, segment);
  matrix_trainer<3, D> projection_fwd(this->projection_fwd), projection_bwd(this->projection_bwd);
  float learning_rate = learning_rate_initial, b1t = 1.f, b2t = 1.f;

  float best_combined_f1 = 0.f; unsigned best_combined_f1_epoch = 0;
  gru_tokenizer_network_trainer best_combined_f1_network;

  size_t training_offset = 0, training_shift;
  vector training_input, instance_input(segment);
  vector training_output, instance_output(segment);
  vector permutation; for (size_t i = 0; i < data.size(); i++) permutation.push_back(permutation.size());
  for (unsigned epoch = 0; epoch < epochs; epoch++) {
    double logprob = 0;
    int total = 0, correct = 0;

    for (int instance = 0, instances = 10000; instance < instances; instance++) {
      // Prepare input instance
      if (training_offset + segment >= training_input.size()) {
        shuffle(permutation.begin(), permutation.end(), generator);
        training_input.clear(); training_output.clear();
        for (auto&& index : permutation) {
          auto& sentence = data[index];
          if (sentence.tokens.empty()) continue;

          training_offset = training_input.size();
          training_input.resize(training_offset + sentence.sentence.size());
          training_output.resize(training_offset + sentence.sentence.size());
          for (size_t i = 0; i < sentence.sentence.size(); i++) {
            training_input[training_offset + i].chr = sentence.sentence[i];
            training_output[training_offset + i].outcome = gru_tokenizer_network::NO_SPLIT;
          }
          for (size_t i = 0; i < sentence.tokens.size(); i++)
            training_output[training_offset + sentence.tokens[i].start + sentence.tokens[i].length - 1].outcome =
                i+1 < sentence.tokens.size() ? gru_tokenizer_network::END_OF_TOKEN : gru_tokenizer_network::END_OF_SENTENCE;
        }
        training_offset = 0;
      }
      copy_n(training_input.begin() + training_offset, segment, instance_input.begin());
      copy_n(training_output.begin() + training_offset, segment, instance_output.begin());

      // Shift training_offset
      for (training_shift = segment - 5; training_shift > segment / 2; training_shift--)
        if (instance_output[training_shift-1].outcome != gru_tokenizer_network::NO_SPLIT || instance_input[training_shift-1].chr == ' ')
          break;
      training_offset += training_shift;

      // Forward pass
      for (unsigned i = 0; i < segment; i++) {
        chosen_embeddings[i] = &embeddings.at(instance_input[i].chr);
        for (unsigned k = 0; k < D; k++)
          embedding_dropouts[i].w[0][k] = dropout && dropout_distribution(generator) ? 0.f : dropout_multiplier;
        for (int j = 0; j < 3; j++)
          instance_output[i].w[j] = projection_fwd.original.b[j];
      }

      for (int dir = 0; dir < 2; dir++) {
        auto& gru = dir == 0 ? gru_fwd : gru_bwd;
        auto& projection = dir == 0 ? projection_fwd : projection_bwd;

        gru.states[0].clear();
        for (size_t i = 0; i < segment; i++) {
          auto& embedding = chosen_embeddings[dir == 0 ? i : segment - 1 - i];
          auto& embedding_dropout = embedding_dropouts[dir == 0 ? i : segment - 1 - i];
          auto& output = instance_output[dir == 0 ? i : segment - 1 - i];

          for (int j = 0; j < D; j++) {
            gru.updates[i].w[0][j] = gru.X_z.original.b[j];
            gru.resets[i].w[0][j] = gru.X_r.original.b[j];
            for (int k = 0; k < D; k++) {
              gru.updates[i].w[0][j] += embedding_dropout.w[0][k] * embedding->original.w[0][k] * gru.X_z.original.w[j][k] + gru.states[i].w[0][k] * gru.H_z.original.w[j][k];
              gru.resets[i].w[0][j] += embedding_dropout.w[0][k] * embedding->original.w[0][k] * gru.X_r.original.w[j][k] + gru.states[i].w[0][k] * gru.H_r.original.w[j][k];
            }
            gru.updates[i].w[0][j] = 1.f / (1.f + exp(-gru.updates[i].w[0][j]));
            gru.resets[i].w[0][j] = 1.f / (1.f + exp(-gru.resets[i].w[0][j]));
            gru.resetstates[i].w[0][j] = gru.resets[i].w[0][j] * gru.states[i].w[0][j];
          }
          for (int j = 0; j < D; j++) {
            gru.candidates[i].w[0][j] = gru.X.original.b[j];
            for (int k = 0; k < D; k++)
              gru.candidates[i].w[0][j] += embedding_dropout.w[0][k] * embedding->original.w[0][k] * gru.X.original.w[j][k] + gru.resetstates[i].w[0][k] * gru.H.original.w[j][k];
            gru.candidates[i].w[0][j] = tanh(gru.candidates[i].w[0][j]);
            gru.states[i+1].w[0][j] = gru.updates[i].w[0][j] * gru.states[i].w[0][j] + (1.f - gru.updates[i].w[0][j]) * gru.candidates[i].w[0][j];
          }

          for (int j = 0; j < D; j++)
            gru.dropouts[i].w[0][j] = dropout && dropout_distribution(generator) ? 0.f : dropout_multiplier * gru.states[i+1].w[0][j];

          for (int j = 0; j < 3; j++)
            for (int k = 0; k < D; k++)
              output.w[j] += projection.original.w[j][k] * gru.dropouts[i].w[0][k];
        }
      }

      for (auto&& output : instance_output) {
        int best = output.w[1] > output.w[0];
        if (output.w[2] > output.w[best]) best = 2;
        float maximum = output.w[best], sum = 0;
        for (int j = 0; j < 3; j++) sum += (output.w[j] = exp(output.w[j] - maximum));
        sum = 1.f / sum;
        for (int j = 0; j < 3; j++) output.w[j] *= sum;

        total++;
        correct += best == output.outcome;
        logprob += log(output.w[output.outcome]);
      }

      // Backward pass
      for (auto&& output : instance_output)
        for (int j = 0; j < 3; j++)
          output.w[j] = (output.outcome == j) - output.w[j];

      for (int dir = 0; dir < 2; dir++) {
        auto& gru = dir == 0 ? gru_fwd : gru_bwd;
        auto& projection = dir == 0 ? projection_fwd : projection_bwd;

        matrix<1, D> state_g, update_g, candidate_g, reset_g, resetstate_g;
        state_g.clear();
        for (size_t i = segment; i--; ) {
          auto& embedding = chosen_embeddings[dir == 0 ? i : segment - 1 - i];
          auto& embedding_dropout = embedding_dropouts[dir == 0 ? i : segment - 1 - i];
          auto& output = instance_output[dir == 0 ? i : segment - 1 - i];

          for (int j = 0; j < D; j++) // These for cycles are swapped because
            for (int k = 0; k < 3; k++) // g++-4.8 generates wrong code otherwise.
              projection.w_g[k][j] += gru.dropouts[i].w[0][j] * output.w[k];

          for (int j = 0; j < D; j++)
            if (gru.dropouts[i].w[0][j])
              for (int k = 0; k < 3; k++)
                state_g.w[0][j] += projection.original.w[k][j] * output.w[k];

          resetstate_g.clear();
          for (int j = 0; j < D; j++) {
            update_g.w[0][j] = state_g.w[0][j] * (gru.states[i].w[0][j] - gru.candidates[i].w[0][j]);
            candidate_g.w[0][j] = state_g.w[0][j] * (1.f - gru.updates[i].w[0][j]);
            state_g.w[0][j] = state_g.w[0][j] * gru.updates[i].w[0][j];

            candidate_g.w[0][j] *= 1 - gru.candidates[i].w[0][j] * gru.candidates[i].w[0][j];
            gru.X.b_g[j] += candidate_g.w[0][j];
            for (int k = 0; k < D; k++) {
              gru.X.w_g[j][k] += candidate_g.w[0][j] * embedding_dropout.w[0][k] * embedding->original.w[0][k];
              gru.H.w_g[j][k] += candidate_g.w[0][j] * gru.resetstates[i].w[0][k];
              embedding->w_g[0][k] += embedding_dropout.w[0][k] * candidate_g.w[0][j] * gru.X.original.w[j][k];
              resetstate_g.w[0][k] += candidate_g.w[0][j] * gru.H.original.w[j][k];
            }
          }
          for (int j = 0; j < D; j++) {
            state_g.w[0][j] += resetstate_g.w[0][j] * gru.resets[i].w[0][j];
            reset_g.w[0][j] = resetstate_g.w[0][j] * gru.states[i].w[0][j];

            update_g.w[0][j] *= gru.updates[i].w[0][j] * (1 - gru.updates[i].w[0][j]);
            reset_g.w[0][j] *= gru.resets[i].w[0][j] * (1 - gru.resets[i].w[0][j]);

            gru.X_z.b_g[j] += update_g.w[0][j];
            gru.X_r.b_g[j] += reset_g.w[0][j];
            for (int k = 0; k < D; k++) {
              gru.X_z.w_g[j][k] += update_g.w[0][j] * embedding_dropout.w[0][k] * embedding->original.w[0][k];
              gru.H_z.w_g[j][k] += update_g.w[0][j] * gru.states[i].w[0][k];
              gru.X_r.w_g[j][k] += reset_g.w[0][j] * embedding_dropout.w[0][k] * embedding->original.w[0][k];
              gru.H_r.w_g[j][k] += reset_g.w[0][j] * gru.states[i].w[0][k];
              embedding->w_g[0][k] += embedding_dropout.w[0][k] * (update_g.w[0][j] * gru.X_z.original.w[j][k] +
                                                                   reset_g.w[0][j] * gru.X_r.original.w[j][k]);
              state_g.w[0][k] += update_g.w[0][j] * gru.H_z.original.w[j][k] + reset_g.w[0][j] * gru.H_r.original.w[j][k];
            }
          }
        }
      }

      // Update the weights
      if (batch_size == 1 ||
          instance+1 == instances ||
          (instance+1) % batch_size == 0) {
        b1t *= 0.9f;
        b2t *= 0.999f;
        float learning_rate_biased = learning_rate * sqrt(1-b2t) / (1-b1t);

        if (batch_size == 1)
          for (auto&& chosen_embedding : chosen_embeddings)
            chosen_embedding->update_weights(learning_rate_biased);
        else
          for (auto&& embedding : embeddings)
            embedding.second.update_weights(learning_rate_biased);
        gru_fwd.update_weights(learning_rate_biased);
        gru_bwd.update_weights(learning_rate_biased);
        projection_fwd.update_weights(learning_rate_biased);
        projection_bwd.update_weights(learning_rate_biased);
      }
    }
    if (learning_rate_final && learning_rate_final != learning_rate_initial)
      learning_rate = exp(((epochs - epoch - 2) * log(learning_rate_initial) + (epoch + 1) * log(learning_rate_final)) / (epochs - 1));

    // Evaluate
    cerr << "Epoch " << epoch+1 << ", logprob: " << scientific << setprecision(4) << logprob
         << ", training acc: " << fixed << setprecision(2) << 100. * correct / double(total) << "%";
    if (!heldout.empty()) {
      f1_info tokens, sentences;
      evaluate(url_email_tokenizer, segment, allow_spaces, heldout, tokens, sentences);
      cerr << ", heldout tokens: " << 100. * tokens.precision << "%P/" << 100. * tokens.recall << "%R/"
           << 100. * tokens.f1 << "%, sentences: " << 100. * sentences.precision << "%P/"
           << 100. * sentences.recall << "%R/" << 100. * sentences.f1 << "%";

      if (early_stopping && sentences.f1 + tokens.f1 > best_combined_f1) {
        best_combined_f1 = sentences.f1 + tokens.f1;
        best_combined_f1_epoch = epoch;
        best_combined_f1_network = *this;
      }
      if (early_stopping && best_combined_f1 && epoch - best_combined_f1_epoch > 30) {
        cerr << endl << "Stopping after 30 iterations of not improving sum of sentence and token f1." << endl;
        break;
      }
    }
    cerr << endl;
  }

  // Choose best network if desired
  if (early_stopping && best_combined_f1) {
    cerr << "Choosing parameters from epoch " << best_combined_f1_epoch+1 << "." << endl;
    this->embeddings = best_combined_f1_network.embeddings;
    this->gru_fwd = best_combined_f1_network.gru_fwd;
    this->gru_bwd = best_combined_f1_network.gru_bwd;
    this->projection_fwd = best_combined_f1_network.projection_fwd;
    this->projection_bwd = best_combined_f1_network.projection_bwd;
  }

  // Encode the network
  enc.add_1B(1);
  enc.add_1B(D);

  enc.add_4B(this->embeddings.size());
  for (auto&& embedding : this->embeddings) {
    enc.add_4B(embedding.first);
    enc.add_data(embedding.second.e.w[0], D);
  }
  save_gru(this->gru_fwd, enc);
  save_gru(this->gru_bwd, enc);
  save_matrix(this->projection_fwd, enc);
  save_matrix(this->projection_bwd, enc);

  return true;
}

template  template
void gru_tokenizer_network_trainer::matrix_trainer::update_weights(float learning_rate) {
  for (int i = 0; i < R; i++) {
    for (int j = 0; j < C; j++) {
      w_m[i][j] = 0.9 * w_m[i][j] + (1-0.9) * w_g[i][j];
      w_v[i][j] = 0.999 * w_v[i][j] + (1-0.999) * w_g[i][j] * w_g[i][j];
      original.w[i][j] += learning_rate * w_m[i][j] / (sqrt(w_v[i][j]) + 1e-8);
    }
    b_m[i] = 0.9 * b_m[i] + (1-0.9) * b_g[i];
    b_v[i] = 0.999 * b_v[i] + (1-0.999) * b_g[i] * b_g[i];
    original.b[i] += learning_rate * b_m[i] / (sqrt(b_v[i]) + 1e-8);
  }

  for (int i = 0; i < R; i++) {
    for (int j = 0; j < C; j++)
      w_g[i][j] = 0.f;
    b_g[i] = 0.f;
  }
}

template
void gru_tokenizer_network_trainer::gru_trainer::update_weights(float learning_rate) {
  X.update_weights(learning_rate);
  X_r.update_weights(learning_rate);
  X_z.update_weights(learning_rate);
  H.update_weights(learning_rate);
  H_r.update_weights(learning_rate);
  H_z.update_weights(learning_rate);
}

template
void gru_tokenizer_network_trainer::evaluate(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, const vector& heldout,
                                                f1_info& tokens_f1, f1_info& sentences_f1) {
  // Generate gold data
  vector gold_sentences, gold_tokens;
  u32string text;
  for (auto&& sentence : heldout) {
    if (sentence.tokens.empty()) continue;

    gold_sentences.emplace_back(text.size() + sentence.tokens.front().start, sentence.tokens.back().start + sentence.tokens.back().length - sentence.tokens.front().start);
    for (auto&& token : sentence.tokens)
      gold_tokens.emplace_back(text.size() + token.start, token.length);
    text.append(sentence.sentence);
  }

  // Generate system data
  vector system_sentences, system_tokens, tokens;
  string text_utf8;

  this->cache_embeddings();
  gru_tokenizer tokenizer(url_email_tokenizer, segment, allow_spaces, *this);
  unilib::utf8::encode(text, text_utf8);
  tokenizer.set_text(text_utf8);

  while (tokenizer.next_sentence(tokens))
    if (!tokens.empty()) {
      system_sentences.emplace_back(tokens.front().start, tokens.back().start + tokens.back().length - tokens.front().start);
      system_tokens.insert(system_tokens.end(), tokens.begin(), tokens.end());
    }

  evaluate_f1(system_tokens, gold_tokens, tokens_f1);
  evaluate_f1(system_sentences, gold_sentences, sentences_f1);
}

template
void gru_tokenizer_network_trainer::evaluate_f1(const vector& system, const vector& gold, f1_info& f1) {
  size_t both = 0;
  for (size_t si = 0, gi = 0; si < system.size() || gi < gold.size(); )
    if (si < system.size() && (gi == gold.size() || system[si].start < gold[gi].start))
      si++;
    else if (gi < gold.size() && (si == system.size() || gold[gi].start < system[si].start))
      gi++;
    else
      both += system[si++].length == gold[gi++].length;

  f1.precision = system.size() ? both / double(system.size()) : 0.;
  f1.recall = gold.size() ? both / double(gold.size()) : 0.;
  f1.f1 = system.size()+gold.size() ? 2 * both / double(system.size() + gold.size()) : 0.;
}

template  template
void gru_tokenizer_network_trainer::random_matrix(matrix& m, mt19937& generator, float range, float bias) {
  uniform_real_distribution uniform(-range, range);
  for (int i = 0; i < R; i++) {
    m.b[i] = bias;
    for (int j = 0; j < C; j++)
      m.w[i][j] = uniform(generator);
  }
}

template
void gru_tokenizer_network_trainer::random_gru(gru& g, mt19937& generator, float range) {
  random_matrix(g.X, generator, range, 0.f);
  random_matrix(g.X_r, generator, range, 1.f);
  random_matrix(g.X_z, generator, range, 1.f);
  random_matrix(g.H, generator, range, 0.f);
  random_matrix(g.H_r, generator, range, 1.f);
  random_matrix(g.H_z, generator, range, 1.f);
}

template  template
void gru_tokenizer_network_trainer::save_matrix(const matrix& m, binary_encoder& enc) {
  for (int i = 0; i < R; i++)
    enc.add_data(m.w[i], C);
  enc.add_data(m.b, R);
}

template
void gru_tokenizer_network_trainer::save_gru(const gru& g, binary_encoder& enc) {
  save_matrix(g.X, enc);
  save_matrix(g.X_r, enc);
  save_matrix(g.X_z, enc);
  save_matrix(g.H, enc);
  save_matrix(g.H_r, enc);
  save_matrix(g.H_z, enc);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/gru_tokenizer_trainer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool gru_tokenizer_trainer::train(unsigned url_email_tokenizer, unsigned segment, bool allow_spaces, unsigned dimension, unsigned epochs,
                                  unsigned batch_size, float learning_rate, float learning_rate_final, float dropout,
                                  float initialization_range, bool early_stopping, const vector& data,
                                  const vector& heldout, ostream& os, string& error) {
  using namespace unilib;

  error.clear();

  // Start encoding the tokenizer
  os.put(2);

  binary_encoder enc;
  enc.add_1B(url_email_tokenizer);
  enc.add_2B(segment);
  enc.add_1B(allow_spaces);

  // Train the GRU network
  if (dimension == 16) {
    gru_tokenizer_network_trainer<16> network;
    if (!network.train(url_email_tokenizer, segment, allow_spaces, epochs, batch_size, learning_rate, learning_rate_final,
                       dropout, initialization_range, early_stopping, data, heldout, enc, error)) return false;
  } else if (dimension == 24) {
    gru_tokenizer_network_trainer<24> network;
    if (!network.train(url_email_tokenizer, segment, allow_spaces, epochs, batch_size, learning_rate, learning_rate_final,
                       dropout, initialization_range, early_stopping, data, heldout, enc, error)) return false;
  } else if (dimension == 64) {
    gru_tokenizer_network_trainer<64> network;
    if (!network.train(url_email_tokenizer, segment, allow_spaces, epochs, batch_size, learning_rate, learning_rate_final,
                       dropout, initialization_range, early_stopping, data, heldout, enc, error)) return false;
  } else {
    return error.assign("Gru tokenizer dimension '").append(to_string(dimension)).append("' is not supported!"), false;
  }

  // Compute best substitutions for every category
  unordered_map> counts;
  for (auto&& sentence : data)
    for (auto&& chr : sentence.sentence)
      counts[unicode::category(chr)][chr]++;

  unordered_map unknown_chars;
  for (auto&& count : counts) {
    char32_t best_chr = 0;
    unsigned best = 0;
    for (auto&& chr : count.second)
      if (chr.second > best)
        best = chr.second, best_chr = chr.first;
    if (best_chr)
      unknown_chars.emplace(count.first, best_chr);
  }
  enc.add_1B(unknown_chars.size());
  for (auto&& unknown_char : unknown_chars) {
    enc.add_4B(unknown_char.first);
    enc.add_4B(unknown_char.second);
  }

  if (!compressor::save(os, enc)) return error.assign("Cannot save gru_tokenizer_factory!"), false;
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/ragel_tokenizer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

static const char _ragel_url_email_cond_offsets[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 2, 3, 3, 4, 5,
	6, 7, 8, 9, 10, 11, 12, 13,
	14, 15, 16
};

static const char _ragel_url_email_cond_lengths[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 1, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 1, 1, 0, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1
};

static const short _ragel_url_email_cond_keys[] = {
	41u, 41u, 47u, 47u, 47u, 47u, 41u, 41u,
	47u, 47u, 47u, 47u, 47u, 47u, 47u, 47u,
	47u, 47u, 47u, 47u, 47u, 47u, 47u, 47u,
	47u, 47u, 47u, 47u, 47u, 47u, 47u, 47u,
	47u, 47u, 0
};

static const char _ragel_url_email_cond_spaces[] = {
	1, 0, 0, 1, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0
};

static const short _ragel_url_email_key_offsets[] = {
	0, 0, 15, 29, 41, 54, 63, 71,
	78, 86, 92, 100, 117, 145, 154, 162,
	171, 179, 188, 196, 204, 215, 225, 233,
	241, 252, 262, 270, 278, 289, 299, 315,
	330, 346, 360, 376, 393, 409, 426, 442,
	459, 475, 491, 510, 528, 544, 560, 579,
	597, 613, 629, 648, 666, 682, 698, 714,
	725, 726, 741, 752, 756, 773, 801, 812,
	823, 834, 848, 861, 879, 893, 908, 926,
	944, 962, 983
};

static const short _ragel_url_email_trans_keys[] = {
	33u, 48u, 49u, 50u, 95u, 36u, 37u, 39u,
	46u, 51u, 57u, 65u, 90u, 97u, 122u, 33u,
	58u, 64u, 95u, 36u, 37u, 39u, 46u, 48u,
	57u, 65u, 90u, 97u, 122u, 33u, 95u, 36u,
	37u, 39u, 46u, 48u, 57u, 65u, 90u, 97u,
	122u, 33u, 64u, 95u, 36u, 37u, 39u, 46u,
	48u, 57u, 65u, 90u, 97u, 122u, 48u, 49u,
	50u, 51u, 57u, 65u, 90u, 97u, 122u, 45u,
	46u, 48u, 57u, 65u, 90u, 97u, 122u, 45u,
	48u, 57u, 65u, 90u, 97u, 122u, 45u, 46u,
	48u, 57u, 65u, 90u, 97u, 122u, 48u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 48u, 57u,
	65u, 90u, 97u, 122u, 33u, 39u, 41u, 61u,
	95u, 36u, 47u, 48u, 57u, 58u, 59u, 63u,
	64u, 65u, 90u, 97u, 122u, 33u, 39u, 40u,
	44u, 46u, 61u, 63u, 95u, 129u, 131u, 135u,
	151u, 809u, 1065u, 36u, 38u, 42u, 57u, 58u,
	59u, 64u, 90u, 97u, 122u, 142u, 143u, 155u,
	159u, 48u, 49u, 50u, 51u, 57u, 65u, 90u,
	97u, 122u, 45u, 46u, 48u, 57u, 65u, 90u,
	97u, 122u, 48u, 49u, 50u, 51u, 57u, 65u,
	90u, 97u, 122u, 45u, 46u, 48u, 57u, 65u,
	90u, 97u, 122u, 48u, 49u, 50u, 51u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 48u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 48u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 53u, 48u,
	52u, 54u, 57u, 65u, 90u, 97u, 122u, 45u,
	46u, 48u, 53u, 54u, 57u, 65u, 90u, 97u,
	122u, 45u, 46u, 48u, 57u, 65u, 90u, 97u,
	122u, 45u, 46u, 48u, 57u, 65u, 90u, 97u,
	122u, 45u, 46u, 53u, 48u, 52u, 54u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 48u, 53u,
	54u, 57u, 65u, 90u, 97u, 122u, 45u, 46u,
	48u, 57u, 65u, 90u, 97u, 122u, 45u, 46u,
	48u, 57u, 65u, 90u, 97u, 122u, 45u, 46u,
	53u, 48u, 52u, 54u, 57u, 65u, 90u, 97u,
	122u, 45u, 46u, 48u, 53u, 54u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 58u, 64u,
	95u, 36u, 37u, 39u, 44u, 48u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 58u, 64u, 95u,
	36u, 37u, 39u, 46u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 58u, 64u, 95u, 36u, 37u,
	39u, 46u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 36u, 37u,
	39u, 44u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 48u, 49u, 50u, 58u, 64u, 95u, 36u,
	37u, 39u, 46u, 51u, 57u, 65u, 90u, 97u,
	122u, 33u, 45u, 46u, 58u, 64u, 95u, 36u,
	37u, 39u, 44u, 48u, 57u, 65u, 90u, 97u,
	122u, 33u, 48u, 49u, 50u, 58u, 64u, 95u,
	36u, 37u, 39u, 46u, 51u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 48u, 49u, 50u, 58u, 64u,
	95u, 36u, 37u, 39u, 46u, 51u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 58u, 64u,
	95u, 36u, 37u, 39u, 44u, 48u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 58u, 64u,
	95u, 36u, 37u, 39u, 44u, 48u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 53u, 58u,
	64u, 95u, 36u, 37u, 39u, 44u, 48u, 52u,
	54u, 57u, 65u, 90u, 97u, 122u, 33u, 45u,
	46u, 58u, 64u, 95u, 36u, 37u, 39u, 44u,
	48u, 53u, 54u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 36u, 37u,
	39u, 44u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 36u, 37u,
	39u, 44u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 53u, 58u, 64u, 95u, 36u,
	37u, 39u, 44u, 48u, 52u, 54u, 57u, 65u,
	90u, 97u, 122u, 33u, 45u, 46u, 58u, 64u,
	95u, 36u, 37u, 39u, 44u, 48u, 53u, 54u,
	57u, 65u, 90u, 97u, 122u, 33u, 45u, 46u,
	58u, 64u, 95u, 36u, 37u, 39u, 44u, 48u,
	57u, 65u, 90u, 97u, 122u, 33u, 45u, 46u,
	58u, 64u, 95u, 36u, 37u, 39u, 44u, 48u,
	57u, 65u, 90u, 97u, 122u, 33u, 45u, 46u,
	53u, 58u, 64u, 95u, 36u, 37u, 39u, 44u,
	48u, 52u, 54u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 36u, 37u,
	39u, 44u, 48u, 53u, 54u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 58u, 64u, 95u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 47u, 95u, 36u, 37u, 39u,
	57u, 65u, 90u, 97u, 122u, 47u, 33u, 48u,
	49u, 50u, 95u, 36u, 37u, 39u, 46u, 51u,
	57u, 65u, 90u, 97u, 122u, 45u, 46u, 58u,
	303u, 559u, 48u, 57u, 65u, 90u, 97u, 122u,
	303u, 559u, 48u, 57u, 33u, 39u, 41u, 61u,
	95u, 36u, 47u, 48u, 57u, 58u, 59u, 63u,
	64u, 65u, 90u, 97u, 122u, 33u, 39u, 40u,
	44u, 46u, 61u, 63u, 95u, 129u, 131u, 135u,
	151u, 809u, 1065u, 36u, 38u, 42u, 57u, 58u,
	59u, 64u, 90u, 97u, 122u, 142u, 143u, 155u,
	159u, 45u, 46u, 58u, 303u, 559u, 48u, 57u,
	65u, 90u, 97u, 122u, 45u, 46u, 58u, 303u,
	559u, 48u, 57u, 65u, 90u, 97u, 122u, 45u,
	46u, 58u, 303u, 559u, 48u, 57u, 65u, 90u,
	97u, 122u, 45u, 46u, 53u, 58u, 303u, 559u,
	48u, 52u, 54u, 57u, 65u, 90u, 97u, 122u,
	45u, 46u, 58u, 303u, 559u, 48u, 53u, 54u,
	57u, 65u, 90u, 97u, 122u, 33u, 45u, 46u,
	58u, 64u, 95u, 303u, 559u, 36u, 37u, 39u,
	44u, 48u, 57u, 65u, 90u, 97u, 122u, 33u,
	95u, 303u, 559u, 36u, 37u, 39u, 46u, 48u,
	57u, 65u, 90u, 97u, 122u, 33u, 64u, 95u,
	303u, 559u, 36u, 37u, 39u, 46u, 48u, 57u,
	65u, 90u, 97u, 122u, 33u, 45u, 46u, 58u,
	64u, 95u, 303u, 559u, 36u, 37u, 39u, 44u,
	48u, 57u, 65u, 90u, 97u, 122u, 33u, 45u,
	46u, 58u, 64u, 95u, 303u, 559u, 36u, 37u,
	39u, 44u, 48u, 57u, 65u, 90u, 97u, 122u,
	33u, 45u, 46u, 58u, 64u, 95u, 303u, 559u,
	36u, 37u, 39u, 44u, 48u, 57u, 65u, 90u,
	97u, 122u, 33u, 45u, 46u, 53u, 58u, 64u,
	95u, 303u, 559u, 36u, 37u, 39u, 44u, 48u,
	52u, 54u, 57u, 65u, 90u, 97u, 122u, 33u,
	45u, 46u, 58u, 64u, 95u, 303u, 559u, 36u,
	37u, 39u, 44u, 48u, 53u, 54u, 57u, 65u,
	90u, 97u, 122u, 0
};

static const char _ragel_url_email_single_lengths[] = {
	0, 5, 4, 2, 3, 3, 2, 1,
	2, 0, 2, 5, 14, 3, 2, 3,
	2, 3, 2, 2, 3, 2, 2, 2,
	3, 2, 2, 2, 3, 2, 6, 5,
	6, 4, 6, 7, 6, 7, 6, 7,
	6, 6, 7, 6, 6, 6, 7, 6,
	6, 6, 7, 6, 6, 6, 6, 3,
	1, 5, 5, 2, 5, 14, 5, 5,
	5, 6, 5, 8, 4, 5, 8, 8,
	8, 9, 8
};

static const char _ragel_url_email_range_lengths[] = {
	0, 5, 5, 5, 5, 3, 3, 3,
	3, 3, 3, 6, 7, 3, 3, 3,
	3, 3, 3, 3, 4, 4, 3, 3,
	4, 4, 3, 3, 4, 4, 5, 5,
	5, 5, 5, 5, 5, 5, 5, 5,
	5, 5, 6, 6, 5, 5, 6, 6,
	5, 5, 6, 6, 5, 5, 5, 4,
	0, 5, 3, 1, 6, 7, 3, 3,
	3, 4, 4, 5, 5, 5, 5, 5,
	5, 6, 6
};

static const short _ragel_url_email_index_offsets[] = {
	0, 0, 11, 21, 29, 38, 45, 51,
	56, 62, 66, 72, 84, 106, 113, 119,
	126, 132, 139, 145, 151, 159, 166, 172,
	178, 186, 193, 199, 205, 213, 220, 232,
	243, 255, 265, 277, 290, 302, 315, 327,
	340, 352, 364, 378, 391, 403, 415, 429,
	442, 454, 466, 480, 493, 505, 517, 529,
	537, 539, 550, 559, 563, 575, 597, 606,
	615, 624, 635, 645, 659, 669, 680, 694,
	708, 722, 738
};

static const char _ragel_url_email_indicies[] = {
	0, 2, 3, 4, 0, 0, 0, 5,
	6, 6, 1, 0, 7, 8, 0, 0,
	0, 0, 0, 0, 1, 9, 9, 9,
	9, 9, 9, 9, 1, 9, 8, 9,
	9, 9, 9, 9, 9, 1, 10, 11,
	12, 13, 14, 14, 1, 15, 16, 14,
	14, 14, 1, 15, 14, 14, 14, 1,
	15, 17, 14, 14, 14, 1, 14, 18,
	18, 1, 15, 17, 14, 19, 19, 1,
	20, 21, 21, 20, 20, 20, 21, 20,
	20, 21, 21, 1, 22, 22, 24, 22,
	22, 23, 22, 23, 23, 23, 23, 23,
	25, 26, 23, 23, 22, 23, 23, 23,
	23, 1, 27, 28, 29, 30, 18, 18,
	1, 15, 31, 14, 14, 14, 1, 32,
	33, 34, 35, 18, 18, 1, 15, 36,
	14, 14, 14, 1, 37, 38, 39, 40,
	18, 18, 1, 15, 36, 35, 14, 14,
	1, 15, 36, 32, 14, 14, 1, 15,
	36, 41, 35, 32, 14, 14, 1, 15,
	36, 32, 14, 14, 14, 1, 15, 31,
	30, 14, 14, 1, 15, 31, 27, 14,
	14, 1, 15, 31, 42, 30, 27, 14,
	14, 1, 15, 31, 27, 14, 14, 14,
	1, 15, 16, 13, 14, 14, 1, 15,
	16, 10, 14, 14, 1, 15, 16, 43,
	13, 10, 14, 14, 1, 15, 16, 10,
	14, 14, 14, 1, 0, 44, 45, 7,
	8, 0, 0, 0, 46, 46, 46, 1,
	0, 44, 7, 8, 0, 0, 0, 46,
	46, 46, 1, 0, 44, 47, 7, 8,
	0, 0, 0, 46, 46, 46, 1, 0,
	7, 8, 0, 0, 0, 46, 48, 48,
	1, 0, 44, 47, 7, 8, 0, 0,
	0, 46, 49, 49, 1, 0, 50, 51,
	52, 7, 8, 0, 0, 0, 53, 48,
	48, 1, 0, 44, 54, 7, 8, 0,
	0, 0, 46, 46, 46, 1, 0, 55,
	56, 57, 7, 8, 0, 0, 0, 58,
	48, 48, 1, 0, 44, 59, 7, 8,
	0, 0, 0, 46, 46, 46, 1, 0,
	60, 61, 62, 7, 8, 0, 0, 0,
	63, 48, 48, 1, 0, 44, 59, 7,
	8, 0, 0, 0, 58, 46, 46, 1,
	0, 44, 59, 7, 8, 0, 0, 0,
	55, 46, 46, 1, 0, 44, 59, 64,
	7, 8, 0, 0, 0, 58, 55, 46,
	46, 1, 0, 44, 59, 7, 8, 0,
	0, 0, 55, 46, 46, 46, 1, 0,
	44, 54, 7, 8, 0, 0, 0, 53,
	46, 46, 1, 0, 44, 54, 7, 8,
	0, 0, 0, 50, 46, 46, 1, 0,
	44, 54, 65, 7, 8, 0, 0, 0,
	53, 50, 46, 46, 1, 0, 44, 54,
	7, 8, 0, 0, 0, 50, 46, 46,
	46, 1, 0, 44, 45, 7, 8, 0,
	0, 0, 5, 46, 46, 1, 0, 44,
	45, 7, 8, 0, 0, 0, 2, 46,
	46, 1, 0, 44, 45, 66, 7, 8,
	0, 0, 0, 5, 2, 46, 46, 1,
	0, 44, 45, 7, 8, 0, 0, 0,
	2, 46, 46, 46, 1, 0, 44, 47,
	7, 8, 0, 0, 0, 46, 67, 67,
	1, 0, 44, 47, 7, 8, 0, 0,
	0, 46, 68, 68, 1, 0, 44, 47,
	69, 8, 0, 0, 0, 46, 68, 68,
	1, 9, 70, 9, 9, 9, 9, 9,
	1, 71, 1, 0, 2, 3, 4, 0,
	0, 0, 5, 46, 46, 1, 15, 17,
	72, 21, 23, 14, 19, 19, 1, 21,
	23, 72, 1, 20, 21, 21, 20, 20,
	20, 21, 20, 20, 21, 21, 1, 22,
	22, 24, 22, 22, 23, 22, 23, 23,
	23, 23, 23, 25, 26, 23, 23, 22,
	23, 23, 23, 23, 1, 15, 17, 72,
	21, 23, 14, 14, 14, 1, 15, 17,
	72, 21, 23, 40, 14, 14, 1, 15,
	17, 72, 21, 23, 37, 14, 14, 1,
	15, 17, 73, 72, 21, 23, 40, 37,
	14, 14, 1, 15, 17, 72, 21, 23,
	37, 14, 14, 14, 1, 0, 44, 47,
	74, 8, 0, 21, 23, 0, 0, 46,
	49, 49, 1, 9, 9, 21, 23, 9,
	9, 75, 9, 9, 1, 9, 8, 9,
	21, 23, 9, 9, 75, 9, 9, 1,
	0, 44, 47, 74, 8, 0, 21, 23,
	0, 0, 46, 46, 46, 1, 0, 44,
	47, 74, 8, 0, 21, 23, 0, 0,
	63, 46, 46, 1, 0, 44, 47, 74,
	8, 0, 21, 23, 0, 0, 60, 46,
	46, 1, 0, 44, 47, 76, 74, 8,
	0, 21, 23, 0, 0, 63, 60, 46,
	46, 1, 0, 44, 47, 74, 8, 0,
	21, 23, 0, 0, 60, 46, 46, 46,
	1, 0
};

static const char _ragel_url_email_trans_targs[] = {
	2, 0, 30, 48, 50, 49, 52, 3,
	5, 4, 6, 26, 28, 27, 8, 7,
	13, 9, 10, 58, 11, 60, 12, 61,
	61, 12, 61, 14, 22, 24, 23, 15,
	16, 18, 20, 19, 17, 62, 63, 65,
	64, 21, 25, 29, 31, 35, 32, 33,
	34, 67, 36, 44, 46, 45, 37, 38,
	40, 42, 41, 39, 70, 71, 73, 72,
	43, 47, 51, 53, 54, 55, 56, 57,
	59, 66, 68, 69, 74
};

static const char _ragel_url_email_trans_actions[] = {
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 1, 0, 1, 0, 1,
	2, 3, 4, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 1, 1, 1,
	1, 0, 0, 0, 0, 0, 0, 0,
	0, 1, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 1, 1, 1, 1,
	0, 0, 0, 0, 0, 0, 0, 0,
	1, 1, 1, 1, 1
};

static const int ragel_url_email_start = 1;

vector ragel_tokenizer::ragel_map;
atomic_flag ragel_tokenizer::ragel_map_flag = ATOMIC_FLAG_INIT;

ragel_tokenizer::ragel_tokenizer(unsigned url_email_tokenizer) : unicode_tokenizer(url_email_tokenizer) {
  initialize_ragel_map();
}

void ragel_tokenizer::initialize_ragel_map() {
  while (ragel_map_flag.test_and_set()) {}
  if (ragel_map.empty()) {
    for (uint8_t ascii = 0; ascii < 128; ascii++)
      ragel_map.push_back(ascii);

    ragel_map_add(U'\u2026', 160); // horizontal ellipsis (TRIPLE DOT)
    ragel_map_add(U'\u2019', 161); // right single quotation mark
    ragel_map_add(U'\u2018', 162); // left single quotation mark
    ragel_map_add(U'\u2010', 163); // hyphen
  }
  ragel_map_flag.clear();
}

void ragel_tokenizer::ragel_map_add(char32_t chr, uint8_t mapping) {
  if (chr >= ragel_map.size())
    ragel_map.resize(chr + 1, 128);
  ragel_map[chr] = mapping;
}

bool ragel_tokenizer::ragel_url_email(unsigned version, const vector& chars, size_t& current, vector& tokens) {
  int cs;

  size_t start = current, end = current, parens = 0;

	{
	cs = ragel_url_email_start;
	}

	{
	int _klen;
	const short *_keys;
	int _trans;
	short _widec;

	if ( ( current) == ( (chars.size() - 1)) )
		goto _test_eof;
	if ( cs == 0 )
		goto _out;
_resume:
	_widec = ( ragel_char(chars[current]));
	_klen = _ragel_url_email_cond_lengths[cs];
	_keys = _ragel_url_email_cond_keys + (_ragel_url_email_cond_offsets[cs]*2);
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				switch ( _ragel_url_email_cond_spaces[_ragel_url_email_cond_offsets[cs] + ((_mid - _keys)>>1)] ) {
	case 0: {
		_widec = (short)(256u + (( ragel_char(chars[current])) - 0u));
		if (
 version >= 2  ) _widec += 256;
		break;
	}
	case 1: {
		_widec = (short)(768u + (( ragel_char(chars[current])) - 0u));
		if (
parens ) _widec += 256;
		break;
	}
				}
				break;
			}
		}
	}

	_keys = _ragel_url_email_trans_keys + _ragel_url_email_key_offsets[cs];
	_trans = _ragel_url_email_index_offsets[cs];

	_klen = _ragel_url_email_single_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + _klen - 1;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + ((_upper-_lower) >> 1);
			if ( _widec < *_mid )
				_upper = _mid - 1;
			else if ( _widec > *_mid )
				_lower = _mid + 1;
			else {
				_trans += (unsigned int)(_mid - _keys);
				goto _match;
			}
		}
		_keys += _klen;
		_trans += _klen;
	}

	_klen = _ragel_url_email_range_lengths[cs];
	if ( _klen > 0 ) {
		const short *_lower = _keys;
		const short *_mid;
		const short *_upper = _keys + (_klen<<1) - 2;
		while (1) {
			if ( _upper < _lower )
				break;

			_mid = _lower + (((_upper-_lower) >> 1) & ~1);
			if ( _widec < _mid[0] )
				_upper = _mid - 2;
			else if ( _widec > _mid[1] )
				_lower = _mid + 2;
			else {
				_trans += (unsigned int)((_mid - _keys)>>1);
				goto _match;
			}
		}
		_trans += _klen;
	}

_match:
	_trans = _ragel_url_email_indicies[_trans];
	cs = _ragel_url_email_trans_targs[_trans];

	if ( _ragel_url_email_trans_actions[_trans] == 0 )
		goto _again;

	switch ( _ragel_url_email_trans_actions[_trans] ) {
	case 3:
	{parens-=!!parens;}
	break;
	case 1:
	{ end = current + 1; }
	break;
	case 2:
	{parens++;}
	{ end = current + 1; }
	break;
	case 4:
	{parens-=!!parens;}
	{ end = current + 1; }
	break;
	}

_again:
	if ( cs == 0 )
		goto _out;
	if ( ++( current) != ( (chars.size() - 1)) )
		goto _resume;
	_test_eof: {}
	_out: {}
	}

  if (end > start) {
    tokens.emplace_back(start, end - start);
    current = end;
    return true;
  } else {
    current = start;
    return false;
  }
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/vertical_tokenizer.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class vertical_tokenizer : public unicode_tokenizer {
 public:
  vertical_tokenizer() : unicode_tokenizer(0) {}

  virtual bool next_sentence(vector& tokens) override;
};

} // namespace morphodita

/////////
// File: morphodita/tokenizer/tokenizer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tokenizer* tokenizer::new_vertical_tokenizer() {
  return new vertical_tokenizer();
}

tokenizer* tokenizer::new_czech_tokenizer() {
  return new czech_tokenizer(czech_tokenizer::CZECH, czech_tokenizer::LATEST);
}

tokenizer* tokenizer::new_english_tokenizer() {
  return new english_tokenizer(english_tokenizer::LATEST);
}

tokenizer* tokenizer::new_generic_tokenizer() {
  return new generic_tokenizer(generic_tokenizer::LATEST);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/tokenizer_ids.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class tokenizer_ids {
 public:
  enum tokenizer_id {
    CZECH = 0,
    ENGLISH = 1,
    GENERIC = 2,
    GRU = 3,
  };

  static bool parse(const string& str, tokenizer_id& id) {
    if (str == "czech") return id = CZECH, true;
    if (str == "english") return id = ENGLISH, true;
    if (str == "generic") return id = GENERIC, true;
    if (str == "gru") return id = GRU, true;
    return false;
  }
};

typedef tokenizer_ids::tokenizer_id tokenizer_id;

} // namespace morphodita

/////////
// File: morphodita/tokenizer/tokenizer_factory.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

tokenizer_factory* tokenizer_factory::load(istream& is) {
  tokenizer_id id = tokenizer_id(is.get());
  switch (id) {
    case tokenizer_ids::GENERIC:
      {
        auto res = new_unique_ptr();
        if (res->load(is)) return res.release();
        break;
      }
    case tokenizer_ids::GRU:
      {
        auto res = new_unique_ptr();
        if (res->load(is)) return res.release();
        break;
      }
    case tokenizer_ids::CZECH:
      {
        auto res = new_unique_ptr();
        if (res->load(is)) return res.release();
        break;
      }
    case tokenizer_ids::ENGLISH:
      break;
  }

  return nullptr;
}

tokenizer_factory* tokenizer_factory::load(const char* fname) {
  ifstream f(path_from_utf8(fname).c_str(), ifstream::binary);
  if (!f) return nullptr;

  return load(f);
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/unicode_tokenizer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

unicode_tokenizer::unicode_tokenizer(unsigned url_email_tokenizer) : url_email_tokenizer(url_email_tokenizer) {
  ragel_tokenizer::initialize_ragel_map();

  set_text(string_piece(nullptr, 0));
}

void unicode_tokenizer::set_text(string_piece text, bool make_copy /*= false*/) {
  using namespace unilib;

  if (make_copy && text.str) {
    text_buffer.assign(text.str, text.len);
    text.str = text_buffer.c_str();
  }
  current = 0;

  chars.clear();
  for (const char* curr_str = text.str; text.len; curr_str = text.str)
    chars.emplace_back(utf8::decode(text.str, text.len), curr_str);
  chars.emplace_back(0, text.str);
}

bool unicode_tokenizer::next_sentence(vector* forms, vector* tokens_ptr) {
  vector& tokens = tokens_ptr ? *tokens_ptr : tokens_buffer;
  tokens.clear();
  if (forms) forms->clear();
  if (current >= chars.size() - 1) return false;

  bool result = next_sentence(tokens);
  if (forms)
    for (auto&& token : tokens)
      forms->emplace_back(chars[token.start].str, chars[token.start + token.length].str - chars[token.start].str);

  return result;
}

bool unicode_tokenizer::tokenize_url_email(vector& tokens) {
  if (current >= chars.size() - 1) return false;

  return url_email_tokenizer ? ragel_tokenizer::ragel_url_email(url_email_tokenizer, chars, current, tokens) : false;
}

bool unicode_tokenizer::emergency_sentence_split(const vector& tokens) {
  using namespace unilib;

  // Implement emergency splitting for large sentences
  return tokens.size() >= 500 ||
         (tokens.size() >= 450 && chars[tokens.back().start].cat & unicode::P) ||
         (tokens.size() >= 400 && chars[tokens.back().start].cat & unicode::Po);
}

bool unicode_tokenizer::is_eos(const vector& tokens, char32_t eos_chr, const unordered_set* abbreviations) {
  using namespace unilib;

  if (eos_chr == '.' && !tokens.empty()) {
    // Ignore one-letter capitals before dot
    if (tokens.back().length == 1 && chars[tokens.back().start].cat & unicode::Lut)
      return false;

    // Ignore specified abbreviations
    if (abbreviations) {
      eos_buffer.clear();
      for (size_t i = 0; i < tokens.back().length; i++)
        utf8::append(eos_buffer, unicode::lowercase(chars[tokens.back().start + i].chr));
      if (abbreviations->count(eos_buffer))
        return false;
    }
  }
  return true;
}

} // namespace morphodita

/////////
// File: morphodita/tokenizer/vertical_tokenizer.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

bool vertical_tokenizer::next_sentence(vector& tokens) {
  if (current >= chars.size() - 1) return false;

  while (true) {
    size_t line_start = current;
    while (current < chars.size() - 1 && chars[current].chr != '\r' && chars[current].chr != '\n') current++;

    size_t line_end = current;
    if (current < chars.size() - 1) {
      current++;
      if (current < chars.size() - 1 &&
          ((chars[current-1].chr == '\r' && chars[current].chr == '\n') ||
           (chars[current-1].chr == '\n' && chars[current].chr == '\r')))
        current++;
    }

    if (line_start < line_end)
      tokens.emplace_back(line_start, line_end - line_start);
    else
      break;
  }

  return true;
}

} // namespace morphodita

/////////
// File: unilib/version.h
/////////

// This file is part of UniLib .
//
// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// UniLib version: 3.3.0
// Unicode version: 15.0.0

namespace unilib {

struct version {
  unsigned major;
  unsigned minor;
  unsigned patch;
  std::string prerelease;

  // Returns current version.
  static version current();
};

} // namespace unilib

/////////
// File: morphodita/version/version.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

class version {
 public:
  unsigned major;
  unsigned minor;
  unsigned patch;
  string prerelease;

  // Returns current MorphoDiTa version.
  static version current();

  // Returns multi-line formated version and copyright string.
  static string version_and_copyright(const string& other_libraries = string());
};

} // namespace morphodita

/////////
// File: morphodita/version/version.cpp
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

version version::current() {
  return {1, 11, 1, "dev"};
}

// Returns multi-line formated version and copyright string.
string version::version_and_copyright(const string& other_libraries) {
  ostringstream info;

  auto morphodita = version::current();
  auto unilib = unilib::version::current();

  info << "MorphoDiTa version " << morphodita.major << '.' << morphodita.minor << '.' << morphodita.patch
       << (morphodita.prerelease.empty() ? "" : "-") << morphodita.prerelease
       << " (using UniLib " << unilib.major << '.' << unilib.minor << '.' << unilib.patch
       << (other_libraries.empty() ? "" : " and ") << other_libraries << ")\n"
          "Copyright 2015 by Institute of Formal and Applied Linguistics, Faculty of\n"
          "Mathematics and Physics, Charles University in Prague, Czech Republic.";

  return info.str();
}

} // namespace morphodita

/////////
// File: parsito/configuration/configuration.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void configuration::init(tree* t) {
  assert(t);

  t->unlink_all_nodes();
  this->t = t;

  stack.clear();
  if (!t->nodes.empty()) stack.push_back(0);

  buffer.clear();
  buffer.reserve(t->nodes.size());
  for (size_t i = t->nodes.size(); i > 1; i--)
    buffer.push_back(i - 1);
}

bool configuration::final() {
  return buffer.empty() && stack.size() <= 1;
}

} // namespace parsito

/////////
// File: parsito/configuration/node_extractor.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class node_extractor {
 public:
  unsigned node_count() const;
  void extract(const configuration& conf, vector& nodes) const;

  bool create(string_piece description, string& error);

 private:
  enum start_t { STACK = 0, BUFFER = 1 };
  enum direction_t { PARENT = 0, CHILD = 1 };
  struct node_selector {
    pair start;
    vector> directions;

    node_selector(start_t start, int start_index) : start(start, start_index) {}
  };

  vector selectors;
};

} // namespace parsito

/////////
// File: parsito/configuration/node_extractor.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

unsigned node_extractor::node_count() const {
  return selectors.size();
}

void node_extractor::extract(const configuration& conf, vector& nodes) const {
  nodes.clear();
  for (auto&& selector : selectors) {
    // Start by locating starting node
    int current = -1;
    switch (selector.start.first) {
      case STACK:
        if (selector.start.second < int(conf.stack.size()))
          current = conf.stack[conf.stack.size() - 1 - selector.start.second];
        break;
      case BUFFER:
        if (selector.start.second < int(conf.buffer.size()))
          current = conf.buffer[conf.buffer.size() - 1 - selector.start.second];
        break;
    }

    // Follow directions to the final node
    if (current >= 0)
      for (auto&& direction : selector.directions) {
        const node& node = conf.t->nodes[current];
        switch (direction.first) {
          case PARENT:
            current = node.head ? node.head : -1;
            break;
          case CHILD:
            current = direction.second >= 0 && direction.second < int(node.children.size()) ?
                        node.children[direction.second] :
                      direction.second < 0 && -direction.second <= int(node.children.size()) ?
                        node.children[node.children.size() + direction.second] :
                        -1;
            break;
        }
        if (current <= 0) break;
      }

    // Add the selected node
    nodes.push_back(current);
  }
}

bool node_extractor::create(string_piece description, string& error) {
  selectors.clear();
  error.clear();

  vector lines, parts, words;
  split(description, '\n', lines);
  for (auto&& line : lines) {
    if (!line.len || line.str[0] == '#') continue;

    // Separate start and directions
    split(line, ',', parts);

    // Parse start
    split(parts[0], ' ', words);
    if (words.size() != 2)
      return error.assign("The node selector '").append(parts[0].str, parts[0].len).append("' on line '").append(line.str, line.len).append("' does not contain two space separated values!"), false;

    start_t start;
    if (words[0] == "stack")
      start = STACK;
    else if (words[0] == "buffer")
      start = BUFFER;
    else
      return error.assign("Cannot parse starting location '").append(words[0].str, words[0].len).append("' on line '").append(line.str, line.len).append(".!"), false;

    int start_index;
    if (!parse_int(words[1], "starting index", start_index, error)) return false;

    selectors.emplace_back(start, start_index);

    // Parse directions
    for (size_t i = 1; i < parts.size(); i++) {
      split(parts[i], ' ', words);
      if (words.empty())
        return error.assign("Empty node selector on line '").append(line.str, line.len).append(".!"), false;

      if (words[0] == "parent") {
        if (words.size() != 1)
          return error.assign("The node selector '").append(parts[i].str, parts[i].len).append("' on line '").append(line.str, line.len).append("' does not contain one space separated value!"), false;
        selectors.back().directions.emplace_back(PARENT, 0);
      } else if (words[0] == "child") {
        if (words.size() != 2)
          return error.assign("The node selector '").append(parts[i].str, parts[i].len).append("' on line '").append(line.str, line.len).append("' does not contain two space separated values!"), false;
        int child_index;
        if (!parse_int(words[1], "child index", child_index, error)) return false;
        selectors.back().directions.emplace_back(CHILD, child_index);
      } else {
        return error.assign("Cannot parse direction location '").append(words[0].str, words[0].len).append("' on line '").append(line.str, line.len).append(".!"), false;
      }
    }
  }

  return true;
}

} // namespace parsito

/////////
// File: parsito/configuration/value_extractor.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class value_extractor {
 public:
  void extract(const node& n, string& value) const;

  bool create(string_piece description, string& error);

 private:
  enum value_t { FORM = 0, LEMMA = 1, LEMMA_ID = 2, TAG = 3, UNIVERSAL_TAG = 4,
    FEATS = 5, UNIVERSAL_TAG_FEATS = 6, DEPREL = 7 };
  value_t selector;
};

} // namespace parsito

/////////
// File: parsito/configuration/value_extractor.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void value_extractor::extract(const node& n, string& value) const {
  switch (selector) {
    case FORM:
      value.assign(n.form);
      break;
    case LEMMA:
      value.assign(n.lemma);
      break;
    case LEMMA_ID:
      if (!n.misc.empty()) {
        // Try finding LId= in misc column
        auto lid = n.misc.find("LId=");
        if (lid != string::npos) {
          lid += 4;

          // Find optional | ending the lemma_id
          auto lid_end = n.misc.find('|', lid);
          if (lid_end == string::npos) lid_end = n.misc.size();

          // Store the lemma_id
          value.assign(n.misc, lid, lid_end - lid);
          break;
        }
      }
      value.assign(n.lemma);
      break;
    case TAG:
      value.assign(n.xpostag);
      break;
    case UNIVERSAL_TAG:
      value.assign(n.upostag);
      break;
    case FEATS:
      value.assign(n.feats);
      break;
    case UNIVERSAL_TAG_FEATS:
      value.assign(n.upostag).append(n.feats);
      break;
    case DEPREL:
      value.assign(n.deprel);
      break;
  }
}

bool value_extractor::create(string_piece description, string& error) {
  error.clear();

  if (description == "form")
    selector = FORM;
  else if (description == "lemma")
    selector = LEMMA;
  else if (description == "lemma_id")
    selector = LEMMA_ID;
  else if (description == "tag")
    selector = TAG;
  else if (description == "universal_tag")
    selector = UNIVERSAL_TAG;
  else if (description == "feats")
    selector = FEATS;
  else if (description == "universal_tag_feats")
    selector = UNIVERSAL_TAG_FEATS;
  else if (description == "deprel")
    selector = DEPREL;
  else
    return error.assign("Cannot parse value selector '").append(description.str, description.len).append("'!"), false;

  return true;
}

} // namespace parsito

/////////
// File: parsito/embedding/embedding.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class embedding {
 public:
  unsigned dimension;

  int lookup_word(const string& word, string& buffer) const;
  int unknown_word() const;
  float* weight(int id); // nullptr for wrong id
  const float* weight(int id) const; // nullpt for wrong id

  bool can_update_weights(int id) const;

  void load(binary_decoder& data);
  void save(binary_encoder& enc) const;

  void create(unsigned dimension, int updatable_index, const vector>>& words, const vector& unknown_weights);
  void export_embeddings(vector>>& words, vector& unknown_weights) const;
 private:
  int updatable_index, unknown_index;

  unordered_map dictionary;
  vector weights;
};

} // namespace parsito

/////////
// File: parsito/embedding/embedding.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

int embedding::lookup_word(const string& word, string& buffer) const {
  using namespace unilib;

  auto it = dictionary.find(word);
  if (it != dictionary.end()) return it->second;

  // We now apply several heuristics to find a match

  // Try locating uppercase/titlecase characters which we could lowercase
  bool first = true;
  unicode::category_t first_category = 0, other_categories = 0;
  for (auto&& chr : utf8::decoder(word)) {
    (first ? first_category : other_categories) |= unicode::category(chr);
    first = false;
  }

  if ((first_category & unicode::Lut) && (other_categories & unicode::Lut)) {
    // Lowercase all characters but the first
    buffer.clear();
    first = true;
    for (auto&& chr : utf8::decoder(word)) {
      utf8::append(buffer, first ? chr : unicode::lowercase(chr));
      first = false;
    }

    it = dictionary.find(buffer);
    if (it != dictionary.end()) return it->second;
  }

  if ((first_category & unicode::Lut) || (other_categories & unicode::Lut)) {
    utf8::map(unicode::lowercase, word, buffer);

    it = dictionary.find(buffer);
    if (it != dictionary.end()) return it->second;
  }

  // If the word starts with digit and contain only digits and non-letter characters
  // i.e. large number, date, time, try replacing it with first digit only.
  if ((first_category & unicode::N) && !(other_categories & unicode::L)) {
    buffer.clear();
    utf8::append(buffer, utf8::first(word));

    it = dictionary.find(buffer);
    if (it != dictionary.end()) return it->second;
  }

  return unknown_index;
}

int embedding::unknown_word() const {
  return unknown_index;
}

float* embedding::weight(int id) {
  if (id < 0 || id * dimension >= weights.size()) return nullptr;
  return weights.data() + id * dimension;
}

const float* embedding::weight(int id) const {
  if (id < 0 || id * dimension >= weights.size()) return nullptr;
  return weights.data() + id * dimension;
}

void embedding::load(binary_decoder& data) {
  // Load dimemsion
  dimension = data.next_4B();

  updatable_index = numeric_limits::max();

  // Load dictionary
  dictionary.clear();
  string word;
  for (unsigned size = data.next_4B(); size; size--) {
    data.next_str(word);
    dictionary.emplace(word, (int)dictionary.size());
  }

  unknown_index = data.next_1B() ? dictionary.size() : -1;

  // Load weights
  weights.resize(dimension * (dictionary.size() + (unknown_index >= 0)));
  memcpy(weights.data(), data.next(weights.size()), sizeof(float) * weights.size());
}

} // namespace parsito

/////////
// File: parsito/embedding/embedding_encode.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void embedding::save(binary_encoder& enc) const {
  // Save dimension and update_weight
  enc.add_4B(dimension);

  // Save the dictionary
  vector words(dictionary.size());
  for (auto&& entry : dictionary) {
    assert(entry.second >= 0 && entry.second < int(dictionary.size()));
    words[entry.second] = entry.first;
  }
  enc.add_4B(dictionary.size());
  for (auto&& word : words)
    enc.add_str(word);

  enc.add_1B(unknown_index >= 0);

  // Save the weights
  enc.add_data(weights);
}

bool embedding::can_update_weights(int id) const {
  return id >= int(updatable_index);
}

void embedding::create(unsigned dimension, int updatable_index, const vector>>& words, const vector& unknown_weights) {
  this->dimension = dimension;
  this->updatable_index = updatable_index;

  dictionary.clear();
  weights.clear();
  for (auto&& word : words) {
    assert(word.second.size() == dimension);
    dictionary.emplace(word.first, (int)dictionary.size());
    weights.insert(weights.end(), word.second.begin(), word.second.end());
  }

  if (unknown_weights.empty()) {
    this->unknown_index = -1;
  } else {
    this->unknown_index = dictionary.size();
    weights.insert(weights.end(), unknown_weights.begin(), unknown_weights.end());
  }
}

void embedding::export_embeddings(vector>>& words, vector& unknown_weights) const {
  words.clear();
  unknown_weights.clear();

  if (dictionary.empty()) return;

  assert(unknown_index < 0 || unknown_index == int(dictionary.size()));

  words.resize(dictionary.size());
  for (auto&& entry : dictionary) {
    words[entry.second].first = entry.first;
    words[entry.second].second.assign(weights.data() + entry.second * dimension, weights.data() + entry.second * dimension + dimension);
  }
  if (unknown_index >= 0)
    unknown_weights.assign(weights.data() + unknown_index * dimension, weights.data() + unknown_index * dimension + dimension);
}

} // namespace parsito

/////////
// File: parsito/network/activation_function.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

struct activation_function {
  enum type { TANH = 0, CUBIC = 1, RELU = 2 };

  static bool create(string_piece name, type& activation) {
    if (name == "tanh") return activation = TANH, true;
    if (name == "cubic") return activation = CUBIC, true;
    if (name == "relu") return activation = RELU, true;
    return false;
  }
};

} // namespace parsito

/////////
// File: parsito/network/neural_network.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class neural_network {
 public:
  typedef vector>> embeddings_cache;

  void propagate(const vector& embeddings, const vector*>& embedding_ids_sequences,
                 vector& hidden_layer, vector& outcomes, const embeddings_cache* cache = nullptr, bool softmax = true) const;

  void load(binary_decoder& data);
  void generate_tanh_cache();
  void generate_embeddings_cache(const vector& embeddings, embeddings_cache& cache, unsigned max_words) const;

 private:
  friend class neural_network_trainer;

  void load_matrix(binary_decoder& data, vector>& m);

  activation_function::type hidden_layer_activation;
  vector> weights[2];

  vector tanh_cache;
};

} // namespace parsito

/////////
// File: parsito/network/neural_network.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void neural_network::load_matrix(binary_decoder& data, vector>& m) {
  unsigned rows = data.next_4B();
  unsigned columns = data.next_4B();

  m.resize(rows);
  for (auto&& row : m) {
    row.resize(columns);
    memcpy(row.data(), data.next(columns), sizeof(float) * columns);
  }
}

void neural_network::load(binary_decoder& data) {
  hidden_layer_activation = activation_function::type(data.next_1B());
  load_matrix(data, weights[0]);
  load_matrix(data, weights[1]);
}

void neural_network::propagate(const vector& embeddings, const vector*>& embedding_ids_sequences,
                               vector& hidden_layer, vector& outcomes, const embeddings_cache* cache, bool softmax) const {
  assert(!weights[0].empty());
  assert(!weights[1].empty());
  for (auto&& embedding_ids : embedding_ids_sequences) if (embedding_ids) assert(embeddings.size() == embedding_ids->size());

  unsigned hidden_layer_size = weights[0].front().size();
  unsigned outcomes_size = weights[1].front().size();

  outcomes.assign(outcomes_size, 0);

  // Hidden layer
  hidden_layer.assign(hidden_layer_size, 0);

  unsigned index = 0;
  for (unsigned sequence = 0; sequence < embedding_ids_sequences.size(); sequence++)
    for (unsigned i = 0; i < embeddings.size(); index += embeddings[i].dimension, i++)
      if (embedding_ids_sequences[sequence] && embedding_ids_sequences[sequence]->at(i) >= 0) {
        unsigned word = embedding_ids_sequences[sequence]->at(i);
        if (cache && i < cache->size() && word < cache->at(i).size()) {
          // Use cache
          const float* precomputed = cache->at(i)[word].data() + sequence * hidden_layer_size;
          for (unsigned j = 0; j < hidden_layer_size; j++)
            hidden_layer[j] += precomputed[j];
        } else {
          // Compute directly
          const float* embedding = embeddings[i].weight(word);
          for (unsigned j = 0; j < embeddings[i].dimension; j++)
            for (unsigned k = 0; k < hidden_layer_size; k++)
              hidden_layer[k] += embedding[j] * weights[0][index + j][k];
        }
      }
  for (unsigned i = 0; i < hidden_layer_size; i++) // Bias
    hidden_layer[i] += weights[0][index][i];

  // Activation function
  switch (hidden_layer_activation) {
    case activation_function::TANH:
      if (!tanh_cache.empty())
        for (auto&& weight : hidden_layer)
          weight = weight <= -10 ? -1 : weight >= 10 ? 1 : tanh_cache[int(weight * 32768 + 10 * 32768)];
      else
        for (auto&& weight : hidden_layer)
          weight = tanh(weight);
      break;
    case activation_function::CUBIC:
      for (auto&& weight : hidden_layer)
        weight = weight * weight * weight;
      break;
    case activation_function::RELU:
      for (auto&& weight : hidden_layer)
        if (weight < 0) weight = 0;
      break;
  }

  for (unsigned i = 0; i < hidden_layer_size; i++)
    for (unsigned j = 0; j < outcomes_size; j++)
      outcomes[j] += hidden_layer[i] * weights[1][i][j];
  for (unsigned i = 0; i < outcomes_size; i++) // Bias
    outcomes[i] += weights[1][hidden_layer_size][i];

  // Softmax if requested
  if (softmax) {
    float max = outcomes[0];
    for (unsigned i = 1; i < outcomes_size; i++) if (outcomes[i] > max) max = outcomes[i];

    float sum = 0;
    for (unsigned i = 0; i < outcomes_size; i++) sum += (outcomes[i] = exp(outcomes[i] - max));
    sum = 1 / sum;

    for (unsigned i = 0; i < outcomes_size; i++) outcomes[i] *= sum;
  }
}

void neural_network::generate_tanh_cache() {
  tanh_cache.resize(2 * 10 * 32768);
  for (unsigned i = 0; i < tanh_cache.size(); i++)
    tanh_cache[i] = tanh(i / 32768.0 - 10);
}

void neural_network::generate_embeddings_cache(const vector& embeddings, embeddings_cache& cache, unsigned max_words) const {
  unsigned embeddings_dim = 0;
  for (auto&& embedding : embeddings) embeddings_dim += embedding.dimension;

  unsigned sequences = weights[0].size() / embeddings_dim;
  assert(sequences * embeddings_dim + 1 == weights[0].size());

  unsigned hidden_layer_size = weights[0].front().size();

  cache.resize(embeddings.size());
  for (unsigned i = 0, weight_index = 0; i < embeddings.size(); weight_index += embeddings[i].dimension, i++) {
    unsigned words = 0;
    while (words < max_words && embeddings[i].weight(words)) words++;

    cache[i].resize(words);
    for (unsigned word = 0; word < words; word++) {
      const float* embedding = embeddings[i].weight(word);

      cache[i][word].assign(sequences * hidden_layer_size, 0);
      for (unsigned sequence = 0, index = weight_index; sequence < sequences; index += embeddings_dim, sequence++)
        for (unsigned j = 0; j < embeddings[i].dimension; j++)
          for (unsigned k = 0; k < hidden_layer_size; k++)
            cache[i][word][sequence * hidden_layer_size + k] += embedding[j] * weights[0][index + j][k];
    }
  }
}

} // namespace parsito

/////////
// File: parsito/network/network_parameters.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

struct network_trainer {
  enum network_trainer_algorithm {
    SGD,
    SGD_MOMENTUM,
    ADAGRAD,
    ADADELTA,
    ADAM,
  };

  network_trainer_algorithm algorithm;
  float learning_rate, learning_rate_final;
  float momentum, momentum2;
  float epsilon;
};

struct network_parameters {
  unsigned iterations;
  int structured_interval;
  unsigned hidden_layer;
  activation_function::type hidden_layer_type;
  network_trainer trainer;
  unsigned batch_size;
  float initialization_range;
  float l1_regularization;
  float l2_regularization;
  float maxnorm_regularization;
  float dropout_hidden, dropout_input;
  bool early_stopping;
};

} // namespace parsito

/////////
// File: parsito/network/neural_network_trainer.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class neural_network_trainer {
 public:
  neural_network_trainer(neural_network& network, unsigned input_size, unsigned output_size,
                         const network_parameters& parameters, mt19937& generator);

  bool next_iteration();

  struct workspace {
    unsigned batch = 0;
    vector outcomes;
    vector hidden_layer;
    vector error_outcomes;
    vector error_hidden;

    // Delta accumulators
    vector> weights_batch[2];
    vector>> error_embedding;
    vector> error_embedding_nonempty;

    // Trainer data
    struct trainer_data {
      float delta = 0;
      float gradient = 0;
    };
    vector> weights_trainer[2];
    vector>> embedding_trainer;

    // Dropout vectors
    vector input_dropout;
    vector hidden_dropout;
    vector hidden_kept;
  };
  void propagate(const vector& embeddings, const vector*>& embedding_ids_sequences, workspace& w) const;
  void backpropagate(vector& embeddings, const vector*>& embedding_ids_sequences, unsigned required_outcome, workspace& w);

  void finalize_sentence();

  void save_network(binary_encoder& enc) const;

 private:
  struct trainer_sgd {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  struct trainer_sgd_momentum {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  struct trainer_adagrad {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  struct trainer_adadelta {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  struct trainer_adam {
    static bool need_trainer_data;
    static inline float delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data);
  };
  template  void backpropagate_template(vector& embeddings, const vector*>& embedding_ids_sequences, unsigned required_outcome, workspace& w);

  void l1_regularize();
  void maxnorm_regularize();

  void save_matrix(const vector>& m, binary_encoder& enc) const;

  neural_network& network;
  mt19937& generator;
  unsigned iteration, iterations, steps;
  network_trainer trainer;
  unsigned batch_size;
  float l1_regularization, l2_regularization, maxnorm_regularization;
  float dropout_hidden, dropout_input;
};

} // namespace parsito

/////////
// File: parsito/network/neural_network_trainer.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

neural_network_trainer::neural_network_trainer(neural_network& network, unsigned input_size, unsigned output_size,
                                               const network_parameters& parameters, mt19937& generator) : network(network), generator(generator) {
  // Initialize hidden layer
  network.hidden_layer_activation = parameters.hidden_layer_type;
  if (parameters.hidden_layer) {
    float uniform_pre_hidden_range = parameters.initialization_range > 0 ? parameters.initialization_range :
        -parameters.initialization_range * sqrt(6.0 / float(input_size + parameters.hidden_layer));
    uniform_real_distribution uniform_pre_hidden(-uniform_pre_hidden_range, uniform_pre_hidden_range);

    network.weights[0].resize(input_size + 1/*bias*/);
    for (auto&& row : network.weights[0]) {
      row.resize(parameters.hidden_layer);
      for (auto&& weight : row)
        weight = uniform_pre_hidden(generator);
    }

    float uniform_post_hidden_range = parameters.initialization_range > 0 ? parameters.initialization_range :
        -parameters.initialization_range * sqrt(6.0 / float(output_size + parameters.hidden_layer));
    uniform_real_distribution uniform_post_hidden(-uniform_post_hidden_range, uniform_post_hidden_range);

    network.weights[1].resize(parameters.hidden_layer + 1/*bias*/);
    for (auto&& row : network.weights[1]) {
      row.resize(output_size);
      for (auto&& weight : row)
        weight = uniform_post_hidden(generator);
    }
  }

  // Store the network_parameters
  iteration = steps = 0;
  iterations = parameters.iterations;
  trainer = parameters.trainer;
  batch_size = parameters.batch_size;
  l1_regularization = parameters.l1_regularization;
  l2_regularization = parameters.l2_regularization;
  maxnorm_regularization = parameters.maxnorm_regularization;
  dropout_hidden = parameters.dropout_hidden;
  dropout_input = parameters.dropout_input;

  // Maxnorm regularize the created weights
  if (maxnorm_regularization) maxnorm_regularize();
}

bool neural_network_trainer::next_iteration() {
  if (iteration++ >= iterations) return false;

  if (trainer.algorithm != network_trainer::ADADELTA)
    if (trainer.learning_rate != trainer.learning_rate_final && iteration > 1)
      trainer.learning_rate =
          exp(((iterations - iteration) * log(trainer.learning_rate) + log(trainer.learning_rate_final)) / (iterations - iteration + 1));

  return true;
}

void neural_network_trainer::propagate(const vector& embeddings, const vector*>& embedding_ids_sequences, workspace& w) const {
  // Initialize dropout if requested
  if (dropout_input) {
    w.input_dropout.resize(network.weights[0].size());
    bernoulli_distribution dropout(dropout_input);
    for (auto&& flag : w.input_dropout)
      flag = dropout(generator);
  }

  if (dropout_hidden) {
    w.hidden_dropout.resize(network.weights[1].size());
    bernoulli_distribution dropout(dropout_hidden);
    for (auto&& flag : w.hidden_dropout)
      flag = dropout(generator);
  }
  w.hidden_kept.clear();
  for (unsigned i = 0; i < network.weights[0].front().size(); i++)
    if (w.hidden_dropout.empty() || !w.hidden_dropout[i])
      w.hidden_kept.push_back(i);

  // Propagate
  unsigned hidden_layer_size = network.weights[0].front().size();
  unsigned outcomes_size = network.weights[1].front().size();

  w.outcomes.assign(outcomes_size, 0);

  // Hidden layer
  w.hidden_layer.assign(hidden_layer_size, 0);

  unsigned index = 0;
  for (auto&& embedding_ids : embedding_ids_sequences)
    // Note: The unnecessary brackets on the following for cycle are needed
    // to compile on VS 2015 Update 3, which otherwise fail to compile it.
    for (unsigned i = 0; i < embeddings.size(); i++) {
      if (embedding_ids && (*embedding_ids)[i] >= 0) {
        const float* embedding = embeddings[i].weight((*embedding_ids)[i]);
        for (unsigned dimension = embeddings[i].dimension; dimension; dimension--, embedding++, index++)
          if (w.input_dropout.empty() || !w.input_dropout[index])
            for (auto&& j : w.hidden_kept)
              w.hidden_layer[j] += *embedding * network.weights[0][index][j];
      } else {
        index += embeddings[i].dimension;
      }
    }
  if (dropout_input) { // Dropout normalization
    float dropout_factor = 1. / (1. - dropout_input);
    for (auto&& i : w.hidden_kept)
      w.hidden_layer[i] *= dropout_factor;
  }
  for (auto&& i : w.hidden_kept) // Bias
    w.hidden_layer[i] += network.weights[0][index][i];

  // Activation function
  switch (network.hidden_layer_activation) {
    case activation_function::TANH:
      for (auto&& weight : w.hidden_layer)
        weight = tanh(weight);
      break;
    case activation_function::CUBIC:
      for (auto&& weight : w.hidden_layer)
        weight = weight * weight * weight;
      break;
    case activation_function::RELU:
      for (auto&& weight : w.hidden_layer)
        if (weight < 0) weight = 0;
      break;
  }
  if (dropout_hidden) { // Dropout normalization
    float dropout_factor = 1. / (1. - dropout_hidden);
    for (auto&& i : w.hidden_kept)
      w.hidden_layer[i] *= dropout_factor;
  }

  for (auto&& i : w.hidden_kept)
    for (unsigned j = 0; j < outcomes_size; j++)
      w.outcomes[j] += w.hidden_layer[i] * network.weights[1][i][j];
  for (unsigned i = 0; i < outcomes_size; i++) // Bias
    w.outcomes[i] += network.weights[1][hidden_layer_size][i];

  // Softmax
  float max = w.outcomes[0];
  for (unsigned i = 1; i < outcomes_size; i++) if (w.outcomes[i] > max) max = w.outcomes[i];

  float sum = 0;
  for (unsigned i = 0; i < outcomes_size; i++) sum += (w.outcomes[i] = exp(w.outcomes[i] - max));
  sum = 1 / sum;

  for (unsigned i = 0; i < outcomes_size; i++) w.outcomes[i] *= sum;
}

// SGD
bool neural_network_trainer::trainer_sgd::need_trainer_data = false;
float neural_network_trainer::trainer_sgd::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& /*data*/) {
  return trainer.learning_rate * gradient;
}

// SGD with momentum
bool neural_network_trainer::trainer_sgd_momentum::need_trainer_data = true;
float neural_network_trainer::trainer_sgd_momentum::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data) {
  data.delta = trainer.momentum * data.delta + trainer.learning_rate * gradient;
  return data.delta;
}

// AdaGrad
bool neural_network_trainer::trainer_adagrad::need_trainer_data = true;
float neural_network_trainer::trainer_adagrad::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data) {
  data.gradient += gradient * gradient;
  return trainer.learning_rate / sqrt(data.gradient + trainer.epsilon) * gradient;
}

// AdaDelta
bool neural_network_trainer::trainer_adadelta::need_trainer_data = true;
float neural_network_trainer::trainer_adadelta::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data) {
  data.gradient = trainer.momentum * data.gradient + (1 - trainer.momentum) * gradient * gradient;
  float delta = sqrt(data.delta + trainer.epsilon) / sqrt(data.gradient + trainer.epsilon) * gradient;
  data.delta = trainer.momentum * data.delta + (1 - trainer.momentum) * delta * delta;
  return delta;
}

// Adam
bool neural_network_trainer::trainer_adam::need_trainer_data = true;
float neural_network_trainer::trainer_adam::delta(float gradient, const network_trainer& trainer, workspace::trainer_data& data) {
  data.gradient = trainer.momentum * data.gradient + (1 - trainer.momentum) * gradient;
  data.delta = trainer.momentum2 * data.delta + (1 - trainer.momentum2) * gradient * gradient;
  return trainer.learning_rate * data.gradient / sqrt(data.delta + trainer.epsilon);
}

// Backpropagation
template
void neural_network_trainer::backpropagate_template(vector& embeddings, const vector*>& embedding_ids_sequences, unsigned required_outcome, workspace& w) {
  size_t hidden_layer_size = network.weights[0].front().size();
  size_t outcomes_size = network.weights[1].front().size();

  // Allocate space for delta accumulators
  if (network.weights[0].size() > w.weights_batch[0].size()) w.weights_batch[0].resize(network.weights[0].size());
  if (network.weights[1].size() > w.weights_batch[1].size()) w.weights_batch[1].resize(network.weights[1].size());
  if (embeddings.size() > w.error_embedding.size()) w.error_embedding.resize(embeddings.size());
  if (embeddings.size() > w.error_embedding_nonempty.size()) w.error_embedding_nonempty.resize(embeddings.size());

  // Allocate space for trainer_data if required)
  workspace::trainer_data none_trainer_data;
  if (TRAINER::need_trainer_data) {
    while (network.weights[0].size() > w.weights_trainer[0].size()) w.weights_trainer[0].emplace_back(network.weights[0].front().size());
    while (network.weights[1].size() > w.weights_trainer[1].size()) w.weights_trainer[1].emplace_back(outcomes_size);
  }

  // Compute error vector
  w.error_outcomes.resize(outcomes_size);
  for (unsigned i = 0; i < outcomes_size; i++)
    w.error_outcomes[i] = (i == required_outcome) - w.outcomes[i];

  // Backpropagate error_outcomes to error_hidden
  w.error_hidden.assign(hidden_layer_size, 0);
  for (auto&& i : w.hidden_kept)
    for (unsigned j = 0; j < outcomes_size; j++)
      w.error_hidden[i] += network.weights[1][i][j] * w.error_outcomes[j];
  // Dropout normalization
  if (dropout_hidden) {
    float dropout_factor = 1. / (1. - dropout_hidden);
    for (auto&& i : w.hidden_kept)
      w.error_hidden[i] *= dropout_factor;
  }

  // Perform activation function derivation
  switch (network.hidden_layer_activation) {
    case activation_function::TANH:
      for (auto&& i : w.hidden_kept)
        w.error_hidden[i] *= 1 - w.hidden_layer[i] * w.hidden_layer[i];
      break;
    case activation_function::CUBIC:
      for (auto&& i : w.hidden_kept) {
        float hidden_layer = cbrt(w.hidden_layer[i]);
        w.error_hidden[i] *= 3 * hidden_layer * hidden_layer;
      }
      break;
    case activation_function::RELU:
      for (auto&& i : w.hidden_kept)
        if (w.hidden_layer[i] <= 0)
          w.error_hidden[i] = 0;
      break;
  }

  // Update weights[1]
  for (auto&& i : w.hidden_kept) {
    if (w.weights_batch[1][i].empty()) w.weights_batch[1][i].resize(outcomes_size);
    for (unsigned j = 0; j < outcomes_size; j++)
      w.weights_batch[1][i][j] += w.hidden_layer[i] * w.error_outcomes[j];
  }
  // Bias
  if (w.weights_batch[1][hidden_layer_size].empty()) w.weights_batch[1][hidden_layer_size].resize(outcomes_size);
  for (unsigned i = 0; i < outcomes_size; i++)
    w.weights_batch[1][hidden_layer_size][i] += w.error_outcomes[i];

  // Dropout normalization
  if (dropout_input) {
    float dropout_factor = 1. / (1. - dropout_input);
    for (auto&& i : w.hidden_kept)
      w.error_hidden[i] *= dropout_factor;
  }
  // Update weights[0] and backpropagate to error_embedding
  unsigned index = 0;
  for (auto&& embedding_ids : embedding_ids_sequences)
    // Note: The unnecessary brackets on the following for cycle are needed
    // to compile on VS 2015 Update 3, which otherwise fail to compile it.
    for (unsigned i = 0; i < embeddings.size(); i++) {
      if (embedding_ids && (*embedding_ids)[i] >= 0) {
        int embedding_id = (*embedding_ids)[i];

        float* error_embedding = nullptr; // Accumulate embedding error if required
        if (embeddings[i].can_update_weights(embedding_id)) {
          if (w.error_embedding[i].size() <= unsigned(embedding_id)) w.error_embedding[i].resize(embedding_id + 1);
          if (w.error_embedding[i][embedding_id].empty()) {
            w.error_embedding[i][embedding_id].assign(embeddings[i].dimension, 0);
            w.error_embedding_nonempty[i].emplace_back(embedding_id);
          }
          error_embedding = w.error_embedding[i][embedding_id].data();
        }

        const float* embedding = embeddings[i].weight(embedding_id);
        for (unsigned dimension = embeddings[i].dimension; dimension; dimension--, index++, embedding++, error_embedding += !!error_embedding)
          if (w.input_dropout.empty() || !w.input_dropout[index]) {
            if (error_embedding)
              for (auto&& j : w.hidden_kept)
                *error_embedding += network.weights[0][index][j] * w.error_hidden[j];
            if (w.weights_batch[0][index].empty()) w.weights_batch[0][index].resize(hidden_layer_size);
            for (auto&& j : w.hidden_kept)
              w.weights_batch[0][index][j] += *embedding * w.error_hidden[j];
          }
      } else {
        index += embeddings[i].dimension;
      }
    }
  // Bias
  {
    float negate_input_dropout = 1. - dropout_hidden;
    if (w.weights_batch[0][index].empty()) w.weights_batch[0][index].resize(hidden_layer_size);
    for (auto&& i : w.hidden_kept)
      w.weights_batch[0][index][i] += w.error_hidden[i] * negate_input_dropout;
  }

  // End if not at the end of the batch
  if (++w.batch < batch_size) return;
  w.batch = 0;

  // Update hidden weights
  if (!network.weights[0].empty())
    for (int i = 0; i < 2; i++) {
      for (unsigned j = 0; j < w.weights_batch[i].size(); j++)
        if (!w.weights_batch[i][j].empty()) {
          for (unsigned k = 0; k < w.weights_batch[i][j].size(); k++)
            network.weights[i][j][k] += TRAINER::delta(w.weights_batch[i][j][k], trainer, TRAINER::need_trainer_data ? w.weights_trainer[i][j][k] : none_trainer_data) - (j+1 == w.weights_batch[i].size() ? /*bias*/ 0. : l2_regularization) * network.weights[i][j][k];
          w.weights_batch[i][j].clear();
        }
    }

  // Update embedding weights using error_embedding
  for (unsigned i = 0; i < embeddings.size(); i++) {
    for (auto&& id : w.error_embedding_nonempty[i]) {
      if (TRAINER::need_trainer_data) {
        if (w.embedding_trainer.size() <= i) w.embedding_trainer.resize(i + 1);
        if (w.embedding_trainer[i].size() <= id) w.embedding_trainer[i].resize(id + 1);
        if (w.embedding_trainer[i][id].size() < embeddings[i].dimension) w.embedding_trainer[i][id].resize(embeddings[i].dimension);
      }
      float* embedding = embeddings[i].weight(id);
      for (unsigned j = 0; j < embeddings[i].dimension; j++)
        embedding[j] += TRAINER::delta(w.error_embedding[i][id][j], trainer, TRAINER::need_trainer_data ? w.embedding_trainer[i][id][j] : none_trainer_data) - l2_regularization * embedding[j];
      w.error_embedding[i][id].clear();
    }
    w.error_embedding_nonempty[i].clear();
  }

  // Maxnorm regularize the updated weights
  if (maxnorm_regularization) maxnorm_regularize();
}

void neural_network_trainer::backpropagate(vector& embeddings, const vector*>& embedding_ids_sequences, unsigned required_outcome, workspace& w) {
  steps++;

  switch (trainer.algorithm) {
    case network_trainer::SGD:
      backpropagate_template(embeddings, embedding_ids_sequences, required_outcome, w);
      return;
    case network_trainer::SGD_MOMENTUM:
      backpropagate_template(embeddings, embedding_ids_sequences, required_outcome, w);
      return;
    case network_trainer::ADAGRAD:
      backpropagate_template(embeddings, embedding_ids_sequences, required_outcome, w);
      return;
    case network_trainer::ADADELTA:
      backpropagate_template(embeddings, embedding_ids_sequences, required_outcome, w);
      return;
    case network_trainer::ADAM:
      float original_learning_rate = trainer.learning_rate;
      trainer.learning_rate *= sqrt(1-pow(trainer.momentum2, steps)) / (1-pow(trainer.momentum, steps));
      backpropagate_template(embeddings, embedding_ids_sequences, required_outcome, w);
      trainer.learning_rate = original_learning_rate;
      return;
  }

  training_failure("Internal error, unsupported trainer!");
}

void neural_network_trainer::l1_regularize() {
  if (!l1_regularization) return;

  for (auto&& weights : network.weights)
    for (unsigned i = 0; i + 1 /*ignore biases*/ < weights.size(); i++) {
      auto& row = weights[i];
      for (auto&& weight : row)
        if (weight < l1_regularization) weight += l1_regularization;
        else if (weight > l1_regularization) weight -= l1_regularization;
        else weight = 0;
    }
}

void neural_network_trainer::maxnorm_regularize() {
  if (!maxnorm_regularization) return;

  for (unsigned i = 0; i < 2; i++)
    for (unsigned j = 0; j < network.weights[i].front().size(); j++) {
      float length = 0;
      for (auto&& row : network.weights[i])
        length += row[j] * row[j];

      if (length > 0 && length > maxnorm_regularization * maxnorm_regularization) {
        float factor = 1 / sqrt(length / (maxnorm_regularization * maxnorm_regularization));
        for (auto&& row : network.weights[i])
          row[j] *= factor;
      }
    }
}

void neural_network_trainer::finalize_sentence() {
  if (l1_regularization) l1_regularize();
}

void neural_network_trainer::save_matrix(const vector>& m, binary_encoder& enc) const {
  enc.add_4B(m.size());
  enc.add_4B(m.empty() ? 0 : m.front().size());

  for (auto&& row : m) {
    assert(row.size() == m.front().size());
    enc.add_data(row);
  }
}

void neural_network_trainer::save_network(binary_encoder& enc) const {
  enc.add_1B(network.hidden_layer_activation);
  save_matrix(network.weights[0], enc);
  save_matrix(network.weights[1], enc);
}

} // namespace parsito

/////////
// File: parsito/transition/transition.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Abstract transition class
class transition {
 public:
  virtual ~transition() {}

  virtual bool applicable(const configuration& conf) const = 0;
  virtual int perform(configuration& conf) const = 0;
};

// Specific transition classes
class transition_left_arc : public transition {
 public:
  transition_left_arc(const string& label) : label(label), label_is_root(label == "root") {}

  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
 private:
  string label;
  bool label_is_root;
};

class transition_right_arc : public transition {
 public:
  transition_right_arc(const string& label) : label(label), label_is_root(label == "root") {}

  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
 private:
  string label;
  bool label_is_root;
};

class transition_shift : public transition {
 public:
  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
};

class transition_swap : public transition {
 public:
  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
};

class transition_left_arc_2 : public transition {
 public:
  transition_left_arc_2(const string& label) : label(label), label_is_root(label == "root") {}

  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
 private:
  string label;
  bool label_is_root;
};

class transition_right_arc_2 : public transition {
 public:
  transition_right_arc_2(const string& label) : label(label), label_is_root(label == "root") {}

  virtual bool applicable(const configuration& conf) const override;
  virtual int perform(configuration& conf) const override;
 private:
  string label;
  bool label_is_root;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_oracle.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_oracle {
 public:
  virtual ~transition_oracle() {}

  struct predicted_transition {
    unsigned best;
    unsigned to_follow;

    predicted_transition(unsigned best, unsigned to_follow) : best(best), to_follow(to_follow) {}
  };

  class tree_oracle {
   public:
    virtual ~tree_oracle() {}

    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const = 0;
    virtual void interesting_transitions(const configuration& conf, vector& transitions) const = 0;
  };

  virtual unique_ptr create_tree_oracle(const tree& gold) const = 0;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_system.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_system {
 public:
  virtual ~transition_system() {}

  virtual unsigned transition_count() const;
  virtual bool applicable(const configuration& conf, unsigned transition) const;
  virtual int perform(configuration& conf, unsigned transition) const;
  virtual transition_oracle* oracle(const string& name) const = 0;

  static transition_system* create(const string& name, const vector& labels);

 protected:
  transition_system(const vector& labels) : labels(labels) {}

  const vector& labels;
  vector> transitions;
};

} // namespace parsito

/////////
// File: parsito/parser/parser_nn.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class parser_nn : public parser {
 public:
  parser_nn(bool versioned);

  virtual void parse(tree& t, unsigned beam_size = 0, double* cost = nullptr) const override;

 protected:
  virtual void load(binary_decoder& data, unsigned cache) override;

 private:
  friend class parser_nn_trainer;
  void parse_greedy(tree& t, double* cost) const;
  void parse_beam_search(tree& t, unsigned beam_size, double* cost) const;

  bool versioned;
  unsigned version;
  bool single_root;
  enum { VERSION_LATEST = 2 };

  vector labels;
  unique_ptr system;

  node_extractor nodes;

  vector values;
  vector embeddings;

  neural_network network;
  neural_network::embeddings_cache embeddings_cache;

  struct workspace {
    workspace(bool single_root) : conf(single_root) {}

    configuration conf;

    string word, word_buffer;
    vector> embeddings;
    vector> embeddings_values;

    vector extracted_nodes;
    vector*> extracted_embeddings;

    vector outcomes, network_buffer;

    // Beam-size structures
    struct beam_size_configuration {
      beam_size_configuration(bool single_root) : conf(single_root) {}

      configuration conf;
      vector heads;
      vector deprels;
      double cost;

      void refresh_tree();
      void save_tree();
    };
    struct beam_size_alternative {
      const beam_size_configuration* bs_conf;
      int transition;
      double cost;
      bool operator<(const beam_size_alternative& other) const { return cost > other.cost; }

      beam_size_alternative(const beam_size_configuration* bs_conf, int transition, double cost)
          : bs_conf(bs_conf), transition(transition), cost(cost) {}
    };
    vector bs_confs[2]; size_t bs_confs_size[2];
    vector bs_alternatives;
  };
  mutable threadsafe_stack workspaces;
};

} // namespace parsito

/////////
// File: parsito/parser/parser.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

parser* parser::load(const char* file, unsigned cache) {
  ifstream in(path_from_utf8(file).c_str(), ifstream::in | ifstream::binary);
  if (!in.is_open()) return nullptr;
  return load(in, cache);
}

parser* parser::load(istream& in, unsigned cache) {
  unique_ptr result;

  binary_decoder data;
  if (!compressor::load(in, data)) return nullptr;

  try {
    string name;
    data.next_str(name);

    result.reset(create(name));
    if (!result) return nullptr;

    result->load(data, cache);
  } catch (binary_decoder_error&) {
    return nullptr;
  }

  return result && data.is_end() ? result.release() : nullptr;
}

parser* parser::create(const string& name) {
  if (name == "nn") return new parser_nn(false);
  if (name == "nn_versioned") return new parser_nn(true);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/parser/parser_nn.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Versions:
// 1: initial version
// 2: add ReLU activation function

parser_nn::parser_nn(bool versioned) : versioned(versioned) {}

void parser_nn::parse(tree& t, unsigned beam_size, double* cost) const {
  if (beam_size > 1)
    parse_beam_search(t, beam_size, cost);
  else
    parse_greedy(t, cost);
}

void parser_nn::parse_greedy(tree& t, double* cost) const {
  assert(system);
  if (cost) *cost = 0.;

  // Retrieve or create workspace
  workspace* w = workspaces.pop();
  if (!w) w = new workspace(single_root);

  // Create configuration
  w->conf.init(&t);

  // Compute embeddings of all nodes
  if (w->embeddings.size() < t.nodes.size()) w->embeddings.resize(t.nodes.size());
  for (size_t i = 0; i < t.nodes.size(); i++) {
    if (w->embeddings[i].size() < embeddings.size()) w->embeddings[i].resize(embeddings.size());
    for (size_t j = 0; j < embeddings.size(); j++) {
      values[j].extract(t.nodes[i], w->word);
      w->embeddings[i][j] = embeddings[j].lookup_word(w->word, w->word_buffer);
    }
  }

  // Compute which transitions to perform and perform them
  int transitions = 0;
  for (; !w->conf.final(); transitions++) {
    // Extract nodes from the configuration
    nodes.extract(w->conf, w->extracted_nodes);
    w->extracted_embeddings.resize(w->extracted_nodes.size());
    for (size_t i = 0; i < w->extracted_nodes.size(); i++)
      w->extracted_embeddings[i] = w->extracted_nodes[i] >= 0 ? &w->embeddings[w->extracted_nodes[i]] : nullptr;

    // Classify using neural network
    network.propagate(embeddings, w->extracted_embeddings, w->network_buffer, w->outcomes, &embeddings_cache, cost ? true : false);

    // Find most probable applicable transition
    int best = -1;
    for (unsigned i = 0; i < w->outcomes.size(); i++)
      if (system->applicable(w->conf, i) && (best < 0 || w->outcomes[i] > w->outcomes[best]))
        best = i;

    // Perform the best transition
    int child = system->perform(w->conf, best);
    if (cost) *cost += log(w->outcomes[best]);

    // If a node was linked, recompute its embeddings as deprel has changed
    if (child >= 0)
      for (size_t i = 0; i < embeddings.size(); i++) {
        values[i].extract(t.nodes[child], w->word);
        w->embeddings[child][i] = embeddings[i].lookup_word(w->word, w->word_buffer);
      }
  }

  if (cost && transitions)
    *cost = *cost / transitions * (t.nodes.size() - 1);

  // Store workspace
  workspaces.push(w);
}

void parser_nn::parse_beam_search(tree& t, unsigned beam_size, double* cost) const {
  assert(system);

  // Retrieve or create workspace
  workspace* w = workspaces.pop();
  if (!w) w = new workspace(single_root);

  // Allocate and initialize configuration
  for (int i = 0; i < 2; i++) {
    while (w->bs_confs[i].size() < beam_size) w->bs_confs[i].emplace_back(single_root);
    while (w->bs_confs[i].size() > beam_size) w->bs_confs[i].pop_back();
    w->bs_confs_size[i] = 0;
  }
  w->bs_confs[0][0].cost = 0;
  w->bs_confs[0][0].conf.init(&t);
  w->bs_confs[0][0].save_tree();
  w->bs_confs_size[0] = 1;

  // Compute embeddings of all nodes
  if (w->embeddings.size() < t.nodes.size()) w->embeddings.resize(t.nodes.size());
  if (w->embeddings_values.size() < t.nodes.size()) w->embeddings_values.resize(t.nodes.size());
  for (size_t i = 0; i < t.nodes.size(); i++) {
    if (w->embeddings[i].size() < embeddings.size()) w->embeddings[i].resize(embeddings.size());
    if (w->embeddings_values[i].size() < embeddings.size()) w->embeddings_values[i].resize(embeddings.size());
    for (size_t j = 0; j < embeddings.size(); j++) {
      values[j].extract(t.nodes[i], w->embeddings_values[i][j]);
      w->embeddings[i][j] = embeddings[j].lookup_word(w->embeddings_values[i][j], w->word_buffer);
    }
  }

  // Compute which transitions to perform and perform them
  size_t iteration = 0;
  for (bool all_final = false; !all_final; iteration++) {
    all_final = true;
    w->bs_alternatives.clear();

    for (size_t c = 0; c < w->bs_confs_size[iteration & 1]; c++) {
      auto& bs_conf = w->bs_confs[iteration & 1][c];

      if (bs_conf.conf.final()) {
        if (w->bs_alternatives.size() == beam_size) {
          if (bs_conf.cost <= w->bs_alternatives[0].cost) continue;
          pop_heap(w->bs_alternatives.begin(), w->bs_alternatives.end());
          w->bs_alternatives.pop_back();
        }
        w->bs_alternatives.emplace_back(&bs_conf, -1, bs_conf.cost);
        push_heap(w->bs_alternatives.begin(), w->bs_alternatives.end());
        continue;
      }
      all_final = false;

      bs_conf.refresh_tree();
      // Update embeddings for all nodes
      for (size_t i = 0; i < t.nodes.size(); i++)
        for (size_t j = 0; j < embeddings.size(); j++) {
          values[j].extract(t.nodes[i], w->word);
          if (w->word != w->embeddings_values[i][j]) {
            w->embeddings[i][j] = embeddings[j].lookup_word(w->word, w->word_buffer);
            w->embeddings_values[i][j].assign(w->word);
          }
        }

      // Extract nodes from the configuration
      nodes.extract(bs_conf.conf, w->extracted_nodes);
      w->extracted_embeddings.resize(w->extracted_nodes.size());
      for (size_t i = 0; i < w->extracted_nodes.size(); i++)
        w->extracted_embeddings[i] = w->extracted_nodes[i] >= 0 ? &w->embeddings[w->extracted_nodes[i]] : nullptr;

      // Classify using neural network
      network.propagate(embeddings, w->extracted_embeddings, w->network_buffer, w->outcomes, &embeddings_cache);

      // Store all alternatives
      for (unsigned i = 0; i < w->outcomes.size(); i++)
        if (system->applicable(bs_conf.conf, i)) {
          double cost = (bs_conf.cost * iteration + log(w->outcomes[i])) / (iteration + 1);
          if (w->bs_alternatives.size() == beam_size) {
            if (cost <= w->bs_alternatives[0].cost) continue;
            pop_heap(w->bs_alternatives.begin(), w->bs_alternatives.end());
            w->bs_alternatives.pop_back();
          }
          w->bs_alternatives.emplace_back(&bs_conf, i, cost);
          push_heap(w->bs_alternatives.begin(), w->bs_alternatives.end());
        }
    }

    w->bs_confs_size[(iteration + 1) & 1] = 0;
    for (auto&& alternative : w->bs_alternatives) {
      auto& bs_conf_new = w->bs_confs[(iteration + 1) & 1][w->bs_confs_size[(iteration + 1) & 1]++];
      bs_conf_new = *alternative.bs_conf;
      bs_conf_new.cost = alternative.cost;
      if (alternative.transition >= 0) {
        bs_conf_new.refresh_tree();
        system->perform(bs_conf_new.conf, alternative.transition);
        bs_conf_new.save_tree();
      }
    }
  }

  // Return the best tree
  size_t best = 0;
  for (size_t i = 1; i < w->bs_confs_size[iteration & 1]; i++)
    if (w->bs_confs[iteration & 1][i].cost > w->bs_confs[iteration & 1][best].cost)
      best = i;
  w->bs_confs[iteration & 1][best].refresh_tree();

  if (cost) *cost = w->bs_confs[iteration & 1][best].cost * (t.nodes.size() - 1);

  // Store workspace
  workspaces.push(w);
}

void parser_nn::workspace::beam_size_configuration::refresh_tree() {
  for (auto&& node : conf.t->nodes) node.children.clear();
  for (size_t i = 0; i < conf.t->nodes.size(); i++) {
    conf.t->nodes[i].head = heads[i];
    conf.t->nodes[i].deprel = deprels[i];
    if (heads[i] >= 0) conf.t->nodes[heads[i]].children.push_back(i);
  }
}

void parser_nn::workspace::beam_size_configuration::save_tree() {
  if (conf.t->nodes.size() > heads.size()) heads.resize(conf.t->nodes.size());
  if (conf.t->nodes.size() > deprels.size()) deprels.resize(conf.t->nodes.size());
  for (size_t i = 0; i < conf.t->nodes.size(); i++) {
    heads[i] = conf.t->nodes[i].head;
    deprels[i] = conf.t->nodes[i].deprel;
  }
}

void parser_nn::load(binary_decoder& data, unsigned cache) {
  string description, error;

  version = versioned ? data.next_1B() : 1;
  if (!(version >= 1 && version <= VERSION_LATEST))
    throw binary_decoder_error("Unrecognized version of the parser_nn model");

  single_root = version >= 2 ? data.next_1B() : false;

  // Load labels
  labels.resize(data.next_2B());
  for (auto&& label : labels)
    data.next_str(label);

  // Load transition system
  string system_name;
  data.next_str(system_name);
  system.reset(transition_system::create(system_name, labels));
  if (!system) throw binary_decoder_error("Cannot load transition system");

  // Load node extractor
  data.next_str(description);
  if (!nodes.create(description, error))
    throw binary_decoder_error(error.c_str());

  // Load value extractors and embeddings
  values.resize(data.next_2B());
  for (auto&& value : values) {
    data.next_str(description);
    if (!value.create(description, error))
      throw binary_decoder_error(error.c_str());
  }

  embeddings.resize(values.size());
  for (auto&& embedding : embeddings)
    embedding.load(data);

  // Load the network
  network.load(data);
  network.generate_tanh_cache();
  network.generate_embeddings_cache(embeddings, embeddings_cache, cache);
}

} // namespace parsito

/////////
// File: parsito/parser/parser_nn_trainer.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class parser_nn_trainer {
 public:
  static void train(const string& transition_system_name, const string& transition_oracle_name, bool single_root,
                    const string& embeddings_description, const string& nodes_description, const network_parameters& parameters,
                    unsigned number_of_threads, const vector& train, const vector& heldout, binary_encoder& enc);
};

} // namespace parsito

/////////
// File: parsito/parser/parser_nn_trainer.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

void parser_nn_trainer::train(const string& transition_system_name, const string& transition_oracle_name, bool single_root,
                              const string& embeddings_description, const string& nodes_description, const network_parameters& parameters,
                              unsigned /*number_of_threads*/, const vector& train, const vector& heldout, binary_encoder& enc) {
  if (train.empty()) training_failure("No training data was given!");

  // Random generator with fixed seed for reproducibility
  mt19937 generator(42);

  // Check that all non-root nodes have heads and nonempty deprel
  for (auto&& tree : train)
    for (auto&& node : tree.nodes)
      if (node.id) {
        if (node.head < 0) training_failure("The node '" << node.form << "' with id " << node.id << " has no head set!");
        if (node.deprel.empty()) training_failure("The node '" << node.form << "' with id " << node.id << " has no deprel set!");
      }

  // Create parser instance to be trained
  parser_nn parser(true); parser.version = parser_nn::VERSION_LATEST;

  // Generate labels for transition system
  unordered_set labels_set;
  for (auto&& tree : train)
    for (auto&& node : tree.nodes)
      if (node.id && !labels_set.count(node.deprel)) {
        labels_set.insert(node.deprel);
        parser.labels.push_back(node.deprel);
      }

  // If single_root, check that exactly root nodes have "root" deprel
  if (single_root) {
    for (auto&& tree : train) {
      unsigned roots = 0;
      for (auto&& node : tree.nodes)
        if (node.id) {
          if (node.head == 0 && node.deprel != "root")
            training_failure("When single root is required, every root node must have 'root' deprel!");
          if (node.head != 0 && node.deprel == "root")
            training_failure("When single root is required, any non-root cannot have 'root' deprel!");
          roots += node.head == 0;
        }
      if (roots != 1)
        training_failure("When single root is required, every training tree must have single root!");
    }

    // Make sure (in case input is really small) there is "root" deprel plus another one
    if (!labels_set.count("root"))
      training_failure("When single root is required, the deprel 'root' must be present!");
    if (labels_set.size() <= 1)
      training_failure("When single root is required, deprel different from 'root' must exist!");
  }

  // Create transition system and transition oracle
  parser.system.reset(transition_system::create(transition_system_name, parser.labels));
  if (!parser.system) training_failure("Cannot create transition system '" << transition_system_name << "'!");

  unique_ptr oracle(parser.system->oracle(transition_oracle_name));
  if (!oracle) training_failure("Cannot create transition oracle '" << transition_oracle_name << "' for transition system '" << transition_system_name << "'!");

  // Create node_extractor
  string error;
  if (!parser.nodes.create(nodes_description, error)) training_failure(error);

  // Load value_extractors and embeddings
  vector value_names;
  vector lines, tokens;
  split(embeddings_description, '\n', lines);
  for (auto&& line : lines) {
    // Ignore empty lines and comments
    if (!line.len || line.str[0] == '#') continue;

    split(line, ' ', tokens);
    if (!(tokens.size() >= 3 && tokens.size() <= 6))
      training_failure("Expected 3 to 6 columns on embedding description line '" << line << "'!");

    value_names.emplace_back(string(tokens[0].str, tokens[0].len));
    parser.values.emplace_back();
    if (!parser.values.back().create(tokens[0], error)) training_failure(error);

    int dimension = parse_int(tokens[1], "embedding dimension");
    int min_count = parse_int(tokens[2], "minimum frequency count");
    unsigned updatable_index = 0;
    unsigned embeddings_from_file = 0;
    string embeddings_from_file_comment;
    vector>> weights;
    unordered_set weights_set;

    // Compute words and counts present in the training data
    string word;
    unordered_map word_counts;
    for (auto&& tree : train)
      for (auto&& node : tree.nodes)
        if (node.id) {
          parser.values.back().extract(node, word);
          word_counts[word]++;
        }

    // Load embedding if it was given
    if (tokens.size() >= 4) {
      int update_weights = tokens.size() >= 5 ? parse_int(tokens[4], "update weights") : 1;
      int max_embeddings = tokens.size() >= 6 ? parse_int(tokens[5], "maximum embeddings count") : numeric_limits::max();
      ifstream in(path_from_utf8(string(tokens[3].str, tokens[3].len)).c_str());
      if (!in.is_open()) training_failure("Cannot load '" << tokens[0] << "' embedding from file '" << tokens[3] << "'!");

      // Load first line containing dictionary size and dimensions
      string line;
      vector parts;
      if (!getline(in, line)) training_failure("Cannot read first line from embedding file '" << tokens[3] << "'!");
      split(line, ' ', parts);
      if (parts.size() != 2) training_failure("Expected two numbers on the first line of embedding file '" << tokens[3] << "'!");
      int file_dimension = parse_int(parts[1], "embedding file dimension");

      if (file_dimension < dimension) training_failure("The embedding file '" << tokens[3] << "' has lower dimension than required!");

      // Generate random projection when smaller dimension is required
      vector> projection;
      if (file_dimension > dimension) {
        embeddings_from_file_comment = "[dim" + to_string(file_dimension) + "->" + to_string(dimension) + "]";

        uniform_real_distribution uniform(0, 1);
        projection.resize(dimension);
        for (auto&& row : projection) {
          row.resize(file_dimension);
          for (auto&& weight : row) weight = uniform(generator);

          double sum = 0;
          for (auto&& weight : row) sum += weight;
          for (auto&& weight : row) weight /= sum;
        }
      }

      // Load input embedding
      vector input_weights(file_dimension);
      vector projected_weights(dimension);
      while (getline(in, line) && int(weights.size()) < max_embeddings) {
        split(line, ' ', parts);
        if (!parts.empty() && !parts.back().len) parts.pop_back(); // Ignore space at the end of line
        if (int(parts.size()) != file_dimension + 1) training_failure("Wrong number of values on line '" << line << "' of embedding file '" << tokens[3]);
        for (int i = 0; i < file_dimension; i++)
          input_weights[i] = parse_double(parts[1 + i], "embedding weight");

        string word(parts[0].str, parts[0].len);

        // For update_weights == 2, ignore embeddings for unknown words
        if (update_weights == 2 && !word_counts.count(word))
          continue;

        for (int i = 0; i < dimension; i++)
          if (file_dimension == dimension) {
            projected_weights[i] = input_weights[i];
          } else {
            projected_weights[i] = 0;
            for (int j = 0; j < file_dimension; j++)
              projected_weights[i] += projection[i][j] * input_weights[j];
          }

        if (!weights_set.count(word)) {
          weights.emplace_back(word, projected_weights);
          weights_set.insert(word);
        }
      }
      embeddings_from_file = weights.size();
      updatable_index = update_weights ? 0 : embeddings_from_file;
    }

    // Add embedding for non-present word with min_count, sorted by count
    {
      vector> count_words;
      for (auto&& word_count : word_counts)
        if (word_count.second >= min_count && !weights_set.count(word_count.first))
          count_words.emplace_back(word_count.second, word_count.first);

      sort(count_words.rbegin(), count_words.rend());

      vector word_weights(dimension);
      uniform_real_distribution uniform(-1, 1);
      for (auto&& count_word : count_words) {
        for (auto&& word_weight : word_weights)
          word_weight = uniform(generator);

        weights.emplace_back(count_word.second, word_weights);
      }
    }

    // If there are unknown words in the training data, create initial embedding
    vector unknown_weights(dimension);
    if (min_count > 1) {
      uniform_real_distribution uniform(-1, 1);

      for (auto&& weight : unknown_weights)
        weight = uniform(generator);
    }

    // Add the embedding
    parser.embeddings.emplace_back();
    parser.embeddings.back().create(dimension, updatable_index, weights, unknown_weights);

    // Count the cover of this embedding
    string buffer;
    unsigned words_total = 0, words_covered = 0, words_covered_from_file = 0;
    for (auto&& tree : train)
      for (auto&& node : tree.nodes)
        if (node.id) {
          parser.values.back().extract(node, word);
          words_total++;
          int word_id = parser.embeddings.back().lookup_word(word, buffer);
          words_covered += word_id != parser.embeddings.back().unknown_word();
          words_covered_from_file += word_id != parser.embeddings.back().unknown_word() && unsigned(word_id) < embeddings_from_file;
        }

    cerr << "Initialized '" << tokens[0] << "' embedding with " << embeddings_from_file << embeddings_from_file_comment
         << "," << weights.size() << " words and " << fixed << setprecision(1) << 100. * words_covered_from_file / words_total
         << "%," << 100. * words_covered / words_total << "% coverage." << endl;
  }

  // Train the network
  unsigned total_dimension = 0, total_nodes = 0;
  for (auto&& embedding : parser.embeddings) total_dimension += embedding.dimension;
  for (auto&& tree : train) total_nodes += tree.nodes.size() - 1;
  auto scaled_parameters = parameters;
  scaled_parameters.l1_regularization /= train.size();
  scaled_parameters.l2_regularization /= total_nodes;
  neural_network_trainer network_trainer(parser.network, total_dimension * parser.nodes.node_count(), parser.system->transition_count(), scaled_parameters, generator);

  neural_network heldout_best_network;
  unsigned heldout_best_correct_labelled = 0, heldout_best_iteration = 0;

  vector permutation;
  for (size_t i = 0; i < train.size(); i++)
    permutation.push_back(permutation.size());

  for (int iteration = 1; network_trainer.next_iteration(); iteration++) {
    // Train on training data
    shuffle(permutation.begin(), permutation.end(), generator);

    atomic atomic_index(0);
    atomic atomic_logprob(0);
    auto training = [&]() {
      tree t;
      configuration conf(single_root);
      string word, word_buffer;
      vector> nodes_embeddings;
      vector extracted_nodes;
      vector*> extracted_embeddings;
      neural_network_trainer::workspace workspace;
      double logprob = 0;

      // Data for structured prediction
      tree t_eval;
      configuration conf_eval(single_root);
      vector> nodes_embeddings_eval;
      vector  extracted_nodes_eval;
      vector*>  extracted_embeddings_eval;
      vector transitions_eval;
      vector hidden_layer_eval, outcomes_eval;

      for (unsigned current_index; (current_index = atomic_index++) < permutation.size();) {
        const tree& gold = train[permutation[current_index]];
        t = gold;
        t.unlink_all_nodes();
        conf.init(&t);

        // Compute embeddings
        if (t.nodes.size() > nodes_embeddings.size()) nodes_embeddings.resize(t.nodes.size());
        for (size_t i = 0; i < t.nodes.size(); i++) {
          nodes_embeddings[i].resize(parser.embeddings.size());
          for (size_t j = 0; j < parser.embeddings.size(); j++) {
            parser.values[j].extract(t.nodes[i], word);
            nodes_embeddings[i][j] = parser.embeddings[j].lookup_word(word, word_buffer);
          }
        }

        // Create tree oracle
        auto tree_oracle = oracle->create_tree_oracle(gold);

        // Train the network
        while (!conf.final()) {
          // Extract nodes
          parser.nodes.extract(conf, extracted_nodes);
          extracted_embeddings.resize(extracted_nodes.size());
          for (size_t i = 0; i < extracted_nodes.size(); i++)
            extracted_embeddings[i] = extracted_nodes[i] >= 0 ? &nodes_embeddings[extracted_nodes[i]] : nullptr;

          // Propagate
          network_trainer.propagate(parser.embeddings, extracted_embeddings, workspace);

          // Find most probable applicable transition
          int network_best = -1;
          for (unsigned i = 0; i < workspace.outcomes.size(); i++)
            if (parser.system->applicable(conf, i) && (network_best < 0 || workspace.outcomes[i] > workspace.outcomes[network_best]))
              network_best = i;

          // Apply the oracle
          auto prediction = tree_oracle->predict(conf, network_best, iteration);

          // If the best transition is applicable, train on it
          if (parser.system->applicable(conf, prediction.best)) {
            // Update logprob
            if (workspace.outcomes[prediction.best])
              logprob += log(workspace.outcomes[prediction.best]);

            // Backpropagate the chosen outcome
            network_trainer.backpropagate(parser.embeddings, extracted_embeddings, prediction.best, workspace);
          }

          // Emergency break if the to_follow transition is not applicable
          if (!parser.system->applicable(conf, prediction.to_follow))
            break;

          // Follow the chosen outcome
          int child = parser.system->perform(conf, prediction.to_follow);

          // If a node was linked, recompute its embeddings as deprel has changed
          if (child >= 0)
            for (size_t i = 0; i < parser.embeddings.size(); i++) {
              parser.values[i].extract(t.nodes[child], word);
              nodes_embeddings[child][i] = parser.embeddings[i].lookup_word(word, word_buffer);
            }
        }
        network_trainer.finalize_sentence();

        // Structured prediction
        if (parameters.structured_interval && (current_index % parameters.structured_interval) == 0) {
          uniform_int_distribution train_distribution(0, train.size() - 1);
          const tree& gold = train[train_distribution(generator)];
          t = gold;
          t.unlink_all_nodes();
          conf.init(&t);

          // Compute embeddings
          if (t.nodes.size() > nodes_embeddings.size()) nodes_embeddings.resize(t.nodes.size());
          for (size_t i = 0; i < t.nodes.size(); i++) {
            nodes_embeddings[i].resize(parser.embeddings.size());
            for (size_t j = 0; j < parser.embeddings.size(); j++) {
              parser.values[j].extract(t.nodes[i], word);
              nodes_embeddings[i][j] = parser.embeddings[j].lookup_word(word, word_buffer);
            }
          }

          // Create tree oracle
          auto tree_oracle = oracle->create_tree_oracle(gold);

          // Train the network
          while (!conf.final()) {
            // Extract nodes
            parser.nodes.extract(conf, extracted_nodes);
            extracted_embeddings.resize(extracted_nodes.size());
            for (size_t i = 0; i < extracted_nodes.size(); i++)
              extracted_embeddings[i] = extracted_nodes[i] >= 0 ? &nodes_embeddings[extracted_nodes[i]] : nullptr;

            // Find the best transition
            int best = 0;
            int best_uas = -1;
            tree_oracle->interesting_transitions(conf, transitions_eval);
            for (auto&& transition : transitions_eval) {
              t_eval = t;
              conf_eval = conf;
              conf_eval.t = &t_eval;
              nodes_embeddings_eval = nodes_embeddings;

              // Perform probed transition
              int child = parser.system->perform(conf_eval, transition);
              if (child >= 0)
                for (size_t i = 0; i < parser.embeddings.size(); i++) {
                  parser.values[i].extract(t_eval.nodes[child], word);
                  nodes_embeddings_eval[child][i] = parser.embeddings[i].lookup_word(word, word_buffer);
                }

              // Train the network
              while (!conf_eval.final()) {
                // Extract nodes
                parser.nodes.extract(conf_eval, extracted_nodes_eval);
                extracted_embeddings_eval.resize(extracted_nodes_eval.size());
                for (size_t i = 0; i < extracted_nodes_eval.size(); i++)
                  extracted_embeddings_eval[i] = extracted_nodes_eval[i] >= 0 ? &nodes_embeddings_eval[extracted_nodes_eval[i]] : nullptr;

                // Classify using neural network
                parser.network.propagate(parser.embeddings, extracted_embeddings_eval, hidden_layer_eval, outcomes_eval, nullptr, false);

                // Find most probable applicable transition
                int network_best = -1;
                for (unsigned i = 0; i < outcomes_eval.size(); i++)
                  if (parser.system->applicable(conf_eval, i) && (network_best < 0 || outcomes_eval[i] > outcomes_eval[network_best]))
                    network_best = i;

                // Perform the best transition
                int child = parser.system->perform(conf_eval, network_best);

                // If a node was linked, recompute its embeddings as deprel has changed
                if (child >= 0)
                  for (size_t i = 0; i < parser.embeddings.size(); i++) {
                    parser.values[i].extract(t_eval.nodes[child], word);
                    nodes_embeddings_eval[child][i] = parser.embeddings[i].lookup_word(word, word_buffer);
                  }
              }

              int uas = 0;
              for (unsigned i = 1; i < gold.nodes.size(); i++)
                uas += gold.nodes[i].head == t_eval.nodes[i].head;

              if (uas > best_uas) best = transition, best_uas = uas;
            }

            // Propagate
            network_trainer.propagate(parser.embeddings, extracted_embeddings, workspace);

            // Backpropagate for the best transition
            if (workspace.outcomes[best])
              logprob += log(workspace.outcomes[best]);
            network_trainer.backpropagate(parser.embeddings, extracted_embeddings, best, workspace);

            //              // Find most probable applicable transition when following network outcome
            //              int network_best = -1;
            //              for (unsigned i = 0; i < workspace.outcomes.size(); i++)
            //                if (parser.system->applicable(conf, i) && (network_best < 0 || workspace.outcomes[i] > workspace.outcomes[network_best]))
            //                  network_best = i;

            // Follow the best outcome
            int child = parser.system->perform(conf, /*network_*/best);

            // If a node was linked, recompute its embeddings as deprel has changed
            if (child >= 0)
              for (size_t i = 0; i < parser.embeddings.size(); i++) {
                parser.values[i].extract(t.nodes[child], word);
                nodes_embeddings[child][i] = parser.embeddings[i].lookup_word(word, word_buffer);
              }
          }
          network_trainer.finalize_sentence();
        }
      }
      for (double old_atomic_logprob = atomic_logprob; atomic_logprob.compare_exchange_weak(old_atomic_logprob, old_atomic_logprob + logprob); ) {}
    };

    cerr << "Iteration " << iteration << ": ";
    training();
    cerr << "training logprob " << scientific << setprecision(4) << atomic_logprob;

    // Evaluate heldout data if present
    if (!heldout.empty()) {
      tree t;
      unsigned total = 0, correct_unlabelled = 0, correct_labelled = 0;
      for (auto&& gold : heldout) {
        t = gold;
        t.unlink_all_nodes();
        parser.parse(t);
        for (size_t i = 1; i < t.nodes.size(); i++) {
          total++;
          correct_unlabelled += t.nodes[i].head == gold.nodes[i].head;
          correct_labelled += t.nodes[i].head == gold.nodes[i].head && t.nodes[i].deprel == gold.nodes[i].deprel;
        }
      }

      cerr << ", heldout UAS " << fixed << setprecision(2) << (100. * correct_unlabelled / total) << "%, LAS " << (100. * correct_labelled / total) << "%";

      if (parameters.early_stopping && correct_labelled > heldout_best_correct_labelled) {
        heldout_best_network = parser.network;
        heldout_best_correct_labelled = correct_labelled;
        heldout_best_iteration = iteration;
      }
    }

    cerr << endl;
  }

  if (parameters.early_stopping && heldout_best_iteration > 0) {
    cerr << "Using early stopping -- choosing network from iteration " << heldout_best_iteration << endl;
    parser.network = heldout_best_network;
  }

  // Encode version
  enc.add_1B(parser.version);

  // Encode single_root
  enc.add_1B(single_root);

  // Encode transition system
  enc.add_2B(parser.labels.size());
  for (auto&& label : parser.labels)
    enc.add_str(label);
  enc.add_str(transition_system_name);

  // Encode nodes selector
  enc.add_str(nodes_description);

  // Encode value extractors and embeddings
  enc.add_2B(value_names.size());
  for (auto&& value_name : value_names)
    enc.add_str(value_name);
  for (auto&& embedding : parser.embeddings)
    embedding.save(enc);

  // Encode the network
  network_trainer.save_network(enc);
}

} // namespace parsito

/////////
// File: parsito/transition/transition.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Left arc
bool transition_left_arc::applicable(const configuration& conf) const {
  if (conf.single_root && label_is_root)
    return false;
  else
    return conf.stack.size() >= 2 && conf.stack[conf.stack.size() - 2];
}

int transition_left_arc::perform(configuration& conf) const {
  assert(applicable(conf));

  int parent = conf.stack.back(); conf.stack.pop_back();
  int child = conf.stack.back(); conf.stack.pop_back();
  conf.stack.push_back(parent);
  conf.t->set_head(child, parent, label);
  return child;
}

// Right arc
bool transition_right_arc::applicable(const configuration& conf) const {
  if (conf.single_root && label_is_root)
    return conf.stack.size() == 2 && conf.buffer.empty();
  else if (conf.single_root) // && !label_is_root
    return conf.stack.size() > 2;
  else
    return conf.stack.size() >= 2;
}

int transition_right_arc::perform(configuration& conf) const {
  assert(applicable(conf));

  int child = conf.stack.back(); conf.stack.pop_back();
  int parent = conf.stack.back();
  conf.t->set_head(child, parent, label);
  return child;
}

// Shift
bool transition_shift::applicable(const configuration& conf) const {
  return !conf.buffer.empty();
}

int transition_shift::perform(configuration& conf) const {
  assert(applicable(conf));

  conf.stack.push_back(conf.buffer.back());
  conf.buffer.pop_back();
  return -1;
}

// Swap
bool transition_swap::applicable(const configuration& conf) const {
  return conf.stack.size() >= 2 && conf.stack[conf.stack.size() - 2] && conf.stack[conf.stack.size() - 2] < conf.stack[conf.stack.size() - 1];
}

int transition_swap::perform(configuration& conf) const {
  assert(applicable(conf));

  int top = conf.stack.back(); conf.stack.pop_back();
  int to_buffer = conf.stack.back(); conf.stack.pop_back();
  conf.stack.push_back(top);
  conf.buffer.push_back(to_buffer);
  return -1;
}

// Left arc 2
bool transition_left_arc_2::applicable(const configuration& conf) const {
  if (conf.single_root && label_is_root)
    return false;
  else
    return conf.stack.size() >= 3 && conf.stack[conf.stack.size() - 3];
}

int transition_left_arc_2::perform(configuration& conf) const {
  assert(applicable(conf));

  int parent = conf.stack.back(); conf.stack.pop_back();
  int ignore = conf.stack.back(); conf.stack.pop_back();
  int child = conf.stack.back(); conf.stack.pop_back();
  conf.stack.push_back(ignore);
  conf.stack.push_back(parent);
  conf.t->set_head(child, parent, label);
  return child;
}

// Right arc 2
bool transition_right_arc_2::applicable(const configuration& conf) const {
  if (conf.single_root && label_is_root)
    return false;
  else if (conf.single_root) // && !label_is_root
    return conf.stack.size() >= 4;
  else
    return conf.stack.size() >= 3;
}

int transition_right_arc_2::perform(configuration& conf) const {
  assert(applicable(conf));

  int child = conf.stack.back(); conf.stack.pop_back();
  int to_buffer = conf.stack.back(); conf.stack.pop_back();
  int parent = conf.stack.back();
  conf.buffer.push_back(to_buffer);
  conf.t->set_head(child, parent, label);
  return child;
}

} // namespace parsito

/////////
// File: parsito/transition/transition_system_link2.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_system_link2 : public transition_system {
 public:
  transition_system_link2(const vector& labels);

  virtual transition_oracle* oracle(const string& name) const override;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_system_projective.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_system_projective : public transition_system {
 public:
  transition_system_projective(const vector& labels);

  virtual transition_oracle* oracle(const string& name) const override;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_system_swap.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

class transition_system_swap : public transition_system {
 public:
  transition_system_swap(const vector& labels);

  virtual transition_oracle* oracle(const string& name) const override;
};

} // namespace parsito

/////////
// File: parsito/transition/transition_system.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

unsigned transition_system::transition_count() const {
  return transitions.size();
}

bool transition_system::applicable(const configuration& conf, unsigned transition) const {
  assert(transition < transitions.size());

  return transitions[transition]->applicable(conf);
}

int transition_system::perform(configuration& conf, unsigned transition) const {
  assert(transition < transitions.size());

  return transitions[transition]->perform(conf);
}

transition_system* transition_system::create(const string& name, const vector& labels) {
  if (name == "projective") return new transition_system_projective(labels);
  if (name == "swap") return new transition_system_swap(labels);
  if (name == "link2") return new transition_system_link2(labels);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/transition/transition_system_link2.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

transition_system_link2::transition_system_link2(const vector& labels) : transition_system(labels) {
  transitions.emplace_back(new transition_shift());
  for (auto&& label : labels) {
    transitions.emplace_back(new transition_left_arc(label));
    transitions.emplace_back(new transition_right_arc(label));
    transitions.emplace_back(new transition_left_arc_2(label));
    transitions.emplace_back(new transition_right_arc_2(label));
  }
}

// Static oracle
class transition_system_link2_oracle_static : public transition_oracle {
 public:
  transition_system_link2_oracle_static(const vector& labels) : labels(labels) {
    for (root_label = 0; root_label < labels.size(); root_label++) if (labels[root_label] == "root") break;
  }

  class tree_oracle_static : public transition_oracle::tree_oracle {
   public:
    tree_oracle_static(const vector& labels, unsigned root_label, const tree& gold) : labels(labels), root_label(root_label), gold(gold) {}
    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const override;
    virtual void interesting_transitions(const configuration& conf, vector& transitions) const override;
   private:
    const vector& labels;
    unsigned root_label;
    const tree& gold;
  };

  virtual unique_ptr create_tree_oracle(const tree& gold) const override;
 private:
  const vector& labels;
  unsigned root_label;
};

unique_ptr transition_system_link2_oracle_static::create_tree_oracle(const tree& gold) const {
  return unique_ptr(new tree_oracle_static(labels, root_label, gold));
}

void transition_system_link2_oracle_static::tree_oracle_static::interesting_transitions(const configuration& conf, vector& transitions) const {
  transitions.clear();

  // Shift
  if (!conf.buffer.empty()) transitions.push_back(0);

  // Arcs
  unsigned parents[4] = {1, 2, 1, 3};
  unsigned children[4] = {2, 1, 3, 1};
  for (int direction = 0; direction < 4; direction++)
    if (conf.stack.size() >= parents[direction] && conf.stack.size() >= children[direction]) {
      int parent = conf.stack[conf.stack.size() - parents[direction]];
      int child = conf.stack[conf.stack.size() - children[direction]];

      // Allow arc_2 only when seeing golden edge.
      if (direction >= 2 && gold.nodes[child].head != parent) continue;

      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          if (!conf.single_root ||
              (i == root_label && conf.stack.size() == 2 && conf.buffer.empty() && direction == 1) ||
              (i != root_label && conf.stack.size() > 2 && direction < 2) ||
              (i != root_label && conf.stack.size() > 3 && direction >= 2))
            transitions.push_back(1 + 4*i + direction);
    }
}

transition_oracle::predicted_transition transition_system_link2_oracle_static::tree_oracle_static::predict(const configuration& conf, unsigned /*network_outcome*/, unsigned /*iteration*/) const {
  // Arcs
  unsigned parents[4] = {1, 2, 1, 3};
  unsigned children[4] = {2, 1, 3, 1};
  for (int direction = 0; direction < 4; direction++)
    if (conf.stack.size() >= parents[direction] && conf.stack.size() >= children[direction]) {
      int parent = conf.stack[conf.stack.size() - parents[direction]];
      int child = conf.stack[conf.stack.size() - children[direction]];

      if (gold.nodes[child].head == parent && gold.nodes[child].children.size() == conf.t->nodes[child].children.size()) {
        for (size_t i = 0; i < labels.size(); i++)
          if (gold.nodes[child].deprel == labels[i])
            return predicted_transition(1 + 4*i + direction, 1 + 4*i + direction);

        assert(!"label was not found");
      }
    }

  // Otherwise, just shift
  return predicted_transition(0, 0);
}

// Oracle factory method
transition_oracle* transition_system_link2::oracle(const string& name) const {
  if (name == "static") return new transition_system_link2_oracle_static(labels);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/transition/transition_system_projective.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

transition_system_projective::transition_system_projective(const vector& labels) : transition_system(labels) {
  transitions.emplace_back(new transition_shift());
  for (auto&& label : labels) {
    transitions.emplace_back(new transition_left_arc(label));
    transitions.emplace_back(new transition_right_arc(label));
  }
}

// Static oracle
class transition_system_projective_oracle_static : public transition_oracle {
 public:
  transition_system_projective_oracle_static(const vector& labels) : labels(labels) {
    for (root_label = 0; root_label < labels.size(); root_label++) if (labels[root_label] == "root") break;
  }

  class tree_oracle_static : public transition_oracle::tree_oracle {
   public:
    tree_oracle_static(const vector& labels, unsigned root_label, const tree& gold) : labels(labels), root_label(root_label), gold(gold) {}
    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const override;
    virtual void interesting_transitions(const configuration& conf, vector& transitions) const override;
   private:
    const vector& labels;
    unsigned root_label;
    const tree& gold;
  };

  virtual unique_ptr create_tree_oracle(const tree& gold) const override;
 private:
  const vector& labels;
  unsigned root_label;
};

unique_ptr transition_system_projective_oracle_static::create_tree_oracle(const tree& gold) const {
  return unique_ptr(new tree_oracle_static(labels, root_label, gold));
}

void transition_system_projective_oracle_static::tree_oracle_static::interesting_transitions(const configuration& conf, vector& transitions) const {
  transitions.clear();
  if (!conf.buffer.empty()) transitions.push_back(0);
  if (conf.stack.size() >= 2)
    for (int direction = 0; direction < 2; direction++) {
      int child = conf.stack[conf.stack.size() - 2 + direction];
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          if (!conf.single_root ||
              (i == root_label && conf.stack.size() == 2 && conf.buffer.empty() && direction == 1) ||
              (i != root_label && conf.stack.size() > 2))
            transitions.push_back(1 + 2*i + direction);
    }
}

transition_oracle::predicted_transition transition_system_projective_oracle_static::tree_oracle_static::predict(const configuration& conf, unsigned /*network_outcome*/, unsigned /*iteration*/) const {
  // Use left if appropriate
  if (conf.stack.size() >= 2) {
    int parent = conf.stack[conf.stack.size() - 1];
    int child = conf.stack[conf.stack.size() - 2];
    if (gold.nodes[child].head == parent) {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return predicted_transition(1 + 2*i, 1 + 2*i);

      assert(!"label was not found");
    }
  }

  // Use right if appropriate
  if (conf.stack.size() >= 2) {
    int child = conf.stack[conf.stack.size() - 1];
    int parent = conf.stack[conf.stack.size() - 2];
    if (gold.nodes[child].head == parent &&
        (conf.buffer.empty() || gold.nodes[child].children.empty() || gold.nodes[child].children.back() < conf.buffer.back())) {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return predicted_transition(1 + 2*i + 1, 1 + 2*i + 1);

      assert(!"label was not found");
    }
  }

  // Otherwise, just shift
  return predicted_transition(0, 0);
}

// Dynamic oracle
class transition_system_projective_oracle_dynamic : public transition_oracle {
 public:
  transition_system_projective_oracle_dynamic(const vector& labels) : labels(labels) {
    for (root_label = 0; root_label < labels.size(); root_label++) if (labels[root_label] == "root") break;
  }

  class tree_oracle_dynamic : public transition_oracle::tree_oracle {
   public:
    tree_oracle_dynamic(const vector& labels, unsigned root_label, const tree& gold) : labels(labels), gold(gold), oracle_static(labels, root_label, gold) {}
    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const override;
    virtual void interesting_transitions(const configuration& conf, vector& transitions) const override;
   private:
    const vector& labels;
    const tree& gold;
    transition_system_projective_oracle_static::tree_oracle_static oracle_static;
  };

  virtual unique_ptr create_tree_oracle(const tree& gold) const override;
 private:
  const vector& labels;
  unsigned root_label;
};

unique_ptr transition_system_projective_oracle_dynamic::create_tree_oracle(const tree& gold) const {
  return unique_ptr(new tree_oracle_dynamic(labels, root_label, gold));
}

void transition_system_projective_oracle_dynamic::tree_oracle_dynamic::interesting_transitions(const configuration& conf, vector& transitions) const {
  oracle_static.interesting_transitions(conf, transitions);
}

transition_oracle::predicted_transition transition_system_projective_oracle_dynamic::tree_oracle_dynamic::predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const {
  // Use static oracle in the first iteration
  if (iteration <= 1)
    return oracle_static.predict(conf, network_outcome, iteration);

  // Use dynamic programming to compute transition leading to best parse tree

  // Start by computing the right stack
  vector right_stack;

  unordered_set right_stack_inserted;
  if (!conf.buffer.empty()) {
    int buffer_start = conf.buffer.back();
    for (size_t i = conf.buffer.size(); i--; ) {
      const auto& node = conf.buffer[i];
      bool to_right_stack = gold.nodes[node].head < buffer_start;
      for (auto&& child : gold.nodes[node].children)
        to_right_stack |= child < buffer_start || right_stack_inserted.count(child);
      if (to_right_stack) {
        right_stack.push_back(node);
        right_stack_inserted.insert(node);
      }
    }
  }

  // Fill the array T from the 2014 Goldberg paper
  class t_representation {
   public:
    t_representation(const vector& stack, const vector& right_stack, const tree& gold, const vector& labels)
        : stack(stack), right_stack(right_stack), gold(gold), labels(labels) {
      for (int i = 0; i < 2; i++) {
        costs[i].reserve((stack.size() + right_stack.size()) * (stack.size() + right_stack.size()));
        transitions[i].reserve((stack.size() + right_stack.size()) * (stack.size() + right_stack.size()));
      }
    }

    void prepare(unsigned diagonal) {
      costs[diagonal & 1].assign((diagonal + 1) * (diagonal + 1), gold.nodes.size() + 1);
      transitions[diagonal & 1].assign((diagonal + 1) * (diagonal + 1), -1);
    }

    int& cost(unsigned i, unsigned j, unsigned h) { return costs[(i+j) & 1][i * (i+j+1) + h]; }
    int& transition(unsigned i, unsigned j, unsigned h) { return transitions[(i+j) & 1][i * (i+j+1) + h]; }

    int node(unsigned i, unsigned /*j*/, unsigned h) const { return h <= i ? stack[stack.size() - 1 - i + h] : right_stack[h - i - 1]; }
    int edge_cost(int parent, int child) const { return gold.nodes[child].head != parent; }
    int which_arc_transition(int parent, int child) const {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return 1 + 2*i + (child > parent);
      assert(!"label was not found");
      return 0; // To keep VS 2015 happy and warning-free
    }

   private:
    const vector& stack;
    const vector& right_stack;
    const tree& gold;
    const vector& labels;
    vector costs[2], transitions[2];
  } t(conf.stack, right_stack, gold, labels);

  t.prepare(0);
  t.cost(0, 0, 0) = 0;
  for (unsigned diagonal = 0; diagonal < conf.stack.size() + right_stack.size(); diagonal++) {
    t.prepare(diagonal + 1);
    for (unsigned i = diagonal > right_stack.size() ? diagonal - right_stack.size() : 0; i <= diagonal && i < conf.stack.size(); i++) {
      unsigned j = diagonal - i;

      // Try extending stack
      if (i+1 < conf.stack.size())
        for (unsigned h = 0; h <= diagonal; h++) {
          int h_node = t.node(i, j, h), new_node = t.node(i+1, j, 0);
          if (new_node && t.cost(i, j, h) + t.edge_cost(h_node, new_node) < t.cost(i+1, j, h+1) + (t.transition(i, j, h) != 0)) {
            t.cost(i+1, j, h+1) = t.cost(i, j, h) + t.edge_cost(h_node, new_node);
            t.transition(i+1, j, h+1) = t.transition(i, j, h) >= 0 ? t.transition(i, j, h) : t.which_arc_transition(h_node, new_node);
          }
          if (t.cost(i, j, h) + t.edge_cost(new_node, h_node) < t.cost(i+1, j, 0) + (t.transition(i, j, h) != 0)) {
            t.cost(i+1, j, 0) = t.cost(i, j, h) + t.edge_cost(new_node, h_node);
            t.transition(i+1, j, 0) = t.transition(i, j, h) >= 0 ? t.transition(i, j, h) : t.which_arc_transition(new_node, h_node);
          }
        }

      // Try extending right_stack
      if (j+1 < right_stack.size() + 1)
        for (unsigned h = 0; h <= diagonal; h++) {
          int h_node = t.node(i, j, h), new_node = t.node(i, j+1, diagonal+1);
          if (t.cost(i, j, h) + t.edge_cost(h_node, new_node) < t.cost(i, j+1, h) + (t.transition(i, j, h) > 0)) {
            t.cost(i, j+1, h) = t.cost(i, j, h) + t.edge_cost(h_node, new_node);
            t.transition(i, j+1, h) = t.transition(i, j, h) >= 0 ? t.transition(i, j, h) : 0;
          }
          if (h_node && t.cost(i, j, h) + t.edge_cost(new_node, h_node) < t.cost(i, j+1, diagonal+1) + (t.transition(i, j, h) > 0)) {
            t.cost(i, j+1, diagonal+1) = t.cost(i, j, h) + t.edge_cost(new_node, h_node);
            t.transition(i, j+1, diagonal+1) = t.transition(i, j, h) >= 0 ? t.transition(i, j, h) : 0;
          }
        }
    }
  }

  return predicted_transition(t.transition(conf.stack.size() - 1, right_stack.size(), 0), network_outcome);
}

// Oracle factory method
transition_oracle* transition_system_projective::oracle(const string& name) const {
  if (name == "static") return new transition_system_projective_oracle_static(labels);
  if (name == "dynamic") return new transition_system_projective_oracle_dynamic(labels);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/transition/transition_system_swap.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

transition_system_swap::transition_system_swap(const vector& labels) : transition_system(labels) {
  transitions.emplace_back(new transition_shift());
  transitions.emplace_back(new transition_swap());
  for (auto&& label : labels) {
    transitions.emplace_back(new transition_left_arc(label));
    transitions.emplace_back(new transition_right_arc(label));
  }
}

// Static oracle
class transition_system_swap_oracle_static : public transition_oracle {
 public:
  transition_system_swap_oracle_static(const vector& labels, bool lazy) : labels(labels), lazy(lazy) {
    for (root_label = 0; root_label < labels.size(); root_label++) if (labels[root_label] == "root") break;
  }

  class tree_oracle_static : public transition_oracle::tree_oracle {
   public:
    tree_oracle_static(const vector& labels, unsigned root_label, const tree& gold, vector&& projective_order, vector&& projective_components)
        : labels(labels), root_label(root_label), gold(gold), projective_order(projective_order), projective_components(projective_components) {}
    virtual predicted_transition predict(const configuration& conf, unsigned network_outcome, unsigned iteration) const override;
    virtual void interesting_transitions(const configuration& conf, vector& transitions) const override;
   private:
    const vector& labels;
    unsigned root_label;
    const tree& gold;
    const vector projective_order;
    const vector projective_components;
  };

  virtual unique_ptr create_tree_oracle(const tree& gold) const override;
 private:
  void create_projective_order(const tree& gold, int node, vector& projective_order, int& projective_index) const;
  void create_projective_component(const tree& gold, int node, vector& projective_components, int component_index) const;

  const vector& labels;
  bool lazy;
  unsigned root_label;
};

unique_ptr transition_system_swap_oracle_static::create_tree_oracle(const tree& gold) const {
  vector projective_order(gold.nodes.size());
  int projective_index;
  create_projective_order(gold, 0, projective_order, projective_index);

  vector projective_components;
  if (lazy) {
    tree_oracle_static projective_oracle(labels, root_label, gold, vector(), vector());
    configuration conf(false);
    tree t = gold;
    transition_system_swap system(labels);

    conf.init(&t);
    while (!conf.final()) {
      auto prediction = projective_oracle.predict(conf, 0, 0);
      if (!system.applicable(conf, prediction.to_follow)) break;
      system.perform(conf, prediction.to_follow);
    }

    projective_components.assign(gold.nodes.size(), 0);
    for (auto&& node : conf.stack)
      if (node)
        create_projective_component(t, node, projective_components, node);
  }

  return unique_ptr(new tree_oracle_static(labels, root_label, gold, move(projective_order), move(projective_components)));
}

void transition_system_swap_oracle_static::create_projective_order(const tree& gold, int node, vector& projective_order, int& projective_index) const {
  unsigned child_index = 0;
  while (child_index < gold.nodes[node].children.size() && gold.nodes[node].children[child_index] < node)
    create_projective_order(gold, gold.nodes[node].children[child_index++], projective_order, projective_index);
  projective_order[node] = projective_index++;
  while (child_index < gold.nodes[node].children.size())
    create_projective_order(gold, gold.nodes[node].children[child_index++], projective_order, projective_index);
}

void transition_system_swap_oracle_static::create_projective_component(const tree& gold, int node, vector& projective_components, int component_index) const {
  projective_components[node] = component_index;
  for (auto&& child : gold.nodes[node].children)
    create_projective_component(gold, child, projective_components, component_index);
}

void transition_system_swap_oracle_static::tree_oracle_static::interesting_transitions(const configuration& conf, vector& transitions) const {
  transitions.clear();
  if (!conf.buffer.empty()) transitions.push_back(0);
  if (conf.stack.size() >= 2) {
    // Swap
    if (!projective_order.empty()) {
      int last = conf.stack[conf.stack.size() - 1];
      int prev = conf.stack[conf.stack.size() - 2];
      if (projective_order[last] < projective_order[prev] &&
          (projective_components.empty() ||
           (conf.buffer.empty() || projective_components[last] != projective_components[conf.buffer.back()])))
        transitions.push_back(1);
    }

    // Arcs
    for (int direction = 0; direction < 2; direction++) {
      int child = conf.stack[conf.stack.size() - 2 + direction];
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          if (!conf.single_root ||
              (i == root_label && conf.stack.size() == 2 && conf.buffer.empty() && direction == 1) ||
              (i != root_label && conf.stack.size() > 2))
            transitions.push_back(2 + 2*i + direction);
    }
  }
}

transition_oracle::predicted_transition transition_system_swap_oracle_static::tree_oracle_static::predict(const configuration& conf, unsigned /*network_outcome*/, unsigned /*iteration*/) const {
  // Use left if appropriate
  if (conf.stack.size() >= 2) {
    int parent = conf.stack[conf.stack.size() - 1];
    int child = conf.stack[conf.stack.size() - 2];
    if (gold.nodes[child].head == parent && gold.nodes[child].children.size() == conf.t->nodes[child].children.size()) {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return predicted_transition(2 + 2*i, 2 + 2*i);

      assert(!"label was not found");
    }
  }

  // Use right if appropriate
  if (conf.stack.size() >= 2) {
    int child = conf.stack[conf.stack.size() - 1];
    int parent = conf.stack[conf.stack.size() - 2];
    if (gold.nodes[child].head == parent && gold.nodes[child].children.size() == conf.t->nodes[child].children.size()) {
      for (size_t i = 0; i < labels.size(); i++)
        if (gold.nodes[child].deprel == labels[i])
          return predicted_transition(2 + 2*i + 1, 2 + 2*i + 1);

      assert(!"label was not found");
    }
  }

  // Use swap if appropriate
  if (conf.stack.size() >= 2 && !projective_order.empty()) {
    int last = conf.stack[conf.stack.size() - 1];
    int prev = conf.stack[conf.stack.size() - 2];
    if (projective_order[last] < projective_order[prev] &&
        (projective_components.empty() ||
         (conf.buffer.empty() || projective_components[last] != projective_components[conf.buffer.back()])))
      return predicted_transition(1, 1);
  }

  // Otherwise, just shift
  return predicted_transition(0, 0);
}

// Oracle factory method
transition_oracle* transition_system_swap::oracle(const string& name) const {
  if (name == "static_eager") return new transition_system_swap_oracle_static(labels, false);
  if (name == "static_lazy") return new transition_system_swap_oracle_static(labels, true);
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/tree/tree.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

const string tree::root_form = "";

tree::tree() {
  clear();
}

bool tree::empty() {
  return nodes.size() == 1;
}

void tree::clear() {
  nodes.clear();
  node& root = add_node(root_form);
  root.lemma = root.upostag = root.xpostag = root.feats = root_form;
}

node& tree::add_node(const string& form) {
  nodes.emplace_back((int)nodes.size(), form);
  return nodes.back();
}

void tree::set_head(int id, int head, const string& deprel) {
  assert(id >= 0 && id < int(nodes.size()));
  assert(head < int(nodes.size()));

  // Remove existing head
  if (nodes[id].head >= 0) {
    auto& children = nodes[nodes[id].head].children;
    for (size_t i = children.size(); i && children[i-1] >= id; i--)
      if (children[i-1] == id) {
        children.erase(children.begin() + i - 1);
        break;
      }
  }

  // Set new head
  nodes[id].head = head;
  nodes[id].deprel = deprel;
  if (head >= 0) {
    auto& children = nodes[head].children;
    size_t i = children.size();
    while (i && children[i-1] > id) i--;
    if (!i || children[i-1] < id) children.insert(children.begin() + i, id);
  }
}

void tree::unlink_all_nodes() {
  for (auto&& node : nodes) {
    node.head = -1;
    node.deprel.clear();
    node.children.clear();
  }
}

} // namespace parsito

/////////
// File: parsito/tree/tree_format.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Input format
class tree_input_format {
 public:
  virtual ~tree_input_format() {}

  virtual bool read_block(istream& in, string& block) const = 0;
  virtual void set_text(string_piece text, bool make_copy = false) = 0;
  virtual bool next_tree(tree& t) = 0;
  const string& last_error() const;

  // Static factory methods
  static tree_input_format* new_input_format(const string& name);
  static tree_input_format* new_conllu_input_format();

 protected:
  string error;
};

// Output format
class tree_output_format {
 public:
  virtual ~tree_output_format() {}

  virtual void write_tree(const tree& t, string& output, const tree_input_format* additional_info = nullptr) const = 0;

  // Static factory methods
  static tree_output_format* new_output_format(const string& name);
  static tree_output_format* new_conllu_output_format();
};

} // namespace parsito

/////////
// File: parsito/tree/tree_format_conllu.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Input CoNLL-U format
class tree_input_format_conllu : public tree_input_format {
 public:
  virtual bool read_block(istream& in, string& block) const override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_tree(tree& t) override;

 private:
  friend class tree_output_format_conllu;
  vector comments;
  vector> multiword_tokens;

  string_piece text;
  string text_copy;
};

// Output CoNLL-U format
class tree_output_format_conllu : public tree_output_format {
 public:
  virtual void write_tree(const tree& t, string& output, const tree_input_format* additional_info = nullptr) const override;

 private:
  static const string underscore;
  const string& underscore_on_empty(const string& str) const { return str.empty() ? underscore : str; }
};

} // namespace parsito

/////////
// File: parsito/tree/tree_format.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

const string& tree_input_format::last_error() const {
  return error;
}

// Input Static factory methods
tree_input_format* tree_input_format::new_conllu_input_format() {
  return new tree_input_format_conllu();
}

tree_input_format* tree_input_format::new_input_format(const string& name) {
  if (name == "conllu") return new_conllu_input_format();
  return nullptr;
}

// Output static factory methods
tree_output_format* tree_output_format::new_conllu_output_format() {
  return new tree_output_format_conllu();
}

tree_output_format* tree_output_format::new_output_format(const string& name) {
  if (name == "conllu") return new_conllu_output_format();
  return nullptr;
}

} // namespace parsito

/////////
// File: parsito/tree/tree_format_conllu.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Input CoNLL-U format

bool tree_input_format_conllu::read_block(istream& in, string& block) const {
  return bool(getpara(in, block));
}

void tree_input_format_conllu::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool tree_input_format_conllu::next_tree(tree& t) {
  error.clear();
  t.clear();
  comments.clear();
  multiword_tokens.clear();
  int last_multiword_token = 0;

  vector tokens, parts;
  while (text.len) {
    // Read line
    string_piece line(text.str, 0);
    while (line.len < text.len && line.str[line.len] != '\n') line.len++;
    text.str += line.len + (line.len < text.len);
    text.len -= line.len + (line.len < text.len);

    // Empty lines denote end of tree, unless at the beginning
    if (!line.len) {
      if (t.empty()) continue;
      break;
    }

    if (*line.str == '#') {
      // Store comments at the beginning and ignore the rest
      if (t.empty()) comments.push_back(line);
      continue;
    }

    // Parse another tree node
    split(line, '\t', tokens);
    if (tokens.size() != 10)
      return error.assign("The CoNLL-U line '").append(line.str, line.len).append("' does not contain 10 columns!") , false;

    // Store and skip multiword tokens
    if (memchr(tokens[0].str, '-', tokens[0].len)) {
      split(tokens[0], '-', parts);
      if (parts.size() != 2)
        return error.assign("Cannot parse ID of multiword token '").append(line.str, line.len).append("'!") , false;
      int from, to;
      if (!parse_int(parts[0], "CoNLL-U id", from, error) || !parse_int(parts[1], "CoNLL-U id", to, error))
        return false;
      if (from != int(t.nodes.size()))
        return error.assign("Incorrect ID '").append(parts[0].str, parts[0].len).append("' of multiword token '").append(line.str, line.len).append("'!"), false;
      if (to < from)
        return error.assign("Incorrect range '").append(tokens[0].str, tokens[0].len).append("' of multiword token '").append(line.str, line.len).append("'!"), false;
      if (from <= last_multiword_token)
        return error.assign("Multiword token '").append(line.str, line.len).append("' overlaps with the previous one!"), false;
      last_multiword_token = to;
      multiword_tokens.emplace_back(from, line);
      continue;
    }

    // Parse node ID and head
    int id;
    if (!parse_int(tokens[0], "CoNLL-U id", id, error))
      return false;
    if (id != int(t.nodes.size()))
      return error.assign("Incorrect ID '").append(tokens[0].str, tokens[0].len).append("' of CoNLL-U line '").append(line.str, line.len).append("'!"), false;

    int head;
    if (tokens[6].len == 1 && tokens[6].str[0] == '_') {
      head = -1;
    } else {
      if (!parse_int(tokens[6], "CoNLL-U head", head, error))
        return false;
      if (head < 0)
        return error.assign("Numeric head value '").append(tokens[0].str, tokens[0].len).append("' cannot be negative!"), false;
    }

    // Add new node
    auto& node = t.add_node(string(tokens[1].str, tokens[1].len));
    if (!(tokens[2].len == 1 && tokens[2].str[0] == '_')) node.lemma.assign(tokens[2].str, tokens[2].len);
    if (!(tokens[3].len == 1 && tokens[3].str[0] == '_')) node.upostag.assign(tokens[3].str, tokens[3].len);
    if (!(tokens[4].len == 1 && tokens[4].str[0] == '_')) node.xpostag.assign(tokens[4].str, tokens[4].len);
    if (!(tokens[5].len == 1 && tokens[5].str[0] == '_')) node.feats.assign(tokens[5].str, tokens[5].len);
    node.head = head;
    if (!(tokens[7].len == 1 && tokens[7].str[0] == '_')) node.deprel.assign(tokens[7].str, tokens[7].len);
    if (!(tokens[8].len == 1 && tokens[8].str[0] == '_')) node.deps.assign(tokens[8].str, tokens[8].len);
    if (!(tokens[9].len == 1 && tokens[9].str[0] == '_')) node.misc.assign(tokens[9].str, tokens[9].len);
  }

  // Check that we got word for the last multiword token
  if (last_multiword_token >= int(t.nodes.size()))
    return error.assign("There are words missing for multiword token '").append(multiword_tokens.back().second.str, multiword_tokens.back().second.len).append("'!"), false;

  // Set heads correctly
  for (auto&& node : t.nodes)
    if (node.id && node.head >= 0) {
      if (node.head >= int(t.nodes.size()))
        return error.assign("Node ID '").append(to_string(node.id)).append("' form '").append(node.form).append("' has too large head: '").append(to_string(node.head)).append("'!"), false;
      t.set_head(node.id, node.head, node.deprel);
    }

  return !t.empty();
}

// Output CoNLL-U format

const string tree_output_format_conllu::underscore = "_";

void tree_output_format_conllu::write_tree(const tree& t, string& output, const tree_input_format* additional_info) const {
  output.clear();

  // Try casting input format to CoNLL-U
  auto input_conllu = dynamic_cast(additional_info);
  size_t input_conllu_multiword_tokens = 0;

  // Comments if present
  if (input_conllu)
    for (auto&& comment : input_conllu->comments)
      output.append(comment.str, comment.len).push_back('\n');

  // Print out the tokens
  for (int i = 1 /*skip the root node*/; i < int(t.nodes.size()); i++) {
    // Write multiword token if present
    if (input_conllu && input_conllu_multiword_tokens < input_conllu->multiword_tokens.size() &&
        i == input_conllu->multiword_tokens[input_conllu_multiword_tokens].first) {
      output.append(input_conllu->multiword_tokens[input_conllu_multiword_tokens].second.str,
                    input_conllu->multiword_tokens[input_conllu_multiword_tokens].second.len).push_back('\n');
      input_conllu_multiword_tokens++;
    }

    // Write the token
    output.append(to_string(i)).push_back('\t');
    output.append(t.nodes[i].form).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].lemma)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].upostag)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].xpostag)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].feats)).push_back('\t');
    output.append(t.nodes[i].head < 0 ? "_" : to_string(t.nodes[i].head)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].deprel)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].deps)).push_back('\t');
    output.append(underscore_on_empty(t.nodes[i].misc)).push_back('\n');
  }
  output.push_back('\n');
}

} // namespace parsito

/////////
// File: parsito/version/version.h
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

struct version {
  unsigned major;
  unsigned minor;
  unsigned patch;
  std::string prerelease;

  // Returns current version.
  static version current();

  // Returns multi-line formated version and copyright string.
  static string version_and_copyright(const string& other_libraries = string());
};

} // namespace parsito

/////////
// File: parsito/version/version.cpp
/////////

// This file is part of Parsito .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace parsito {

// Returns current version.
version version::current() {
  return {1, 1, 1, "devel"};
}

// Returns multi-line formated version and copyright string.
string version::version_and_copyright(const string& other_libraries) {
  ostringstream info;

  auto parsito = version::current();
  auto unilib = unilib::version::current();

  info << "Parsito version " << parsito.major << '.' << parsito.minor << '.' << parsito.patch
       << (parsito.prerelease.empty() ? "" : "-") << parsito.prerelease
       << " (using UniLib " << unilib.major << '.' << unilib.minor << '.' << unilib.patch
       << (other_libraries.empty() ? "" : " and ") << other_libraries << ")\n"
          "Copyright 2015 by Institute of Formal and Applied Linguistics, Faculty of\n"
          "Mathematics and Physics, Charles University in Prague, Czech Republic.";

  return info.str();
}

} // namespace parsito

/////////
// File: sentence/input_format.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string input_format::CONLLU_V1 = "v1";
const string input_format::CONLLU_V2 = "v2";
const string input_format::GENERIC_TOKENIZER_NORMALIZED_SPACES = "normalized_spaces";
const string input_format::GENERIC_TOKENIZER_PRESEGMENTED = "presegmented";
const string input_format::GENERIC_TOKENIZER_RANGES = "ranges";

// CoNLL-U input format
class input_format_conllu : public input_format {
 public:
  input_format_conllu(unsigned version) : version(version) {}

  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id = string_piece()) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  unsigned version;
  string_piece text;
  string text_copy;

  static const string columns[10];
};

const string input_format_conllu::columns[10] = {"ID", "FORM", "LEMMA",
  "UPOS", "XPOS", "FEATS", "HEAD", "DEPREL", "DEPS", "MISC"};

bool input_format_conllu::read_block(istream& is, string& block) const {
  return bool(getpara(is, block));
}

void input_format_conllu::reset_document(string_piece /*id*/) {
  set_text("");
}

void input_format_conllu::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool input_format_conllu::next_sentence(sentence& s, string& error) {
  error.clear();
  s.clear();
  int last_multiword_token = 0;

  vector tokens, parts;
  while (text.len) {
    // Read line
    string_piece line(text.str, 0);
    while (line.len < text.len && (line.str[line.len] != '\r' && line.str[line.len] != '\n')) line.len++;

    text.str += line.len, text.len -= line.len;
    if (text.len >= 2 && text.str[0] == '\r' && text.str[1] == '\n')
      text.str += 2, text.len -= 2;
    else if (text.len && *text.str == '\n')
      text.str++, text.len--;

    // Empty lines denote end of tree, unless at the beginning
    if (!line.len) {
      if (s.empty()) continue;
      break;
    }

    if (*line.str == '#') {
      // Store comments at the beginning and ignore the rest
      if (s.empty()) s.comments.emplace_back(line.str, line.len);
      continue;
    }

    // Parse the line
    split(line, '\t', tokens);
    if (tokens.size() != 10)
      return error.assign("The CoNLL-U line '").append(line.str, line.len).append("' does not contain 10 columns!") , false;

    // Check that no column is empty and contains no spaces (except FORM, LEMMA and MISC in version >= 2)
    for (int i = 0; i < 10; i++) {
      if (!tokens[i].len)
        return error.assign("The CoNLL-U line '").append(line.str, line.len).append("' contains empty column ").append(columns[i]).append("!"), false;
      if ((version < 2 || (i != 1 && i != 2 && i != 9)) && memchr(tokens[i].str, ' ', tokens[i].len) != NULL)
        return error.assign("The CoNLL-U line '").append(line.str, line.len).append("' contains spaces in column ").append(columns[i]).append("!"), false;
    }

    // Handle multiword tokens
    if (memchr(tokens[0].str, '-', tokens[0].len)) {
      split(tokens[0], '-', parts);
      if (parts.size() != 2)
        return error.assign("Cannot parse ID of multiword token '").append(line.str, line.len).append("'!") , false;
      int from, to;
      if (!parse_int(parts[0], "CoNLL-U id", from, error) || !parse_int(parts[1], "CoNLL-U id", to, error))
        return false;
      if (from != int(s.words.size()))
        return error.assign("Incorrect ID '").append(parts[0].str, parts[0].len).append("' of multiword token '").append(line.str, line.len).append("'!"), false;
      if (to < from)
        return error.assign("Incorrect range '").append(tokens[0].str, tokens[0].len).append("' of multiword token '").append(line.str, line.len).append("'!"), false;
      if (from <= last_multiword_token)
        return error.assign("Multiword token '").append(line.str, line.len).append("' overlaps with the previous one!"), false;
      last_multiword_token = to;
      for (int i = 2; i < 9; i++)
        if (tokens[i].len != 1 || tokens[i].str[0] != '_')
          return error.assign("Column ").append(columns[i]).append(" of an multi-word token '").append(line.str, line.len).append("' is not an empty!"), false;
      s.multiword_tokens.emplace_back(from, to, tokens[1], tokens[9].len == 1 && tokens[9].str[0] == '_' ? string_piece() : tokens[9]);
      continue;
    }

    // Handle empty nodes
    if (version >= 2)
      if (memchr(tokens[0].str, '.', tokens[0].len)) {
        split(tokens[0], '.', parts);
        if (parts.size() != 2)
          return error.assign("Cannot parse ID of empty node '").append(line.str, line.len).append("'!") , false;
        int id, index;
        if (!parse_int(parts[0], "CoNLL-U empty node id", id, error) || !parse_int(parts[1], "CoNLL-U empty node index", index, error))
          return false;
        if (id != int(s.words.size()) - 1)
          return error.assign("Incorrect ID '").append(parts[0].str, parts[0].len).append("' of empty node token '").append(line.str, line.len).append("'!"), false;
        if (!((s.empty_nodes.empty() && index == 1) || (!s.empty_nodes.empty() && s.empty_nodes.back().id < id && index == 1) ||
             (!s.empty_nodes.empty() && s.empty_nodes.back().id == id && index == s.empty_nodes.back().index + 1)))
          return error.assign("Incorrect ID index '").append(parts[1].str, parts[1].len).append("' of empty node token '").append(line.str, line.len).append("'!"), false;
        for (int i = 6; i < 8; i++)
          if (tokens[i].len != 1 || tokens[i].str[0] != '_')
            return error.assign("Column ").append(columns[i]).append(" of an empty node token '").append(line.str, line.len).append("' is not an empty!"), false;

        s.empty_nodes.emplace_back(id, index);
        s.empty_nodes.back().form.assign(tokens[1].str, tokens[1].len);
        s.empty_nodes.back().lemma.assign(tokens[2].str, tokens[2].len);
        if (!(tokens[3].len == 1 && tokens[3].str[0] == '_')) s.empty_nodes.back().upostag.assign(tokens[3].str, tokens[3].len);
        if (!(tokens[4].len == 1 && tokens[4].str[0] == '_')) s.empty_nodes.back().xpostag.assign(tokens[4].str, tokens[4].len);
        if (!(tokens[5].len == 1 && tokens[5].str[0] == '_')) s.empty_nodes.back().feats.assign(tokens[5].str, tokens[5].len);
        if (!(tokens[8].len == 1 && tokens[8].str[0] == '_')) s.empty_nodes.back().deps.assign(tokens[8].str, tokens[8].len);
        if (!(tokens[9].len == 1 && tokens[9].str[0] == '_')) s.empty_nodes.back().misc.assign(tokens[9].str, tokens[9].len);
        continue;
      }

    // Parse word ID and head
    int id;
    if (!parse_int(tokens[0], "CoNLL-U id", id, error))
      return false;
    if (id != int(s.words.size()))
      return error.assign("Incorrect ID '").append(tokens[0].str, tokens[0].len).append("' of CoNLL-U line '").append(line.str, line.len).append("'!"), false;

    int head;
    if (tokens[6].len == 1 && tokens[6].str[0] == '_') {
      head = -1;
    } else {
      if (!parse_int(tokens[6], "CoNLL-U head", head, error))
        return false;
      if (head < 0)
        return error.assign("Numeric head value '").append(tokens[0].str, tokens[0].len).append("' cannot be negative!"), false;
    }

    // Add new word
    auto& word = s.add_word(tokens[1]);
    word.lemma.assign(tokens[2].str, tokens[2].len);
    if (!(tokens[3].len == 1 && tokens[3].str[0] == '_')) word.upostag.assign(tokens[3].str, tokens[3].len);
    if (!(tokens[4].len == 1 && tokens[4].str[0] == '_')) word.xpostag.assign(tokens[4].str, tokens[4].len);
    if (!(tokens[5].len == 1 && tokens[5].str[0] == '_')) word.feats.assign(tokens[5].str, tokens[5].len);
    word.head = head;
    if (!(tokens[7].len == 1 && tokens[7].str[0] == '_')) word.deprel.assign(tokens[7].str, tokens[7].len);
    if (!(tokens[8].len == 1 && tokens[8].str[0] == '_')) word.deps.assign(tokens[8].str, tokens[8].len);
    if (!(tokens[9].len == 1 && tokens[9].str[0] == '_')) word.misc.assign(tokens[9].str, tokens[9].len);
  }

  // Check that we got word for the last multiword token
  if (last_multiword_token >= int(s.words.size()))
    return error.assign("There are words missing for multiword token '").append(s.multiword_tokens.back().form).append("'!"), false;

  // Set heads correctly
  for (auto&& word : s.words)
    if (word.id && word.head >= 0) {
      if (word.head >= int(s.words.size()))
        return error.assign("Node ID '").append(to_string(word.id)).append("' form '").append(word.form).append("' has too large head: '").append(to_string(word.head)).append("'!"), false;
      s.set_head(word.id, word.head, word.deprel);
    }

  return !s.empty();
}

// Horizontal input format
class input_format_horizontal : public input_format {
 public:
  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id = string_piece()) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  string_piece text;
  string text_copy;
  bool new_document = true;
  string document_id;
  unsigned preceeding_newlines = 2;
  unsigned sentence_id = 1;
};

bool input_format_horizontal::read_block(istream& is, string& block) const {
  if (getline(is, block))
    return block.push_back('\n'), true;
  return false;
}

void input_format_horizontal::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  preceeding_newlines = 2;
  sentence_id = 1;
  set_text("");
}

void input_format_horizontal::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool input_format_horizontal::next_sentence(sentence& s, string& error) {
  error.clear();
  s.clear();

  // Skip spaces and newlines
  while (text.len && (*text.str == ' ' || *text.str == '\t' || *text.str == '\r' || *text.str == '\n')) {
    preceeding_newlines += *text.str == '\n';
    text.str++, text.len--;
  }

  // Read space (and tab) separated words
  while (text.len && *text.str != '\r' && *text.str != '\n') {
    string_piece word = text;

    // Slurp the word
    while (text.len && *text.str != ' ' && *text.str != '\t' && *text.str != '\r' && *text.str != '\n')
      text.str++, text.len--;
    word.len = text.str - word.str;
    s.add_word(word);

    // Replace  s by regular spaces
    if (s.words.back().form.find("\302\240") != string::npos) {
      string& form = s.words.back().form;
      size_t form_len = 0;
      for (size_t i = 0; i < form.size(); i++) {
        if (form_len && form[form_len-1] == '\302' && form[i] == '\240')
          form[form_len - 1] = ' ';
        else
          form[form_len++] = form[i];
      }
      form.resize(form_len);
    }

    // Skip spaces
    while (text.len && (*text.str == ' ' || *text.str == '\t'))
      text.str++, text.len--;
  }

  if (!s.empty()) {
    // Mark new document if needed
    if (new_document)
      s.set_new_doc(true, document_id);
    new_document = false;

    // Mark new paragraph if needed
    if (preceeding_newlines >= 2)
      s.set_new_par(true);
    preceeding_newlines = 0;

    // Sentence id
    s.set_sent_id(to_string(sentence_id++));
  }

  return !s.empty();
}

// Vertical input format
class input_format_vertical : public input_format {
 public:
  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id = string_piece()) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  string_piece text;
  string text_copy;
  bool new_document = true;
  string document_id;
  unsigned preceeding_newlines = 2;
  unsigned sentence_id = 1;
};

bool input_format_vertical::read_block(istream& is, string& block) const {
  return bool(getpara(is, block));
}

void input_format_vertical::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  preceeding_newlines = 2;
  sentence_id = 1;
  set_text("");
}

void input_format_vertical::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool input_format_vertical::next_sentence(sentence& s, string& error) {
  error.clear();
  s.clear();

  // Skip tabs and newlines
  while (text.len && (*text.str == '\t' || *text.str == '\r' || *text.str == '\n')) {
    preceeding_newlines += *text.str == '\n';
    text.str++, text.len--;
  }

  // Read first word without tabs on every line
  while (text.len && *text.str != '\r' && *text.str != '\n') {
    string_piece word = text;

    // Slurp the word
    while (text.len && *text.str != '\t' && *text.str != '\r' && *text.str != '\n')
      text.str++, text.len--;
    word.len = text.str - word.str;
    s.add_word(word);

    // Skip spaces till end of line
    while (text.len && *text.str != '\r' && *text.str != '\n')
      text.str++, text.len--;

    // Skip one new line
    if (text.len >= 2 && text.str[0] == '\r' && text.str[1] == '\n')
      text.str += 2, text.len -= 2;
    else if (text.len && *text.str == '\n')
      text.str++, text.len--;

    // Skip tabs on the beginning of the line
    while (text.len && *text.str == '\t')
      text.str++, text.len--;
  }

  if (!s.empty()) {
    // Mark new document if needed
    if (new_document)
      s.set_new_doc(true, document_id);
    new_document = false;

    // Mark new paragraph if needed
    if (preceeding_newlines >= 2)
      s.set_new_par(true);
    preceeding_newlines = 0;

    // Sentence id
    s.set_sent_id(to_string(sentence_id++));
  }

  return !s.empty();
}

// Presegmented tokenizer
class input_format_presegmented_tokenizer : public input_format {
 public:
  input_format_presegmented_tokenizer(input_format* tokenizer) : tokenizer(tokenizer) {}

  virtual bool read_block(istream& is, string& block) const override;
  virtual void reset_document(string_piece id) override;
  virtual void set_text(string_piece text, bool make_copy = false) override;
  virtual bool next_sentence(sentence& s, string& error) override;

 private:
  unique_ptr tokenizer;
  string_piece text;
  string text_copy;
  bool new_document = true;
  string document_id;
  unsigned preceeding_newlines = 2;
  unsigned sentence_id = 1;
};

bool input_format_presegmented_tokenizer::read_block(istream& is, string& block) const {
  if (getline(is, block))
    return block.push_back('\n'), true;
  return false;
}

void input_format_presegmented_tokenizer::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  preceeding_newlines = 2;
  sentence_id = 1;
  tokenizer->reset_document();
  set_text("");
}

void input_format_presegmented_tokenizer::set_text(string_piece text, bool make_copy) {
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }
  this->text = text;
}

bool input_format_presegmented_tokenizer::next_sentence(sentence& s, string& error) {
  error.clear();
  s.clear();

  sentence partial;
  unsigned following_newlines = 0;
  while (text.len && s.empty()) {
    // Move next line from `text' to `line', including leading and following newlines
    string_piece line(text.str, 0);
    while (line.len < text.len && (line.str[line.len] == '\n' || line.str[line.len] == '\r')) {
      preceeding_newlines += line.str[line.len] == '\n';
      line.len++;
    }
    while (line.len < text.len && (line.str[line.len] != '\n' && line.str[line.len] != '\r'))
      line.len++;
    while (line.len < text.len && (line.str[line.len] == '\n' || line.str[line.len] == '\r')) {
      following_newlines += line.str[line.len] == '\n';
      line.len++;
    }
    text.str += line.len, text.len -= line.len;

    // Add all tokens from the line to `s'
    tokenizer->set_text(line, false);
    while (tokenizer->next_sentence(partial, error)) {
      // Append words
      size_t words = s.words.size() - 1;
      for (size_t i = 1; i < partial.words.size(); i++) {
        s.words.push_back(move(partial.words[i]));
        s.words.back().id += words;
        if (s.words.back().head > 0) s.words.back().head += words;
      }

      // Append multiword_tokens
      for (auto&& multiword_token : partial.multiword_tokens) {
        s.multiword_tokens.push_back(move(multiword_token));
        s.multiword_tokens.back().id_first += words;
        s.multiword_tokens.back().id_last += words;
      }

      // Append empty nodes
      for (auto&& empty_node : partial.empty_nodes) {
        s.empty_nodes.push_back(move(empty_node));
        s.empty_nodes.back().id += words;
      }
    }
    if (!error.empty()) return false;

    if (s.empty()) {
      preceeding_newlines += following_newlines;
      following_newlines = 0;
    }
  }

  if (!s.empty()) {
  // Mark new document if needed
    if (new_document)
      s.set_new_doc(true, document_id);
    new_document = false;

    // Mark new paragraph if needed
    if (preceeding_newlines >= 2)
      s.set_new_par(true);
    preceeding_newlines = following_newlines;

    // Sentence id
    s.set_sent_id(to_string(sentence_id++));

    // Fill "# text" comment
    s.comments.emplace_back("# text = ");
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j].form : (const token&)s.words[i].form;
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;

      s.comments.back().append(tok.form);
      if (i+1 < s.words.size() && tok.get_space_after()) s.comments.back().push_back(' ');
    }
  }

  return !s.empty();
}

// Static factory methods
input_format* input_format::new_conllu_input_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  unsigned version = 2;
  if (parsed_options.count(CONLLU_V1))
    version = 1;
  if (parsed_options.count(CONLLU_V2))
    version = 2;

  return new input_format_conllu(version);
}

input_format* input_format::new_generic_tokenizer_input_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  bool normalized_spaces = parsed_options.count(GENERIC_TOKENIZER_NORMALIZED_SPACES);
  bool token_ranges = parsed_options.count(GENERIC_TOKENIZER_RANGES);

  input_format* result = new morphodita_tokenizer_wrapper(morphodita::tokenizer::new_generic_tokenizer(), nullptr, normalized_spaces, token_ranges);
  return (parsed_options.count(GENERIC_TOKENIZER_PRESEGMENTED) && result) ? input_format::new_presegmented_tokenizer(result) : result;
}

input_format* input_format::new_horizontal_input_format(const string& /*options*/) {
  return new input_format_horizontal();
}

input_format* input_format::new_vertical_input_format(const string& /*options*/) {
  return new input_format_vertical();
}

input_format* input_format::new_input_format(const string& name) {
  size_t equal = name.find('=');
  size_t name_len = equal != string::npos ? equal : name.size();
  size_t option_offset = equal != string::npos ? equal + 1 : name.size();

  if (name.compare(0, name_len, "conllu") == 0) return new_conllu_input_format(name.substr(option_offset));
  if (name.compare(0, name_len, "generic_tokenizer") == 0) return new_generic_tokenizer_input_format(name.substr(option_offset));
  if (name.compare(0, name_len, "horizontal") == 0) return new_horizontal_input_format(name.substr(option_offset));
  if (name.compare(0, name_len, "vertical") == 0) return new_vertical_input_format(name.substr(option_offset));
  return nullptr;
}

input_format* input_format::new_presegmented_tokenizer(input_format* tokenizer) {
  return new input_format_presegmented_tokenizer(tokenizer);
}

/////////
// File: utils/xml_encoded.h
/////////

// This file is part of UFAL C++ Utils .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace utils {

//
// Declarations
//

// Print xml content while encoding <>& and optionally " using XML entities.
class xml_encoded {
 public:
  xml_encoded(string_piece str, bool encode_quot = false) : str(str), encode_quot(encode_quot) {}

  friend ostream& operator<<(ostream& os, xml_encoded data);
 private:
  string_piece str;
  bool encode_quot;
};

inline ostream& operator<<(ostream& os, xml_encoded data);

//
// Definitions
//

ostream& operator<<(ostream& os, xml_encoded data) {
  string_piece& str = data.str;
  const char* to_print = str.str;

  while (str.len) {
    while (str.len && *str.str != '<' && *str.str != '>' && *str.str != '&' && (!data.encode_quot || *str.str != '"'))
      str.str++, str.len--;

    if (str.len) {
      if (to_print < str.str) os.write(to_print, str.str - to_print);
      os << (*str.str == '<' ? "<" : *str.str == '>' ? ">" : *str.str == '&' ? "&" : """);
      str.str++, str.len--;
      to_print = str.str;
    }
  }

  if (to_print < str.str) os.write(to_print, str.str - to_print);

  return os;
}

} // namespace utils

/////////
// File: sentence/output_format.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string output_format::CONLLU_V1 = "v1";
const string output_format::CONLLU_V2 = "v2";
const string output_format::HORIZONTAL_PARAGRAPHS = "paragraphs";
const string output_format::PLAINTEXT_NORMALIZED_SPACES = "normalized_spaces";
const string output_format::VERTICAL_PARAGRAPHS = "paragraphs";

// CoNLL-U output format
class output_format_conllu : public output_format {
 public:
  output_format_conllu(unsigned version) : version(version) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;

 private:
  unsigned version;
  static const string underscore;
  const string& underscore_on_empty(const string& str) const { return str.empty() ? underscore : str; }
  ostream& write_with_spaces(ostream& os, const string& str);
};

const string output_format_conllu::underscore = "_";

void output_format_conllu::write_sentence(const sentence& s, ostream& os) {
  // Comments
  for (auto&& comment : s.comments)
    os << comment << '\n';

  // Words and multiword tokens
  size_t multiword_token = 0, empty_node = 0;
  for (int i = 0; i < int(s.words.size()); i++) {
    // Write non-root nodes
    if (i > 0) {
      // Multiword token if present
      if (multiword_token < s.multiword_tokens.size() &&
          i == s.multiword_tokens[multiword_token].id_first) {
        os << s.multiword_tokens[multiword_token].id_first << '-'
           << s.multiword_tokens[multiword_token].id_last << '\t';
        write_with_spaces(os, s.multiword_tokens[multiword_token].form) << "\t_\t_\t_\t_\t_\t_\t_\t"
           << underscore_on_empty(s.multiword_tokens[multiword_token].misc) << '\n';
        multiword_token++;
      }

      // Write the word
      os << i << '\t';
      write_with_spaces(os, s.words[i].form) << '\t';
      write_with_spaces(os, underscore_on_empty(s.words[i].lemma)) << '\t'
         << underscore_on_empty(s.words[i].upostag) << '\t'
         << underscore_on_empty(s.words[i].xpostag) << '\t'
         << underscore_on_empty(s.words[i].feats) << '\t';
      if (s.words[i].head < 0) os << '_'; else os << s.words[i].head; os << '\t'
         << underscore_on_empty(s.words[i].deprel) << '\t'
         << underscore_on_empty(s.words[i].deps) << '\t'
         << underscore_on_empty(s.words[i].misc) << '\n';
    }

    // Empty nodes
    if (version >= 2)
      for (; empty_node < s.empty_nodes.size() && i == s.empty_nodes[empty_node].id; empty_node++) {
        os << i << '.' << s.empty_nodes[empty_node].index << '\t'
           << s.empty_nodes[empty_node].form << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].lemma) << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].upostag) << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].xpostag) << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].feats) << '\t'
           << "_\t"
           << "_\t"
           << underscore_on_empty(s.empty_nodes[empty_node].deps) << '\t'
           << underscore_on_empty(s.empty_nodes[empty_node].misc) << '\n';
      }
  }
  os << endl;
}

ostream& output_format_conllu::write_with_spaces(ostream& os, const string& str) {
  if (version >= 2 || str.find(' ') == string::npos)
    os << str;
  else
    for (auto&& chr : str)
      os << (chr == ' ' ? '_' : chr);

  return os;
}

// EPE output format
class output_format_epe : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& os) override;

 private:
  class json_builder {
   public:
    json_builder& object() { comma(); json.push_back('{'); stack.push_back('}'); return *this; }
    json_builder& array() { comma(); json.push_back('['); stack.push_back(']'); return *this; }
    json_builder& close() { if (!stack.empty()) { json.push_back(stack.back()); stack.pop_back(); } comma_needed = true; return *this; }
    json_builder& key(string_piece name) { comma(); string(name); json.push_back(':'); return *this; }
    json_builder& value(string_piece value) { comma(); string(value); comma_needed=true; return *this; }
    json_builder& value(size_t value) { comma(); number(value); comma_needed=true; return *this; }
    json_builder& value_true() { comma(); json.push_back('t'); json.push_back('r'); json.push_back('u'); json.push_back('e'); comma_needed=true; return *this; }

    string_piece current() const { return string_piece(json.data(), json.size()); }
    void clear() { json.clear(); stack.clear(); comma_needed=false; }

   private:
    void comma() {
      if (comma_needed) {
        json.push_back(',');
        json.push_back(' ');
      }
      comma_needed = false;
    }
    void string(string_piece str) {
      json.push_back('"');
      for (; str.len; str.str++, str.len--)
        switch (*str.str) {
          case '"': json.push_back('\\'); json.push_back('\"'); break;
          case '\\': json.push_back('\\'); json.push_back('\\'); break;
          case '\b': json.push_back('\\'); json.push_back('b'); break;
          case '\f': json.push_back('\\'); json.push_back('f'); break;
          case '\n': json.push_back('\\'); json.push_back('n'); break;
          case '\r': json.push_back('\\'); json.push_back('r'); break;
          case '\t': json.push_back('\\'); json.push_back('t'); break;
          default:
            if (((unsigned char)*str.str) < 32) {
              json.push_back('u'); json.push_back('0'); json.push_back('0'); json.push_back('0' + (*str.str >> 4)); json.push_back("0123456789ABCDEF"[*str.str & 0xF]);
            } else {
              json.push_back(*str.str);
            }
        }
      json.push_back('"');
    }
    void number(size_t value) {
      size_t start_size = json.size();
      for (; value || start_size == json.size(); value /= 10)
        json.push_back('0' + (value % 10));
      reverse(json.begin() + start_size, json.end());
    }

    std::vector json;
    std::vector stack;
    bool comma_needed = false;
  } json;

  vector feats;
  size_t sentences = 0;
};

void output_format_epe::write_sentence(const sentence& s, ostream& os) {
  json.object().key("id").value(++sentences).key("nodes").array();

  for (size_t i = 1; i < s.words.size(); i++) {
    json.object().key("id").value(i).key("form").value(s.words[i].form);

    size_t start, end;
    if (s.words[i].get_token_range(start, end))
      json.key("start").value(start).key("end").value(end);
    if (s.words[i].head == 0)
      json.key("top").value_true();

    json.key("properties").object()
        .key("lemma").value(s.words[i].lemma)
        .key("upos").value(s.words[i].upostag)
        .key("xpos").value(s.words[i].xpostag);
    split(s.words[i].feats, '|', feats);
    for (auto&& feat : feats) {
      string_piece key(feat.str, 0);
      while (key.len < feat.len && key.str[key.len] != '=')
        key.len++;
      if (key.len + 1 < feat.len)
        json.key(key).value(string_piece(key.str + key.len + 1, feat.len - key.len - 1));
    }
    json.close();

    if (!s.words[i].children.empty()) {
      json.key("edges").array();
      for (auto&& child : s.words[i].children)
        json.object().key("label").value(s.words[child].deprel).key("target").value(child).close();
      json.close();
    }

    json.close();
  }
  json.close().close();

  string_piece current = json.current();
  os.write(current.str, current.len).put('\n');
  json.clear();
}

void output_format_epe::finish_document(ostream& /*os*/) {
  sentences = 0;
}

// Matxin output format
class output_format_matxin : public output_format {
 public:
  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& os) override;

 private:
  void write_node(const sentence& s, int node, string& pad, ostream& os);

  int sentences = 0;
};

void output_format_matxin::write_sentence(const sentence& s, ostream& os) {
  if (!sentences) {
    os << "";
  }
  os << "\n\n";

  string pad;
  for (auto&& node : s.words[0].children)
    write_node(s, node, pad, os);

  os << "" << endl;
}

void output_format_matxin::finish_document(ostream& os) {
  os << "\n";

  sentences = 0;
}

void output_format_matxin::write_node(const sentence& s, int node, string& pad, ostream& os) {
  //
  pad.push_back(' ');

  os << pad << "
     << "\" form=\"" << xml_encoded(s.words[node].form, true)
     << "\" lem=\"" << xml_encoded(s.words[node].lemma, true)
     << "\" mi=\"" << xml_encoded(s.words[node].feats, true)
     << "\" si=\"" << xml_encoded(s.words[node].deprel, true) << '"';

  if (s.words[node].children.empty()) {
    os << "/>\n";
  } else {
    os << ">\n";
    for (auto&& child : s.words[node].children)
      write_node(s, child, pad, os);
    os << pad << "\n";
  }

  pad.pop_back();
}

// Horizontal output format
class output_format_horizontal : public output_format {
 public:
  output_format_horizontal(bool paragraphs) : paragraphs(paragraphs), empty(true) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& /*os*/) override { empty = true; }

 private:
  bool paragraphs;
  bool empty;
};

void output_format_horizontal::write_sentence(const sentence& s, ostream& os) {
  if (paragraphs && !empty && (s.get_new_doc() || s.get_new_par()))
    os << '\n';
  empty = false;

  string line;
  for (size_t i = 1; i < s.words.size(); i++) {
    // Append word, but replace spaces by  s
    for (auto&& chr : s.words[i].form)
      if (chr == ' ')
        line.append("\302\240");
      else
        line.push_back(chr);

    if (i+1 < s.words.size())
      line.push_back(' ');
  }
  os << line << endl;
}

// Plaintext output format
class output_format_plaintext : public output_format {
 public:
  output_format_plaintext(bool normalized): normalized(normalized), empty(true) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& /*os*/) override { empty = true; }
 private:
  bool normalized;
  bool empty;
};

void output_format_plaintext::write_sentence(const sentence& s, ostream& os) {
  if (normalized) {
    if (!empty && (s.get_new_doc() || s.get_new_par()))
      os << '\n';
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j] : (const token&)s.words[i];
      os << tok.form;
      if (i+1 < s.words.size() && tok.get_space_after())
        os << ' ';
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;
    }
    os << endl;
  } else {
    string spaces;
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j] : (const token&)s.words[i];
      tok.get_spaces_before(spaces); os << spaces;
      tok.get_spaces_in_token(spaces); os << (!spaces.empty() ? spaces : tok.form);
      tok.get_spaces_after(spaces); os << spaces;
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;
    }
    os << flush;
  }
  empty = false;
}

// Vertical output format
class output_format_vertical : public output_format {
 public:
  output_format_vertical(bool paragraphs) : paragraphs(paragraphs), empty(true) {}

  virtual void write_sentence(const sentence& s, ostream& os) override;
  virtual void finish_document(ostream& /*os*/) override { empty = true; }

 private:
  bool paragraphs;
  bool empty;
};

void output_format_vertical::write_sentence(const sentence& s, ostream& os) {
  if (paragraphs && !empty && (s.get_new_doc() || s.get_new_par()))
    os << '\n';
  empty = false;

  for (size_t i = 1; i < s.words.size(); i++)
    os << s.words[i].form << '\n';
  os << endl;
}

// Static factory methods
output_format* output_format::new_conllu_output_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  unsigned version = 2;
  if (parsed_options.count(CONLLU_V1))
    version = 1;
  if (parsed_options.count(CONLLU_V2))
    version = 2;

  return new output_format_conllu(version);
}

output_format* output_format::new_epe_output_format(const string& /*options*/) {
  return new output_format_epe();
}

output_format* output_format::new_matxin_output_format(const string& /*options*/) {
  return new output_format_matxin();
}

output_format* output_format::new_horizontal_output_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  return new output_format_horizontal(parsed_options.count(HORIZONTAL_PARAGRAPHS));
}

output_format* output_format::new_plaintext_output_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  return new output_format_plaintext(parsed_options.count(PLAINTEXT_NORMALIZED_SPACES));
}

output_format* output_format::new_vertical_output_format(const string& options) {
  named_values::map parsed_options;
  string parse_error;
  if (!named_values::parse(options, parsed_options, parse_error))
    return nullptr;

  return new output_format_vertical(parsed_options.count(VERTICAL_PARAGRAPHS));
}

output_format* output_format::new_output_format(const string& name) {
  size_t equal = name.find('=');
  size_t name_len = equal != string::npos ? equal : name.size();
  size_t option_offset = equal != string::npos ? equal + 1 : name.size();

  if (name.compare(0, name_len, "conllu") == 0) return new_conllu_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "epe") == 0) return new_epe_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "matxin") == 0) return new_matxin_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "horizontal") == 0) return new_horizontal_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "plaintext") == 0) return new_plaintext_output_format(name.substr(option_offset));
  if (name.compare(0, name_len, "vertical") == 0) return new_vertical_output_format(name.substr(option_offset));
  return nullptr;
}

/////////
// File: sentence/sentence.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string sentence::root_form = "";

sentence::sentence() {
  clear();
}

bool sentence::empty() {
  return words.size() == 1;
}

void sentence::clear() {
  words.clear();
  multiword_tokens.clear();
  empty_nodes.clear();
  comments.clear();

  word& root = add_word(root_form);
  root.lemma = root.upostag = root.xpostag = root.feats = root_form;
}

word& sentence::add_word(string_piece form) {
  words.emplace_back((int)words.size(), form);
  return words.back();
}

void sentence::set_head(int id, int head, const string& deprel) {
  assert(id >= 0 && id < int(words.size()));
  assert(head < int(words.size()));

  // Remove existing head
  if (words[id].head >= 0) {
    auto& children = words[words[id].head].children;
    for (size_t i = children.size(); i && children[i-1] >= id; i--)
      if (children[i-1] == id) {
        children.erase(children.begin() + i - 1);
        break;
      }
  }

  // Set new head
  words[id].head = head;
  words[id].deprel = deprel;
  if (head >= 0) {
    auto& children = words[head].children;
    size_t i = children.size();
    while (i && children[i-1] > id) i--;
    if (!i || children[i-1] < id) children.insert(children.begin() + i, id);
  }
}

void sentence::unlink_all_words() {
  for (auto&& word : words) {
    word.head = -1;
    word.deprel.clear();
    word.children.clear();
  }
}

bool sentence::get_new_doc(string* id) const {
  if (get_comment("newdoc id", id))
    return true;
  return get_comment("newdoc", id);
}

void sentence::set_new_doc(bool new_doc, string_piece id) {
  remove_comment("newdoc");
  remove_comment("newdoc id");

  if (new_doc && id.len)
    set_comment("newdoc id", id);
  else if (new_doc)
    set_comment("newdoc");
}

bool sentence::get_new_par(string* id) const {
  if (get_comment("newpar id", id))
    return true;
  return get_comment("newpar", id);
}

void sentence::set_new_par(bool new_par, string_piece id) {
  remove_comment("newpar");
  remove_comment("newpar id");

  if (new_par && id.len)
    set_comment("newpar id", id);
  else if (new_par)
    set_comment("newpar");
}

bool sentence::get_sent_id(string& id) const {
  id.clear();

  return get_comment("sent_id", &id);
}

void sentence::set_sent_id(string_piece id) {
  remove_comment("sent_id");

  if (id.len)
    set_comment("sent_id", id);
}

bool sentence::get_text(string& text) const {
  text.clear();

  return get_comment("text", &text);
}

void sentence::set_text(string_piece text) {
  remove_comment("text");

  if (text.len)
    set_comment("text", text);
}

bool sentence::get_comment(string_piece name, string* value) const {
  for (auto&& comment : comments)
    if (comment[0] == '#') {
      // Skip spaces
      unsigned j = 1;
      while (j < comment.size() && (comment[j] == ' ' || comment[j] == '\t')) j++;

      // Try matching the name
      if (j + name.len <= comment.size() && comment.compare(j, name.len, name.str, name.len) == 0) {
        j += name.len;
        while (j < comment.size() && (comment[j] == ' ' || comment[j] == '\t')) j++;
        if (j < comment.size() && comment[j] == '=') {
          //We have a value
          j++;
          while (j < comment.size() && (comment[j] == ' ' || comment[j] == '\t')) j++;
          if (value) value->assign(comment, j, comment.size() - j);
        } else {
          // No value
          if (value) value->clear();
        }

        return true;
      }
    }

  return false;
}

void sentence::remove_comment(string_piece name) {
  for (unsigned i = comments.size(); i--; )
    if (comments[i][0] == '#') {
      // Skip spaces
      unsigned j = 1;
      while (j < comments[i].size() && (comments[i][j] == ' ' || comments[i][j] == '\t')) j++;

      // Remove matching comments
      if (j + name.len <= comments[i].size() && comments[i].compare(j, name.len, name.str, name.len) == 0)
        comments.erase(comments.begin() + i);
    }
}

void sentence::set_comment(string_piece name, string_piece value) {
  remove_comment(name);

  string comment;
  comment.append("# ").append(name.str, name.len);
  if (value.len) {
    comment.append(" = ");
    for (size_t i = 0; i < value.len; i++)
      comment.push_back(value.str[i] == '\r' || value.str[i] == '\n' ? ' ' : value.str[i]);
  }
  comments.push_back(move(comment));
}

/////////
// File: sentence/token.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2017 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

token::token(string_piece form, string_piece misc) {
  if (form.len) this->form.assign(form.str, form.len);
  if (misc.len) this->misc.assign(misc.str, misc.len);
}

// CoNLL-U defined SpaceAfter=No feature
bool token::get_space_after() const {
  string_piece value;

  return !(get_misc_field("SpaceAfter", value) && value.len == 2 && memcmp(value.str, "No", 2) == 0);
}

void token::set_space_after(bool space_after) {
  if (space_after)
    remove_misc_field("SpaceAfter");
  else
    start_misc_field("SpaceAfter").append("No");
}

// UDPipe-specific all-spaces-preserving SpacesBefore and SpacesAfter features
void token::get_spaces_before(string& spaces_before) const {
  string_piece value;

  if (get_misc_field("SpacesBefore", value))
    unescape_spaces(value, spaces_before);
  else
    spaces_before.clear();
}

void token::set_spaces_before(string_piece spaces_before) {
  if (spaces_before.len == 0)
    remove_misc_field("SpacesBefore");
  else
    append_escaped_spaces(spaces_before, start_misc_field("SpacesBefore"));
}

void token::get_spaces_after(string& spaces_after) const {
  string_piece value;

  if (get_misc_field("SpacesAfter", value))
    unescape_spaces(value, spaces_after);
  else
    spaces_after.assign(get_space_after() ? " " : "");
}

void token::set_spaces_after(string_piece spaces_after) {
  if (spaces_after.len == 0) {
    set_space_after(false);
    remove_misc_field("SpacesAfter");
  } else if (spaces_after.len == 1 && spaces_after.str[0] == ' ') {
    set_space_after(true);
    remove_misc_field("SpacesAfter");
  } else {
    set_space_after(true);
    append_escaped_spaces(spaces_after, start_misc_field("SpacesAfter"));
  }
}

void token::get_spaces_in_token(string& spaces_in_token) const {
  string_piece value;

  if (get_misc_field("SpacesInToken", value))
    unescape_spaces(value, spaces_in_token);
  else
    spaces_in_token.clear();
}

void token::set_spaces_in_token(string_piece spaces_in_token) {
  if (spaces_in_token.len == 0)
    remove_misc_field("SpacesInToken");
  else
    append_escaped_spaces(spaces_in_token, start_misc_field("SpacesInToken"));
}

// UDPipe-specific TokenRange feature
bool token::get_token_range(size_t& start, size_t& end) const {
  string_piece value;

  if (!get_misc_field("TokenRange", value)) return false;

  start = 0;
  while (value.len && value.str[0] >= '0' && value.str[0] <= '9') {
    if (start > (numeric_limits::max() - (value.str[0] - '0')) / 10)
      return false;
    start = 10 * start + (value.str[0] - '0');
    value.str++, value.len--;
  }

  if (value.len == 0 || value.str[0] != ':') return false;
  value.str++, value.len--;

  end = 0;
  while (value.len && value.str[0] >= '0' && value.str[0] <= '9') {
    if (end > (numeric_limits::max() - (value.str[0] - '0')) / 10)
      return false;
    end = 10 * end + (value.str[0] - '0');
    value.str++, value.len--;
  }

  return true;
}

void token::set_token_range(size_t start, size_t end) {
  if (start == size_t(string::npos))
    remove_misc_field("TokenRange");
  else
    start_misc_field("TokenRange").append(to_string(start)).append(1, ':').append(to_string(end));
}

// Private MISC field helpers
bool token::get_misc_field(string_piece name, string_piece& value) const {
  for (size_t index = 0; index < misc.size(); ) {
    if (misc.compare(index, name.len, name.str, name.len) == 0 && misc[index + name.len] == '=') {
      index += name.len + 1;
      value.str = misc.c_str() + index;
      value.len = misc.find('|', index);
      value.len = (value.len == size_t(string::npos) ? misc.size() : value.len) - index;
      return true;
    }
    index = misc.find('|', index);
    if (index != size_t(string::npos)) index++;
  }
  return false;
}

void token::remove_misc_field(string_piece name) {
  for (size_t index = 0; index < misc.size(); )
    if (misc.compare(index, name.len, name.str, name.len) == 0 && misc[index + name.len] == '=') {
      size_t end_index = misc.find('|', index + name.len + 1);
      if (end_index == size_t(string::npos)) end_index = misc.size();

      // Be careful to delete at most one neighboring '|'
      if (index)
        misc.erase(index - 1, end_index - (index - 1));
      else
        misc.erase(index, end_index + (end_index < misc.size() ? 1 : 0) - index);
    } else {
      index = misc.find('|', index);
      if (index != size_t(string::npos)) index++;
    }
}

string& token::start_misc_field(string_piece name) {
  remove_misc_field(name);
  if (!misc.empty()) misc.push_back('|');
  misc.append(name.str, name.len).push_back('=');
  return misc;
}

void token::append_escaped_spaces(string_piece spaces, string& escaped_spaces) const {
  for (unsigned i = 0; i < spaces.len; i++)
    switch (spaces.str[i]) {
      case ' ':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('s'); break;
      case '|':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('p'); break;
      case '\t':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('t'); break;
      case '\r':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('r'); break;
      case '\n':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('n'); break;
      case '\\':
        escaped_spaces.push_back('\\'); escaped_spaces.push_back('\\'); break;
      default:
        escaped_spaces.push_back(spaces.str[i]);
    }
}

void token::unescape_spaces(string_piece escaped_spaces, string& spaces) const {
  spaces.clear();

  for (unsigned i = 0; i < escaped_spaces.len; i++)
    if (escaped_spaces.str[i] != '\\' || i+1 >= escaped_spaces.len)
      spaces.push_back(escaped_spaces.str[i]);
    else switch (escaped_spaces.str[++i]) {
      case 's':
        spaces.push_back(' '); break;
      case 'p':
        spaces.push_back('|'); break;
      case 't':
        spaces.push_back('\t'); break;
      case 'r':
        spaces.push_back('\r'); break;
      case 'n':
        spaces.push_back('\n'); break;
      case '\\':
        spaces.push_back('\\'); break;
      default:
        spaces.push_back(escaped_spaces.str[i - 1]);
        spaces.push_back(escaped_spaces.str[i]);
    }
}

/////////
// File: tokenizer/detokenizer.h
/////////

// This file is part of UDPipe .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class detokenizer {
 public:
  detokenizer(const string& plain_text);

  void detokenize(sentence& s) const;
 private:
  enum { LOWERCASE, CATEGORIZE, TOTAL };

  int difference(const string& left, const string& right, bool separate, int mode) const;

  static string perform_lowercase(const string& input);
  static string perform_categorize(const string& input);
  bool has_letters(const string& word) const;
  bool only_digits(const string& word) const;

  class suffix_array {
   public:
    suffix_array(const string& str);
    suffix_array(suffix_array&& other) = default;

    unsigned count(const string& data) const;

   private:
    vector sa;

    struct suffix_compare {
      suffix_compare(const string& str) : str(str) {}
      bool operator()(unsigned a, unsigned b) const { return str.compare(a, string::npos, str, b, string::npos) < 0; }
     private:
      const string& str;
    } suffix_comparator;

    struct suffix_lower_find {
      suffix_lower_find(const string& str) : str(str) {}
      bool operator()(unsigned a, const string& data) const { return str.compare(a, data.size(), data) < 0; }

     private:
      const string& str;
    } suffix_lower_finder;

    struct suffix_upper_find {
      suffix_upper_find(const string& str) : str(str) {}
      bool operator()(const string& data, unsigned a) const { return str.compare(a, data.size(), data) > 0; }

     private:
      const string& str;
    } suffix_upper_finder;
  };

  string data_lowercased, data_categorized;
  suffix_array sa_lowercased, sa_categorized;
};

/////////
// File: tokenizer/detokenizer.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

detokenizer::detokenizer(const string& plain_text)
    : data_lowercased(perform_lowercase(plain_text)), data_categorized(perform_categorize(plain_text)),
    sa_lowercased(data_lowercased), sa_categorized(data_categorized) {}

void detokenizer::detokenize(sentence& s) const {
  token* previous_tok = nullptr;
  for (size_t i = 1, j = 0; i < s.words.size(); i++) {
    token* tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (token*)&s.multiword_tokens[j] : (token*)&s.words[i];

    if (previous_tok) {
      // Should we add SpaceAfter=No to the previous form?
      int score = difference(previous_tok->form, tok->form, true, LOWERCASE);
      if (!score) score = has_letters(previous_tok->form) && has_letters(tok->form) ? -1 : 0;
      if (!score) score = only_digits(previous_tok->form) && only_digits(tok->form) ? -1 : 0;
      if (!score) score = difference(previous_tok->form, tok->form, false, LOWERCASE);
      if (!score) score = difference(previous_tok->form, tok->form, false, CATEGORIZE);
      if (!score) score = difference(previous_tok->form, tok->form, true, CATEGORIZE);

      if (score > 0)
        previous_tok->set_space_after(false);
    }

    // Remove the SpaceAfter attribute on current token
    tok->set_space_after(true);
    previous_tok = tok;

    if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
      i = s.multiword_tokens[j++].id_last;
  }
}

int detokenizer::difference(const string& left, const string& right, bool separate, int mode) const {
  auto& func = mode == LOWERCASE ? perform_lowercase : perform_categorize;
  auto& sa = mode == LOWERCASE ? sa_lowercased : sa_categorized;

  string left_mapped = func(left);
  string right_mapped = func(right);
  string pattern;

  pattern.assign(separate?" ":"").append(left_mapped).append(right_mapped).append(separate?" ":"");
  int together = sa.count(pattern);

  pattern.assign(separate?" ":"").append(left_mapped).append(" ").append(right_mapped).append(separate?" ":"");
  int apart = sa.count(pattern);

  return together - apart;
}

string detokenizer::perform_lowercase(const string& input) {
  using namespace unilib;

  string output;
  for (auto&& chr : utf8::decoder(input))
    utf8::append(output, unicode::lowercase(chr));
  return output;
}

string detokenizer::perform_categorize(const string& input) {
  using namespace unilib;

  string output;
  for (auto&& chr : utf8::decoder(input)) {
    auto category = unicode::category(chr);
    if (category & unicode::C) output.push_back('C');
    if (category & unicode::L) output.push_back('L');
    if (category & unicode::M) output.push_back('M');
    if (category & unicode::N) output.push_back('N');
    if (category & unicode::Pc) output.push_back('c');
    if (category & unicode::Pd) output.push_back('d');
    if (category & unicode::Pe) output.push_back('e');
    if (category & unicode::Pf) output.push_back('f');
    if (category & unicode::Pi) output.push_back('i');
    if (category & unicode::Po) output.push_back('o');
    if (category & unicode::Ps) output.push_back('s');
    if (category & unicode::S) output.push_back('S');
    if (category & unicode::Zl) output.push_back('Z');
    if (category & unicode::Zp) output.push_back('z');
    if (category & unicode::Zs) output.push_back(' ');
  }
  return output;
}

bool detokenizer::has_letters(const string& word) const {
  using namespace unilib;

  for (auto&& chr : utf8::decoder(word))
    if (unicode::category(chr) & unicode::L)
      return true;
  return false;
}

bool detokenizer::only_digits(const string& word) const {
  using namespace unilib;

  for (auto&& chr : utf8::decoder(word))
    if (unicode::category(chr) & ~unicode::N)
      return false;
  return true;
}

detokenizer::suffix_array::suffix_array(const string& str) : suffix_comparator(str), suffix_lower_finder(str), suffix_upper_finder(str) {
  sa.reserve(str.size());
  for (unsigned i = 0; i < str.size(); i++)
    sa.push_back(i);

  sort(sa.begin(), sa.end(), suffix_comparator);
}

unsigned detokenizer::suffix_array::count(const string& data) const {
  auto lower_it = lower_bound(sa.begin(), sa.end(), data, suffix_lower_finder);
  auto upper_it = upper_bound(sa.begin(), sa.end(), data, suffix_upper_finder);
  return upper_it - lower_it;
}

/////////
// File: tokenizer/morphodita_tokenizer_wrapper.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

morphodita_tokenizer_wrapper::morphodita_tokenizer_wrapper(morphodita::tokenizer* tokenizer, const multiword_splitter* splitter,
                                                           bool normalized_spaces, bool token_ranges)
  : tokenizer(tokenizer), splitter(splitter), normalized_spaces(normalized_spaces), token_ranges(token_ranges) {}

bool morphodita_tokenizer_wrapper::read_block(istream& is, string& block) const {
  return bool(getpara(is, block));
}

void morphodita_tokenizer_wrapper::reset_document(string_piece id) {
  new_document = true;
  document_id.assign(id.str, id.len);
  preceeding_newlines = 2;
  sentence_id = 1;
  set_text("");
  unicode_offset = 0;
  text_unicode_length = 0;
  saved_spaces.clear();
}

void morphodita_tokenizer_wrapper::set_text(string_piece text, bool make_copy) {
  // Start by skipping spaces and copying them to saved_spaces
  string_piece following;
  for (char32_t chr;
       text.len && (following = text, chr = unilib::utf8::decode(following.str, following.len),
                    (unilib::unicode::category(chr) & unilib::unicode::Zs) || chr == '\r' || chr == '\n' || chr == '\t');
       text = following, unicode_offset++)
    saved_spaces.append(text.str, following.str - text.str);

  // Offset unicode_offset by length of previous text, update text_unicode_length for the new text
  unicode_offset += text_unicode_length;
  text_unicode_length = 0;
  for (following = text; following.len; unilib::utf8::decode(following.str, following.len))
    text_unicode_length++;

  // Copy the text to local storage if needed
  if (make_copy) {
    text_copy.assign(text.str, text.len);
    text = string_piece(text_copy.c_str(), text_copy.size());
  }

  // Store the text locally and in the morphodita::tokenizer
  this->text = text;
  tokenizer->set_text(this->text, false);

}

bool morphodita_tokenizer_wrapper::next_sentence(sentence& s, string& error) {
  unsigned following_newlines = 0;

  s.clear();
  error.clear();

  if (tokenizer->next_sentence(&forms, token_ranges ? &tokens : nullptr)) {
    // The forms returned by GRU tokenizer *should not* start/end with spaces,
    // but we trim them anyway (including all "remove empty forms/sentences" machinery).
    for (size_t i = 0; i < forms.size(); i++) {
      while (forms[i].len && (forms[i].str[0] == '\r' || forms[i].str[0] == '\n' ||
                              forms[i].str[0] == '\t' || forms[i].str[0] == ' '))
        forms[i].str++, forms[i].len--;
      while (forms[i].len && (forms[i].str[forms[i].len-1] == '\r' || forms[i].str[forms[i].len-1] == '\n' ||
                              forms[i].str[forms[i].len-1] == '\t' || forms[i].str[forms[i].len-1] == ' '))
        forms[i].len--;
      if (!forms[i].len)
        forms.erase(forms.begin() + i--);
    }
    if (!forms.size()) return next_sentence(s, error);

    for (size_t i = 0; i < forms.size(); i++) {
      // The form might contain spaces, even '\r', '\n' or '\t',
      // which we change to space. We also normalize multiple spaces to one.
      tok.form.clear();
      for (size_t j = 0; j < forms[i].len; j++) {
        char chr = forms[i].str[j];
        if (chr == '\r' || chr == '\n' || chr == '\t') chr = ' ';
        if (chr != ' ' || tok.form.empty() || tok.form.back() != ' ')
          tok.form.push_back(chr);
      }

      // Track pre-sentence spaces and store SpacesBefore
      if (i == 0) {
        if (forms[0].str > text.str)
          saved_spaces.append(text.str, forms[0].str - text.str);
        preceeding_newlines += count(saved_spaces.begin(), saved_spaces.end(), '\n');
      }
      if (!normalized_spaces) {
        tok.set_spaces_before(i == 0 ? saved_spaces : "");
      }
      saved_spaces.clear();

      // Track post-sentence spaces and store SpaceAfter, SpacesInToken and SpacesAfter
      if (i+1 == forms.size()) {
        text.len -= forms[i].str + forms[i].len - text.str;
        text.str = forms[i].str + forms[i].len;

        string_piece following;
        for (char32_t chr; text.len && (following = text, chr = unilib::utf8::decode(following.str, following.len),
                                        (unilib::unicode::category(chr) & unilib::unicode::Zs) || chr == '\r' || chr == '\n' || chr == '\t'); text = following)
          saved_spaces.append(text.str, following.str - text.str);

        following_newlines += count(saved_spaces.begin(), saved_spaces.end(), '\n');
      }
      if (normalized_spaces) {
        tok.set_space_after(i+1 == forms.size() ? !saved_spaces.empty() : forms[i+1].str > forms[i].str + forms[i].len);
      } else {
        tok.set_spaces_in_token(tok.form.size() != forms[i].len ? forms[i] : "");
        tok.set_spaces_after(i+1 == forms.size() ? saved_spaces : string_piece(forms[i].str + forms[i].len, forms[i+1].str - forms[i].str - forms[i].len));
      }
      saved_spaces.clear();

      // Store TokenRange if requested
      if (token_ranges)
        tok.set_token_range(unicode_offset + tokens[i].start, unicode_offset + tokens[i].start + tokens[i].length);

      if (splitter)
        splitter->append_token(tok.form, tok.misc, s);
      else
        s.add_word(tok.form).misc.assign(tok.misc);
    }

    // Mark new document if needed
    if (new_document) {
      s.set_new_doc(true, document_id);
      new_document = false;
    }

    // Mark new paragraph if needed
    if (preceeding_newlines >= 2)
      s.set_new_par(true);
    preceeding_newlines = following_newlines;

    s.set_sent_id(to_string(sentence_id++));

    // Fill "# text" comment
    s.comments.emplace_back("# text = ");
    for (size_t i = 1, j = 0; i < s.words.size(); i++) {
      const token& tok = j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i) ? (const token&)s.multiword_tokens[j].form : (const token&)s.words[i].form;
      if (j < s.multiword_tokens.size() && s.multiword_tokens[j].id_first == int(i))
        i = s.multiword_tokens[j++].id_last;

      s.comments.back().append(tok.form);
      if (i+1 < s.words.size() && tok.get_space_after()) s.comments.back().push_back(' ');
    }

    return true;
  }

  // Save unused text parts.
  if (text.len) {
    saved_spaces.append(text.str, text.len);
    text.str += text.len;
    text.len = 0;
  }

  return false;
}

/////////
// File: tokenizer/multiword_splitter.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

void multiword_splitter::append_token(string_piece token, string_piece misc, sentence& s) const {
  using namespace unilib;

  // Buffer
  s.add_word();
  string& buffer = s.words.back().form;

  // Lowercase the token
  utf8::map(unicode::lowercase, token.str, token.len, buffer);
  reverse(buffer.begin(), buffer.end());

  // Try finding lowercased version in the full_rules
  size_t prefix_len = 0;
  auto it = full_rules.find(buffer);

  if (it == full_rules.end()) {
    if (version >= 2) {
      string& suffix = s.words.back().misc;
      // Try searching suffix_rules if needed
      while (suffix.size() + 1 < buffer.size()) {
        suffix.push_back(buffer[suffix.size()]);

        auto suffix_it = suffix_rules.find(suffix);
        if (suffix_it == suffix_rules.end())
          break;

        if (!suffix_it->second.words.empty()) {
          it = suffix_it;
          prefix_len = buffer.size() - suffix.size();
        }
      }
      suffix.clear();
    }

    if (!prefix_len) {
      // No match
      s.words.back().form.assign(token.str, token.len);
      if (misc.len) s.words.back().misc.assign(misc.str, misc.len);
      return;
    }
  }

  // Determine casing
  enum { UC_FIRST, UC_ALL, UC_OTHER }; int casing = UC_OTHER;

  if (unicode::category(utf8::first(token.str, token.len)) & unicode::Lut) {
    casing = UC_ALL;
    for (auto&& chr : utf8::decoder(token.str, token.len))
      if (unicode::category(chr) & (unicode::L & ~unicode::Lut)) { casing = UC_FIRST; break; }
  }

  // Fill the multiword token
  s.multiword_tokens.emplace_back(s.words.back().id, s.words.back().id + (int)it->second.words.size() - 1, token, misc);

  s.words.back().form.clear();
  if (prefix_len) {
    // Note that prefix_len is measured in byte length of lowercased characters
    string_piece suffix(token);
    while (s.words.back().form.size() < prefix_len && suffix.len)
      utf8::append(s.words.back().form, unicode::lowercase(utf8::decode(suffix.str, suffix.len)));
    s.words.back().form.assign(token.str, token.len - suffix.len);
  }
  for (auto&& chr : utf8::decoder(it->second.words[0]))
    utf8::append(s.words.back().form, casing == UC_ALL || (casing == UC_FIRST && s.words.back().form.empty()) ? unicode::uppercase(chr) : chr);

  for (size_t i = 1; i < it->second.words.size(); i++)
    if (casing != UC_ALL) {
      s.add_word(it->second.words[i]);
    } else {
      s.add_word();
      utf8::map(unicode::uppercase, it->second.words[i], s.words.back().form);
    }
}

multiword_splitter* multiword_splitter::load(istream& is) {
  char version;
  if (!is.get(version)) return nullptr;
  if (!(version >= 1 && version <= VERSION_LATEST)) return nullptr;

  binary_decoder data;
  if (!compressor::load(is, data)) return nullptr;

  unique_ptr splitter(new multiword_splitter(version));
  try {
    for (unsigned full_rules = data.next_4B(); full_rules; full_rules--) {
      string full_rule;
      data.next_str(full_rule);
      reverse(full_rule.begin(), full_rule.end());

      // Add the full_rule and its words
      auto& info = splitter->full_rules[full_rule];
      for (unsigned words = data.next_1B(); words; words--) {
        info.words.emplace_back();
        data.next_str(info.words.back());
      }
      if (info.words.empty()) return nullptr;
    }

    if (version >= 2)
      for (unsigned suffix_rules = data.next_4B(); suffix_rules; suffix_rules--) {
        string suffix_rule;
        data.next_str(suffix_rule);
        reverse(suffix_rule.begin(), suffix_rule.end());

        // Add the suffix_rule and its words
        auto& info = splitter->suffix_rules[suffix_rule];
        for (unsigned words = data.next_1B(); words; words--) {
          info.words.emplace_back();
          data.next_str(info.words.back());
        }
        if (info.words.empty()) return nullptr;

        // Add prefixes of the suffix with empty data
        if (!suffix_rule.empty())
          for (suffix_rule.pop_back(); !suffix_rule.empty(); suffix_rule.pop_back())
            splitter->suffix_rules[suffix_rule];
      }
  } catch (binary_decoder_error&) {
    return nullptr;
  }

  return data.is_end() ? splitter.release() : nullptr;
}

/////////
// File: tokenizer/multiword_splitter_trainer.h
/////////

// This file is part of UDPipe .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class multiword_splitter_trainer {
 public:
  static bool train(const vector& data, ostream& os, string& error);
};

/////////
// File: tokenizer/multiword_splitter_trainer.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2016 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

bool multiword_splitter_trainer::train(const vector& data, ostream& os, string& error) {
  using namespace unilib;
  error.clear();

  // Train
  struct rule_info {
    vector words;
    unsigned count = 0;
  };
  map full_rules, suffix_rules;

  // Full rules
  string lc_form;
  vector lc_words;
  for (auto&& sentence : data)
    for (auto&& multiword : sentence.multiword_tokens) {
      utf8::map(unicode::lowercase, multiword.form, lc_form);
      lc_words.clear();
      for (int i = multiword.id_first; i <= multiword.id_last; i++)
        utf8::map(unicode::lowercase, sentence.words[i].form, (lc_words.emplace_back(), lc_words.back()));

      auto& info = full_rules[lc_form];
      if (info.words.empty())
        info.words.assign(lc_words.begin(), lc_words.end());
      info.count += lc_words == info.words;
      if (!info.count) full_rules.erase(lc_form);
    }

  // Remove the full rules which trigger too negatively
  for (auto&& sentence : data)
    for (size_t i = 1, j = 0; i < sentence.words.size(); i++) {
      if (j < sentence.multiword_tokens.size() && sentence.multiword_tokens[j].id_first == int(i)) {
        i = sentence.multiword_tokens[j++].id_last;
        continue;
      }

      utf8::map(unicode::lowercase, sentence.words[i].form, lc_form);
      auto it = full_rules.find(lc_form);
      if (it != full_rules.end())
        if (!--it->second.count)
          full_rules.erase(it);
    }

  // Suffix rules
  for (auto&& full_rule : full_rules) {
    size_t prefix_match = 0;
    while (prefix_match < full_rule.first.size() && prefix_match < full_rule.second.words[0].size()) prefix_match++;
    for (; prefix_match; prefix_match--)
      if (((unsigned char)full_rule.first[prefix_match]) < 0x80 || ((unsigned char)full_rule.first[prefix_match]) >= 0xC0) {
        lc_form.assign(full_rule.first, prefix_match, string::npos);
        lc_words.assign(full_rule.second.words.begin(), full_rule.second.words.end());
        lc_words[0].erase(0, prefix_match);

        auto& info = suffix_rules[lc_form];
        if (info.words.empty())
          info.words.assign(lc_words.begin(), lc_words.end());
        info.count += lc_words == info.words;
        if (!info.count) suffix_rules.erase(lc_form);
      }
  }

  // Remove the suffix rules which trigger too negatively
  for (auto&& sentence : data)
    for (size_t i = 1, j = 0; i < sentence.words.size(); i++) {
      if (j < sentence.multiword_tokens.size() && sentence.multiword_tokens[j].id_first == int(i)) {
        i = sentence.multiword_tokens[j++].id_last;
        continue;
      }

      utf8::map(unicode::lowercase, sentence.words[i].form, lc_form);
      while (lc_form.size() > 1) {
        lc_form.erase(0, 1);
        auto it = suffix_rules.find(lc_form);
        if (it != suffix_rules.end()) {
          if (it->second.count <= 10)
            suffix_rules.erase(it);
          else
            it->second.count -= 10;
        }
      }
    }

  // Encode
  binary_encoder enc;
  enc.add_4B(full_rules.size());
  for (auto&& full_rule : full_rules) {
    enc.add_str(full_rule.first);
    enc.add_1B(full_rule.second.words.size());
    for (auto& word : full_rule.second.words)
      enc.add_str(word);
  }
  enc.add_4B(suffix_rules.size());
  for (auto&& suffix_rule : suffix_rules) {
    enc.add_str(suffix_rule.first);
    enc.add_1B(suffix_rule.second.words.size());
    for (auto& word : suffix_rule.second.words)
      enc.add_str(word);
  }

  // Save
  os.put(multiword_splitter::VERSION_LATEST);
  if (!compressor::save(os, enc)) return error.assign("Cannot encode multiword_splitter!"), false;

  return true;
}

/////////
// File: trainer/trainer.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class trainer {
 public:
  static bool train(const string& method, const vector& train, const vector& heldout,
                    const string& tokenizer, const string& tagger, const string& parser, ostream& os, string& error);

  static const string DEFAULT;
  static const string NONE;

 protected:
  static unsigned hyperparameter_integer(unsigned run, unsigned index, unsigned minimum, unsigned maximum);
  static double hyperparameter_uniform(unsigned run, unsigned index, double minimum, double maximum);
  static double hyperparameter_logarithmic(unsigned run, unsigned index, double minimum, double maximum);

 private:
  static double rnd(unsigned run, unsigned index);
};

/////////
// File: trainer/trainer_morphodita_parsito.h
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

class trainer_morphodita_parsito : public trainer {
 public:
  static bool train(const vector& training, const vector& heldout,
                    const string& tokenizer, const string& tagger, const string& parser, ostream& os, string& error);

 private:
  static bool train_tokenizer(const vector& training, const vector& heldout,
                              const string& options, ostream& os, string& error);
  static bool train_tagger(const vector& training, const vector& heldout,
                           const string& options, ostream& os, string& error);
  static bool train_parser(const vector& training, const vector& heldout,
                           const string& options, const string& tagger_model, ostream& os, string& error);

  // Generic model methods
  enum model_type { TOKENIZER_MODEL, TAGGER_MODEL, PARSER_MODEL };
  static bool load_model(const string& data, model_type model, string_piece& range);
  static const string& model_normalize_form(string_piece form, string& output);
  static const string& model_normalize_lemma(string_piece lemma, string& output);
  static void model_fill_word_analysis(const morphodita::tagged_lemma& analysis, bool upostag, int lemma, bool xpostag, bool feats, word& word);

  // Tagger-specific model methods
  static bool train_tagger_model(const vector& training, const vector& heldout,
                                 unsigned model, unsigned models, const named_values::map& tagger, ostream& os, string& error);
  static bool can_combine_tag(const word& w, string& error);
  static const string& combine_tag(const word& w, bool xpostag, bool feats, string& combined_tag);
  static const string& most_frequent_tag(const vector& data, const string& upostag, bool xpostag, bool feats, string& combined_tag);
  static const string& combine_lemma(const word& w, int use_lemma, string& combined_lemma, const unordered_set& flat_lemmas = unordered_set());

  // Generic options handling
  static const string& option_str(const named_values::map& options, const string& name, int model = -1);
  static bool option_int(const named_values::map& options, const string& name, int& value, string& error, int model = -1);
  static bool option_bool(const named_values::map& options, const string& name, bool& value, string& error, int model = -1);
  static bool option_double(const named_values::map& options, const string& name, double& value, string& error, int model = -1);

  // Various string data
  static const string empty_string;
  static const string tag_separators;
  static const string tagger_features_tagger;
  static const string tagger_features_lemmatizer;
  static const string parser_nodes;
};

/////////
// File: trainer/trainer.cpp
/////////

// This file is part of UDPipe .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

const string trainer::DEFAULT;
const string trainer::NONE = "none";

bool trainer::train(const string& method, const vector& training, const vector& heldout,
                    const string& tokenizer, const string& tagger, const string& parser, ostream& os, string& error) {
  error.clear();

  stringstream os_buffer;
  os_buffer.put(method.size());
  os_buffer.write(method.c_str(), method.size());

  try {
    if (method == "morphodita_parsito") {
      if (!trainer_morphodita_parsito::train(training, heldout, tokenizer, tagger, parser, os_buffer, error))
        return false;
    } else {
      error.assign("Unknown UDPipe method '").append(method).append("'!");
      return false;
    }
  } catch (training_error& e) {
    error.assign(e.what());
    return false;
  }

  os << os_buffer.rdbuf();
  return true;
}

unsigned trainer::hyperparameter_integer(unsigned run, unsigned index, unsigned minimum, unsigned maximum) {
  return minimum + int((maximum - minimum + 1) * rnd(run, index));
}

double trainer::hyperparameter_uniform(unsigned run, unsigned index, double minimum, double maximum) {
  return minimum + (maximum - minimum) * rnd(run, index);
}

double trainer::hyperparameter_logarithmic(unsigned run, unsigned index, double minimum, double maximum) {
  return exp(log(minimum) + (log(maximum) - log(minimum)) * rnd(run, index));
}

double trainer::rnd(unsigned run, unsigned index) {
  uint32_t state = 12345U;
  for (unsigned i = 0; i < 10; i++)
    state = state * 1103515245U + run * 19999999U + index * 1000000007U + 12345U;
  return (state >> 16) / double(1<<16);
}

/////////
// File: morphodita/tagger/elementary_features_encoder.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

template
inline bool elementary_features::save(ostream& os) {
  binary_encoder enc;

  enc.add_1B(maps.size());
  for (auto&& map : maps)
    map.save(enc);

  return compressor::save(os, enc);
}

} // namespace morphodita

/////////
// File: morphodita/tagger/feature_sequences_encoder.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

template
void feature_sequences::parse(int window_size, istream& is) {
  unordered_map elementary_map;
  for (auto&& description : ElementaryFeatures::descriptions)
    if (!elementary_map.emplace(description.name, description).second)
      training_failure("Repeated elementary feature with name " << description.name << '!');

  string line;
  vector tokens;
  while (getline(is, line)) {
    split(line, ',', tokens);
    if (tokens.empty()) training_failure("Feature sequence cannot be empty!");

    bool contains_only_current = false;
    sequences.emplace_back();
    for (auto&& token : tokens) {
      vector parts;
      split(token, ' ', parts);
      if (parts.size() != 2) training_failure("Cannot parse feature sequence element '" << token << "'!");
      auto it = elementary_map.find(parts[0]);
      if (it == elementary_map.end()) training_failure("Unknown elementary feature '" << parts[0] << "' used in feature sequence '" << token << "'!");

      auto& desc = it->second;
      int sequence_index = parse_int(parts[1].c_str(), "sequence_index");
      if (desc.type == DYNAMIC && sequence_index != 0) training_failure("Nonzero sequence index " << sequence_index << " of dynamic elementary feature '" << desc.name << "'!");
      if (desc.type == PER_TAG && (sequence_index > 0 || sequence_index <= -window_size)) training_failure("Wrong sequence index " << sequence_index << " of per-tag elementary feature '" << desc.name << "'!");
      if (desc.range == ONLY_CURRENT && sequence_index != 0) training_failure("Nonzero sequence index " << sequence_index << " of elementary feature '" << desc.name << "' requiring zero offset!");

      sequences.back().elements.emplace_back(it->second.type, it->second.index, sequence_index);
      if (desc.type == DYNAMIC) sequences.back().dependant_range = max(sequences.back().dependant_range, window_size + 1);
      if (desc.type == PER_TAG) sequences.back().dependant_range = max(sequences.back().dependant_range, 1 - sequence_index);
      contains_only_current |= desc.range == ONLY_CURRENT;
    }
    if (contains_only_current && sequences.back().dependant_range > 1) training_failure("Feature sequence '" << line << "' contains both a non-local elementary feature and exclusively-local elementary feature!");
  }

  stable_sort(sequences.begin(), sequences.end(), [](const feature_sequence& a, const feature_sequence& b) { return a.dependant_range > b.dependant_range; });
  scores.resize(sequences.size());
}

template
inline bool feature_sequences::save(ostream& os) {
  if (!elementary.save(os)) return false;

  binary_encoder enc;
  enc.add_1B(sequences.size());
  for (auto&& sequence : sequences) {
    enc.add_4B(sequence.dependant_range);
    enc.add_1B(sequence.elements.size());
    for (auto&& element : sequence.elements) {
      enc.add_4B(element.type);
      enc.add_4B(element.elementary_index);
      enc.add_4B(element.sequence_index);
    }
  }

  enc.add_1B(scores.size());
  for (auto&& score : scores)
    score.save(enc);

  return compressor::save(os, enc);
}

} // namespace morphodita

/////////
// File: morphodita/tagger/training_maps.h
/////////

// This file is part of MorphoDiTa .
//
// Copyright 2015 Institute of Formal and Applied Linguistics, Faculty of
// Mathematics and Physics, Charles University in Prague, Czech Republic.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

namespace morphodita {

// Declarations
class training_elementary_feature_map {
 public:
  inline elementary_feature_value value(const char* feature, int len) const;
  mutable unordered_map map = {{"", elementary_feature_empty}};
 private:
  mutable string key;
};

class training_feature_sequence_map {
 public:
  struct info {
    // We deliberately use feature_sequence*s*_score to check for overflow
    feature_sequences_score alpha = 0;
    feature_sequences_score gamma = 0;
    int last_gamma_update = 0;
  };

  inline feature_sequence_score score(const char* feature, int len) const;
  mutable unordered_map map;
 private:
  mutable string key;
};

template