kitten-text-processing 0.1.0__py3-none-any.whl

kitten_text_processing/_token_parser.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import string
+from collections import OrderedDict
+from typing import Dict, List, Union
+
+PRESERVE_ORDER_KEY = "preserve_order"
+EOS = "<EOS>"
+
+
+class TokenParser:
+    """
+    Parses tokenized/classified text, e.g. 'tokens { money { integer: "20" currency: "$" } } tokens { name: "left"}'
+
+    Args
+        text: tokenized text
+    """
+
+    def __call__(self, text):
+        """
+        Setup function
+
+        Args:
+            text: text to be parsed
+
+        """
+        self.text = text
+        self.len_text = len(text)
+        self.char = text[0]  # cannot handle empty string
+        self.index = 0
+
+    def parse(self) -> List[dict]:
+        """
+        Main function. Implements grammar:
+        A -> space F space F space F ... space
+
+        Returns list of dictionaries
+        """
+        l = list()
+        while self.parse_ws():
+            token = self.parse_token()
+            if not token:
+                break
+            l.append(token)
+        return l
+
+    def parse_token(self) -> Dict[str, Union[str, dict]]:
+        """
+        Implements grammar:
+        F-> no_space KG no_space
+
+        Returns: K, G as dictionary values
+        """
+        d = OrderedDict()
+        key = self.parse_string_key()
+        if key is None:
+            return None
+        self.parse_ws()
+        if key == PRESERVE_ORDER_KEY:
+            self.parse_char(":")
+            self.parse_ws()
+            value = self.parse_chars("true")
+        else:
+            value = self.parse_token_value()
+
+        d[key] = value
+        return d
+
+    def parse_token_value(self) -> Union[str, dict]:
+        """
+        Implements grammar:
+        G-> no_space :"VALUE" no_space | no_space {A} no_space
+
+        Returns: string or dictionary
+        """
+        if self.char == ":":
+            self.parse_char(":")
+            self.parse_ws()
+            self.parse_char("\"")
+            value_string = self.parse_string_value()
+            self.parse_char("\"")
+            return value_string
+        elif self.char == "{":
+            d = OrderedDict()
+            self.parse_char("{")
+            list_token_dicts = self.parse()
+            # flatten tokens
+            for tok_dict in list_token_dicts:
+                for k, v in tok_dict.items():
+                    d[k] = v
+            self.parse_char("}")
+            return d
+        else:
+            raise ValueError()
+
+    def parse_char(self, exp) -> bool:
+        """
+        Parses character
+
+        Args:
+            exp: character to read in
+
+        Returns true if successful
+        """
+        assert self.char == exp
+        self.read()
+        return True
+
+    def parse_chars(self, exp) -> bool:
+        """
+        Parses characters
+
+        Args:
+            exp: characters to read in
+
+        Returns true if successful
+        """
+        ok = False
+        for x in exp:
+            ok |= self.parse_char(x)
+        return ok
+
+    def parse_string_key(self) -> str:
+        """
+        Parses string key, can only contain ascii and '_' characters
+
+        Returns parsed string key
+        """
+        assert self.char not in string.whitespace and self.char != EOS
+        incl_criterium = string.ascii_letters + "_"
+        l = []
+        while self.char in incl_criterium:
+            l.append(self.char)
+            if not self.read():
+                raise ValueError()
+
+        if not l:
+            return None
+        return "".join(l)
+
+    def parse_string_value(self) -> str:
+        """
+        Parses string value, ends with quote followed by space
+
+        Returns parsed string value
+        """
+        # assert self.char not in string.whitespace and self.char != EOS
+        assert self.char != EOS
+        l = []
+        while self.char != "\"" or self.text[self.index + 1] != " ":
+            l.append(self.char)
+            if not self.read():
+                raise ValueError()
+
+        if not l:
+            return None
+        return "".join(l)
+
+    def parse_ws(self):
+        """
+        Deletes whitespaces.
+
+        Returns true if not EOS after parsing
+        """
+        not_eos = self.char != EOS
+        while not_eos and self.char == " ":
+            not_eos = self.read()
+        return not_eos
+
+    def read(self):
+        """
+        Reads in next char.
+
+        Returns true if not EOS
+        """
+        if self.index < self.len_text - 1:  # should be unique
+            self.index += 1
+            self.char = self.text[self.index]
+            return True
+        self.char = EOS
+        return False

kitten_text_processing/_unicode_data.py

@@ -0,0 +1,6 @@
+# Character data from sacremoses 0.2.0 (MIT).
+# Copyright (c) 2020 alvations. See LICENSE-SACREMOSES.
+# Generated by tools/export_detokenizer.py; do not use Python Unicode classes
+# here: their membership changes with the interpreter version.
+ALPHA_RANGES = ((65, 90), (97, 122), (170, 170), (181, 181), (186, 186), (192, 214), (216, 246), (248, 705), (710, 721), (736, 740), (748, 748), (750, 750), (837, 837), (880, 884), (886, 887), (890, 893), (895, 895), (902, 902), (904, 906), (908, 908), (910, 929), (931, 1013), (1015, 1153), (1162, 1327), (1329, 1366), (1369, 1369), (1377, 1415), (1456, 1469), (1471, 1471), (1473, 1474), (1476, 1477), (1479, 1479), (1488, 1514), (1520, 1522), (1552, 1562), (1568, 1623), (1625, 1631), (1646, 1747), (1749, 1756), (1761, 1768), (1773, 1775), (1786, 1788), (1791, 1791), (1808, 1855), (1869, 1969), (1994, 2026), (2036, 2037), (2042, 2042), (2048, 2071), (2074, 2092), (2112, 2136), (2208, 2226), (2276, 2281), (2288, 2363), (2365, 2380), (2382, 2384), (2389, 2403), (2417, 2435), (2437, 2444), (2447, 2448), (2451, 2472), (2474, 2480), (2482, 2482), (2486, 2489), (2493, 2500), (2503, 2504), (2507, 2508), (2510, 2510), (2519, 2519), (2524, 2525), (2527, 2531), (2544, 2545), (2561, 2563), (2565, 2570), (2575, 2576), (2579, 2600), (2602, 2608), (2610, 2611), (2613, 2614), (2616, 2617), (2622, 2626), (2631, 2632), (2635, 2636), (2641, 2641), (2649, 2652), (2654, 2654), (2672, 2677), (2689, 2691), (2693, 2701), (2703, 2705), (2707, 2728), (2730, 2736), (2738, 2739), (2741, 2745), (2749, 2757), (2759, 2761), (2763, 2764), (2768, 2768), (2784, 2787), (2817, 2819), (2821, 2828), (2831, 2832), (2835, 2856), (2858, 2864), (2866, 2867), (2869, 2873), (2877, 2884), (2887, 2888), (2891, 2892), (2902, 2903), (2908, 2909), (2911, 2915), (2929, 2929), (2946, 2947), (2949, 2954), (2958, 2960), (2962, 2965), (2969, 2970), (2972, 2972), (2974, 2975), (2979, 2980), (2984, 2986), (2990, 3001), (3006, 3010), (3014, 3016), (3018, 3020), (3024, 3024), (3031, 3031), (3072, 3075), (3077, 3084), (3086, 3088), (3090, 3112), (3114, 3129), (3133, 3140), (3142, 3144), (3146, 3148), (3157, 3158), (3160, 3161), (3168, 3171), (3201, 3203), (3205, 3212), (3214, 3216), (3218, 3240), (3242, 3251), (3253, 3257), (3261, 3268), (3270, 3272), (3274, 3276), (3285, 3286), (3294, 3294), (3296, 3299), (3313, 3314), (3329, 3331), (3333, 3340), (3342, 3344), (3346, 3386), (3389, 3396), (3398, 3400), (3402, 3404), (3406, 3406), (3415, 3415), (3424, 3427), (3450, 3455), (3458, 3459), (3461, 3478), (3482, 3505), (3507, 3515), (3517, 3517), (3520, 3526), (3535, 3540), (3542, 3542), (3544, 3551), (3570, 3571), (3585, 3642), (3648, 3654), (3661, 3661), (3713, 3714), (3716, 3716), (3719, 3720), (3722, 3722), (3725, 3725), (3732, 3735), (3737, 3743), (3745, 3747), (3749, 3749), (3751, 3751), (3754, 3755), (3757, 3769), (3771, 3773), (3776, 3780), (3782, 3782), (3789, 3789), (3804, 3807), (3840, 3840), (3904, 3911), (3913, 3948), (3953, 3969), (3976, 3991), (3993, 4028), (4096, 4150), (4152, 4152), (4155, 4159), (4176, 4194), (4197, 4200), (4206, 4230), (4238, 4238), (4252, 4253), (4256, 4293), (4295, 4295), (4301, 4301), (4304, 4346), (4348, 4680), (4682, 4685), (4688, 4694), (4696, 4696), (4698, 4701), (4704, 4744), (4746, 4749), (4752, 4784), (4786, 4789), (4792, 4798), (4800, 4800), (4802, 4805), (4808, 4822), (4824, 4880), (4882, 4885), (4888, 4954), (4959, 4959), (4992, 5007), (5024, 5108), (5121, 5740), (5743, 5759), (5761, 5786), (5792, 5866), (5870, 5880), (5888, 5900), (5902, 5907), (5920, 5939), (5952, 5971), (5984, 5996), (5998, 6000), (6002, 6003), (6016, 6067), (6070, 6088), (6103, 6103), (6108, 6108), (6176, 6263), (6272, 6314), (6320, 6389), (6400, 6430), (6432, 6443), (6448, 6456), (6480, 6509), (6512, 6516), (6528, 6571), (6576, 6601), (6656, 6683), (6688, 6750), (6753, 6772), (6823, 6823), (6912, 6963), (6965, 6979), (6981, 6987), (7040, 7081), (7084, 7087), (7098, 7141), (7143, 7153), (7168, 7221), (7245, 7247), (7258, 7293), (7401, 7404), (7406, 7411), (7413, 7414), (7424, 7615), (7655, 7668), (7680, 7957), (7960, 7965), (7968, 8005), (8008, 8013), (8016, 8023), (8025, 8025), (8027, 8027), (8029, 8029), (8031, 8061), (8064, 8116), (8118, 8124), (8126, 8126), (8130, 8132), (8134, 8140), (8144, 8147), (8150, 8155), (8160, 8172), (8178, 8180), (8182, 8188), (8305, 8305), (8319, 8319), (8336, 8348), (8450, 8450), (8455, 8455), (8458, 8467), (8469, 8469), (8473, 8477), (8484, 8484), (8486, 8486), (8488, 8488), (8490, 8493), (8495, 8505), (8508, 8511), (8517, 8521), (8526, 8526), (8544, 8584), (9398, 9449), (9676, 9676), (11264, 11310), (11312, 11358), (11360, 11492), (11499, 11502), (11506, 11507), (11520, 11557), (11559, 11559), (11565, 11565), (11568, 11623), (11631, 11631), (11648, 11670), (11680, 11686), (11688, 11694), (11696, 11702), (11704, 11710), (11712, 11718), (11720, 11726), (11728, 11734), (11736, 11742), (11744, 11775), (11823, 11823), (12294, 12294), (12337, 12341), (12348, 12348), (12353, 12438), (12445, 12447), (12449, 12538), (12540, 12543), (12549, 12589), (12593, 12686), (12704, 12730), (12784, 12799), (40960, 42124), (42192, 42237), (42240, 42508), (42512, 42527), (42538, 42539), (42560, 42606), (42612, 42619), (42623, 42653), (42655, 42735), (42775, 42783), (42786, 42888), (42891, 42894), (42896, 42925), (42928, 42929), (42999, 43009), (43011, 43013), (43015, 43018), (43020, 43047), (43072, 43123), (43136, 43203), (43250, 43255), (43259, 43259), (43274, 43306), (43312, 43346), (43360, 43388), (43392, 43442), (43444, 43455), (43471, 43471), (43488, 43492), (43494, 43503), (43514, 43518), (43520, 43574), (43584, 43597), (43616, 43638), (43642, 43642), (43646, 43710), (43712, 43712), (43714, 43714), (43739, 43741), (43744, 43759), (43762, 43765), (43777, 43782), (43785, 43790), (43793, 43798), (43808, 43814), (43816, 43822), (43824, 43866), (43868, 43871), (43876, 43877), (43968, 44010), (55216, 55238), (55243, 55291), (64256, 64262), (64275, 64279), (64285, 64296), (64298, 64310), (64312, 64316), (64318, 64318), (64320, 64321), (64323, 64324), (64326, 64433), (64467, 64829), (64848, 64911), (64914, 64967), (65008, 65019), (65136, 65140), (65142, 65276), (65313, 65338), (65345, 65370), (65382, 65470), (65474, 65479), (65482, 65487), (65490, 65495), (65498, 65500))
+CURRENCY_SYMBOLS = '$\xa2\xa3\xa4\xa5\u058f\u060b\u09f2\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0\u20a1\u20a2\u20a3\u20a4\u20a5\u20a6\u20a7\u20a8\u20a9\u20aa\u20ab\u20ac\u20ad\u20ae\u20af\u20b0\u20b1\u20b2\u20b3\u20b4\u20b5\u20b6\u20b7\u20b8\u20b9\u20ba\u20bb\u20bc\u20bd\ua838\ufdfc\ufe69\uff04\uffe0\uffe1\uffe5\uffe6\n'

kitten_text_processing/data/ar/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "ar",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 92384,
+      "arcs": 132720,
+      "bytes": 329776,
+      "sha256": "94f247b4bcb0a0b7520fb913a1cdea17f222777e9cf793e12a6ad661c85a91f0"
+    },
+    "verbalizer": {
+      "states": 1169,
+      "arcs": 13706,
+      "bytes": 7339,
+      "sha256": "ade78ab86fc5aab227dfa9559c3b9b0523d3684710ec0ef78d44965c1a476a07"
+    }
+  }
+}

kitten_text_processing/data/de/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "de",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 282068,
+      "arcs": 358293,
+      "bytes": 945232,
+      "sha256": "3a2204972a86803f74814e9d75fc973da79d17621322145d0ff699e5629a4b9b"
+    },
+    "verbalizer": {
+      "states": 18553,
+      "arcs": 144309,
+      "bytes": 93586,
+      "sha256": "1b4859aa4464771c2bf65fedf34c7ed3092926a6a19d6a11c4de2d77f1541c19"
+    }
+  }
+}

kitten_text_processing/data/en/metadata.json

@@ -0,0 +1,26 @@
+{
+  "language": "en",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 161202,
+      "arcs": 345794,
+      "bytes": 760818,
+      "sha256": "8bd9b7ccca17c15c68d6efce6ebd080ee676ebacf25087e60be73b94a23636f3"
+    },
+    "verbalizer": {
+      "states": 4261,
+      "arcs": 94434,
+      "bytes": 32685,
+      "sha256": "85e4dc57484408fe0675e59cc050d92d7e51bbb38b07164387ca7d1e40336070"
+    },
+    "post": {
+      "states": 825,
+      "arcs": 20059,
+      "bytes": 8244,
+      "sha256": "32f2438d980349dfe82f2deaa03675376322d0daf67929df865d9903f66447c7"
+    }
+  }
+}

kitten_text_processing/data/es/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "es",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 372349,
+      "arcs": 494951,
+      "bytes": 1346937,
+      "sha256": "315cfefbdc715ba8f3bf457740fdb935496713a498090283c506c88b73b2a393"
+    },
+    "verbalizer": {
+      "states": 29193,
+      "arcs": 2081240,
+      "bytes": 444594,
+      "sha256": "b226133e13d07d31edb1ba3b0c12c1c13cf240a70881987441c87b2d67092328"
+    }
+  }
+}

kitten_text_processing/data/fr/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "fr",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 24323,
+      "arcs": 74655,
+      "bytes": 113030,
+      "sha256": "80ec6891d990e10a89afa2c2617b6c77ef35a70056012807b798bcd0f95edee9"
+    },
+    "verbalizer": {
+      "states": 2718,
+      "arcs": 36183,
+      "bytes": 16129,
+      "sha256": "2bd95415de7ccf2c15210852340f1d1f0197f398b89e5cf363482309da254694"
+    }
+  }
+}

kitten_text_processing/data/hi/metadata.json

@@ -0,0 +1,26 @@
+{
+  "language": "hi",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 186723,
+      "arcs": 1720742,
+      "bytes": 908026,
+      "sha256": "747fda009853de6436e3ada16fdd76c2730693988346cf9e3cb5698f2f901f5d"
+    },
+    "verbalizer": {
+      "states": 15437,
+      "arcs": 47530,
+      "bytes": 65079,
+      "sha256": "52ba87bd7512b2a4e41021974cd7d5cc02044ec11d595b46ed9140f06156bdd0"
+    },
+    "post": {
+      "states": 134,
+      "arcs": 1638,
+      "bytes": 2030,
+      "sha256": "00dd2b470b2b515f7a5c4d53070750b258ff7ecbafb1b3db49351454d02a6743"
+    }
+  }
+}

kitten_text_processing/data/hu/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "hu",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 222065,
+      "arcs": 309003,
+      "bytes": 850637,
+      "sha256": "208bdad64d16ae9aebdad9d43b5069140d249d9fc44473714ee3f38782a920cd"
+    },
+    "verbalizer": {
+      "states": 3402,
+      "arcs": 39461,
+      "bytes": 21536,
+      "sha256": "8609658bea017f989dfb6c6edec8d1d09dc38d8e35d73634d23f457456c905d8"
+    }
+  }
+}

kitten_text_processing/data/hy/metadata.json

@@ -0,0 +1,24 @@
+{
+  "language": "hy",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 43320,
+      "arcs": 56726,
+      "bytes": 164382,
+      "sha256": "17ff890785d42cab0634d55ef3f658f16f9ef58be27888cd7aae35efd1be9723"
+    },
+    "verbalizer": {
+      "states": 1388,
+      "arcs": 28989,
+      "bytes": 10435,
+      "sha256": "cb607e29d5ceb223cb096e56f1374f57c2d1a884dc97bee47b0703b98b1e2092"
+    }
+  },
+  "input_cases": [
+    "cased",
+    "lower_cased"
+  ]
+}

kitten_text_processing/data/it/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "it",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 122500,
+      "arcs": 180660,
+      "bytes": 462184,
+      "sha256": "5e4dbf418f43635e2ed09f91d440e225c269b0050392e6f436b55a45b794e9b2"
+    },
+    "verbalizer": {
+      "states": 6594,
+      "arcs": 380908,
+      "bytes": 72448,
+      "sha256": "7f35057b51f6b79d02396bd7ae8802ffd67aebcc3fc37dda4cd7ad95fa385512"
+    }
+  }
+}

kitten_text_processing/data/ja/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "ja",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 12191,
+      "arcs": 26585,
+      "bytes": 48132,
+      "sha256": "129cb5fdca07e690f66aaad393e2a1f47076db5fa757bfb7e1a3732a1c802eb9"
+    },
+    "verbalizer": {
+      "states": 551,
+      "arcs": 13030,
+      "bytes": 4932,
+      "sha256": "307528a9972a15dd1055d127bddfe281e6a944402770c863945ce6db2bafd0e8"
+    }
+  }
+}

kitten_text_processing/data/ko/metadata.json

@@ -0,0 +1,24 @@
+{
+  "language": "ko",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 21569,
+      "arcs": 35529,
+      "bytes": 85990,
+      "sha256": "2e30b9182335d252a18739f3172ba15f6224f8651c7e0fc06557fc4b506f9b2b"
+    },
+    "verbalizer": {
+      "states": 127474,
+      "arcs": 261026,
+      "bytes": 489629,
+      "sha256": "8241032a8afb4a760dc8136ee18dce9a980b065e34c717023fcdff9e0288feb6"
+    }
+  },
+  "input_cases": [
+    "cased",
+    "lower_cased"
+  ]
+}

kitten_text_processing/data/pt/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "pt",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 163637,
+      "arcs": 275786,
+      "bytes": 677349,
+      "sha256": "404372f3a1b2ad4f62965e1aaaef5886127b5abf7a5a36f6e20cd53bec173742"
+    },
+    "verbalizer": {
+      "states": 4512,
+      "arcs": 118175,
+      "bytes": 35239,
+      "sha256": "219e2245769e353e54e7636777badd9d5f6372bed0ada70c41f75534e1871945"
+    }
+  }
+}

kitten_text_processing/data/ru/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "ru",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": false,
+  "graphs": {
+    "tagger": {
+      "states": 1238499,
+      "arcs": 1551178,
+      "bytes": 4900950,
+      "sha256": "f03a59df77a437582abb2a418756da7b63f3bb02b8d4919341e82cc5325b38b0"
+    },
+    "verbalizer": {
+      "states": 406,
+      "arcs": 5849,
+      "bytes": 3519,
+      "sha256": "9840715b6aa31a346f48216bddf6ada155c375faecf488b16db7f1fc9bdba8be"
+    }
+  }
+}

kitten_text_processing/data/rw/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "rw",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 6187,
+      "arcs": 80659,
+      "bytes": 40693,
+      "sha256": "a2bc6b0992a07449e9f7a51c8605185ff5c358dfe317e44fe42485e12fb583f8"
+    },
+    "verbalizer": {
+      "states": 127,
+      "arcs": 2782,
+      "bytes": 1808,
+      "sha256": "68e99ef18e0826c4f1d6f7a19e1f7580f38852c14181357d115e91a1fead83e0"
+    }
+  }
+}

kitten_text_processing/data/sv/metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "sv",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 165824,
+      "arcs": 288626,
+      "bytes": 699416,
+      "sha256": "3296a2542d37ed8c38c0ab80a1462f21ae4272de02e44973cf89bfc610d61c83"
+    },
+    "verbalizer": {
+      "states": 2937,
+      "arcs": 47251,
+      "bytes": 19640,
+      "sha256": "0b595c4a367fa360345a7cac8e32965aa7e27fa4092744056d173ea35a8ce5a9"
+    }
+  }
+}

kitten_text_processing/data/vi/metadata.json

@@ -0,0 +1,26 @@
+{
+  "language": "vi",
+  "nemo_version": "1.2.0",
+  "input_case": "cased",
+  "deterministic": true,
+  "graphs": {
+    "tagger": {
+      "states": 169635,
+      "arcs": 227401,
+      "bytes": 658968,
+      "sha256": "3c481d67ed42ab7bbd8fe6c94d42ea216d466fd3285a5b5d54dab8364ee29614"
+    },
+    "verbalizer": {
+      "states": 22570,
+      "arcs": 84460,
+      "bytes": 98357,
+      "sha256": "98eb5b1409814078288ec9b56318089f9cba718a9c69e4a0de4bcd5fbebac30d"
+    },
+    "post": {
+      "states": 11,
+      "arcs": 856,
+      "bytes": 930,
+      "sha256": "4accd4cccf90d6425391c518150ad81f27c162c1503d571f97bce34f8fdf2749"
+    }
+  }
+}