indicate 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. indicate/__init__.py +3 -0
  2. indicate/data/model/hindi_to_english/english_tokens.json +1 -0
  3. indicate/data/model/hindi_to_english/hindi_tokens.json +1 -0
  4. indicate/data/model/hindi_to_english/saved_weights/decoder.weights.h5 +0 -0
  5. indicate/data/model/hindi_to_english/saved_weights/encoder.weights.h5 +0 -0
  6. indicate/decoder.py +62 -0
  7. indicate/encoder.py +24 -0
  8. indicate/hindi2english.py +138 -0
  9. indicate/logging.py +6 -0
  10. indicate/notebooks/Data_Preparation.ipynb +1 -0
  11. indicate/notebooks/Inference.ipynb +1 -0
  12. indicate/notebooks/Inference_latest.ipynb +2331 -0
  13. indicate/notebooks/Train.ipynb +1 -0
  14. indicate/notebooks/Train_latest.ipynb +1746 -0
  15. indicate/notebooks/Transliterate.ipynb +2501 -0
  16. indicate/notebooks/transliterate_cricket.ipynb +305 -0
  17. indicate/tests/__init__.py +12 -0
  18. indicate/tests/test_010_hindi_translate.py +20 -0
  19. indicate/transliterate.py +28 -0
  20. indicate/utils.py +86 -0
  21. indicate-0.2.0.dist-info/LICENSE +21 -0
  22. indicate-0.2.0.dist-info/METADATA +139 -0
  23. indicate-0.2.0.dist-info/RECORD +26 -0
  24. indicate-0.2.0.dist-info/WHEEL +6 -0
  25. indicate-0.2.0.dist-info/entry_points.txt +2 -0
  26. indicate-0.2.0.dist-info/top_level.txt +1 -0
indicate/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .transliterate import hindi2english
2
+
3
+ __all__ = ["hindi2english"]
indicate/data/model/hindi_to_english/english_tokens.json ADDED
@@ -0,0 +1 @@
1
+ "{\"class_name\": \"Tokenizer\", \"config\": {\"num_words\": null, \"filters\": \"\", \"lower\": true, \"split\": \" \", \"char_level\": true, \"oov_token\": null, \"document_count\": 371035, \"word_counts\": \"{\\\"^\\\": 371037, \\\"l\\\": 97394, \\\"a\\\": 566344, \\\"t\\\": 105572, \\\"e\\\": 127758, \\\" \\\": 13427, \\\"s\\\": 125548, \\\"u\\\": 88436, \\\"b\\\": 63551, \\\"o\\\": 86032, \\\"d\\\": 115107, \\\"h\\\": 165603, \\\"c\\\": 38730, \\\"n\\\": 174122, \\\"r\\\": 187969, \\\"$\\\": 371035, \\\"p\\\": 51055, \\\"i\\\": 198549, \\\"k\\\": 90447, \\\"m\\\": 90714, \\\"w\\\": 16948, \\\"y\\\": 59782, \\\"j\\\": 37472, \\\"g\\\": 55940, \\\"-\\\": 56, \\\"f\\\": 10867, \\\"q\\\": 1414, \\\"z\\\": 5302, \\\"v\\\": 53230, \\\":\\\": 8, \\\"(\\\": 106, \\\")\\\": 106, \\\"x\\\": 1128, \\\"/\\\": 42, \\\"@\\\": 25, \\\"\\\\u012b\\\": 1068, \\\"\\\\u0101\\\": 2775, \\\"'\\\": 95, \\\"\\\\u200c\\\": 99, \\\"\\\\u1e0d\\\": 580, \\\"\\\\u1e47\\\": 207, \\\"\\\\u0113\\\": 174, \\\"\\\\u1e6d\\\": 390, \\\"\\\\u016b\\\": 169, \\\"0\\\": 201, \\\"\\\\u1e45\\\": 116, \\\"\\\\u010d\\\": 1, \\\"\\\\u0962\\\": 1, \\\"\\\\u1e5b\\\": 113, \\\"\\\\u0442\\\": 1, \\\"\\\\u0438\\\": 1, \\\"\\\\u043f\\\": 1, \\\"\\\\u200d\\\": 2, \\\".\\\": 100, \\\"\\\\u1e63\\\": 64, \\\"\\\\u1e43\\\": 28, \\\"\\\\u015b\\\": 76, \\\"8\\\": 65, \\\"\\\\u1e35\\\": 2, \\\"\\\\u1e96\\\": 2, \\\"\\\\u0323\\\": 57, \\\"\\\\u1e49\\\": 1, \\\"\\\\u1e41\\\": 45, \\\"\\\\u0308\\\": 2, \\\"3\\\": 3, \\\"[\\\": 7, \\\"\\\\u0304\\\": 264, \\\"\\\\u094c\\\": 1, \\\"!\\\": 1, \\\"\\\\u2019\\\": 3, \\\"4\\\": 19, \\\"\\\\\\\"\\\": 33, \\\"\\\\u0341\\\": 2, \\\"\\\\u0942\\\": 1, \\\"\\\\u1ea1\\\": 2, \\\"\\\\u0303\\\": 12, \\\"\\\\u0310\\\": 20, \\\"\\\\u032e\\\": 2, \\\"\\\\u0951\\\": 1, \\\"]\\\": 4, \\\"*\\\": 7, \\\">\\\": 3, \\\"9\\\": 9, \\\"\\\\u014d\\\": 57, \\\"\\\\u0970\\\": 2, \\\"1\\\": 1, \\\"7\\\": 11, \\\"5\\\": 4, \\\"\\\\u0966\\\": 5, \\\"\\\\u0131\\\": 1, \\\"\\\\u00bb\\\": 4, \\\",\\\": 19, \\\"\\\\u1ebd\\\": 3, \\\"\\\\u00a3\\\": 1, \\\"\\\\u00e9\\\": 18, \\\"\\\\u0930\\\": 3, \\\"\\\\u0947\\\": 3, \\\"\\\\u030a\\\": 21, \\\"\\\\u0167\\\": 1, \\\"\\\\u0302\\\": 1, \\\"\\\\u0331\\\": 2, \\\"\\\\u1e33\\\": 5, \\\"\\\\u1e11\\\": 1, \\\"\\\\u1e25\\\": 7, \\\"\\\\u0921\\\": 1, \\\"\\\\u093f\\\": 1, \\\"\\\\u092f\\\": 1, \\\"\\\\u093e\\\": 1, \\\"\\\\u096e\\\": 2, \\\"\\\\u096f\\\": 1, \\\"\\\\u01f0\\\": 1, \\\"\\\\u0307\\\": 20, \\\"\\\\u0306\\\": 2, \\\"\\\\u02bc\\\": 14, \\\"\\\\u1e95\\\": 2, \\\"\\\\u0121\\\": 1, \\\"\\\\u1eb9\\\": 2, \\\"\\\\u0301\\\": 6, \\\"\\\\u1e37\\\": 2, \\\"\\\\u1e61\\\": 2, \\\"_\\\": 10, \\\"~\\\": 1, \\\"6\\\": 3, \\\"\\\\u0309\\\": 1, \\\"2\\\": 3, \\\"\\\\u0965\\\": 1, \\\"\\\\u0902\\\": 1, \\\"\\\\u0314\\\": 2, \\\"\\\\u0312\\\": 2, \\\"|\\\": 2, \\\"\\\\u00f1\\\": 2, \\\"\\\\u030d\\\": 1, \\\"\\\\u0324\\\": 1, \\\"\\\\u0952\\\": 1, \\\"\\\\u200b\\\": 2, \\\"\\\\u0325\\\": 2}\", \"word_docs\": \"{\\\"c\\\": 35979, \\\" \\\": 9557, \\\"n\\\": 148241, \\\"r\\\": 166211, \\\"a\\\": 311632, \\\"h\\\": 145053, \\\"^\\\": 371035, \\\"u\\\": 81144, \\\"s\\\": 112443, \\\"b\\\": 59957, \\\"t\\\": 93685, \\\"$\\\": 371035, \\\"e\\\": 104259, \\\"o\\\": 75066, \\\"d\\\": 103891, \\\"l\\\": 88175, \\\"k\\\": 83051, \\\"i\\\": 171439, \\\"p\\\": 47318, \\\"m\\\": 82811, \\\"w\\\": 16782, \\\"y\\\": 57479, \\\"j\\\": 35887, \\\"g\\\": 52738, \\\"-\\\": 51, \\\"f\\\": 10365, \\\"q\\\": 1388, \\\"z\\\": 5110, \\\"v\\\": 50618, \\\":\\\": 8, \\\")\\\": 106, \\\"(\\\": 106, \\\"x\\\": 1120, \\\"/\\\": 42, \\\"@\\\": 25, \\\"\\\\u012b\\\": 1020, \\\"\\\\u0101\\\": 2313, \\\"'\\\": 94, \\\"\\\\u200c\\\": 89, \\\"\\\\u1e0d\\\": 546, \\\"\\\\u1e47\\\": 203, \\\"\\\\u1e6d\\\": 353, \\\"\\\\u0113\\\": 168, \\\"\\\\u016b\\\": 168, \\\"0\\\": 182, \\\"\\\\u1e45\\\": 111, \\\"\\\\u010d\\\": 1, \\\"\\\\u0962\\\": 1, \\\"\\\\u1e5b\\\": 113, \\\"\\\\u0438\\\": 1, \\\"\\\\u0442\\\": 1, \\\"\\\\u043f\\\": 1, \\\"\\\\u200d\\\": 2, \\\".\\\": 84, \\\"\\\\u1e63\\\": 63, \\\"\\\\u1e43\\\": 28, \\\"\\\\u015b\\\": 75, \\\"8\\\": 64, \\\"\\\\u1e35\\\": 2, \\\"\\\\u1e96\\\": 2, \\\"\\\\u0323\\\": 52, \\\"\\\\u1e49\\\": 1, \\\"\\\\u1e41\\\": 45, \\\"\\\\u0308\\\": 2, \\\"3\\\": 3, \\\"[\\\": 7, \\\"\\\\u0304\\\": 213, \\\"\\\\u094c\\\": 1, \\\"!\\\": 1, \\\"\\\\u2019\\\": 3, \\\"4\\\": 14, \\\"\\\\\\\"\\\": 32, \\\"\\\\u0341\\\": 2, \\\"\\\\u0942\\\": 1, \\\"\\\\u1ea1\\\": 2, \\\"\\\\u0303\\\": 12, \\\"\\\\u0310\\\": 18, \\\"\\\\u032e\\\": 2, \\\"\\\\u0951\\\": 1, \\\"]\\\": 4, \\\"*\\\": 7, \\\">\\\": 3, \\\"9\\\": 9, \\\"\\\\u014d\\\": 56, \\\"\\\\u0970\\\": 2, \\\"7\\\": 8, \\\"1\\\": 1, \\\"5\\\": 3, \\\"\\\\u0966\\\": 5, \\\"\\\\u0131\\\": 1, \\\"\\\\u00bb\\\": 4, \\\",\\\": 1, \\\"\\\\u1ebd\\\": 3, \\\"\\\\u00a3\\\": 1, \\\"\\\\u00e9\\\": 18, \\\"\\\\u0930\\\": 3, \\\"\\\\u0947\\\": 3, \\\"\\\\u030a\\\": 18, \\\"\\\\u0167\\\": 1, \\\"\\\\u0302\\\": 1, \\\"\\\\u0331\\\": 2, \\\"\\\\u1e33\\\": 5, \\\"\\\\u1e11\\\": 1, \\\"\\\\u1e25\\\": 7, \\\"\\\\u0921\\\": 1, \\\"\\\\u092f\\\": 1, \\\"\\\\u093f\\\": 1, \\\"\\\\u093e\\\": 1, \\\"\\\\u096e\\\": 2, \\\"\\\\u096f\\\": 1, \\\"\\\\u01f0\\\": 1, \\\"\\\\u0307\\\": 20, \\\"\\\\u0306\\\": 2, \\\"\\\\u02bc\\\": 14, \\\"\\\\u1e95\\\": 2, \\\"\\\\u0121\\\": 1, \\\"\\\\u1eb9\\\": 2, \\\"\\\\u0301\\\": 6, \\\"\\\\u1e37\\\": 2, \\\"\\\\u1e61\\\": 2, \\\"_\\\": 10, \\\"~\\\": 1, \\\"6\\\": 3, \\\"\\\\u0309\\\": 1, \\\"2\\\": 3, \\\"\\\\u0965\\\": 1, \\\"\\\\u0902\\\": 1, \\\"\\\\u0314\\\": 2, \\\"\\\\u0312\\\": 2, \\\"|\\\": 1, \\\"\\\\u00f1\\\": 2, \\\"\\\\u030d\\\": 1, \\\"\\\\u0324\\\": 1, \\\"\\\\u0952\\\": 1, \\\"\\\\u200b\\\": 2, \\\"\\\\u0325\\\": 2}\", \"index_docs\": \"{\\\"22\\\": 35979, \\\"25\\\": 9557, \\\"6\\\": 148241, \\\"5\\\": 166211, \\\"1\\\": 311632, \\\"7\\\": 145053, \\\"2\\\": 371035, \\\"15\\\": 81144, \\\"9\\\": 112443, \\\"17\\\": 59957, \\\"11\\\": 93685, \\\"3\\\": 371035, \\\"8\\\": 104259, \\\"16\\\": 75066, \\\"10\\\": 103891, \\\"12\\\": 88175, \\\"14\\\": 83051, \\\"4\\\": 171439, \\\"21\\\": 47318, \\\"13\\\": 82811, \\\"24\\\": 16782, \\\"18\\\": 57479, \\\"23\\\": 35887, \\\"19\\\": 52738, \\\"51\\\": 51, \\\"26\\\": 10365, \\\"29\\\": 1388, \\\"27\\\": 5110, \\\"20\\\": 50618, \\\"68\\\": 8, \\\"42\\\": 106, \\\"41\\\": 106, \\\"30\\\": 1120, \\\"53\\\": 42, \\\"56\\\": 25, \\\"31\\\": 1020, \\\"28\\\": 2313, \\\"45\\\": 94, \\\"44\\\": 89, \\\"32\\\": 546, \\\"35\\\": 203, \\\"33\\\": 353, \\\"37\\\": 168, \\\"38\\\": 168, \\\"36\\\": 182, \\\"39\\\": 111, \\\"107\\\": 1, \\\"108\\\": 1, \\\"40\\\": 113, \\\"110\\\": 1, \\\"109\\\": 1, \\\"111\\\": 1, \\\"86\\\": 2, \\\"43\\\": 84, \\\"48\\\": 63, \\\"55\\\": 28, \\\"46\\\": 75, \\\"47\\\": 64, \\\"87\\\": 2, \\\"88\\\": 2, \\\"49\\\": 52, \\\"112\\\": 1, \\\"52\\\": 45, \\\"89\\\": 2, \\\"78\\\": 3, \\\"69\\\": 7, \\\"34\\\": 213, \\\"113\\\": 1, \\\"114\\\": 1, \\\"79\\\": 3, \\\"60\\\": 14, \\\"54\\\": 32, \\\"90\\\": 2, \\\"115\\\": 1, \\\"91\\\": 2, \\\"64\\\": 12, \\\"58\\\": 18, \\\"92\\\": 2, \\\"116\\\": 1, \\\"75\\\": 4, \\\"70\\\": 7, \\\"80\\\": 3, \\\"67\\\": 9, \\\"50\\\": 56, \\\"93\\\": 2, \\\"65\\\": 8, \\\"117\\\": 1, \\\"76\\\": 3, \\\"73\\\": 5, \\\"118\\\": 1, \\\"77\\\": 4, \\\"61\\\": 1, \\\"81\\\": 3, \\\"119\\\": 1, \\\"62\\\": 18, \\\"82\\\": 3, \\\"83\\\": 3, \\\"57\\\": 18, \\\"120\\\": 1, \\\"121\\\": 1, \\\"94\\\": 2, \\\"74\\\": 5, \\\"122\\\": 1, \\\"71\\\": 7, \\\"123\\\": 1, \\\"125\\\": 1, \\\"124\\\": 1, \\\"126\\\": 1, \\\"95\\\": 2, \\\"127\\\": 1, \\\"128\\\": 1, \\\"59\\\": 20, \\\"96\\\": 2, \\\"63\\\": 14, \\\"97\\\": 2, \\\"129\\\": 1, \\\"98\\\": 2, \\\"72\\\": 6, \\\"99\\\": 2, \\\"100\\\": 2, \\\"66\\\": 10, \\\"130\\\": 1, \\\"84\\\": 3, \\\"131\\\": 1, \\\"85\\\": 3, \\\"132\\\": 1, \\\"133\\\": 1, \\\"101\\\": 2, \\\"102\\\": 2, \\\"103\\\": 1, \\\"104\\\": 2, \\\"134\\\": 1, \\\"135\\\": 1, \\\"136\\\": 1, \\\"105\\\": 2, \\\"106\\\": 2}\", \"index_word\": \"{\\\"1\\\": \\\"a\\\", \\\"2\\\": \\\"^\\\", \\\"3\\\": \\\"$\\\", \\\"4\\\": \\\"i\\\", \\\"5\\\": \\\"r\\\", \\\"6\\\": \\\"n\\\", \\\"7\\\": \\\"h\\\", \\\"8\\\": \\\"e\\\", \\\"9\\\": \\\"s\\\", \\\"10\\\": \\\"d\\\", \\\"11\\\": \\\"t\\\", \\\"12\\\": \\\"l\\\", \\\"13\\\": \\\"m\\\", \\\"14\\\": \\\"k\\\", \\\"15\\\": \\\"u\\\", \\\"16\\\": \\\"o\\\", \\\"17\\\": \\\"b\\\", \\\"18\\\": \\\"y\\\", \\\"19\\\": \\\"g\\\", \\\"20\\\": \\\"v\\\", \\\"21\\\": \\\"p\\\", \\\"22\\\": \\\"c\\\", \\\"23\\\": \\\"j\\\", \\\"24\\\": \\\"w\\\", \\\"25\\\": \\\" \\\", \\\"26\\\": \\\"f\\\", \\\"27\\\": \\\"z\\\", \\\"28\\\": \\\"\\\\u0101\\\", \\\"29\\\": \\\"q\\\", \\\"30\\\": \\\"x\\\", \\\"31\\\": \\\"\\\\u012b\\\", \\\"32\\\": \\\"\\\\u1e0d\\\", \\\"33\\\": \\\"\\\\u1e6d\\\", \\\"34\\\": \\\"\\\\u0304\\\", \\\"35\\\": \\\"\\\\u1e47\\\", \\\"36\\\": \\\"0\\\", \\\"37\\\": \\\"\\\\u0113\\\", \\\"38\\\": \\\"\\\\u016b\\\", \\\"39\\\": \\\"\\\\u1e45\\\", \\\"40\\\": \\\"\\\\u1e5b\\\", \\\"41\\\": \\\"(\\\", \\\"42\\\": \\\")\\\", \\\"43\\\": \\\".\\\", \\\"44\\\": \\\"\\\\u200c\\\", \\\"45\\\": \\\"'\\\", \\\"46\\\": \\\"\\\\u015b\\\", \\\"47\\\": \\\"8\\\", \\\"48\\\": \\\"\\\\u1e63\\\", \\\"49\\\": \\\"\\\\u0323\\\", \\\"50\\\": \\\"\\\\u014d\\\", \\\"51\\\": \\\"-\\\", \\\"52\\\": \\\"\\\\u1e41\\\", \\\"53\\\": \\\"/\\\", \\\"54\\\": \\\"\\\\\\\"\\\", \\\"55\\\": \\\"\\\\u1e43\\\", \\\"56\\\": \\\"@\\\", \\\"57\\\": \\\"\\\\u030a\\\", \\\"58\\\": \\\"\\\\u0310\\\", \\\"59\\\": \\\"\\\\u0307\\\", \\\"60\\\": \\\"4\\\", \\\"61\\\": \\\",\\\", \\\"62\\\": \\\"\\\\u00e9\\\", \\\"63\\\": \\\"\\\\u02bc\\\", \\\"64\\\": \\\"\\\\u0303\\\", \\\"65\\\": \\\"7\\\", \\\"66\\\": \\\"_\\\", \\\"67\\\": \\\"9\\\", \\\"68\\\": \\\":\\\", \\\"69\\\": \\\"[\\\", \\\"70\\\": \\\"*\\\", \\\"71\\\": \\\"\\\\u1e25\\\", \\\"72\\\": \\\"\\\\u0301\\\", \\\"73\\\": \\\"\\\\u0966\\\", \\\"74\\\": \\\"\\\\u1e33\\\", \\\"75\\\": \\\"]\\\", \\\"76\\\": \\\"5\\\", \\\"77\\\": \\\"\\\\u00bb\\\", \\\"78\\\": \\\"3\\\", \\\"79\\\": \\\"\\\\u2019\\\", \\\"80\\\": \\\">\\\", \\\"81\\\": \\\"\\\\u1ebd\\\", \\\"82\\\": \\\"\\\\u0930\\\", \\\"83\\\": \\\"\\\\u0947\\\", \\\"84\\\": \\\"6\\\", \\\"85\\\": \\\"2\\\", \\\"86\\\": \\\"\\\\u200d\\\", \\\"87\\\": \\\"\\\\u1e35\\\", \\\"88\\\": \\\"\\\\u1e96\\\", \\\"89\\\": \\\"\\\\u0308\\\", \\\"90\\\": \\\"\\\\u0341\\\", \\\"91\\\": \\\"\\\\u1ea1\\\", \\\"92\\\": \\\"\\\\u032e\\\", \\\"93\\\": \\\"\\\\u0970\\\", \\\"94\\\": \\\"\\\\u0331\\\", \\\"95\\\": \\\"\\\\u096e\\\", \\\"96\\\": \\\"\\\\u0306\\\", \\\"97\\\": \\\"\\\\u1e95\\\", \\\"98\\\": \\\"\\\\u1eb9\\\", \\\"99\\\": \\\"\\\\u1e37\\\", \\\"100\\\": \\\"\\\\u1e61\\\", \\\"101\\\": \\\"\\\\u0314\\\", \\\"102\\\": \\\"\\\\u0312\\\", \\\"103\\\": \\\"|\\\", \\\"104\\\": \\\"\\\\u00f1\\\", \\\"105\\\": \\\"\\\\u200b\\\", \\\"106\\\": \\\"\\\\u0325\\\", \\\"107\\\": \\\"\\\\u010d\\\", \\\"108\\\": \\\"\\\\u0962\\\", \\\"109\\\": \\\"\\\\u0442\\\", \\\"110\\\": \\\"\\\\u0438\\\", \\\"111\\\": \\\"\\\\u043f\\\", \\\"112\\\": \\\"\\\\u1e49\\\", \\\"113\\\": \\\"\\\\u094c\\\", \\\"114\\\": \\\"!\\\", \\\"115\\\": \\\"\\\\u0942\\\", \\\"116\\\": \\\"\\\\u0951\\\", \\\"117\\\": \\\"1\\\", \\\"118\\\": \\\"\\\\u0131\\\", \\\"119\\\": \\\"\\\\u00a3\\\", \\\"120\\\": \\\"\\\\u0167\\\", \\\"121\\\": \\\"\\\\u0302\\\", \\\"122\\\": \\\"\\\\u1e11\\\", \\\"123\\\": \\\"\\\\u0921\\\", \\\"124\\\": \\\"\\\\u093f\\\", \\\"125\\\": \\\"\\\\u092f\\\", \\\"126\\\": \\\"\\\\u093e\\\", \\\"127\\\": \\\"\\\\u096f\\\", \\\"128\\\": \\\"\\\\u01f0\\\", \\\"129\\\": \\\"\\\\u0121\\\", \\\"130\\\": \\\"~\\\", \\\"131\\\": \\\"\\\\u0309\\\", \\\"132\\\": \\\"\\\\u0965\\\", \\\"133\\\": \\\"\\\\u0902\\\", \\\"134\\\": \\\"\\\\u030d\\\", \\\"135\\\": \\\"\\\\u0324\\\", \\\"136\\\": \\\"\\\\u0952\\\"}\", \"word_index\": \"{\\\"a\\\": 1, \\\"^\\\": 2, \\\"$\\\": 3, \\\"i\\\": 4, \\\"r\\\": 5, \\\"n\\\": 6, \\\"h\\\": 7, \\\"e\\\": 8, \\\"s\\\": 9, \\\"d\\\": 10, \\\"t\\\": 11, \\\"l\\\": 12, \\\"m\\\": 13, \\\"k\\\": 14, \\\"u\\\": 15, \\\"o\\\": 16, \\\"b\\\": 17, \\\"y\\\": 18, \\\"g\\\": 19, \\\"v\\\": 20, \\\"p\\\": 21, \\\"c\\\": 22, \\\"j\\\": 23, \\\"w\\\": 24, \\\" \\\": 25, \\\"f\\\": 26, \\\"z\\\": 27, \\\"\\\\u0101\\\": 28, \\\"q\\\": 29, \\\"x\\\": 30, \\\"\\\\u012b\\\": 31, \\\"\\\\u1e0d\\\": 32, \\\"\\\\u1e6d\\\": 33, \\\"\\\\u0304\\\": 34, \\\"\\\\u1e47\\\": 35, \\\"0\\\": 36, \\\"\\\\u0113\\\": 37, \\\"\\\\u016b\\\": 38, \\\"\\\\u1e45\\\": 39, \\\"\\\\u1e5b\\\": 40, \\\"(\\\": 41, \\\")\\\": 42, \\\".\\\": 43, \\\"\\\\u200c\\\": 44, \\\"'\\\": 45, \\\"\\\\u015b\\\": 46, \\\"8\\\": 47, \\\"\\\\u1e63\\\": 48, \\\"\\\\u0323\\\": 49, \\\"\\\\u014d\\\": 50, \\\"-\\\": 51, \\\"\\\\u1e41\\\": 52, \\\"/\\\": 53, \\\"\\\\\\\"\\\": 54, \\\"\\\\u1e43\\\": 55, \\\"@\\\": 56, \\\"\\\\u030a\\\": 57, \\\"\\\\u0310\\\": 58, \\\"\\\\u0307\\\": 59, \\\"4\\\": 60, \\\",\\\": 61, \\\"\\\\u00e9\\\": 62, \\\"\\\\u02bc\\\": 63, \\\"\\\\u0303\\\": 64, \\\"7\\\": 65, \\\"_\\\": 66, \\\"9\\\": 67, \\\":\\\": 68, \\\"[\\\": 69, \\\"*\\\": 70, \\\"\\\\u1e25\\\": 71, \\\"\\\\u0301\\\": 72, \\\"\\\\u0966\\\": 73, \\\"\\\\u1e33\\\": 74, \\\"]\\\": 75, \\\"5\\\": 76, \\\"\\\\u00bb\\\": 77, \\\"3\\\": 78, \\\"\\\\u2019\\\": 79, \\\">\\\": 80, \\\"\\\\u1ebd\\\": 81, \\\"\\\\u0930\\\": 82, \\\"\\\\u0947\\\": 83, \\\"6\\\": 84, \\\"2\\\": 85, \\\"\\\\u200d\\\": 86, \\\"\\\\u1e35\\\": 87, \\\"\\\\u1e96\\\": 88, \\\"\\\\u0308\\\": 89, \\\"\\\\u0341\\\": 90, \\\"\\\\u1ea1\\\": 91, \\\"\\\\u032e\\\": 92, \\\"\\\\u0970\\\": 93, \\\"\\\\u0331\\\": 94, \\\"\\\\u096e\\\": 95, \\\"\\\\u0306\\\": 96, \\\"\\\\u1e95\\\": 97, \\\"\\\\u1eb9\\\": 98, \\\"\\\\u1e37\\\": 99, \\\"\\\\u1e61\\\": 100, \\\"\\\\u0314\\\": 101, \\\"\\\\u0312\\\": 102, \\\"|\\\": 103, \\\"\\\\u00f1\\\": 104, \\\"\\\\u200b\\\": 105, \\\"\\\\u0325\\\": 106, \\\"\\\\u010d\\\": 107, \\\"\\\\u0962\\\": 108, \\\"\\\\u0442\\\": 109, \\\"\\\\u0438\\\": 110, \\\"\\\\u043f\\\": 111, \\\"\\\\u1e49\\\": 112, \\\"\\\\u094c\\\": 113, \\\"!\\\": 114, \\\"\\\\u0942\\\": 115, \\\"\\\\u0951\\\": 116, \\\"1\\\": 117, \\\"\\\\u0131\\\": 118, \\\"\\\\u00a3\\\": 119, \\\"\\\\u0167\\\": 120, \\\"\\\\u0302\\\": 121, \\\"\\\\u1e11\\\": 122, \\\"\\\\u0921\\\": 123, \\\"\\\\u093f\\\": 124, \\\"\\\\u092f\\\": 125, \\\"\\\\u093e\\\": 126, \\\"\\\\u096f\\\": 127, \\\"\\\\u01f0\\\": 128, \\\"\\\\u0121\\\": 129, \\\"~\\\": 130, \\\"\\\\u0309\\\": 131, \\\"\\\\u0965\\\": 132, \\\"\\\\u0902\\\": 133, \\\"\\\\u030d\\\": 134, \\\"\\\\u0324\\\": 135, \\\"\\\\u0952\\\": 136}\"}}"
indicate/data/model/hindi_to_english/hindi_tokens.json ADDED
@@ -0,0 +1 @@
1
+ "{\"class_name\": \"Tokenizer\", \"config\": {\"num_words\": null, \"filters\": \"\", \"lower\": true, \"split\": \" \", \"char_level\": true, \"oov_token\": null, \"document_count\": 371035, \"word_counts\": \"{\\\"\\\\u092a\\\": 48661, \\\"\\\\u094d\\\": 132066, \\\"\\\\u0930\\\": 183826, \\\"\\\\u092f\\\": 63421, \\\"\\\\u093e\\\": 255343, \\\"\\\\u0924\\\": 53540, \\\" \\\": 13335, \\\"\\\\u0938\\\": 89177, \\\"\\\\u0941\\\": 53687, \\\"\\\\u092c\\\": 49756, \\\"\\\\u094b\\\": 62453, \\\"\\\\u0927\\\": 14001, \\\"\\\\u091a\\\": 21283, \\\"\\\\u0928\\\": 111193, \\\"\\\\u0926\\\": 53529, \\\"\\\\u0935\\\": 71167, \\\"\\\\u0917\\\": 48119, \\\"\\\\u0940\\\": 90822, \\\"\\\\u093f\\\": 99462, \\\"\\\\u0915\\\": 87916, \\\"\\\\u0902\\\": 53814, \\\"\\\\u0921\\\": 46250, \\\"\\\\u0942\\\": 19176, \\\"\\\\u092e\\\": 88455, \\\"\\\\u0943\\\": 2590, \\\"\\\\u0939\\\": 37877, \\\"\\\\u0936\\\": 27978, \\\"\\\\u0923\\\": 12432, \\\"\\\\u0949\\\": 2302, \\\"\\\\u0906\\\": 9046, \\\"\\\\u0932\\\": 94662, \\\"\\\\u091c\\\": 43773, \\\"\\\\u0947\\\": 86705, \\\"\\\\u0916\\\": 14695, \\\"\\\\u0905\\\": 15423, \\\"\\\\u095f\\\": 66, \\\"\\\\u0925\\\": 8804, \\\".\\\": 167, \\\"\\\\u092d\\\": 13296, \\\"\\\\u0937\\\": 6529, \\\"-\\\": 29, \\\"\\\\u092b\\\": 12701, \\\"\\\\u0903\\\": 319, \\\"\\\\u0907\\\": 10028, \\\"\\\\u091f\\\": 35686, \\\"\\\\u0901\\\": 2662, \\\"\\\\u0909\\\": 6042, \\\"\\\\u094c\\\": 9456, \\\":\\\": 38, \\\"\\\\u0908\\\": 8960, \\\"\\\\u0918\\\": 5106, \\\"\\\\u0948\\\": 15975, \\\"\\\\u09fa\\\": 13, \\\"\\\\u09b2\\\": 3, \\\"\\\\u09be\\\": 7, \\\"\\\\u093c\\\": 13222, \\\"\\\\u095c\\\": 51, \\\"\\\\u0920\\\": 5112, \\\"\\\\u095b\\\": 56, \\\"\\\\u0913\\\": 1602, \\\"\\\\u0959\\\": 3, \\\"\\\\u091d\\\": 2512, \\\"\\\\u0922\\\": 3540, \\\"\\\\u091b\\\": 3887, \\\"\\\\u090f\\\": 9715, \\\"\\\\u090b\\\": 182, \\\"\\\\u0958\\\": 10, \\\"\\\\u090a\\\": 587, \\\"\\\\u0919\\\": 35, \\\"\\\\u091e\\\": 493, \\\"(\\\": 103, \\\")\\\": 103, \\\"\\\\u0910\\\": 566, \\\"@\\\": 22, \\\"\\\\u0964\\\": 133, \\\"\\\\u0946\\\": 9, \\\"\\\\u094a\\\": 4, \\\"\\\\u090d\\\": 3, \\\"\\\\u095e\\\": 12, \\\"\\\\u0929\\\": 5, \\\"3\\\": 1, \\\"2\\\": 2, \\\",\\\": 3, \\\"s\\\": 66, \\\"m\\\": 35, \\\"\\\\u0933\\\": 16, \\\"a\\\": 121, \\\"r\\\": 53, \\\"t\\\": 30, \\\"h\\\": 33, \\\"i\\\": 44, \\\"n\\\": 42, \\\"k\\\": 25, \\\"p\\\": 24, \\\"y\\\": 13, \\\"d\\\": 20, \\\"e\\\": 24, \\\"g\\\": 8, \\\"\\\\u0931\\\": 9, \\\"\\\\u0945\\\": 18, \\\"b\\\": 11, \\\"u\\\": 23, \\\"v\\\": 23, \\\"c\\\": 10, \\\"\\\\u0bae\\\": 1, \\\"\\\\u0ba3\\\": 2, \\\"\\\\u0bcd\\\": 5, \\\"\\\\u0bbe\\\": 2, \\\"\\\\u0b99\\\": 1, \\\"\\\\u0b95\\\": 1, \\\"\\\\u0b9f\\\": 2, \\\"\\\\u0bbf\\\": 1, \\\"\\\\u200d\\\": 2425, \\\"l\\\": 15, \\\"z\\\": 3, \\\"j\\\": 19, \\\"\\\\u090e\\\": 6, \\\"\\\\u0911\\\": 148, \\\"\\\\u00e0\\\": 1, \\\"\\\\u00b7\\\": 1, \\\"\\\\u0b9a\\\": 1, \\\"\\\\u0bc1\\\": 1, \\\"\\\\u0ba8\\\": 1, \\\"\\\\u0ba4\\\": 1, \\\"\\\\u0bb0\\\": 2, \\\"\\\\u0b9c\\\": 1, \\\"\\\\u0ba9\\\": 1, \\\"\\\\u095a\\\": 2, \\\"\\\\u0986\\\": 1, \\\"\\\\u09ac\\\": 1, \\\"\\\\u09cd\\\": 4, \\\"\\\\u09a6\\\": 4, \\\"\\\\u09c1\\\": 2, \\\"\\\\u0996\\\": 1, \\\"\\\\u09c7\\\": 1, \\\"\\\\u0995\\\": 2, \\\"\\\\u09ae\\\": 3, \\\"\\\\u099c\\\": 3, \\\"\\\\u09f0\\\": 1, \\\"\\\\u0981\\\": 1, \\\"\\\\u2019\\\": 2, \\\"\\\\u0954\\\": 1, \\\"\\\\u09a8\\\": 4, \\\"\\\\u09bf\\\": 3, \\\"\\\\u09b0\\\": 3, \\\"\\\\u09cb\\\": 1, \\\"\\\\u099a\\\": 1, \\\"\\\\u0985\\\": 1, \\\"\\\\u09a7\\\": 1, \\\"\\\\u09c0\\\": 1, \\\"/\\\": 23, \\\"o\\\": 33, \\\"\\\\u200c\\\": 1464, \\\"\\\\u090c\\\": 1, \\\"\\\\u0914\\\": 342, \\\"\\\\u095d\\\": 3, \\\"`\\\": 1, \\\"\\\\u0970\\\": 2, \\\"w\\\": 1, \\\"\\\\t\\\": 1, \\\"\\\\u09b8\\\": 1, \\\"\\\\u09b9\\\": 1, \\\"\\\\u0989\\\": 1, \\\"'\\\": 62, \\\"\\\\u0950\\\": 1, \\\"\\\\u0966\\\": 958, \\\"\\\\u0951\\\": 166, \\\"\\\\u0952\\\": 287, \\\"\\\\u096e\\\": 94, \\\"\\\\u00a3\\\": 5, \\\"\\\\u0969\\\": 4, \\\"[\\\": 17, \\\"\\\\u0965\\\": 25, \\\"\\\\u096a\\\": 24, \\\"\\\\u0644\\\": 1, \\\"\\\\u201c\\\": 16, \\\"\\\\u201d\\\": 27, \\\"\\\\u00bb\\\": 20, \\\"0\\\": 3, \\\"]\\\": 7, \\\"\\\\\\\"\\\": 8, \\\"\\\\u200b\\\": 2, \\\"*\\\": 14, \\\"\\\\u096f\\\": 13, \\\"\\\\u0967\\\": 1, \\\"\\\\u096d\\\": 11, \\\"\\\\u096b\\\": 4, \\\"\\\\ud804\\\\ude00\\\": 1, \\\"8\\\": 1, \\\"\\\\u0440\\\": 1, \\\"\\\\u062d\\\": 1, \\\"\\\\ufffd\\\": 3, \\\"_\\\": 17, \\\"\\\\u096c\\\": 3, \\\"\\\\u0968\\\": 3, \\\"\\\\u0962\\\": 1, \\\"\\\\u00ab\\\": 1, \\\"\\\\u093d\\\": 1, \\\"$\\\": 1}\", \"word_docs\": \"{\\\"\\\\u0926\\\": 49570, \\\"\\\\u092f\\\": 60978, \\\"\\\\u092c\\\": 47035, \\\" \\\": 8972, \\\"\\\\u0924\\\": 49068, \\\"\\\\u094d\\\": 110099, \\\"\\\\u0941\\\": 50002, \\\"\\\\u0938\\\": 82466, \\\"\\\\u0928\\\": 98547, \\\"\\\\u094b\\\": 59126, \\\"\\\\u093e\\\": 209484, \\\"\\\\u092a\\\": 45436, \\\"\\\\u091a\\\": 20552, \\\"\\\\u0930\\\": 163008, \\\"\\\\u0927\\\": 13742, \\\"\\\\u0921\\\": 44151, \\\"\\\\u0942\\\": 18858, \\\"\\\\u0917\\\": 45605, \\\"\\\\u0935\\\": 67226, \\\"\\\\u0940\\\": 85789, \\\"\\\\u093f\\\": 90833, \\\"\\\\u0915\\\": 80680, \\\"\\\\u0902\\\": 51724, \\\"\\\\u0939\\\": 37214, \\\"\\\\u0943\\\": 2576, \\\"\\\\u0936\\\": 27173, \\\"\\\\u092e\\\": 80927, \\\"\\\\u0949\\\": 2288, \\\"\\\\u0923\\\": 12299, \\\"\\\\u0906\\\": 8762, \\\"\\\\u0932\\\": 87457, \\\"\\\\u091c\\\": 41968, \\\"\\\\u0916\\\": 14403, \\\"\\\\u0947\\\": 80702, \\\"\\\\u095f\\\": 59, \\\"\\\\u0905\\\": 15341, \\\"\\\\u0925\\\": 8668, \\\"\\\\u092d\\\": 13069, \\\"\\\\u0937\\\": 6461, \\\".\\\": 139, \\\"-\\\": 28, \\\"\\\\u092b\\\": 12373, \\\"\\\\u0903\\\": 316, \\\"\\\\u0907\\\": 9745, \\\"\\\\u091f\\\": 32031, \\\"\\\\u0901\\\": 2649, \\\"\\\\u0909\\\": 6024, \\\"\\\\u094c\\\": 9438, \\\"\\\\u0908\\\": 8713, \\\":\\\": 38, \\\"\\\\u0918\\\": 4981, \\\"\\\\u0948\\\": 15790, \\\"\\\\u09fa\\\": 13, \\\"\\\\u09b2\\\": 2, \\\"\\\\u09be\\\": 4, \\\"\\\\u093c\\\": 12695, \\\"\\\\u095c\\\": 50, \\\"\\\\u0920\\\": 4976, \\\"\\\\u095b\\\": 50, \\\"\\\\u0913\\\": 1599, \\\"\\\\u0959\\\": 3, \\\"\\\\u091d\\\": 2342, \\\"\\\\u0922\\\": 3340, \\\"\\\\u091b\\\": 3821, \\\"\\\\u090f\\\": 8277, \\\"\\\\u090b\\\": 182, \\\"\\\\u0958\\\": 10, \\\"\\\\u090a\\\": 587, \\\"\\\\u0919\\\": 34, \\\"\\\\u091e\\\": 480, \\\"(\\\": 103, \\\")\\\": 103, \\\"\\\\u0910\\\": 556, \\\"@\\\": 22, \\\"\\\\u0964\\\": 131, \\\"\\\\u0946\\\": 9, \\\"\\\\u094a\\\": 4, \\\"\\\\u090d\\\": 3, \\\"\\\\u095e\\\": 12, \\\"\\\\u0929\\\": 5, \\\"3\\\": 1, \\\",\\\": 3, \\\"2\\\": 2, \\\"s\\\": 59, \\\"m\\\": 31, \\\"\\\\u0933\\\": 12, \\\"a\\\": 50, \\\"r\\\": 44, \\\"i\\\": 32, \\\"t\\\": 23, \\\"h\\\": 25, \\\"k\\\": 22, \\\"n\\\": 27, \\\"p\\\": 20, \\\"y\\\": 11, \\\"e\\\": 16, \\\"d\\\": 19, \\\"g\\\": 8, \\\"\\\\u0931\\\": 9, \\\"\\\\u0945\\\": 18, \\\"b\\\": 11, \\\"u\\\": 19, \\\"v\\\": 20, \\\"c\\\": 10, \\\"\\\\u0bcd\\\": 2, \\\"\\\\u0bbf\\\": 1, \\\"\\\\u0b95\\\": 1, \\\"\\\\u0ba3\\\": 1, \\\"\\\\u0b9f\\\": 1, \\\"\\\\u0bae\\\": 1, \\\"\\\\u0b99\\\": 1, \\\"\\\\u0bbe\\\": 2, \\\"\\\\u200d\\\": 2288, \\\"z\\\": 2, \\\"l\\\": 12, \\\"j\\\": 17, \\\"\\\\u090e\\\": 5, \\\"\\\\u0911\\\": 147, \\\"\\\\u00e0\\\": 1, \\\"\\\\u00b7\\\": 1, \\\"\\\\u0b9a\\\": 1, \\\"\\\\u0b9c\\\": 1, \\\"\\\\u0ba4\\\": 1, \\\"\\\\u0bc1\\\": 1, \\\"\\\\u0ba9\\\": 1, \\\"\\\\u0ba8\\\": 1, \\\"\\\\u0bb0\\\": 1, \\\"\\\\u095a\\\": 2, \\\"\\\\u0996\\\": 1, \\\"\\\\u09cd\\\": 3, \\\"\\\\u09ac\\\": 1, \\\"\\\\u09ae\\\": 2, \\\"\\\\u0995\\\": 2, \\\"\\\\u09c7\\\": 1, \\\"\\\\u09c1\\\": 1, \\\"\\\\u0986\\\": 1, \\\"\\\\u099c\\\": 2, \\\"\\\\u09f0\\\": 1, \\\"\\\\u09a6\\\": 2, \\\"\\\\u0981\\\": 1, \\\"\\\\u2019\\\": 2, \\\"\\\\u0954\\\": 1, \\\"\\\\u09a8\\\": 2, \\\"\\\\u099a\\\": 1, \\\"\\\\u09a7\\\": 1, \\\"\\\\u09cb\\\": 1, \\\"\\\\u0985\\\": 1, \\\"\\\\u09c0\\\": 1, \\\"\\\\u09b0\\\": 1, \\\"\\\\u09bf\\\": 2, \\\"/\\\": 23, \\\"o\\\": 30, \\\"\\\\u200c\\\": 1439, \\\"\\\\u090c\\\": 1, \\\"\\\\u0914\\\": 342, \\\"\\\\u095d\\\": 3, \\\"`\\\": 1, \\\"\\\\u0970\\\": 1, \\\"w\\\": 1, \\\"\\\\t\\\": 1, \\\"\\\\u09b8\\\": 1, \\\"\\\\u0989\\\": 1, \\\"\\\\u09b9\\\": 1, \\\"'\\\": 62, \\\"\\\\u0950\\\": 1, \\\"\\\\u0966\\\": 923, \\\"\\\\u0951\\\": 166, \\\"\\\\u0952\\\": 287, \\\"\\\\u096e\\\": 93, \\\"\\\\u00a3\\\": 5, \\\"\\\\u0969\\\": 4, \\\"[\\\": 17, \\\"\\\\u0965\\\": 20, \\\"\\\\u096a\\\": 19, \\\"\\\\u0644\\\": 1, \\\"\\\\u201c\\\": 16, \\\"\\\\u201d\\\": 27, \\\"\\\\u00bb\\\": 20, \\\"0\\\": 2, \\\"]\\\": 7, \\\"\\\\\\\"\\\": 8, \\\"\\\\u200b\\\": 2, \\\"*\\\": 14, \\\"\\\\u096f\\\": 13, \\\"\\\\u096d\\\": 8, \\\"\\\\u0967\\\": 1, \\\"\\\\u096b\\\": 3, \\\"\\\\ud804\\\\ude00\\\": 1, \\\"8\\\": 1, \\\"\\\\u0440\\\": 1, \\\"\\\\u062d\\\": 1, \\\"\\\\ufffd\\\": 3, \\\"_\\\": 17, \\\"\\\\u096c\\\": 3, \\\"\\\\u0968\\\": 3, \\\"\\\\u0962\\\": 1, \\\"\\\\u00ab\\\": 1, \\\"\\\\u093d\\\": 1, \\\"$\\\": 1}\", \"index_docs\": \"{\\\"18\\\": 49570, \\\"13\\\": 60978, \\\"19\\\": 47035, \\\"33\\\": 8972, \\\"17\\\": 49068, \\\"3\\\": 110099, \\\"16\\\": 50002, \\\"8\\\": 82466, \\\"4\\\": 98547, \\\"14\\\": 59126, \\\"1\\\": 209484, \\\"20\\\": 45436, \\\"27\\\": 20552, \\\"2\\\": 163008, \\\"32\\\": 13742, \\\"22\\\": 44151, \\\"28\\\": 18858, \\\"21\\\": 45605, \\\"12\\\": 67226, \\\"7\\\": 85789, \\\"5\\\": 90833, \\\"10\\\": 80680, \\\"15\\\": 51724, \\\"24\\\": 37214, \\\"51\\\": 2576, \\\"26\\\": 27173, \\\"9\\\": 80927, \\\"54\\\": 2288, \\\"37\\\": 12299, \\\"41\\\": 8762, \\\"6\\\": 87457, \\\"23\\\": 41968, \\\"31\\\": 14403, \\\"11\\\": 80702, \\\"73\\\": 59, \\\"30\\\": 15341, \\\"43\\\": 8668, \\\"34\\\": 13069, \\\"44\\\": 6461, \\\"65\\\": 139, \\\"87\\\": 28, \\\"36\\\": 12373, \\\"62\\\": 316, \\\"38\\\": 9745, \\\"25\\\": 32031, \\\"50\\\": 2649, \\\"45\\\": 6024, \\\"40\\\": 9438, \\\"42\\\": 8713, \\\"81\\\": 38, \\\"47\\\": 4981, \\\"29\\\": 15790, \\\"108\\\": 13, \\\"132\\\": 2, \\\"120\\\": 4, \\\"35\\\": 12695, \\\"78\\\": 50, \\\"46\\\": 4976, \\\"76\\\": 50, \\\"55\\\": 1599, \\\"133\\\": 3, \\\"52\\\": 2342, \\\"49\\\": 3340, \\\"48\\\": 3821, \\\"39\\\": 8277, \\\"64\\\": 182, \\\"114\\\": 10, \\\"58\\\": 587, \\\"82\\\": 34, \\\"60\\\": 480, \\\"70\\\": 103, \\\"71\\\": 103, \\\"59\\\": 556, \\\"97\\\": 22, \\\"68\\\": 131, \\\"116\\\": 9, \\\"126\\\": 4, \\\"134\\\": 3, \\\"111\\\": 12, \\\"123\\\": 5, \\\"157\\\": 1, \\\"135\\\": 3, \\\"146\\\": 2, \\\"74\\\": 59, \\\"83\\\": 31, \\\"104\\\": 12, \\\"69\\\": 50, \\\"77\\\": 44, \\\"79\\\": 32, \\\"86\\\": 23, \\\"84\\\": 25, \\\"89\\\": 22, \\\"80\\\": 27, \\\"91\\\": 20, \\\"109\\\": 11, \\\"92\\\": 16, \\\"98\\\": 19, \\\"118\\\": 8, \\\"117\\\": 9, \\\"101\\\": 18, \\\"112\\\": 11, \\\"94\\\": 19, \\\"95\\\": 20, \\\"115\\\": 10, \\\"124\\\": 2, \\\"161\\\": 1, \\\"160\\\": 1, \\\"147\\\": 1, \\\"149\\\": 1, \\\"158\\\": 1, \\\"159\\\": 1, \\\"148\\\": 2, \\\"53\\\": 2288, \\\"136\\\": 2, \\\"106\\\": 12, \\\"100\\\": 17, \\\"122\\\": 5, \\\"67\\\": 147, \\\"162\\\": 1, \\\"163\\\": 1, \\\"164\\\": 1, \\\"168\\\": 1, \\\"167\\\": 1, \\\"165\\\": 1, \\\"169\\\": 1, \\\"166\\\": 1, \\\"150\\\": 1, \\\"151\\\": 2, \\\"172\\\": 1, \\\"127\\\": 3, \\\"171\\\": 1, \\\"137\\\": 2, \\\"153\\\": 2, \\\"173\\\": 1, \\\"152\\\": 1, \\\"170\\\": 1, \\\"138\\\": 2, \\\"174\\\": 1, \\\"128\\\": 2, \\\"175\\\": 1, \\\"154\\\": 2, \\\"176\\\": 1, \\\"129\\\": 2, \\\"178\\\": 1, \\\"180\\\": 1, \\\"177\\\": 1, \\\"179\\\": 1, \\\"181\\\": 1, \\\"140\\\": 1, \\\"139\\\": 2, \\\"96\\\": 23, \\\"85\\\": 30, \\\"56\\\": 1439, \\\"182\\\": 1, \\\"61\\\": 342, \\\"141\\\": 3, \\\"183\\\": 1, \\\"155\\\": 1, \\\"184\\\": 1, \\\"185\\\": 1, \\\"186\\\": 1, \\\"188\\\": 1, \\\"187\\\": 1, \\\"75\\\": 62, \\\"189\\\": 1, \\\"57\\\": 923, \\\"66\\\": 166, \\\"63\\\": 287, \\\"72\\\": 93, \\\"125\\\": 5, \\\"130\\\": 4, \\\"102\\\": 17, \\\"90\\\": 20, \\\"93\\\": 19, \\\"190\\\": 1, \\\"105\\\": 16, \\\"88\\\": 27, \\\"99\\\": 20, \\\"142\\\": 2, \\\"121\\\": 7, \\\"119\\\": 8, \\\"156\\\": 2, \\\"107\\\": 14, \\\"110\\\": 13, \\\"113\\\": 8, \\\"191\\\": 1, \\\"131\\\": 3, \\\"192\\\": 1, \\\"193\\\": 1, \\\"194\\\": 1, \\\"195\\\": 1, \\\"143\\\": 3, \\\"103\\\": 17, \\\"144\\\": 3, \\\"145\\\": 3, \\\"196\\\": 1, \\\"197\\\": 1, \\\"198\\\": 1, \\\"199\\\": 1}\", \"index_word\": \"{\\\"1\\\": \\\"\\\\u093e\\\", \\\"2\\\": \\\"\\\\u0930\\\", \\\"3\\\": \\\"\\\\u094d\\\", \\\"4\\\": \\\"\\\\u0928\\\", \\\"5\\\": \\\"\\\\u093f\\\", \\\"6\\\": \\\"\\\\u0932\\\", \\\"7\\\": \\\"\\\\u0940\\\", \\\"8\\\": \\\"\\\\u0938\\\", \\\"9\\\": \\\"\\\\u092e\\\", \\\"10\\\": \\\"\\\\u0915\\\", \\\"11\\\": \\\"\\\\u0947\\\", \\\"12\\\": \\\"\\\\u0935\\\", \\\"13\\\": \\\"\\\\u092f\\\", \\\"14\\\": \\\"\\\\u094b\\\", \\\"15\\\": \\\"\\\\u0902\\\", \\\"16\\\": \\\"\\\\u0941\\\", \\\"17\\\": \\\"\\\\u0924\\\", \\\"18\\\": \\\"\\\\u0926\\\", \\\"19\\\": \\\"\\\\u092c\\\", \\\"20\\\": \\\"\\\\u092a\\\", \\\"21\\\": \\\"\\\\u0917\\\", \\\"22\\\": \\\"\\\\u0921\\\", \\\"23\\\": \\\"\\\\u091c\\\", \\\"24\\\": \\\"\\\\u0939\\\", \\\"25\\\": \\\"\\\\u091f\\\", \\\"26\\\": \\\"\\\\u0936\\\", \\\"27\\\": \\\"\\\\u091a\\\", \\\"28\\\": \\\"\\\\u0942\\\", \\\"29\\\": \\\"\\\\u0948\\\", \\\"30\\\": \\\"\\\\u0905\\\", \\\"31\\\": \\\"\\\\u0916\\\", \\\"32\\\": \\\"\\\\u0927\\\", \\\"33\\\": \\\" \\\", \\\"34\\\": \\\"\\\\u092d\\\", \\\"35\\\": \\\"\\\\u093c\\\", \\\"36\\\": \\\"\\\\u092b\\\", \\\"37\\\": \\\"\\\\u0923\\\", \\\"38\\\": \\\"\\\\u0907\\\", \\\"39\\\": \\\"\\\\u090f\\\", \\\"40\\\": \\\"\\\\u094c\\\", \\\"41\\\": \\\"\\\\u0906\\\", \\\"42\\\": \\\"\\\\u0908\\\", \\\"43\\\": \\\"\\\\u0925\\\", \\\"44\\\": \\\"\\\\u0937\\\", \\\"45\\\": \\\"\\\\u0909\\\", \\\"46\\\": \\\"\\\\u0920\\\", \\\"47\\\": \\\"\\\\u0918\\\", \\\"48\\\": \\\"\\\\u091b\\\", \\\"49\\\": \\\"\\\\u0922\\\", \\\"50\\\": \\\"\\\\u0901\\\", \\\"51\\\": \\\"\\\\u0943\\\", \\\"52\\\": \\\"\\\\u091d\\\", \\\"53\\\": \\\"\\\\u200d\\\", \\\"54\\\": \\\"\\\\u0949\\\", \\\"55\\\": \\\"\\\\u0913\\\", \\\"56\\\": \\\"\\\\u200c\\\", \\\"57\\\": \\\"\\\\u0966\\\", \\\"58\\\": \\\"\\\\u090a\\\", \\\"59\\\": \\\"\\\\u0910\\\", \\\"60\\\": \\\"\\\\u091e\\\", \\\"61\\\": \\\"\\\\u0914\\\", \\\"62\\\": \\\"\\\\u0903\\\", \\\"63\\\": \\\"\\\\u0952\\\", \\\"64\\\": \\\"\\\\u090b\\\", \\\"65\\\": \\\".\\\", \\\"66\\\": \\\"\\\\u0951\\\", \\\"67\\\": \\\"\\\\u0911\\\", \\\"68\\\": \\\"\\\\u0964\\\", \\\"69\\\": \\\"a\\\", \\\"70\\\": \\\"(\\\", \\\"71\\\": \\\")\\\", \\\"72\\\": \\\"\\\\u096e\\\", \\\"73\\\": \\\"\\\\u095f\\\", \\\"74\\\": \\\"s\\\", \\\"75\\\": \\\"'\\\", \\\"76\\\": \\\"\\\\u095b\\\", \\\"77\\\": \\\"r\\\", \\\"78\\\": \\\"\\\\u095c\\\", \\\"79\\\": \\\"i\\\", \\\"80\\\": \\\"n\\\", \\\"81\\\": \\\":\\\", \\\"82\\\": \\\"\\\\u0919\\\", \\\"83\\\": \\\"m\\\", \\\"84\\\": \\\"h\\\", \\\"85\\\": \\\"o\\\", \\\"86\\\": \\\"t\\\", \\\"87\\\": \\\"-\\\", \\\"88\\\": \\\"\\\\u201d\\\", \\\"89\\\": \\\"k\\\", \\\"90\\\": \\\"\\\\u0965\\\", \\\"91\\\": \\\"p\\\", \\\"92\\\": \\\"e\\\", \\\"93\\\": \\\"\\\\u096a\\\", \\\"94\\\": \\\"u\\\", \\\"95\\\": \\\"v\\\", \\\"96\\\": \\\"/\\\", \\\"97\\\": \\\"@\\\", \\\"98\\\": \\\"d\\\", \\\"99\\\": \\\"\\\\u00bb\\\", \\\"100\\\": \\\"j\\\", \\\"101\\\": \\\"\\\\u0945\\\", \\\"102\\\": \\\"[\\\", \\\"103\\\": \\\"_\\\", \\\"104\\\": \\\"\\\\u0933\\\", \\\"105\\\": \\\"\\\\u201c\\\", \\\"106\\\": \\\"l\\\", \\\"107\\\": \\\"*\\\", \\\"108\\\": \\\"\\\\u09fa\\\", \\\"109\\\": \\\"y\\\", \\\"110\\\": \\\"\\\\u096f\\\", \\\"111\\\": \\\"\\\\u095e\\\", \\\"112\\\": \\\"b\\\", \\\"113\\\": \\\"\\\\u096d\\\", \\\"114\\\": \\\"\\\\u0958\\\", \\\"115\\\": \\\"c\\\", \\\"116\\\": \\\"\\\\u0946\\\", \\\"117\\\": \\\"\\\\u0931\\\", \\\"118\\\": \\\"g\\\", \\\"119\\\": \\\"\\\\\\\"\\\", \\\"120\\\": \\\"\\\\u09be\\\", \\\"121\\\": \\\"]\\\", \\\"122\\\": \\\"\\\\u090e\\\", \\\"123\\\": \\\"\\\\u0929\\\", \\\"124\\\": \\\"\\\\u0bcd\\\", \\\"125\\\": \\\"\\\\u00a3\\\", \\\"126\\\": \\\"\\\\u094a\\\", \\\"127\\\": \\\"\\\\u09cd\\\", \\\"128\\\": \\\"\\\\u09a6\\\", \\\"129\\\": \\\"\\\\u09a8\\\", \\\"130\\\": \\\"\\\\u0969\\\", \\\"131\\\": \\\"\\\\u096b\\\", \\\"132\\\": \\\"\\\\u09b2\\\", \\\"133\\\": \\\"\\\\u0959\\\", \\\"134\\\": \\\"\\\\u090d\\\", \\\"135\\\": \\\",\\\", \\\"136\\\": \\\"z\\\", \\\"137\\\": \\\"\\\\u09ae\\\", \\\"138\\\": \\\"\\\\u099c\\\", \\\"139\\\": \\\"\\\\u09bf\\\", \\\"140\\\": \\\"\\\\u09b0\\\", \\\"141\\\": \\\"\\\\u095d\\\", \\\"142\\\": \\\"0\\\", \\\"143\\\": \\\"\\\\ufffd\\\", \\\"144\\\": \\\"\\\\u096c\\\", \\\"145\\\": \\\"\\\\u0968\\\", \\\"146\\\": \\\"2\\\", \\\"147\\\": \\\"\\\\u0ba3\\\", \\\"148\\\": \\\"\\\\u0bbe\\\", \\\"149\\\": \\\"\\\\u0b9f\\\", \\\"150\\\": \\\"\\\\u0bb0\\\", \\\"151\\\": \\\"\\\\u095a\\\", \\\"152\\\": \\\"\\\\u09c1\\\", \\\"153\\\": \\\"\\\\u0995\\\", \\\"154\\\": \\\"\\\\u2019\\\", \\\"155\\\": \\\"\\\\u0970\\\", \\\"156\\\": \\\"\\\\u200b\\\", \\\"157\\\": \\\"3\\\", \\\"158\\\": \\\"\\\\u0bae\\\", \\\"159\\\": \\\"\\\\u0b99\\\", \\\"160\\\": \\\"\\\\u0b95\\\", \\\"161\\\": \\\"\\\\u0bbf\\\", \\\"162\\\": \\\"\\\\u00e0\\\", \\\"163\\\": \\\"\\\\u00b7\\\", \\\"164\\\": \\\"\\\\u0b9a\\\", \\\"165\\\": \\\"\\\\u0bc1\\\", \\\"166\\\": \\\"\\\\u0ba8\\\", \\\"167\\\": \\\"\\\\u0ba4\\\", \\\"168\\\": \\\"\\\\u0b9c\\\", \\\"169\\\": \\\"\\\\u0ba9\\\", \\\"170\\\": \\\"\\\\u0986\\\", \\\"171\\\": \\\"\\\\u09ac\\\", \\\"172\\\": \\\"\\\\u0996\\\", \\\"173\\\": \\\"\\\\u09c7\\\", \\\"174\\\": \\\"\\\\u09f0\\\", \\\"175\\\": \\\"\\\\u0981\\\", \\\"176\\\": \\\"\\\\u0954\\\", \\\"177\\\": \\\"\\\\u09cb\\\", \\\"178\\\": \\\"\\\\u099a\\\", \\\"179\\\": \\\"\\\\u0985\\\", \\\"180\\\": \\\"\\\\u09a7\\\", \\\"181\\\": \\\"\\\\u09c0\\\", \\\"182\\\": \\\"\\\\u090c\\\", \\\"183\\\": \\\"`\\\", \\\"184\\\": \\\"w\\\", \\\"185\\\": \\\"\\\\t\\\", \\\"186\\\": \\\"\\\\u09b8\\\", \\\"187\\\": \\\"\\\\u09b9\\\", \\\"188\\\": \\\"\\\\u0989\\\", \\\"189\\\": \\\"\\\\u0950\\\", \\\"190\\\": \\\"\\\\u0644\\\", \\\"191\\\": \\\"\\\\u0967\\\", \\\"192\\\": \\\"\\\\ud804\\\\ude00\\\", \\\"193\\\": \\\"8\\\", \\\"194\\\": \\\"\\\\u0440\\\", \\\"195\\\": \\\"\\\\u062d\\\", \\\"196\\\": \\\"\\\\u0962\\\", \\\"197\\\": \\\"\\\\u00ab\\\", \\\"198\\\": \\\"\\\\u093d\\\", \\\"199\\\": \\\"$\\\"}\", \"word_index\": \"{\\\"\\\\u093e\\\": 1, \\\"\\\\u0930\\\": 2, \\\"\\\\u094d\\\": 3, \\\"\\\\u0928\\\": 4, \\\"\\\\u093f\\\": 5, \\\"\\\\u0932\\\": 6, \\\"\\\\u0940\\\": 7, \\\"\\\\u0938\\\": 8, \\\"\\\\u092e\\\": 9, \\\"\\\\u0915\\\": 10, \\\"\\\\u0947\\\": 11, \\\"\\\\u0935\\\": 12, \\\"\\\\u092f\\\": 13, \\\"\\\\u094b\\\": 14, \\\"\\\\u0902\\\": 15, \\\"\\\\u0941\\\": 16, \\\"\\\\u0924\\\": 17, \\\"\\\\u0926\\\": 18, \\\"\\\\u092c\\\": 19, \\\"\\\\u092a\\\": 20, \\\"\\\\u0917\\\": 21, \\\"\\\\u0921\\\": 22, \\\"\\\\u091c\\\": 23, \\\"\\\\u0939\\\": 24, \\\"\\\\u091f\\\": 25, \\\"\\\\u0936\\\": 26, \\\"\\\\u091a\\\": 27, \\\"\\\\u0942\\\": 28, \\\"\\\\u0948\\\": 29, \\\"\\\\u0905\\\": 30, \\\"\\\\u0916\\\": 31, \\\"\\\\u0927\\\": 32, \\\" \\\": 33, \\\"\\\\u092d\\\": 34, \\\"\\\\u093c\\\": 35, \\\"\\\\u092b\\\": 36, \\\"\\\\u0923\\\": 37, \\\"\\\\u0907\\\": 38, \\\"\\\\u090f\\\": 39, \\\"\\\\u094c\\\": 40, \\\"\\\\u0906\\\": 41, \\\"\\\\u0908\\\": 42, \\\"\\\\u0925\\\": 43, \\\"\\\\u0937\\\": 44, \\\"\\\\u0909\\\": 45, \\\"\\\\u0920\\\": 46, \\\"\\\\u0918\\\": 47, \\\"\\\\u091b\\\": 48, \\\"\\\\u0922\\\": 49, \\\"\\\\u0901\\\": 50, \\\"\\\\u0943\\\": 51, \\\"\\\\u091d\\\": 52, \\\"\\\\u200d\\\": 53, \\\"\\\\u0949\\\": 54, \\\"\\\\u0913\\\": 55, \\\"\\\\u200c\\\": 56, \\\"\\\\u0966\\\": 57, \\\"\\\\u090a\\\": 58, \\\"\\\\u0910\\\": 59, \\\"\\\\u091e\\\": 60, \\\"\\\\u0914\\\": 61, \\\"\\\\u0903\\\": 62, \\\"\\\\u0952\\\": 63, \\\"\\\\u090b\\\": 64, \\\".\\\": 65, \\\"\\\\u0951\\\": 66, \\\"\\\\u0911\\\": 67, \\\"\\\\u0964\\\": 68, \\\"a\\\": 69, \\\"(\\\": 70, \\\")\\\": 71, \\\"\\\\u096e\\\": 72, \\\"\\\\u095f\\\": 73, \\\"s\\\": 74, \\\"'\\\": 75, \\\"\\\\u095b\\\": 76, \\\"r\\\": 77, \\\"\\\\u095c\\\": 78, \\\"i\\\": 79, \\\"n\\\": 80, \\\":\\\": 81, \\\"\\\\u0919\\\": 82, \\\"m\\\": 83, \\\"h\\\": 84, \\\"o\\\": 85, \\\"t\\\": 86, \\\"-\\\": 87, \\\"\\\\u201d\\\": 88, \\\"k\\\": 89, \\\"\\\\u0965\\\": 90, \\\"p\\\": 91, \\\"e\\\": 92, \\\"\\\\u096a\\\": 93, \\\"u\\\": 94, \\\"v\\\": 95, \\\"/\\\": 96, \\\"@\\\": 97, \\\"d\\\": 98, \\\"\\\\u00bb\\\": 99, \\\"j\\\": 100, \\\"\\\\u0945\\\": 101, \\\"[\\\": 102, \\\"_\\\": 103, \\\"\\\\u0933\\\": 104, \\\"\\\\u201c\\\": 105, \\\"l\\\": 106, \\\"*\\\": 107, \\\"\\\\u09fa\\\": 108, \\\"y\\\": 109, \\\"\\\\u096f\\\": 110, \\\"\\\\u095e\\\": 111, \\\"b\\\": 112, \\\"\\\\u096d\\\": 113, \\\"\\\\u0958\\\": 114, \\\"c\\\": 115, \\\"\\\\u0946\\\": 116, \\\"\\\\u0931\\\": 117, \\\"g\\\": 118, \\\"\\\\\\\"\\\": 119, \\\"\\\\u09be\\\": 120, \\\"]\\\": 121, \\\"\\\\u090e\\\": 122, \\\"\\\\u0929\\\": 123, \\\"\\\\u0bcd\\\": 124, \\\"\\\\u00a3\\\": 125, \\\"\\\\u094a\\\": 126, \\\"\\\\u09cd\\\": 127, \\\"\\\\u09a6\\\": 128, \\\"\\\\u09a8\\\": 129, \\\"\\\\u0969\\\": 130, \\\"\\\\u096b\\\": 131, \\\"\\\\u09b2\\\": 132, \\\"\\\\u0959\\\": 133, \\\"\\\\u090d\\\": 134, \\\",\\\": 135, \\\"z\\\": 136, \\\"\\\\u09ae\\\": 137, \\\"\\\\u099c\\\": 138, \\\"\\\\u09bf\\\": 139, \\\"\\\\u09b0\\\": 140, \\\"\\\\u095d\\\": 141, \\\"0\\\": 142, \\\"\\\\ufffd\\\": 143, \\\"\\\\u096c\\\": 144, \\\"\\\\u0968\\\": 145, \\\"2\\\": 146, \\\"\\\\u0ba3\\\": 147, \\\"\\\\u0bbe\\\": 148, \\\"\\\\u0b9f\\\": 149, \\\"\\\\u0bb0\\\": 150, \\\"\\\\u095a\\\": 151, \\\"\\\\u09c1\\\": 152, \\\"\\\\u0995\\\": 153, \\\"\\\\u2019\\\": 154, \\\"\\\\u0970\\\": 155, \\\"\\\\u200b\\\": 156, \\\"3\\\": 157, \\\"\\\\u0bae\\\": 158, \\\"\\\\u0b99\\\": 159, \\\"\\\\u0b95\\\": 160, \\\"\\\\u0bbf\\\": 161, \\\"\\\\u00e0\\\": 162, \\\"\\\\u00b7\\\": 163, \\\"\\\\u0b9a\\\": 164, \\\"\\\\u0bc1\\\": 165, \\\"\\\\u0ba8\\\": 166, \\\"\\\\u0ba4\\\": 167, \\\"\\\\u0b9c\\\": 168, \\\"\\\\u0ba9\\\": 169, \\\"\\\\u0986\\\": 170, \\\"\\\\u09ac\\\": 171, \\\"\\\\u0996\\\": 172, \\\"\\\\u09c7\\\": 173, \\\"\\\\u09f0\\\": 174, \\\"\\\\u0981\\\": 175, \\\"\\\\u0954\\\": 176, \\\"\\\\u09cb\\\": 177, \\\"\\\\u099a\\\": 178, \\\"\\\\u0985\\\": 179, \\\"\\\\u09a7\\\": 180, \\\"\\\\u09c0\\\": 181, \\\"\\\\u090c\\\": 182, \\\"`\\\": 183, \\\"w\\\": 184, \\\"\\\\t\\\": 185, \\\"\\\\u09b8\\\": 186, \\\"\\\\u09b9\\\": 187, \\\"\\\\u0989\\\": 188, \\\"\\\\u0950\\\": 189, \\\"\\\\u0644\\\": 190, \\\"\\\\u0967\\\": 191, \\\"\\\\ud804\\\\ude00\\\": 192, \\\"8\\\": 193, \\\"\\\\u0440\\\": 194, \\\"\\\\u062d\\\": 195, \\\"\\\\u0962\\\": 196, \\\"\\\\u00ab\\\": 197, \\\"\\\\u093d\\\": 198, \\\"$\\\": 199}\"}}"
indicate/decoder.py ADDED
@@ -0,0 +1,62 @@
1
+ import tensorflow as tf
2
+
3
+ from .logging import get_logger
4
+
5
+ logger = get_logger()
6
+
7
+ class Decoder(tf.keras.Model):
8
+ def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz,
9
+ max_length_input, max_length_output, attention_type='luong'):
10
+ super(Decoder, self).__init__()
11
+ self.batch_sz = batch_sz
12
+ self.dec_units = dec_units
13
+ self.attention_type = attention_type
14
+ self.max_length_input = max_length_input
15
+ self.max_length_output = max_length_output
16
+
17
+ # Embedding Layer
18
+ self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)
19
+
20
+ # Final Dense layer on which softmax will be applied
21
+ self.fc = tf.keras.layers.Dense(vocab_size)
22
+
23
+ # Define the fundamental cell for decoder recurrent structure
24
+ self.decoder_rnn_cell = tf.keras.layers.LSTMCell(self.dec_units)
25
+ self.decoder_rnn = tf.keras.layers.RNN(self.decoder_rnn_cell, return_sequences=True, return_state=True)
26
+
27
+ # Create attention mechanism
28
+ # For Luong attention: project the query to match the encoder output's dimension (dec_units)
29
+ if self.attention_type == 'luong':
30
+ self.query_layer = tf.keras.layers.Dense(dec_units)
31
+
32
+ self.attention_mechanism = self.build_attention_mechanism(None)
33
+
34
+ def build_attention_mechanism(self, memory):
35
+ if self.attention_type == 'bahdanau':
36
+ return tf.keras.layers.AdditiveAttention()
37
+ else:
38
+ return tf.keras.layers.Attention()
39
+
40
+ def call(self, inputs, initial_state, encoder_outputs):
41
+ # Get the embeddings of the inputs
42
+ x = self.embedding(inputs)
43
+
44
+ if self.attention_type == 'luong':
45
+ query = self.query_layer(x)
46
+ else:
47
+ query = x
48
+
49
+ # Feature dimensions to the attention layer
50
+ attention_output, attention_weights = self.attention_mechanism([query, encoder_outputs], return_attention_scores=True)
51
+ # Concatenate the attention output with the LSTM Cell output
52
+ lstm_input = tf.concat([attention_output, x], axis=-1)
53
+ # Process through the LSTM cell
54
+ outputs, state_h, state_c = self.decoder_rnn(lstm_input, initial_state)
55
+ # Pass through the final dense layer
56
+ outputs = self.fc(outputs)
57
+ return outputs, (state_h, state_c), attention_weights
58
+
59
+ def build_initial_state(self, batch_sz, encoder_state):
60
+ hidden_state, cell_state = encoder_state
61
+ return [hidden_state, cell_state]
62
+
indicate/encoder.py ADDED
@@ -0,0 +1,24 @@
1
+ import tensorflow as tf
2
+
3
+ class Encoder(tf.keras.Model):
4
+ def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):
5
+ super(Encoder, self).__init__()
6
+ self.batch_sz = batch_sz
7
+ self.enc_units = enc_units
8
+ self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)
9
+
10
+ ##-------- LSTM layer in Encoder ------- ##
11
+ self.lstm_layer = tf.keras.layers.LSTM(self.enc_units,
12
+ return_sequences=True,
13
+ return_state=True,
14
+ recurrent_initializer='glorot_uniform')
15
+
16
+
17
+
18
+ def call(self, x, hidden, training=None):
19
+ x = self.embedding(x)
20
+ output, h, c = self.lstm_layer(x, initial_state = hidden)
21
+ return output, h, c
22
+
23
+ def initialize_hidden_state(self):
24
+ return [tf.zeros((self.batch_sz, self.enc_units)), tf.zeros((self.batch_sz, self.enc_units))]
indicate/hindi2english.py ADDED
@@ -0,0 +1,138 @@
1
+ import json
2
+ from .logging import get_logger
3
+ from pkg_resources import resource_filename
4
+
5
+ import tensorflow as tf
6
+
7
+ from .encoder import Encoder
8
+ from .decoder import Decoder
9
+ from .utils import translate
10
+
11
+ from func_timeout import func_timeout, FunctionTimedOut
12
+
13
+
14
+ logger = get_logger()
15
+
16
+
17
+ class HindiToEnglish():
18
+ MODELFN = "data/model/hindi_to_english/saved_weights/"
19
+ INPUT_VOCAB = "data/model/hindi_to_english/hindi_tokens.json"
20
+ TARGET_VOCAB = "data/model/hindi_to_english/english_tokens.json"
21
+
22
+ embedding_dim = 256
23
+ units = 1024
24
+ BATCH_SIZE = 64
25
+ BUFFER_SIZE = 120000
26
+
27
+ max_length_input = 47
28
+ max_length_output = 173
29
+ START_TOKEN = "^"
30
+ END_TOKEN = "$"
31
+
32
+ weights_loaded = False
33
+
34
+ input_lang_tokenizer = None
35
+ target_lang_tokenizer = None
36
+ encoder = None
37
+ decoder = None
38
+
39
+ @classmethod
40
+ def get_model_path(cls):
41
+ return resource_filename(__name__, cls.MODELFN)
42
+
43
+ @classmethod
44
+ def get_input_vocab(cls):
45
+ return resource_filename(__name__, cls.INPUT_VOCAB)
46
+
47
+ @classmethod
48
+ def get_target_vocab(cls):
49
+ return resource_filename(__name__, cls.TARGET_VOCAB)
50
+
51
+ @classmethod
52
+ def transliterate(cls, input: str) -> str:
53
+ """
54
+ Transliterate from Hindi to English.
55
+
56
+ Args:
57
+ input (str): Hindi text
58
+ Returns:
59
+ output (str): English text
60
+ """
61
+
62
+ if not cls.weights_loaded:
63
+ cls.model_path = cls.get_model_path()
64
+ cls.input_vocab = cls.get_input_vocab()
65
+ cls.target_vocab = cls.get_target_vocab()
66
+
67
+ with open(cls.get_input_vocab(), "r", encoding="utf-8") as f:
68
+ cls.input_lang_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(json.load(f))
69
+
70
+ with open(cls.get_target_vocab(), "r", encoding="utf-8") as f:
71
+ cls.target_lang_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(json.load(f))
72
+
73
+ vocab_inp_size = len(cls.input_lang_tokenizer.word_index) + 1
74
+ vocab_tar_size = len(cls.target_lang_tokenizer.word_index) + 1
75
+
76
+ cls.encoder = Encoder(vocab_inp_size, cls.embedding_dim, cls.units, cls.BATCH_SIZE)
77
+ cls.decoder = Decoder(
78
+ vocab_tar_size,
79
+ cls.embedding_dim,
80
+ cls.units,
81
+ cls.BATCH_SIZE,
82
+ cls.max_length_input,
83
+ cls.max_length_output,
84
+ "luong",
85
+ )
86
+
87
+ # This is needed to load waits from checkpoint
88
+ example_input_batch = tf.random.uniform(shape=[cls.BATCH_SIZE, cls.max_length_input])
89
+ sample_hidden = cls.encoder.initialize_hidden_state()
90
+ sample_output, sample_h, sample_c = cls.encoder(example_input_batch, sample_hidden)
91
+
92
+ sample_x = tf.random.uniform((cls.BATCH_SIZE, cls.max_length_output))
93
+ initial_state = cls.decoder.build_initial_state(cls.BATCH_SIZE, [sample_h, sample_c])
94
+ cls.decoder(sample_x, initial_state, sample_output)
95
+
96
+ logger.debug(f"Restoring model weights from {cls.model_path}")
97
+ # restore weights from checkpoint
98
+ cls.encoder.load_weights(f"{cls.model_path}/encoder.weights.h5")
99
+ cls.decoder.load_weights(f"{cls.model_path}/decoder.weights.h5")
100
+
101
+ cls.weights_loaded = True
102
+
103
+ # Predictions
104
+ # this is needed because sometimes BasicDecoder is going into loop
105
+ # and not exiting for sentences like 'पति स्व.गंगा भगत'
106
+ # if space is there then split by space and pass each sentence to translate
107
+ if " " in input:
108
+ input = input.split(" ")
109
+ else:
110
+ input = [input]
111
+
112
+ # translate each sentence
113
+ output = []
114
+ for inp in input:
115
+ target = ""
116
+ try:
117
+ target = func_timeout(
118
+ 10,
119
+ translate,
120
+ args=(
121
+ inp,
122
+ cls.units,
123
+ cls.input_lang_tokenizer,
124
+ cls.target_lang_tokenizer,
125
+ cls.encoder,
126
+ cls.decoder,
127
+ cls.max_length_input,
128
+ ),
129
+ )
130
+
131
+ logger.debug(f"Model predicted {target}")
132
+ except FunctionTimedOut as fex:
133
+ logger.error(f"Not able to transliterate {input} within 10 seconds, exiting with {fex}")
134
+ except Exception as exe:
135
+ logger.error(f"Not able to transliterate {input} due to {exe}")
136
+ output.append(target)
137
+
138
+ return " ".join(output)
indicate/logging.py ADDED
@@ -0,0 +1,6 @@
1
+ import logging
2
+
3
+
4
+ def get_logger():
5
+ logger = logging.getLogger("indicate")
6
+ return logger
indicate/notebooks/Data_Preparation.ipynb ADDED
@@ -0,0 +1 @@
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Data_Preparation.ipynb","provenance":[],"collapsed_sections":[],"mount_file_id":"1p9NfePf6vwm-GJZWdjiSoMaFHiGJkJlh","authorship_tag":"ABX9TyMn/2ZghbbgCTNlb8PV4dF4"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","source":["import pandas as pd"],"metadata":{"id":"IrF60G4Jjv_j","executionInfo":{"status":"ok","timestamp":1639445375817,"user_tz":480,"elapsed":236,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":5,"outputs":[]},{"cell_type":"markdown","source":["# Preprocess Data"],"metadata":{"id":"beYvfAz0jPcW"}},{"cell_type":"code","execution_count":3,"metadata":{"id":"ZlXTX-oUiiaZ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1639445348945,"user_tz":480,"elapsed":265,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"357012f2-78e6-42de-a536-fefe2ed305b2"},"outputs":[{"output_type":"stream","name":"stdout","text":["/content/drive/MyDrive/Colab/Transliterate/Main/data\n"]}],"source":["%cd /content/drive/MyDrive/Colab/Transliterate/Main/data"]},{"cell_type":"markdown","source":["## Affadavits Dataset"],"metadata":{"id":"nWxjcJLDjU1q"}},{"cell_type":"code","source":["affidavits_df = pd.read_csv('affidavits.csv')"],"metadata":{"id":"Lc1mwo9YjLxx","executionInfo":{"status":"ok","timestamp":1639445380212,"user_tz":480,"elapsed":1944,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["affidavits_df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":625},"id":"0jlbAsbFjz2_","executionInfo":{"status":"ok","timestamp":1639445402207,"user_tz":480,"elapsed":308,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"9a3f2abd-ac36-4dfe-b3e4-1c32c723e2fa"},"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>address</th>\n"," <th>affidavit_filename_with_unique_id</th>\n"," <th>age</th>\n"," <th>application_uploaded</th>\n"," <th>assembly_constituency</th>\n"," <th>current_status</th>\n"," <th>dropdown_value</th>\n"," <th>fathers_or_husbands_name_english</th>\n"," <th>fathers_or_husbands_name_hindi</th>\n"," <th>name</th>\n"," <th>name_english</th>\n"," <th>name_hindi</th>\n"," <th>party_name</th>\n"," <th>photo_filename_with_unique_id</th>\n"," <th>photo_url</th>\n"," <th>state</th>\n"," <th>unique_id</th>\n"," <th>url</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>O/5, BANKIM MUKHERJEE SARANI, DURGAPUR COLONY,...</td>\n"," <td>NaN</td>\n"," <td>57</td>\n"," <td>7th April, 2021</td>\n"," <td>Rashbehari</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>LATE SUBODH CHANDRA DAS</td>\n"," <td>प्रयात सुबोध चन्द्र दास</td>\n"," <td>SWAPAN DAS</td>\n"," <td>SWAPAN DAS</td>\n"," <td>स्वपन दास</td>\n"," <td>Bahujan Mukti Party</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>West Bengal</td>\n"," <td>1</td>\n"," <td>NaN</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>F-301, PRANTIK PALLY, KOLKATA – 700042</td>\n"," <td>NaN</td>\n"," <td>41</td>\n"," <td>7th April, 2021</td>\n"," <td>Rashbehari</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>LATE PINAKI KUNDU</td>\n"," <td>स्वर्गीय पिनाकि कुंडू</td>\n"," <td>PREETAM KUNDU</td>\n"," <td>PREETAM KUNDU</td>\n"," <td>प्रीतम कुंडू</td>\n"," <td>Independent</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>West Bengal</td>\n"," <td>2</td>\n"," <td>NaN</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>Vill- Debidaspur, P.O.- Kankuria, P.S.- Samser...</td>\n"," <td>NaN</td>\n"," <td>60</td>\n"," <td>26th April, 2021</td>\n"," <td>Samserganj</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>Late Nur Mohammad Biswas</td>\n"," <td>मृत नूर महम्मद बिश्वास</td>\n"," <td>Zaidur Rahaman</td>\n"," <td>Zaidur Rahaman</td>\n"," <td>जैदुर रहमान</td>\n"," <td>Indian National Congress</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>West Bengal</td>\n"," <td>3</td>\n"," <td>NaN</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>F-301, PRANTIK PALLY, KOLKATA – 700042</td>\n"," <td>NaN</td>\n"," <td>41</td>\n"," <td>7th April, 2021</td>\n"," <td>Rashbehari</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>LATE PINAKI KUNDU</td>\n"," <td>स्वर्गीय पिनाकि कुंडू</td>\n"," <td>PREETAM KUNDU</td>\n"," <td>PREETAM KUNDU</td>\n"," <td>प्रीतम कुंडू</td>\n"," <td>Independent</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>West Bengal</td>\n"," <td>4</td>\n"," <td>NaN</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>VILL-FULBARI, P.O-KAMLABARI, P.S-ENGLISH BAZAR...</td>\n"," <td>NaN</td>\n"," <td>42</td>\n"," <td>6th April, 2021</td>\n"," <td>Manikchak</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>LT. ALI MOHAMMAD</td>\n"," <td>स्वर्गीय आली महममद</td>\n"," <td>ALAM MOTTAKIN</td>\n"," <td>ALAM MOTTAKIN</td>\n"," <td>आलम मोत्ताकिन</td>\n"," <td>Indian National Congress</td>\n"," <td>NaN</td>\n"," <td>NaN</td>\n"," <td>West Bengal</td>\n"," <td>5</td>\n"," <td>NaN</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" address ... url\n","0 O/5, BANKIM MUKHERJEE SARANI, DURGAPUR COLONY,... ... NaN\n","1 F-301, PRANTIK PALLY, KOLKATA – 700042 ... NaN\n","2 Vill- Debidaspur, P.O.- Kankuria, P.S.- Samser... ... NaN\n","3 F-301, PRANTIK PALLY, KOLKATA – 700042 ... NaN\n","4 VILL-FULBARI, P.O-KAMLABARI, P.S-ENGLISH BAZAR... ... NaN\n","\n","[5 rows x 18 columns]"]},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["af_df = affidavits_df[['fathers_or_husbands_name_hindi', 'fathers_or_husbands_name_english', 'name_hindi', 'name_english']]"],"metadata":{"id":"OJpV9Hj1j5mB","executionInfo":{"status":"ok","timestamp":1639445414752,"user_tz":480,"elapsed":231,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":8,"outputs":[]},{"cell_type":"code","source":["# split and merge\n","input_target_df = pd.concat([\n"," af_df[['fathers_or_husbands_name_hindi','fathers_or_husbands_name_english']].rename(columns={\"fathers_or_husbands_name_hindi\": \"hindi\", \"fathers_or_husbands_name_english\": \"english\"}), \n"," af_df[['name_hindi','name_english']].rename(columns={\"name_hindi\": \"hindi\", \"name_english\": \"english\"})\n"," ], ignore_index=True)"],"metadata":{"id":"B6QrX0kdj9J_","executionInfo":{"status":"ok","timestamp":1639445427681,"user_tz":480,"elapsed":229,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":9,"outputs":[]},{"cell_type":"code","source":["input_target_df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"VFlD36zbj_xS","executionInfo":{"status":"ok","timestamp":1639445436096,"user_tz":480,"elapsed":241,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"6a03f66f-fe98-4a43-bef1-34178e91bee7"},"execution_count":10,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>hindi</th>\n"," <th>english</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>प्रयात सुबोध चन्द्र दास</td>\n"," <td>LATE SUBODH CHANDRA DAS</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>स्वर्गीय पिनाकि कुंडू</td>\n"," <td>LATE PINAKI KUNDU</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>मृत नूर महम्मद बिश्वास</td>\n"," <td>Late Nur Mohammad Biswas</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>स्वर्गीय पिनाकि कुंडू</td>\n"," <td>LATE PINAKI KUNDU</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>स्वर्गीय आली महममद</td>\n"," <td>LT. ALI MOHAMMAD</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" hindi english\n","0 प्रयात सुबोध चन्द्र दास LATE SUBODH CHANDRA DAS\n","1 स्वर्गीय पिनाकि कुंडू LATE PINAKI KUNDU\n","2 मृत नूर महम्मद बिश्वास Late Nur Mohammad Biswas\n","3 स्वर्गीय पिनाकि कुंडू LATE PINAKI KUNDU\n","4 स्वर्गीय आली महममद LT. ALI MOHAMMAD"]},"metadata":{},"execution_count":10}]},{"cell_type":"markdown","source":["## Cricket Dataset"],"metadata":{"id":"oJ6GzSQWkEV9"}},{"cell_type":"code","source":["cricket_df = pd.read_json('players_with_hindi_names.json')"],"metadata":{"id":"ZTkmkFCDkB_W","executionInfo":{"status":"ok","timestamp":1639445551287,"user_tz":480,"elapsed":1400,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":13,"outputs":[]},{"cell_type":"code","source":["cricket_df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"E-pZzI1mkeOD","executionInfo":{"status":"ok","timestamp":1639445563393,"user_tz":480,"elapsed":258,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"b0cedbaa-4c50-419c-decb-1c5b5fc49914"},"execution_count":14,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>id</th>\n"," <th>hindi_name</th>\n"," <th>hindi_long_name</th>\n"," <th>english_name</th>\n"," <th>gender</th>\n"," <th>date_of_birth</th>\n"," <th>country_id</th>\n"," <th>slug</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>90143</td>\n"," <td>आर गुरबाज़</td>\n"," <td>रहमानउल्लाह गुरबाज़</td>\n"," <td>Rahmanullah Gurbaz</td>\n"," <td>M</td>\n"," <td>{'year': 2001, 'month': 11, 'date': 28}</td>\n"," <td>40</td>\n"," <td>rahmanullah-gurbaz</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>54018</td>\n"," <td>एम रिज़वान</td>\n"," <td>मोहम्मद रिज़वान</td>\n"," <td>Mohammad Rizwan</td>\n"," <td>M</td>\n"," <td>{'year': 1992, 'month': 6, 'date': 1}</td>\n"," <td>7</td>\n"," <td>mohammad-rizwan</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>47686</td>\n"," <td>एस मक़सूद</td>\n"," <td>सोहेब मक़सूद</td>\n"," <td>Sohaib Maqsood</td>\n"," <td>M</td>\n"," <td>{'year': 1987, 'month': 4, 'date': 15}</td>\n"," <td>7</td>\n"," <td>sohaib-maqsood</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>53649</td>\n"," <td>आर आर रुसो</td>\n"," <td>राइली रुसो</td>\n"," <td>Rossouw, RR</td>\n"," <td>M</td>\n"," <td>{'year': 1989, 'month': 10, 'date': 9}</td>\n"," <td>3</td>\n"," <td>rilee-rossouw</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>72393</td>\n"," <td>एस हेटमायर</td>\n"," <td>शिमरॉन हेटमायर</td>\n"," <td>Hetmyer, SO</td>\n"," <td>M</td>\n"," <td>{'year': 1996, 'month': 12, 'date': 26}</td>\n"," <td>4</td>\n"," <td>shimron-hetmyer</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" id hindi_name ... country_id slug\n","0 90143 आर गुरबाज़ ... 40 rahmanullah-gurbaz\n","1 54018 एम रिज़वान ... 7 mohammad-rizwan\n","2 47686 एस मक़सूद ... 7 sohaib-maqsood\n","3 53649 आर आर रुसो ... 3 rilee-rossouw\n","4 72393 एस हेटमायर ... 4 shimron-hetmyer\n","\n","[5 rows x 8 columns]"]},"metadata":{},"execution_count":14}]},{"cell_type":"code","source":["input_target_df = pd.concat([\n"," input_target_df, \n"," cricket_df[['hindi_long_name', 'english_name']].rename(columns={\"hindi_long_name\": \"hindi\", \"english_name\": \"english\"}) \n"," ], ignore_index=True)"],"metadata":{"id":"CJKRQ0m7mnd2","executionInfo":{"status":"ok","timestamp":1639446127845,"user_tz":480,"elapsed":277,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":15,"outputs":[]},{"cell_type":"markdown","source":["## Dakshina Dataset"],"metadata":{"id":"NFJM4f3mmslv"}},{"cell_type":"code","source":["dakshina_df = pd.read_csv('hi.translit.sampled.train.tsv', sep='\\t', names=[\"hindi\", \"english\", \"freq\"])"],"metadata":{"id":"LKJSEHvFmqwC","executionInfo":{"status":"ok","timestamp":1639446213633,"user_tz":480,"elapsed":885,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["dakshina_df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"BH-i7ayBnHaV","executionInfo":{"status":"ok","timestamp":1639446249893,"user_tz":480,"elapsed":256,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"9c88ee33-8d7c-4c28-ff5a-ee3dc684cf83"},"execution_count":19,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>hindi</th>\n"," <th>english</th>\n"," <th>freq</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>अं</td>\n"," <td>an</td>\n"," <td>3</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>अंकगणित</td>\n"," <td>ankganit</td>\n"," <td>3</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>अंकल</td>\n"," <td>uncle</td>\n"," <td>4</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>अंकुर</td>\n"," <td>ankur</td>\n"," <td>4</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>अंकुरण</td>\n"," <td>ankuran</td>\n"," <td>3</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" hindi english freq\n","0 अं an 3\n","1 अंकगणित ankganit 3\n","2 अंकल uncle 4\n","3 अंकुर ankur 4\n","4 अंकुरण ankuran 3"]},"metadata":{},"execution_count":19}]},{"cell_type":"code","source":["input_target_df = pd.concat([\n"," input_target_df, \n"," dakshina_df[['hindi', 'english']]\n"," ], ignore_index=True)"],"metadata":{"id":"T2rN_ELJnKLc","executionInfo":{"status":"ok","timestamp":1639446267309,"user_tz":480,"elapsed":353,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":21,"outputs":[]},{"cell_type":"markdown","source":["## IIT Bombay English-Hindi Corpus\n"],"metadata":{"id":"t5-IPwRanbSw"}},{"cell_type":"code","source":["!head -10 en-hi.mined-pairs"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DGpuko8jnsXJ","executionInfo":{"status":"ok","timestamp":1639446426217,"user_tz":480,"elapsed":814,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"d4069b91-5484-4067-efc0-3bf4962ea98a"},"execution_count":24,"outputs":[{"output_type":"stream","name":"stdout","text":["aabu\tआबू\n","aacharan\tआचरण\n","aadat\tआदत\n","aadha\tआद्य\n","aadhaar\tआधार\n","aadhar\tआधार\n","aadhi\tआधी\n","aadi\tआदि\n","aadim\tआदितम\n","aadiparva\tआदिपर्व\n"]}]},{"cell_type":"code","source":["iitb_df = pd.read_csv('en-hi.mined-pairs', sep='\\t', names=[\"english\", \"hindi\"])"],"metadata":{"id":"Xz-62JW5nNCI","executionInfo":{"status":"ok","timestamp":1639446489028,"user_tz":480,"elapsed":1124,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":25,"outputs":[]},{"cell_type":"code","source":["iitb_df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"kuhrS-cfoCyv","executionInfo":{"status":"ok","timestamp":1639446498391,"user_tz":480,"elapsed":251,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"de38f25c-3a19-407b-b2c9-0efa41d4e074"},"execution_count":26,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>english</th>\n"," <th>hindi</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>aabu</td>\n"," <td>आबू</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>aacharan</td>\n"," <td>आचरण</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>aadat</td>\n"," <td>आदत</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>aadha</td>\n"," <td>आद्य</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>aadhaar</td>\n"," <td>आधार</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" english hindi\n","0 aabu आबू\n","1 aacharan आचरण\n","2 aadat आदत\n","3 aadha आद्य\n","4 aadhaar आधार"]},"metadata":{},"execution_count":26}]},{"cell_type":"code","source":["input_target_df = pd.concat([\n"," input_target_df, \n"," iitb_df[['hindi', 'english']]\n"," ], ignore_index=True)"],"metadata":{"id":"mjNS5TrHnPdH","executionInfo":{"status":"ok","timestamp":1639446544053,"user_tz":480,"elapsed":228,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":28,"outputs":[]},{"cell_type":"markdown","source":["## Merged Dataset"],"metadata":{"id":"LWvAfmoSoR8H"}},{"cell_type":"code","source":["input_target_df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"SzLvMGP9oWZN","executionInfo":{"status":"ok","timestamp":1639446582714,"user_tz":480,"elapsed":259,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"dbff608a-4fb5-4ca0-de48-ad9b610a2a82"},"execution_count":29,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>hindi</th>\n"," <th>english</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>प्रयात सुबोध चन्द्र दास</td>\n"," <td>LATE SUBODH CHANDRA DAS</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>स्वर्गीय पिनाकि कुंडू</td>\n"," <td>LATE PINAKI KUNDU</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>मृत नूर महम्मद बिश्वास</td>\n"," <td>Late Nur Mohammad Biswas</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>स्वर्गीय पिनाकि कुंडू</td>\n"," <td>LATE PINAKI KUNDU</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>स्वर्गीय आली महममद</td>\n"," <td>LT. ALI MOHAMMAD</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" hindi english\n","0 प्रयात सुबोध चन्द्र दास LATE SUBODH CHANDRA DAS\n","1 स्वर्गीय पिनाकि कुंडू LATE PINAKI KUNDU\n","2 मृत नूर महम्मद बिश्वास Late Nur Mohammad Biswas\n","3 स्वर्गीय पिनाकि कुंडू LATE PINAKI KUNDU\n","4 स्वर्गीय आली महममद LT. ALI MOHAMMAD"]},"metadata":{},"execution_count":29}]},{"cell_type":"code","source":["input_target_df.info()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_Cy-tDc2oaDg","executionInfo":{"status":"ok","timestamp":1639446628489,"user_tz":480,"elapsed":232,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"8d9bc8b4-312f-4b88-b5ba-1f7da2945e98"},"execution_count":30,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 146830 entries, 0 to 146829\n","Data columns (total 2 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 hindi 141649 non-null object\n"," 1 english 146827 non-null object\n","dtypes: object(2)\n","memory usage: 2.2+ MB\n"]}]},{"cell_type":"markdown","source":["## Drop NA"],"metadata":{"id":"MkEKfr38oze3"}},{"cell_type":"code","source":["input_target_df = input_target_df.dropna()"],"metadata":{"id":"fD8i56U3olt3","executionInfo":{"status":"ok","timestamp":1639446641623,"user_tz":480,"elapsed":247,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":31,"outputs":[]},{"cell_type":"code","source":["input_target_df.info()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0wbKrbJrooM-","executionInfo":{"status":"ok","timestamp":1639446650064,"user_tz":480,"elapsed":250,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"8edb0b24-a8c1-42b6-cf33-2b0b95af29d3"},"execution_count":32,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 141646 entries, 0 to 146829\n","Data columns (total 2 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 hindi 141646 non-null object\n"," 1 english 141646 non-null object\n","dtypes: object(2)\n","memory usage: 3.2+ MB\n"]}]},{"cell_type":"markdown","source":["# Strip and make lower case"],"metadata":{"id":"Pih_ac29o3G4"}},{"cell_type":"code","source":["# strip and make lower case\n","input_target_df['hindi'] = input_target_df['hindi'].str.strip() # hindi has no caps\n","input_target_df['english'] = input_target_df['english'].str.strip().str.lower()"],"metadata":{"id":"-lBCfhlEo7hA","executionInfo":{"status":"ok","timestamp":1639446734482,"user_tz":480,"elapsed":730,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":33,"outputs":[]},{"cell_type":"code","source":["input_target_df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"bD_F-VOro_lg","executionInfo":{"status":"ok","timestamp":1639446750405,"user_tz":480,"elapsed":246,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"5fa40a2b-6b16-4af6-b80c-229702c86088"},"execution_count":34,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n"," .dataframe tbody tr th:only-of-type {\n"," vertical-align: middle;\n"," }\n","\n"," .dataframe tbody tr th {\n"," vertical-align: top;\n"," }\n","\n"," .dataframe thead th {\n"," text-align: right;\n"," }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n"," <thead>\n"," <tr style=\"text-align: right;\">\n"," <th></th>\n"," <th>hindi</th>\n"," <th>english</th>\n"," </tr>\n"," </thead>\n"," <tbody>\n"," <tr>\n"," <th>0</th>\n"," <td>प्रयात सुबोध चन्द्र दास</td>\n"," <td>late subodh chandra das</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>स्वर्गीय पिनाकि कुंडू</td>\n"," <td>late pinaki kundu</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>मृत नूर महम्मद बिश्वास</td>\n"," <td>late nur mohammad biswas</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>स्वर्गीय पिनाकि कुंडू</td>\n"," <td>late pinaki kundu</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>स्वर्गीय आली महममद</td>\n"," <td>lt. ali mohammad</td>\n"," </tr>\n"," </tbody>\n","</table>\n","</div>"],"text/plain":[" hindi english\n","0 प्रयात सुबोध चन्द्र दास late subodh chandra das\n","1 स्वर्गीय पिनाकि कुंडू late pinaki kundu\n","2 मृत नूर महम्मद बिश्वास late nur mohammad biswas\n","3 स्वर्गीय पिनाकि कुंडू late pinaki kundu\n","4 स्वर्गीय आली महममद lt. ali mohammad"]},"metadata":{},"execution_count":34}]},{"cell_type":"markdown","source":["# Strip out any abbreviated names"],"metadata":{"id":"Tagytgs8pFjD"}},{"cell_type":"code","source":["# Strip out any abbreviated names\n","input_target_df = input_target_df[~input_target_df['english'].str.contains('\\.')]\n","input_target_df = input_target_df[~input_target_df['english'].str.contains(',')]"],"metadata":{"id":"Lbt3MOhcpDKI","executionInfo":{"status":"ok","timestamp":1639446795337,"user_tz":480,"elapsed":307,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":36,"outputs":[]},{"cell_type":"code","source":["input_target_df.info()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7T5ga_kZpOch","executionInfo":{"status":"ok","timestamp":1639446808450,"user_tz":480,"elapsed":244,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"b578aab2-ba80-44ed-946e-e2b69fcfa421"},"execution_count":37,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 132820 entries, 0 to 146829\n","Data columns (total 2 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 hindi 132820 non-null object\n"," 1 english 132820 non-null object\n","dtypes: object(2)\n","memory usage: 3.0+ MB\n"]}]},{"cell_type":"markdown","source":["## Drop duplicates"],"metadata":{"id":"2mmlxltdpTSL"}},{"cell_type":"code","source":["# drop duplicates\n","input_target_df = input_target_df.drop_duplicates()"],"metadata":{"id":"xZjz-h6UpRkd","executionInfo":{"status":"ok","timestamp":1639446843399,"user_tz":480,"elapsed":261,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":38,"outputs":[]},{"cell_type":"code","source":["input_target_df.info()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"FcKgzbtkpadI","executionInfo":{"status":"ok","timestamp":1639446851505,"user_tz":480,"elapsed":242,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"6b8d8032-5796-4cd3-a991-cd0ff0dbe13b"},"execution_count":39,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 116401 entries, 0 to 146829\n","Data columns (total 2 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 hindi 116401 non-null object\n"," 1 english 116401 non-null object\n","dtypes: object(2)\n","memory usage: 2.7+ MB\n"]}]},{"cell_type":"markdown","source":["## Remove if target has non-english"],"metadata":{"id":"sIJGFq8spgMT"}},{"cell_type":"code","source":["# Remove if target has non-english \n","def isEnglish(s):\n"," try:\n"," s.encode(encoding='utf-8').decode('ascii')\n"," except UnicodeDecodeError:\n"," return False\n"," else:\n"," return True\n","\n","input_target_df = input_target_df[input_target_df['english'].apply(isEnglish)]"],"metadata":{"id":"MIkpmCEOpdic","executionInfo":{"status":"ok","timestamp":1639446894575,"user_tz":480,"elapsed":258,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":40,"outputs":[]},{"cell_type":"code","source":["input_target_df.info()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"jYAQdoN0pmEx","executionInfo":{"status":"ok","timestamp":1639446904570,"user_tz":480,"elapsed":298,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"68ebbc9a-0742-48ee-ae4d-440afae9ca70"},"execution_count":41,"outputs":[{"output_type":"stream","name":"stdout","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 116021 entries, 0 to 146680\n","Data columns (total 2 columns):\n"," # Column Non-Null Count Dtype \n","--- ------ -------------- ----- \n"," 0 hindi 116021 non-null object\n"," 1 english 116021 non-null object\n","dtypes: object(2)\n","memory usage: 2.7+ MB\n"]}]},{"cell_type":"markdown","source":["## Save merged dataset"],"metadata":{"id":"bau0xcdWp1y-"}},{"cell_type":"code","source":["input_target_df.to_csv('merged_hi_to_en.csv', index=False)"],"metadata":{"id":"Ki17J624puzR","executionInfo":{"status":"ok","timestamp":1639447067926,"user_tz":480,"elapsed":252,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}}},"execution_count":42,"outputs":[]},{"cell_type":"code","source":["!head -10 merged_hi_to_en.csv"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"xDumbxudqQfV","executionInfo":{"status":"ok","timestamp":1639447087625,"user_tz":480,"elapsed":241,"user":{"displayName":"Rajashekar Chintalapati","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhvaKS4oQw2sBCyEe0cxq9rGBNCg4w2UoWOgVaD660=s64","userId":"03596288833202137831"}},"outputId":"28cee470-340f-4103-d872-1bcb6e26bcff"},"execution_count":43,"outputs":[{"output_type":"stream","name":"stdout","text":["hindi,english\n","प्रयात सुबोध चन्द्र दास,late subodh chandra das\n","स्वर्गीय पिनाकि कुंडू,late pinaki kundu\n","मृत नूर महम्मद बिश्वास,late nur mohammad biswas\n","नारायण चंद्र डॉं,narayan chandra dawn\n","आब्दुल मन्नान,abdul mannan\n","जमादार सेख,jamadar sk\n","असित बरण ब्य़ानार्जी,asit baran banerjee\n","नंदा दुलाल नाथ,nanda dulal nath\n","स्व. बिभूति भूषण दासगुप्ता,late bibhuti bhushan dasgupta\n"]}]}]}