indicate 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,305 @@
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "metadata": {},
7
+ "outputs": [],
8
+ "source": [
9
+ "import pandas as pd"
10
+ ]
11
+ },
12
+ {
13
+ "cell_type": "code",
14
+ "execution_count": 2,
15
+ "metadata": {},
16
+ "outputs": [],
17
+ "source": [
18
+ "df = pd.read_json('../data/players_with_hindi_names.json')"
19
+ ]
20
+ },
21
+ {
22
+ "cell_type": "code",
23
+ "execution_count": 3,
24
+ "metadata": {},
25
+ "outputs": [
26
+ {
27
+ "data": {
28
+ "text/html": [
29
+ "<div>\n",
30
+ "<style scoped>\n",
31
+ " .dataframe tbody tr th:only-of-type {\n",
32
+ " vertical-align: middle;\n",
33
+ " }\n",
34
+ "\n",
35
+ " .dataframe tbody tr th {\n",
36
+ " vertical-align: top;\n",
37
+ " }\n",
38
+ "\n",
39
+ " .dataframe thead th {\n",
40
+ " text-align: right;\n",
41
+ " }\n",
42
+ "</style>\n",
43
+ "<table border=\"1\" class=\"dataframe\">\n",
44
+ " <thead>\n",
45
+ " <tr style=\"text-align: right;\">\n",
46
+ " <th></th>\n",
47
+ " <th>id</th>\n",
48
+ " <th>hindi_name</th>\n",
49
+ " <th>hindi_long_name</th>\n",
50
+ " <th>english_name</th>\n",
51
+ " <th>gender</th>\n",
52
+ " <th>date_of_birth</th>\n",
53
+ " <th>country_id</th>\n",
54
+ " <th>slug</th>\n",
55
+ " </tr>\n",
56
+ " </thead>\n",
57
+ " <tbody>\n",
58
+ " <tr>\n",
59
+ " <th>0</th>\n",
60
+ " <td>90143</td>\n",
61
+ " <td>आर गुरबाज़</td>\n",
62
+ " <td>रहमानउल्लाह गुरबाज़</td>\n",
63
+ " <td>Rahmanullah Gurbaz</td>\n",
64
+ " <td>M</td>\n",
65
+ " <td>{'year': 2001, 'month': 11, 'date': 28}</td>\n",
66
+ " <td>40</td>\n",
67
+ " <td>rahmanullah-gurbaz</td>\n",
68
+ " </tr>\n",
69
+ " <tr>\n",
70
+ " <th>1</th>\n",
71
+ " <td>54018</td>\n",
72
+ " <td>एम रिज़वान</td>\n",
73
+ " <td>मोहम्मद रिज़वान</td>\n",
74
+ " <td>Mohammad Rizwan</td>\n",
75
+ " <td>M</td>\n",
76
+ " <td>{'year': 1992, 'month': 6, 'date': 1}</td>\n",
77
+ " <td>7</td>\n",
78
+ " <td>mohammad-rizwan</td>\n",
79
+ " </tr>\n",
80
+ " <tr>\n",
81
+ " <th>2</th>\n",
82
+ " <td>47686</td>\n",
83
+ " <td>एस मक़सूद</td>\n",
84
+ " <td>सोहेब मक़सूद</td>\n",
85
+ " <td>Sohaib Maqsood</td>\n",
86
+ " <td>M</td>\n",
87
+ " <td>{'year': 1987, 'month': 4, 'date': 15}</td>\n",
88
+ " <td>7</td>\n",
89
+ " <td>sohaib-maqsood</td>\n",
90
+ " </tr>\n",
91
+ " <tr>\n",
92
+ " <th>3</th>\n",
93
+ " <td>53649</td>\n",
94
+ " <td>आर आर रुसो</td>\n",
95
+ " <td>राइली रुसो</td>\n",
96
+ " <td>Rossouw, RR</td>\n",
97
+ " <td>M</td>\n",
98
+ " <td>{'year': 1989, 'month': 10, 'date': 9}</td>\n",
99
+ " <td>3</td>\n",
100
+ " <td>rilee-rossouw</td>\n",
101
+ " </tr>\n",
102
+ " <tr>\n",
103
+ " <th>4</th>\n",
104
+ " <td>72393</td>\n",
105
+ " <td>एस हेटमायर</td>\n",
106
+ " <td>शिमरॉन हेटमायर</td>\n",
107
+ " <td>Hetmyer, SO</td>\n",
108
+ " <td>M</td>\n",
109
+ " <td>{'year': 1996, 'month': 12, 'date': 26}</td>\n",
110
+ " <td>4</td>\n",
111
+ " <td>shimron-hetmyer</td>\n",
112
+ " </tr>\n",
113
+ " </tbody>\n",
114
+ "</table>\n",
115
+ "</div>"
116
+ ],
117
+ "text/plain": [
118
+ " id hindi_name hindi_long_name english_name gender \\\n",
119
+ "0 90143 आर गुरबाज़ रहमानउल्लाह गुरबाज़ Rahmanullah Gurbaz M \n",
120
+ "1 54018 एम रिज़वान मोहम्मद रिज़वान Mohammad Rizwan M \n",
121
+ "2 47686 एस मक़सूद सोहेब मक़सूद Sohaib Maqsood M \n",
122
+ "3 53649 आर आर रुसो राइली रुसो Rossouw, RR M \n",
123
+ "4 72393 एस हेटमायर शिमरॉन हेटमायर Hetmyer, SO M \n",
124
+ "\n",
125
+ " date_of_birth country_id slug \n",
126
+ "0 {'year': 2001, 'month': 11, 'date': 28} 40 rahmanullah-gurbaz \n",
127
+ "1 {'year': 1992, 'month': 6, 'date': 1} 7 mohammad-rizwan \n",
128
+ "2 {'year': 1987, 'month': 4, 'date': 15} 7 sohaib-maqsood \n",
129
+ "3 {'year': 1989, 'month': 10, 'date': 9} 3 rilee-rossouw \n",
130
+ "4 {'year': 1996, 'month': 12, 'date': 26} 4 shimron-hetmyer "
131
+ ]
132
+ },
133
+ "execution_count": 3,
134
+ "metadata": {},
135
+ "output_type": "execute_result"
136
+ }
137
+ ],
138
+ "source": [
139
+ "df.head()"
140
+ ]
141
+ },
142
+ {
143
+ "cell_type": "code",
144
+ "execution_count": 14,
145
+ "metadata": {},
146
+ "outputs": [
147
+ {
148
+ "name": "stdout",
149
+ "output_type": "stream",
150
+ "text": [
151
+ "Number of samples: 2512\n",
152
+ "Number of unique input tokens: 452\n",
153
+ "Number of unique output tokens: 457\n",
154
+ "Max sequence length for inputs: 22\n",
155
+ "Max sequence length for outputs: 26\n"
156
+ ]
157
+ }
158
+ ],
159
+ "source": [
160
+ "from keras.models import Model\n",
161
+ "from keras.layers import Input, LSTM, Dense\n",
162
+ "import numpy as np\n",
163
+ "\n",
164
+ "batch_size = 64 # Batch size for training.\n",
165
+ "epochs = 10 # Number of epochs to train for.\n",
166
+ "latent_dim = 256 # Latent dimensionality of the encoding space.\n",
167
+ "num_samples = 1000 # Number of samples to train on.\n",
168
+ "# Path to the data txt file on disk.\n",
169
+ "\n",
170
+ "# Vectorize the data.\n",
171
+ "input_texts = df.hindi_name.tolist()\n",
172
+ "target_texts = df.english_name.tolist()\n",
173
+ "input_characters = set()\n",
174
+ "target_characters = set()\n",
175
+ "\n",
176
+ "input_list = df['hindi_name'].tolist()\n",
177
+ "target_list = df['english_name'].tolist()\n",
178
+ "\n",
179
+ "for char in input_list:\n",
180
+ " if char not in input_characters:\n",
181
+ " input_characters.add(char)\n",
182
+ "for char in target_list:\n",
183
+ " if char not in target_characters:\n",
184
+ " target_characters.add(char)\n",
185
+ " \n",
186
+ "\n",
187
+ "input_characters = sorted(list(input_characters))\n",
188
+ "target_characters = sorted(list(target_characters))\n",
189
+ "num_encoder_tokens = len(input_characters)\n",
190
+ "num_decoder_tokens = len(target_characters)\n",
191
+ "max_encoder_seq_length = max([len(txt) for txt in input_texts])\n",
192
+ "max_decoder_seq_length = max([len(txt) for txt in target_texts])\n",
193
+ "\n",
194
+ "print('Number of samples:', len(input_texts))\n",
195
+ "print('Number of unique input tokens:', num_encoder_tokens)\n",
196
+ "print('Number of unique output tokens:', num_decoder_tokens)\n",
197
+ "print('Max sequence length for inputs:', max_encoder_seq_length)\n",
198
+ "print('Max sequence length for outputs:', max_decoder_seq_length)"
199
+ ]
200
+ },
201
+ {
202
+ "cell_type": "code",
203
+ "execution_count": 15,
204
+ "metadata": {},
205
+ "outputs": [
206
+ {
207
+ "ename": "KeyError",
208
+ "evalue": "'आ'",
209
+ "output_type": "error",
210
+ "traceback": [
211
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
212
+ "\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)",
213
+ "\u001b[1;32m<ipython-input-15-094370a4c3f2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m 16\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0minput_text\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_text\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_texts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_texts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 17\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchar\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m \u001b[0mencoder_input_data\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_token_index\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mchar\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 19\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchar\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 20\u001b[0m \u001b[1;31m# decoder_target_data is ahead of decoder_input_data by one timestep\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
214
+ "\u001b[1;31mKeyError\u001b[0m: 'आ'"
215
+ ]
216
+ }
217
+ ],
218
+ "source": [
219
+ "input_token_index = dict(\n",
220
+ " [(char, i) for i, char in enumerate(input_characters)])\n",
221
+ "target_token_index = dict(\n",
222
+ " [(char, i) for i, char in enumerate(target_characters)])\n",
223
+ "\n",
224
+ "encoder_input_data = np.zeros(\n",
225
+ " (len(input_texts), max_encoder_seq_length, num_encoder_tokens),\n",
226
+ " dtype='float32')\n",
227
+ "decoder_input_data = np.zeros(\n",
228
+ " (len(input_texts), max_decoder_seq_length, num_decoder_tokens),\n",
229
+ " dtype='float32')\n",
230
+ "decoder_target_data = np.zeros(\n",
231
+ " (len(input_texts), max_decoder_seq_length, num_decoder_tokens),\n",
232
+ " dtype='float32')\n",
233
+ "\n",
234
+ "for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):\n",
235
+ " for t, char in enumerate(input_text):\n",
236
+ " encoder_input_data[i, t, input_token_index[char]] = 1.\n",
237
+ " for t, char in enumerate(target_text):\n",
238
+ " # decoder_target_data is ahead of decoder_input_data by one timestep\n",
239
+ " decoder_input_data[i, t, target_token_index[char]] = 1.\n",
240
+ " if t > 0:\n",
241
+ " # decoder_target_data will be ahead by one timestep\n",
242
+ " # and will not include the start character.\n",
243
+ " decoder_target_data[i, t - 1, target_token_index[char]] = 1.\n",
244
+ "\n",
245
+ "# Define an input sequence and process it.\n",
246
+ "encoder_inputs = Input(shape=(None, num_encoder_tokens))\n",
247
+ "encoder = LSTM(latent_dim, return_state=True)\n",
248
+ "encoder_outputs, state_h, state_c = encoder(encoder_inputs)\n",
249
+ "# We discard `encoder_outputs` and only keep the states.\n",
250
+ "encoder_states = [state_h, state_c]\n",
251
+ "# Set up the decoder, using `encoder_states` as initial state.\n",
252
+ "decoder_inputs = Input(shape=(None, num_decoder_tokens))\n",
253
+ "# We set up our decoder to return full output sequences,\n",
254
+ "# and to return internal states as well. We don't use the\n",
255
+ "# return states in the training model, but we will use them in inference.\n",
256
+ "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)\n",
257
+ "decoder_outputs, _, _ = decoder_lstm(decoder_inputs,\n",
258
+ " initial_state=encoder_states)\n",
259
+ "decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
260
+ "decoder_outputs = decoder_dense(decoder_outputs)\n",
261
+ "\n",
262
+ "# Define the model that will turn\n",
263
+ "# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
264
+ "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
265
+ "\n",
266
+ "# Run training\n",
267
+ "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')\n",
268
+ "model.fit([encoder_input_data, decoder_input_data], decoder_target_data,\n",
269
+ " batch_size=batch_size,\n",
270
+ " epochs=epochs,\n",
271
+ " validation_split=0.2)\n",
272
+ "# Save model\n",
273
+ "model.save('s2s.h5')"
274
+ ]
275
+ },
276
+ {
277
+ "cell_type": "code",
278
+ "execution_count": null,
279
+ "metadata": {},
280
+ "outputs": [],
281
+ "source": []
282
+ }
283
+ ],
284
+ "metadata": {
285
+ "kernelspec": {
286
+ "display_name": "Python 3",
287
+ "language": "python",
288
+ "name": "python3"
289
+ },
290
+ "language_info": {
291
+ "codemirror_mode": {
292
+ "name": "ipython",
293
+ "version": 3
294
+ },
295
+ "file_extension": ".py",
296
+ "mimetype": "text/x-python",
297
+ "name": "python",
298
+ "nbconvert_exporter": "python",
299
+ "pygments_lexer": "ipython3",
300
+ "version": "3.6.4"
301
+ }
302
+ },
303
+ "nbformat": 4,
304
+ "nbformat_minor": 2
305
+ }
@@ -0,0 +1,12 @@
1
+ import sys
2
+ from contextlib import contextmanager
3
+ from io import StringIO
4
+
5
+
6
+ @contextmanager
7
+ def capture(command, *args, **kwargs):
8
+ out, sys.stdout = sys.stdout, StringIO()
9
+ command(*args, **kwargs)
10
+ sys.stdout.seek(0)
11
+ yield sys.stdout.read()
12
+ sys.stdout = out
@@ -0,0 +1,20 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ """
5
+ Test Hindi to English translation
6
+ """
7
+
8
+ import unittest
9
+
10
+ from indicate.hindi2english import HindiToEnglish
11
+
12
+ class TestHindiToEnglish(unittest.TestCase):
13
+ def test_hindi_to_english(self):
14
+ test_inputs = ["राजशेखर चिंतालपति","गौरव सूद"]
15
+ test_outputs = ["rajshekhar chintalpati","gaurav sood"]
16
+ for hindi, english in zip(test_inputs, test_outputs):
17
+ self.assertEqual(HindiToEnglish.transliterate(hindi), english)
18
+
19
+ if __name__ == "__main__":
20
+ unittest.main()
@@ -0,0 +1,28 @@
1
+ import sys
2
+ import argparse
3
+
4
+ from .hindi2english import HindiToEnglish
5
+
6
+ hindi2english = HindiToEnglish.transliterate
7
+
8
+
9
+ def main(argv=sys.argv[1:]):
10
+ title = "Transliterate from Hindi to English"
11
+ parser = argparse.ArgumentParser(description=title)
12
+ parser.add_argument("--type", default=None, help="hin2eng")
13
+ parser.add_argument("--input", default=None, help="Hindi text")
14
+ args = parser.parse_args(argv)
15
+ print(args)
16
+ if not args.input or not args.type:
17
+ return -1
18
+
19
+ output = ""
20
+ if args.type == "hin2eng":
21
+ output = hindi2english(args.input)
22
+ print(output)
23
+
24
+ return 0
25
+
26
+
27
+ if __name__ == "__main__":
28
+ sys.exit(main())
indicate/utils.py ADDED
@@ -0,0 +1,86 @@
1
+ import tensorflow as tf
2
+
3
+ def sequence_to_chars(tokenizer, sequence):
4
+ """Convert a sequence of indices back to characters."""
5
+ word_index = tokenizer.word_index
6
+ reverse_map = {val: key for key, val in word_index.items()}
7
+ retext = ''
8
+
9
+ # Convert tensor to numpy array if it's a tensor
10
+ if tf.is_tensor(sequence):
11
+ sequence = sequence.numpy()
12
+
13
+ for q in sequence:
14
+ q_int = int(q) # Convert to integer
15
+ if q_int != 0: # Skip padding token
16
+ retext += reverse_map[q_int]
17
+ return retext
18
+
19
+ def evaluate_sentence(sentence, units, input_lang_tokenizer, target_lang_tokenizer,
20
+ encoder, decoder, max_length_input):
21
+ """Evaluate/translate a single sentence."""
22
+ # Convert input sentence to token indices
23
+ inputs = [input_lang_tokenizer.word_index[i] for i in sentence]
24
+ inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],
25
+ maxlen=max_length_input,
26
+ padding='post')
27
+ inputs = tf.convert_to_tensor(inputs)
28
+
29
+ # Use the same batch size as decoder.batch_sz for consistency
30
+ inference_batch_size = inputs.shape[0]
31
+
32
+ inputs = tf.tile(inputs, [inference_batch_size, 1]) # Replicate input to match batch size
33
+
34
+ # Initialize encoder state
35
+ enc_start_state = [tf.zeros((inference_batch_size, units)),
36
+ tf.zeros((inference_batch_size, units))]
37
+
38
+ # Get encoder output
39
+ enc_out, enc_h, enc_c = encoder(inputs, enc_start_state)
40
+
41
+ # Initialize decoder with encoder's final state
42
+ dec_state = [enc_h, enc_c]
43
+
44
+ # Prepare decoder input with start token
45
+ dec_input = tf.fill([inference_batch_size, 1],
46
+ target_lang_tokenizer.word_index['^'])
47
+
48
+ outputs = []
49
+
50
+ # Decoding loop
51
+ for t in range(decoder.max_length_output):
52
+ # Call decoder
53
+ predictions, dec_state, _ = decoder(dec_input, dec_state, enc_out)
54
+
55
+ # Get predicted token (use first batch item since all are the same)
56
+ predicted_id = tf.argmax(predictions, axis=-1).numpy()[0, -1]
57
+
58
+ # Append prediction
59
+ outputs.append(predicted_id)
60
+
61
+ # Break if end token is predicted
62
+ if predicted_id == target_lang_tokenizer.word_index['$']:
63
+ break
64
+
65
+ # Update decoder input
66
+ dec_input = tf.fill([inference_batch_size, 1], predicted_id)
67
+
68
+ return tf.convert_to_tensor(outputs)
69
+
70
+
71
+ def translate(sentence, units, input_lang_tokenizer, target_lang_tokenizer,
72
+ encoder, decoder, max_length_input):
73
+ """Translate a sentence from source to target language."""
74
+ result = evaluate_sentence(sentence,
75
+ units,
76
+ input_lang_tokenizer,
77
+ target_lang_tokenizer,
78
+ encoder,
79
+ decoder,
80
+ max_length_input)
81
+
82
+ # Convert the output tokens back to characters
83
+ translated_text = sequence_to_chars(target_lang_tokenizer, result)
84
+
85
+ # Remove the end token and return
86
+ return translated_text.strip('$')
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2021 Rajasekhar Chintalapati and Gaurav Sood
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,139 @@
1
+ Metadata-Version: 2.2
2
+ Name: indicate
3
+ Version: 0.2.0
4
+ Summary: Transliterations to/from Indian languages
5
+ Home-page: https://github.com/in-rolls/indicate
6
+ Author: Rajashekar Chintalapati, Gaurav Sood
7
+ Author-email: rajshekar.ch@gmail.com, gsood07@gmail.com
8
+ License: MIT
9
+ Keywords: transliterate indic hindi english
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3.6
14
+ Classifier: Programming Language :: Python :: 3.7
15
+ Classifier: Programming Language :: Python :: 3.8
16
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
17
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
18
+ Classifier: Topic :: Utilities
19
+ License-File: LICENSE
20
+ Requires-Dist: tqdm
21
+ Requires-Dist: func-timeout
22
+ Requires-Dist: tensorflow==2.18.0
23
+ Provides-Extra: dev
24
+ Requires-Dist: check-manifest; extra == "dev"
25
+ Provides-Extra: test
26
+ Requires-Dist: coverage; extra == "test"
27
+ Dynamic: author
28
+ Dynamic: author-email
29
+ Dynamic: classifier
30
+ Dynamic: description
31
+ Dynamic: home-page
32
+ Dynamic: keywords
33
+ Dynamic: license
34
+ Dynamic: provides-extra
35
+ Dynamic: requires-dist
36
+ Dynamic: summary
37
+
38
+ ==================================================
39
+ Indicate: Transliterate Indic Languages to English
40
+ ==================================================
41
+
42
+ .. image:: https://app.travis-ci.com/in-rolls/indicate.svg?branch=master
43
+ :target: https://travis-ci.org/in-rolls/indicate
44
+ .. image:: https://img.shields.io/pypi/v/indicate.svg
45
+ :target: https://pypi.python.org/pypi/indicate
46
+ .. image:: https://readthedocs.org/projects/indicate/badge/?version=latest
47
+ :target: http://notnews.readthedocs.io/en/latest/?badge=latest
48
+ :alt: Documentation Status
49
+ .. image:: https://static.pepy.tech/badge/indicate
50
+ :target: https://pepy.tech/project/indicate
51
+
52
+ Transliterations to/from Indian languages are still generally low quality. One problem is access to data. Another is that there is no standard transliteration.
53
+ For Hindi--English, we build novel dataset for names using the ESPNcricinfo. For instance, see `here <https://www.espncricinfo.com/hindi/series/pakistan-tour-of-england-2021-1239529/england-vs-pakistan-1st-odi-1239537/full-scorecard>`__ for hindi version of the `english scorecard <https://www.espncricinfo.com/series/pakistan-tour-of-england-2021-1239529/england-vs-pakistan-1st-odi-1239537/full-scorecard>`__.
54
+ We also create a dataset from `election affidavits <https://affidavit.eci.gov.in/CandidateCustomFilter>`__
55
+ We also exploit the `Google Dakshina dataset <https://github.com/google-research-datasets/dakshina>`__.
56
+
57
+ To overcome the fact that there isn't one standard way of transliteration, we provide k-best transliterations.
58
+
59
+ Install
60
+ -------
61
+ We strongly recommend installing `indicate` inside a Python virtual environment
62
+ (see `venv documentation <https://docs.python.org/3/library/venv.html#creating-virtual-environments>`__)
63
+
64
+ ::
65
+
66
+ pip install indicate
67
+
68
+ General API
69
+ -----------
70
+ 1. transliterate.hindi2english will take Hindi text and translate into English.
71
+
72
+ Examples
73
+ --------
74
+ ::
75
+
76
+ from indicate import transliterate
77
+ english_translated = transliterate.hindi2english("हिंदी")
78
+ print(english_translated)
79
+
80
+ output -
81
+ hindi
82
+
83
+ Functions
84
+ ----------
85
+ We expose 1 function, which will take Hindi text and transliterate it to English.
86
+
87
+ - **transliterate.hindi2english(input)**
88
+
89
+ - What it does:
90
+
91
+ - Converts given hindi text into English alphabet
92
+
93
+ - Output
94
+
95
+ - Returns text in English
96
+
97
+ Data
98
+ ----
99
+ The datasets used to train the model:
100
+
101
+ - `Indian Election affidavits <https://affidavit.eci.gov.in/CandidateCustomFilter>`__
102
+
103
+ - `Google Dakshina dataset <https://github.com/google-research-datasets/dakshina>`__
104
+
105
+ - `ESPN Cric Info <https://www.espncricinfo.com/hindi/series/pakistan-tour-of-england-2021-1239529/england-vs-pakistan-1st-odi-1239537/full-scorecard>`__ for hindi version of the `english scorecard <https://www.espncricinfo.com/series/pakistan-tour-of-england-2021-1239529/england-vs-pakistan-1st-odi-1239537/full-scorecard>`__.
106
+
107
+ - `IIT Bombay English-Hindi Corpus <https://www.cfilt.iitb.ac.in/iitb_parallel/>`__
108
+
109
+ Evaluation
110
+ ----------
111
+ Model was evaluated on test dataset of Google Dakshina dataset, Model predicted 73.64% exact matches.
112
+ `Indic-trans <https://github.com/libindic/indic-trans>`__ predicted 63.12% exact matches on Google Dakshina dataset.
113
+ Below is the edit distance metrics on test dataset (0.0 mean exact match, the farther away from 0.0,
114
+ the difference is more between predicted text and actual text)
115
+
116
+ .. image:: https://github.com/in-rolls/indicate/raw/master/images/h2e_ed.png
117
+ :width: 400
118
+ :alt: Edit distance metrics of model on Google Dakshina test dataset
119
+
120
+
121
+ Authors
122
+ -------
123
+
124
+ Rajashekar Chintalapati and Gaurav Sood
125
+
126
+ Contributor Code of Conduct
127
+ ---------------------------------
128
+
129
+ The project welcomes contributions from everyone! In fact, it depends on
130
+ it. To maintain this welcoming atmosphere, and to collaborate in a fun
131
+ and productive way, we expect contributors to the project to abide by
132
+ the `Contributor Code of
133
+ Conduct <http://contributor-covenant.org/version/1/0/0/>`__.
134
+
135
+ License
136
+ ----------
137
+
138
+ The package is released under the `MIT
139
+ License <https://opensource.org/licenses/MIT>`__.
@@ -0,0 +1,26 @@
1
+ indicate/__init__.py,sha256=89udvCQOFZ2icsiF0Nmk_KeUWLq3vhQxzA38Hz1ARWA,70
2
+ indicate/decoder.py,sha256=_u7npwJlmF2N--bfwb6hok3QF0ezObj9FyOjhr5ENjY,2567
3
+ indicate/encoder.py,sha256=pkFv7rggaISgdHMQflfi019gbp9FVjsU3ex_-eBqGtI,910
4
+ indicate/hindi2english.py,sha256=T4lcR8NpHhr7-Gq4XFOJJbGSFyvv57AOsUnvt9nuAII,4677
5
+ indicate/logging.py,sha256=AGzjUiPusLS7QJnMdjHDSWRkxC_WoL4jKv0_b2YuhJ0,96
6
+ indicate/transliterate.py,sha256=jrVfz7VaR2p-FthcMXFSDym-UmU9WeFu30ZrXzAzR34,667
7
+ indicate/utils.py,sha256=gD7WjTlBBsh9tPXi8i8wO3d6JDYCtp-rnUNyOnZN-04,3168
8
+ indicate/data/model/hindi_to_english/english_tokens.json,sha256=g_pssuCQyXnE05DRVqing852wHMcxZ-eT5753feCTJs,14480
9
+ indicate/data/model/hindi_to_english/hindi_tokens.json,sha256=mMZYNM8HkKz7eCHFgHGe8KEKbrXMGz-Mq3v-zQ2lJvU,22545
10
+ indicate/data/model/hindi_to_english/saved_weights/decoder.weights.h5,sha256=e-An6CE6yVi2J8Y8cGn7adogLhCD3UVPqNxf8kbX75o,39543312
11
+ indicate/data/model/hindi_to_english/saved_weights/encoder.weights.h5,sha256=6sd66LpwQlCXbK6od_hd2tR_KOzGHG0Q5PqLk7-h7gE,21208200
12
+ indicate/notebooks/Data_Preparation.ipynb,sha256=O65ytgmQRxujLS5LShxFtTnXr4ZTockDkHEPub0cGiw,37341
13
+ indicate/notebooks/Inference.ipynb,sha256=Y2k7oHAfQP579XTB-KL3ahiQTGdFygidBCqNNr6LqEI,72057
14
+ indicate/notebooks/Inference_latest.ipynb,sha256=2cGLK-KHWg3gKAJcABhZbEEuo6uoP2rzWAQt-Q8U9Jc,95628
15
+ indicate/notebooks/Train.ipynb,sha256=ZmG5iZ_n79FSBbyJBNOkTdmz_7JkNnWkng9A6wQ8U-w,58879
16
+ indicate/notebooks/Train_latest.ipynb,sha256=m91MAI2zxCKvts_QVgjc_mWXYi3Wb9a5Dur8EymCIRc,69089
17
+ indicate/notebooks/Transliterate.ipynb,sha256=u4MZ-9za3S0Dvz87QtKZnh1CIo8uYGR2RYYCj7hlA0I,308328
18
+ indicate/notebooks/transliterate_cricket.ipynb,sha256=Dr0iu3r0XwIF0a_DwQqQZ7qauKk4AfLkFEWjglRDb58,13072
19
+ indicate/tests/__init__.py,sha256=ezvdIE58d5GooM9wrJ51pNk9U3pb-LWMWE2nKYqiQP0,275
20
+ indicate/tests/test_010_hindi_translate.py,sha256=69XHcpDlRtOw9gMYv11cSBBOSsG31q_pfqi0mT9K1EE,590
21
+ indicate-0.2.0.dist-info/LICENSE,sha256=iEky6BOuLfXUmOQ67jpWOyqapJVj37qeXqUIHIhVHYM,1106
22
+ indicate-0.2.0.dist-info/METADATA,sha256=wxXC9ZmUTgWUZ6ysC17hCIlzHnT5W8_ph7b5MXnirv8,5269
23
+ indicate-0.2.0.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
24
+ indicate-0.2.0.dist-info/entry_points.txt,sha256=LrGK3DYYXfR5vWPG5RvW9wj7R6ZoSfa0iuxvXIkmZec,62
25
+ indicate-0.2.0.dist-info/top_level.txt,sha256=MH_uxn6O8-g_WCij25wOlx2cNuv9-mScUuZDBPSb2AA,9
26
+ indicate-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,6 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (75.8.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py2-none-any
5
+ Tag: py3-none-any
6
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ hindi2english = indicate.transliterate:main
@@ -0,0 +1 @@
1
+ indicate