indicate 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indicate/__init__.py +3 -0
- indicate/data/model/hindi_to_english/english_tokens.json +1 -0
- indicate/data/model/hindi_to_english/hindi_tokens.json +1 -0
- indicate/data/model/hindi_to_english/saved_weights/decoder.weights.h5 +0 -0
- indicate/data/model/hindi_to_english/saved_weights/encoder.weights.h5 +0 -0
- indicate/decoder.py +62 -0
- indicate/encoder.py +24 -0
- indicate/hindi2english.py +138 -0
- indicate/logging.py +6 -0
- indicate/notebooks/Data_Preparation.ipynb +1 -0
- indicate/notebooks/Inference.ipynb +1 -0
- indicate/notebooks/Inference_latest.ipynb +2331 -0
- indicate/notebooks/Train.ipynb +1 -0
- indicate/notebooks/Train_latest.ipynb +1746 -0
- indicate/notebooks/Transliterate.ipynb +2501 -0
- indicate/notebooks/transliterate_cricket.ipynb +305 -0
- indicate/tests/__init__.py +12 -0
- indicate/tests/test_010_hindi_translate.py +20 -0
- indicate/transliterate.py +28 -0
- indicate/utils.py +86 -0
- indicate-0.2.0.dist-info/LICENSE +21 -0
- indicate-0.2.0.dist-info/METADATA +139 -0
- indicate-0.2.0.dist-info/RECORD +26 -0
- indicate-0.2.0.dist-info/WHEEL +6 -0
- indicate-0.2.0.dist-info/entry_points.txt +2 -0
- indicate-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
{
|
|
2
|
+
"cells": [
|
|
3
|
+
{
|
|
4
|
+
"cell_type": "code",
|
|
5
|
+
"execution_count": 1,
|
|
6
|
+
"metadata": {},
|
|
7
|
+
"outputs": [],
|
|
8
|
+
"source": [
|
|
9
|
+
"import pandas as pd"
|
|
10
|
+
]
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"cell_type": "code",
|
|
14
|
+
"execution_count": 2,
|
|
15
|
+
"metadata": {},
|
|
16
|
+
"outputs": [],
|
|
17
|
+
"source": [
|
|
18
|
+
"df = pd.read_json('../data/players_with_hindi_names.json')"
|
|
19
|
+
]
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"cell_type": "code",
|
|
23
|
+
"execution_count": 3,
|
|
24
|
+
"metadata": {},
|
|
25
|
+
"outputs": [
|
|
26
|
+
{
|
|
27
|
+
"data": {
|
|
28
|
+
"text/html": [
|
|
29
|
+
"<div>\n",
|
|
30
|
+
"<style scoped>\n",
|
|
31
|
+
" .dataframe tbody tr th:only-of-type {\n",
|
|
32
|
+
" vertical-align: middle;\n",
|
|
33
|
+
" }\n",
|
|
34
|
+
"\n",
|
|
35
|
+
" .dataframe tbody tr th {\n",
|
|
36
|
+
" vertical-align: top;\n",
|
|
37
|
+
" }\n",
|
|
38
|
+
"\n",
|
|
39
|
+
" .dataframe thead th {\n",
|
|
40
|
+
" text-align: right;\n",
|
|
41
|
+
" }\n",
|
|
42
|
+
"</style>\n",
|
|
43
|
+
"<table border=\"1\" class=\"dataframe\">\n",
|
|
44
|
+
" <thead>\n",
|
|
45
|
+
" <tr style=\"text-align: right;\">\n",
|
|
46
|
+
" <th></th>\n",
|
|
47
|
+
" <th>id</th>\n",
|
|
48
|
+
" <th>hindi_name</th>\n",
|
|
49
|
+
" <th>hindi_long_name</th>\n",
|
|
50
|
+
" <th>english_name</th>\n",
|
|
51
|
+
" <th>gender</th>\n",
|
|
52
|
+
" <th>date_of_birth</th>\n",
|
|
53
|
+
" <th>country_id</th>\n",
|
|
54
|
+
" <th>slug</th>\n",
|
|
55
|
+
" </tr>\n",
|
|
56
|
+
" </thead>\n",
|
|
57
|
+
" <tbody>\n",
|
|
58
|
+
" <tr>\n",
|
|
59
|
+
" <th>0</th>\n",
|
|
60
|
+
" <td>90143</td>\n",
|
|
61
|
+
" <td>आर गुरबाज़</td>\n",
|
|
62
|
+
" <td>रहमानउल्लाह गुरबाज़</td>\n",
|
|
63
|
+
" <td>Rahmanullah Gurbaz</td>\n",
|
|
64
|
+
" <td>M</td>\n",
|
|
65
|
+
" <td>{'year': 2001, 'month': 11, 'date': 28}</td>\n",
|
|
66
|
+
" <td>40</td>\n",
|
|
67
|
+
" <td>rahmanullah-gurbaz</td>\n",
|
|
68
|
+
" </tr>\n",
|
|
69
|
+
" <tr>\n",
|
|
70
|
+
" <th>1</th>\n",
|
|
71
|
+
" <td>54018</td>\n",
|
|
72
|
+
" <td>एम रिज़वान</td>\n",
|
|
73
|
+
" <td>मोहम्मद रिज़वान</td>\n",
|
|
74
|
+
" <td>Mohammad Rizwan</td>\n",
|
|
75
|
+
" <td>M</td>\n",
|
|
76
|
+
" <td>{'year': 1992, 'month': 6, 'date': 1}</td>\n",
|
|
77
|
+
" <td>7</td>\n",
|
|
78
|
+
" <td>mohammad-rizwan</td>\n",
|
|
79
|
+
" </tr>\n",
|
|
80
|
+
" <tr>\n",
|
|
81
|
+
" <th>2</th>\n",
|
|
82
|
+
" <td>47686</td>\n",
|
|
83
|
+
" <td>एस मक़सूद</td>\n",
|
|
84
|
+
" <td>सोहेब मक़सूद</td>\n",
|
|
85
|
+
" <td>Sohaib Maqsood</td>\n",
|
|
86
|
+
" <td>M</td>\n",
|
|
87
|
+
" <td>{'year': 1987, 'month': 4, 'date': 15}</td>\n",
|
|
88
|
+
" <td>7</td>\n",
|
|
89
|
+
" <td>sohaib-maqsood</td>\n",
|
|
90
|
+
" </tr>\n",
|
|
91
|
+
" <tr>\n",
|
|
92
|
+
" <th>3</th>\n",
|
|
93
|
+
" <td>53649</td>\n",
|
|
94
|
+
" <td>आर आर रुसो</td>\n",
|
|
95
|
+
" <td>राइली रुसो</td>\n",
|
|
96
|
+
" <td>Rossouw, RR</td>\n",
|
|
97
|
+
" <td>M</td>\n",
|
|
98
|
+
" <td>{'year': 1989, 'month': 10, 'date': 9}</td>\n",
|
|
99
|
+
" <td>3</td>\n",
|
|
100
|
+
" <td>rilee-rossouw</td>\n",
|
|
101
|
+
" </tr>\n",
|
|
102
|
+
" <tr>\n",
|
|
103
|
+
" <th>4</th>\n",
|
|
104
|
+
" <td>72393</td>\n",
|
|
105
|
+
" <td>एस हेटमायर</td>\n",
|
|
106
|
+
" <td>शिमरॉन हेटमायर</td>\n",
|
|
107
|
+
" <td>Hetmyer, SO</td>\n",
|
|
108
|
+
" <td>M</td>\n",
|
|
109
|
+
" <td>{'year': 1996, 'month': 12, 'date': 26}</td>\n",
|
|
110
|
+
" <td>4</td>\n",
|
|
111
|
+
" <td>shimron-hetmyer</td>\n",
|
|
112
|
+
" </tr>\n",
|
|
113
|
+
" </tbody>\n",
|
|
114
|
+
"</table>\n",
|
|
115
|
+
"</div>"
|
|
116
|
+
],
|
|
117
|
+
"text/plain": [
|
|
118
|
+
" id hindi_name hindi_long_name english_name gender \\\n",
|
|
119
|
+
"0 90143 आर गुरबाज़ रहमानउल्लाह गुरबाज़ Rahmanullah Gurbaz M \n",
|
|
120
|
+
"1 54018 एम रिज़वान मोहम्मद रिज़वान Mohammad Rizwan M \n",
|
|
121
|
+
"2 47686 एस मक़सूद सोहेब मक़सूद Sohaib Maqsood M \n",
|
|
122
|
+
"3 53649 आर आर रुसो राइली रुसो Rossouw, RR M \n",
|
|
123
|
+
"4 72393 एस हेटमायर शिमरॉन हेटमायर Hetmyer, SO M \n",
|
|
124
|
+
"\n",
|
|
125
|
+
" date_of_birth country_id slug \n",
|
|
126
|
+
"0 {'year': 2001, 'month': 11, 'date': 28} 40 rahmanullah-gurbaz \n",
|
|
127
|
+
"1 {'year': 1992, 'month': 6, 'date': 1} 7 mohammad-rizwan \n",
|
|
128
|
+
"2 {'year': 1987, 'month': 4, 'date': 15} 7 sohaib-maqsood \n",
|
|
129
|
+
"3 {'year': 1989, 'month': 10, 'date': 9} 3 rilee-rossouw \n",
|
|
130
|
+
"4 {'year': 1996, 'month': 12, 'date': 26} 4 shimron-hetmyer "
|
|
131
|
+
]
|
|
132
|
+
},
|
|
133
|
+
"execution_count": 3,
|
|
134
|
+
"metadata": {},
|
|
135
|
+
"output_type": "execute_result"
|
|
136
|
+
}
|
|
137
|
+
],
|
|
138
|
+
"source": [
|
|
139
|
+
"df.head()"
|
|
140
|
+
]
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"cell_type": "code",
|
|
144
|
+
"execution_count": 14,
|
|
145
|
+
"metadata": {},
|
|
146
|
+
"outputs": [
|
|
147
|
+
{
|
|
148
|
+
"name": "stdout",
|
|
149
|
+
"output_type": "stream",
|
|
150
|
+
"text": [
|
|
151
|
+
"Number of samples: 2512\n",
|
|
152
|
+
"Number of unique input tokens: 452\n",
|
|
153
|
+
"Number of unique output tokens: 457\n",
|
|
154
|
+
"Max sequence length for inputs: 22\n",
|
|
155
|
+
"Max sequence length for outputs: 26\n"
|
|
156
|
+
]
|
|
157
|
+
}
|
|
158
|
+
],
|
|
159
|
+
"source": [
|
|
160
|
+
"from keras.models import Model\n",
|
|
161
|
+
"from keras.layers import Input, LSTM, Dense\n",
|
|
162
|
+
"import numpy as np\n",
|
|
163
|
+
"\n",
|
|
164
|
+
"batch_size = 64 # Batch size for training.\n",
|
|
165
|
+
"epochs = 10 # Number of epochs to train for.\n",
|
|
166
|
+
"latent_dim = 256 # Latent dimensionality of the encoding space.\n",
|
|
167
|
+
"num_samples = 1000 # Number of samples to train on.\n",
|
|
168
|
+
"# Path to the data txt file on disk.\n",
|
|
169
|
+
"\n",
|
|
170
|
+
"# Vectorize the data.\n",
|
|
171
|
+
"input_texts = df.hindi_name.tolist()\n",
|
|
172
|
+
"target_texts = df.english_name.tolist()\n",
|
|
173
|
+
"input_characters = set()\n",
|
|
174
|
+
"target_characters = set()\n",
|
|
175
|
+
"\n",
|
|
176
|
+
"input_list = df['hindi_name'].tolist()\n",
|
|
177
|
+
"target_list = df['english_name'].tolist()\n",
|
|
178
|
+
"\n",
|
|
179
|
+
"for char in input_list:\n",
|
|
180
|
+
" if char not in input_characters:\n",
|
|
181
|
+
" input_characters.add(char)\n",
|
|
182
|
+
"for char in target_list:\n",
|
|
183
|
+
" if char not in target_characters:\n",
|
|
184
|
+
" target_characters.add(char)\n",
|
|
185
|
+
" \n",
|
|
186
|
+
"\n",
|
|
187
|
+
"input_characters = sorted(list(input_characters))\n",
|
|
188
|
+
"target_characters = sorted(list(target_characters))\n",
|
|
189
|
+
"num_encoder_tokens = len(input_characters)\n",
|
|
190
|
+
"num_decoder_tokens = len(target_characters)\n",
|
|
191
|
+
"max_encoder_seq_length = max([len(txt) for txt in input_texts])\n",
|
|
192
|
+
"max_decoder_seq_length = max([len(txt) for txt in target_texts])\n",
|
|
193
|
+
"\n",
|
|
194
|
+
"print('Number of samples:', len(input_texts))\n",
|
|
195
|
+
"print('Number of unique input tokens:', num_encoder_tokens)\n",
|
|
196
|
+
"print('Number of unique output tokens:', num_decoder_tokens)\n",
|
|
197
|
+
"print('Max sequence length for inputs:', max_encoder_seq_length)\n",
|
|
198
|
+
"print('Max sequence length for outputs:', max_decoder_seq_length)"
|
|
199
|
+
]
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
"cell_type": "code",
|
|
203
|
+
"execution_count": 15,
|
|
204
|
+
"metadata": {},
|
|
205
|
+
"outputs": [
|
|
206
|
+
{
|
|
207
|
+
"ename": "KeyError",
|
|
208
|
+
"evalue": "'आ'",
|
|
209
|
+
"output_type": "error",
|
|
210
|
+
"traceback": [
|
|
211
|
+
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
|
212
|
+
"\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)",
|
|
213
|
+
"\u001b[1;32m<ipython-input-15-094370a4c3f2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m 16\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0minput_text\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_text\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_texts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_texts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 17\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchar\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m \u001b[0mencoder_input_data\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_token_index\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mchar\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 19\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchar\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 20\u001b[0m \u001b[1;31m# decoder_target_data is ahead of decoder_input_data by one timestep\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
|
|
214
|
+
"\u001b[1;31mKeyError\u001b[0m: 'आ'"
|
|
215
|
+
]
|
|
216
|
+
}
|
|
217
|
+
],
|
|
218
|
+
"source": [
|
|
219
|
+
"input_token_index = dict(\n",
|
|
220
|
+
" [(char, i) for i, char in enumerate(input_characters)])\n",
|
|
221
|
+
"target_token_index = dict(\n",
|
|
222
|
+
" [(char, i) for i, char in enumerate(target_characters)])\n",
|
|
223
|
+
"\n",
|
|
224
|
+
"encoder_input_data = np.zeros(\n",
|
|
225
|
+
" (len(input_texts), max_encoder_seq_length, num_encoder_tokens),\n",
|
|
226
|
+
" dtype='float32')\n",
|
|
227
|
+
"decoder_input_data = np.zeros(\n",
|
|
228
|
+
" (len(input_texts), max_decoder_seq_length, num_decoder_tokens),\n",
|
|
229
|
+
" dtype='float32')\n",
|
|
230
|
+
"decoder_target_data = np.zeros(\n",
|
|
231
|
+
" (len(input_texts), max_decoder_seq_length, num_decoder_tokens),\n",
|
|
232
|
+
" dtype='float32')\n",
|
|
233
|
+
"\n",
|
|
234
|
+
"for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):\n",
|
|
235
|
+
" for t, char in enumerate(input_text):\n",
|
|
236
|
+
" encoder_input_data[i, t, input_token_index[char]] = 1.\n",
|
|
237
|
+
" for t, char in enumerate(target_text):\n",
|
|
238
|
+
" # decoder_target_data is ahead of decoder_input_data by one timestep\n",
|
|
239
|
+
" decoder_input_data[i, t, target_token_index[char]] = 1.\n",
|
|
240
|
+
" if t > 0:\n",
|
|
241
|
+
" # decoder_target_data will be ahead by one timestep\n",
|
|
242
|
+
" # and will not include the start character.\n",
|
|
243
|
+
" decoder_target_data[i, t - 1, target_token_index[char]] = 1.\n",
|
|
244
|
+
"\n",
|
|
245
|
+
"# Define an input sequence and process it.\n",
|
|
246
|
+
"encoder_inputs = Input(shape=(None, num_encoder_tokens))\n",
|
|
247
|
+
"encoder = LSTM(latent_dim, return_state=True)\n",
|
|
248
|
+
"encoder_outputs, state_h, state_c = encoder(encoder_inputs)\n",
|
|
249
|
+
"# We discard `encoder_outputs` and only keep the states.\n",
|
|
250
|
+
"encoder_states = [state_h, state_c]\n",
|
|
251
|
+
"# Set up the decoder, using `encoder_states` as initial state.\n",
|
|
252
|
+
"decoder_inputs = Input(shape=(None, num_decoder_tokens))\n",
|
|
253
|
+
"# We set up our decoder to return full output sequences,\n",
|
|
254
|
+
"# and to return internal states as well. We don't use the\n",
|
|
255
|
+
"# return states in the training model, but we will use them in inference.\n",
|
|
256
|
+
"decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)\n",
|
|
257
|
+
"decoder_outputs, _, _ = decoder_lstm(decoder_inputs,\n",
|
|
258
|
+
" initial_state=encoder_states)\n",
|
|
259
|
+
"decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
|
|
260
|
+
"decoder_outputs = decoder_dense(decoder_outputs)\n",
|
|
261
|
+
"\n",
|
|
262
|
+
"# Define the model that will turn\n",
|
|
263
|
+
"# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
|
|
264
|
+
"model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
|
|
265
|
+
"\n",
|
|
266
|
+
"# Run training\n",
|
|
267
|
+
"model.compile(optimizer='rmsprop', loss='categorical_crossentropy')\n",
|
|
268
|
+
"model.fit([encoder_input_data, decoder_input_data], decoder_target_data,\n",
|
|
269
|
+
" batch_size=batch_size,\n",
|
|
270
|
+
" epochs=epochs,\n",
|
|
271
|
+
" validation_split=0.2)\n",
|
|
272
|
+
"# Save model\n",
|
|
273
|
+
"model.save('s2s.h5')"
|
|
274
|
+
]
|
|
275
|
+
},
|
|
276
|
+
{
|
|
277
|
+
"cell_type": "code",
|
|
278
|
+
"execution_count": null,
|
|
279
|
+
"metadata": {},
|
|
280
|
+
"outputs": [],
|
|
281
|
+
"source": []
|
|
282
|
+
}
|
|
283
|
+
],
|
|
284
|
+
"metadata": {
|
|
285
|
+
"kernelspec": {
|
|
286
|
+
"display_name": "Python 3",
|
|
287
|
+
"language": "python",
|
|
288
|
+
"name": "python3"
|
|
289
|
+
},
|
|
290
|
+
"language_info": {
|
|
291
|
+
"codemirror_mode": {
|
|
292
|
+
"name": "ipython",
|
|
293
|
+
"version": 3
|
|
294
|
+
},
|
|
295
|
+
"file_extension": ".py",
|
|
296
|
+
"mimetype": "text/x-python",
|
|
297
|
+
"name": "python",
|
|
298
|
+
"nbconvert_exporter": "python",
|
|
299
|
+
"pygments_lexer": "ipython3",
|
|
300
|
+
"version": "3.6.4"
|
|
301
|
+
}
|
|
302
|
+
},
|
|
303
|
+
"nbformat": 4,
|
|
304
|
+
"nbformat_minor": 2
|
|
305
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from io import StringIO
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@contextmanager
|
|
7
|
+
def capture(command, *args, **kwargs):
|
|
8
|
+
out, sys.stdout = sys.stdout, StringIO()
|
|
9
|
+
command(*args, **kwargs)
|
|
10
|
+
sys.stdout.seek(0)
|
|
11
|
+
yield sys.stdout.read()
|
|
12
|
+
sys.stdout = out
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
Test Hindi to English translation
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import unittest
|
|
9
|
+
|
|
10
|
+
from indicate.hindi2english import HindiToEnglish
|
|
11
|
+
|
|
12
|
+
class TestHindiToEnglish(unittest.TestCase):
|
|
13
|
+
def test_hindi_to_english(self):
|
|
14
|
+
test_inputs = ["राजशेखर चिंतालपति","गौरव सूद"]
|
|
15
|
+
test_outputs = ["rajshekhar chintalpati","gaurav sood"]
|
|
16
|
+
for hindi, english in zip(test_inputs, test_outputs):
|
|
17
|
+
self.assertEqual(HindiToEnglish.transliterate(hindi), english)
|
|
18
|
+
|
|
19
|
+
if __name__ == "__main__":
|
|
20
|
+
unittest.main()
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import argparse
|
|
3
|
+
|
|
4
|
+
from .hindi2english import HindiToEnglish
|
|
5
|
+
|
|
6
|
+
hindi2english = HindiToEnglish.transliterate
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def main(argv=sys.argv[1:]):
|
|
10
|
+
title = "Transliterate from Hindi to English"
|
|
11
|
+
parser = argparse.ArgumentParser(description=title)
|
|
12
|
+
parser.add_argument("--type", default=None, help="hin2eng")
|
|
13
|
+
parser.add_argument("--input", default=None, help="Hindi text")
|
|
14
|
+
args = parser.parse_args(argv)
|
|
15
|
+
print(args)
|
|
16
|
+
if not args.input or not args.type:
|
|
17
|
+
return -1
|
|
18
|
+
|
|
19
|
+
output = ""
|
|
20
|
+
if args.type == "hin2eng":
|
|
21
|
+
output = hindi2english(args.input)
|
|
22
|
+
print(output)
|
|
23
|
+
|
|
24
|
+
return 0
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
if __name__ == "__main__":
|
|
28
|
+
sys.exit(main())
|
indicate/utils.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import tensorflow as tf
|
|
2
|
+
|
|
3
|
+
def sequence_to_chars(tokenizer, sequence):
|
|
4
|
+
"""Convert a sequence of indices back to characters."""
|
|
5
|
+
word_index = tokenizer.word_index
|
|
6
|
+
reverse_map = {val: key for key, val in word_index.items()}
|
|
7
|
+
retext = ''
|
|
8
|
+
|
|
9
|
+
# Convert tensor to numpy array if it's a tensor
|
|
10
|
+
if tf.is_tensor(sequence):
|
|
11
|
+
sequence = sequence.numpy()
|
|
12
|
+
|
|
13
|
+
for q in sequence:
|
|
14
|
+
q_int = int(q) # Convert to integer
|
|
15
|
+
if q_int != 0: # Skip padding token
|
|
16
|
+
retext += reverse_map[q_int]
|
|
17
|
+
return retext
|
|
18
|
+
|
|
19
|
+
def evaluate_sentence(sentence, units, input_lang_tokenizer, target_lang_tokenizer,
|
|
20
|
+
encoder, decoder, max_length_input):
|
|
21
|
+
"""Evaluate/translate a single sentence."""
|
|
22
|
+
# Convert input sentence to token indices
|
|
23
|
+
inputs = [input_lang_tokenizer.word_index[i] for i in sentence]
|
|
24
|
+
inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],
|
|
25
|
+
maxlen=max_length_input,
|
|
26
|
+
padding='post')
|
|
27
|
+
inputs = tf.convert_to_tensor(inputs)
|
|
28
|
+
|
|
29
|
+
# Use the same batch size as decoder.batch_sz for consistency
|
|
30
|
+
inference_batch_size = inputs.shape[0]
|
|
31
|
+
|
|
32
|
+
inputs = tf.tile(inputs, [inference_batch_size, 1]) # Replicate input to match batch size
|
|
33
|
+
|
|
34
|
+
# Initialize encoder state
|
|
35
|
+
enc_start_state = [tf.zeros((inference_batch_size, units)),
|
|
36
|
+
tf.zeros((inference_batch_size, units))]
|
|
37
|
+
|
|
38
|
+
# Get encoder output
|
|
39
|
+
enc_out, enc_h, enc_c = encoder(inputs, enc_start_state)
|
|
40
|
+
|
|
41
|
+
# Initialize decoder with encoder's final state
|
|
42
|
+
dec_state = [enc_h, enc_c]
|
|
43
|
+
|
|
44
|
+
# Prepare decoder input with start token
|
|
45
|
+
dec_input = tf.fill([inference_batch_size, 1],
|
|
46
|
+
target_lang_tokenizer.word_index['^'])
|
|
47
|
+
|
|
48
|
+
outputs = []
|
|
49
|
+
|
|
50
|
+
# Decoding loop
|
|
51
|
+
for t in range(decoder.max_length_output):
|
|
52
|
+
# Call decoder
|
|
53
|
+
predictions, dec_state, _ = decoder(dec_input, dec_state, enc_out)
|
|
54
|
+
|
|
55
|
+
# Get predicted token (use first batch item since all are the same)
|
|
56
|
+
predicted_id = tf.argmax(predictions, axis=-1).numpy()[0, -1]
|
|
57
|
+
|
|
58
|
+
# Append prediction
|
|
59
|
+
outputs.append(predicted_id)
|
|
60
|
+
|
|
61
|
+
# Break if end token is predicted
|
|
62
|
+
if predicted_id == target_lang_tokenizer.word_index['$']:
|
|
63
|
+
break
|
|
64
|
+
|
|
65
|
+
# Update decoder input
|
|
66
|
+
dec_input = tf.fill([inference_batch_size, 1], predicted_id)
|
|
67
|
+
|
|
68
|
+
return tf.convert_to_tensor(outputs)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def translate(sentence, units, input_lang_tokenizer, target_lang_tokenizer,
|
|
72
|
+
encoder, decoder, max_length_input):
|
|
73
|
+
"""Translate a sentence from source to target language."""
|
|
74
|
+
result = evaluate_sentence(sentence,
|
|
75
|
+
units,
|
|
76
|
+
input_lang_tokenizer,
|
|
77
|
+
target_lang_tokenizer,
|
|
78
|
+
encoder,
|
|
79
|
+
decoder,
|
|
80
|
+
max_length_input)
|
|
81
|
+
|
|
82
|
+
# Convert the output tokens back to characters
|
|
83
|
+
translated_text = sequence_to_chars(target_lang_tokenizer, result)
|
|
84
|
+
|
|
85
|
+
# Remove the end token and return
|
|
86
|
+
return translated_text.strip('$')
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2021 Rajasekhar Chintalapati and Gaurav Sood
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: indicate
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Transliterations to/from Indian languages
|
|
5
|
+
Home-page: https://github.com/in-rolls/indicate
|
|
6
|
+
Author: Rajashekar Chintalapati, Gaurav Sood
|
|
7
|
+
Author-email: rajshekar.ch@gmail.com, gsood07@gmail.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Keywords: transliterate indic hindi english
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
Classifier: Topic :: Utilities
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: tqdm
|
|
21
|
+
Requires-Dist: func-timeout
|
|
22
|
+
Requires-Dist: tensorflow==2.18.0
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: check-manifest; extra == "dev"
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: coverage; extra == "test"
|
|
27
|
+
Dynamic: author
|
|
28
|
+
Dynamic: author-email
|
|
29
|
+
Dynamic: classifier
|
|
30
|
+
Dynamic: description
|
|
31
|
+
Dynamic: home-page
|
|
32
|
+
Dynamic: keywords
|
|
33
|
+
Dynamic: license
|
|
34
|
+
Dynamic: provides-extra
|
|
35
|
+
Dynamic: requires-dist
|
|
36
|
+
Dynamic: summary
|
|
37
|
+
|
|
38
|
+
==================================================
|
|
39
|
+
Indicate: Transliterate Indic Languages to English
|
|
40
|
+
==================================================
|
|
41
|
+
|
|
42
|
+
.. image:: https://app.travis-ci.com/in-rolls/indicate.svg?branch=master
|
|
43
|
+
:target: https://travis-ci.org/in-rolls/indicate
|
|
44
|
+
.. image:: https://img.shields.io/pypi/v/indicate.svg
|
|
45
|
+
:target: https://pypi.python.org/pypi/indicate
|
|
46
|
+
.. image:: https://readthedocs.org/projects/indicate/badge/?version=latest
|
|
47
|
+
:target: http://notnews.readthedocs.io/en/latest/?badge=latest
|
|
48
|
+
:alt: Documentation Status
|
|
49
|
+
.. image:: https://static.pepy.tech/badge/indicate
|
|
50
|
+
:target: https://pepy.tech/project/indicate
|
|
51
|
+
|
|
52
|
+
Transliterations to/from Indian languages are still generally low quality. One problem is access to data. Another is that there is no standard transliteration.
|
|
53
|
+
For Hindi--English, we build novel dataset for names using the ESPNcricinfo. For instance, see `here <https://www.espncricinfo.com/hindi/series/pakistan-tour-of-england-2021-1239529/england-vs-pakistan-1st-odi-1239537/full-scorecard>`__ for hindi version of the `english scorecard <https://www.espncricinfo.com/series/pakistan-tour-of-england-2021-1239529/england-vs-pakistan-1st-odi-1239537/full-scorecard>`__.
|
|
54
|
+
We also create a dataset from `election affidavits <https://affidavit.eci.gov.in/CandidateCustomFilter>`__
|
|
55
|
+
We also exploit the `Google Dakshina dataset <https://github.com/google-research-datasets/dakshina>`__.
|
|
56
|
+
|
|
57
|
+
To overcome the fact that there isn't one standard way of transliteration, we provide k-best transliterations.
|
|
58
|
+
|
|
59
|
+
Install
|
|
60
|
+
-------
|
|
61
|
+
We strongly recommend installing `indicate` inside a Python virtual environment
|
|
62
|
+
(see `venv documentation <https://docs.python.org/3/library/venv.html#creating-virtual-environments>`__)
|
|
63
|
+
|
|
64
|
+
::
|
|
65
|
+
|
|
66
|
+
pip install indicate
|
|
67
|
+
|
|
68
|
+
General API
|
|
69
|
+
-----------
|
|
70
|
+
1. transliterate.hindi2english will take Hindi text and translate into English.
|
|
71
|
+
|
|
72
|
+
Examples
|
|
73
|
+
--------
|
|
74
|
+
::
|
|
75
|
+
|
|
76
|
+
from indicate import transliterate
|
|
77
|
+
english_translated = transliterate.hindi2english("हिंदी")
|
|
78
|
+
print(english_translated)
|
|
79
|
+
|
|
80
|
+
output -
|
|
81
|
+
hindi
|
|
82
|
+
|
|
83
|
+
Functions
|
|
84
|
+
----------
|
|
85
|
+
We expose 1 function, which will take Hindi text and transliterate it to English.
|
|
86
|
+
|
|
87
|
+
- **transliterate.hindi2english(input)**
|
|
88
|
+
|
|
89
|
+
- What it does:
|
|
90
|
+
|
|
91
|
+
- Converts given hindi text into English alphabet
|
|
92
|
+
|
|
93
|
+
- Output
|
|
94
|
+
|
|
95
|
+
- Returns text in English
|
|
96
|
+
|
|
97
|
+
Data
|
|
98
|
+
----
|
|
99
|
+
The datasets used to train the model:
|
|
100
|
+
|
|
101
|
+
- `Indian Election affidavits <https://affidavit.eci.gov.in/CandidateCustomFilter>`__
|
|
102
|
+
|
|
103
|
+
- `Google Dakshina dataset <https://github.com/google-research-datasets/dakshina>`__
|
|
104
|
+
|
|
105
|
+
- `ESPN Cric Info <https://www.espncricinfo.com/hindi/series/pakistan-tour-of-england-2021-1239529/england-vs-pakistan-1st-odi-1239537/full-scorecard>`__ for hindi version of the `english scorecard <https://www.espncricinfo.com/series/pakistan-tour-of-england-2021-1239529/england-vs-pakistan-1st-odi-1239537/full-scorecard>`__.
|
|
106
|
+
|
|
107
|
+
- `IIT Bombay English-Hindi Corpus <https://www.cfilt.iitb.ac.in/iitb_parallel/>`__
|
|
108
|
+
|
|
109
|
+
Evaluation
|
|
110
|
+
----------
|
|
111
|
+
Model was evaluated on test dataset of Google Dakshina dataset, Model predicted 73.64% exact matches.
|
|
112
|
+
`Indic-trans <https://github.com/libindic/indic-trans>`__ predicted 63.12% exact matches on Google Dakshina dataset.
|
|
113
|
+
Below is the edit distance metrics on test dataset (0.0 mean exact match, the farther away from 0.0,
|
|
114
|
+
the difference is more between predicted text and actual text)
|
|
115
|
+
|
|
116
|
+
.. image:: https://github.com/in-rolls/indicate/raw/master/images/h2e_ed.png
|
|
117
|
+
:width: 400
|
|
118
|
+
:alt: Edit distance metrics of model on Google Dakshina test dataset
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
Authors
|
|
122
|
+
-------
|
|
123
|
+
|
|
124
|
+
Rajashekar Chintalapati and Gaurav Sood
|
|
125
|
+
|
|
126
|
+
Contributor Code of Conduct
|
|
127
|
+
---------------------------------
|
|
128
|
+
|
|
129
|
+
The project welcomes contributions from everyone! In fact, it depends on
|
|
130
|
+
it. To maintain this welcoming atmosphere, and to collaborate in a fun
|
|
131
|
+
and productive way, we expect contributors to the project to abide by
|
|
132
|
+
the `Contributor Code of
|
|
133
|
+
Conduct <http://contributor-covenant.org/version/1/0/0/>`__.
|
|
134
|
+
|
|
135
|
+
License
|
|
136
|
+
----------
|
|
137
|
+
|
|
138
|
+
The package is released under the `MIT
|
|
139
|
+
License <https://opensource.org/licenses/MIT>`__.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
indicate/__init__.py,sha256=89udvCQOFZ2icsiF0Nmk_KeUWLq3vhQxzA38Hz1ARWA,70
|
|
2
|
+
indicate/decoder.py,sha256=_u7npwJlmF2N--bfwb6hok3QF0ezObj9FyOjhr5ENjY,2567
|
|
3
|
+
indicate/encoder.py,sha256=pkFv7rggaISgdHMQflfi019gbp9FVjsU3ex_-eBqGtI,910
|
|
4
|
+
indicate/hindi2english.py,sha256=T4lcR8NpHhr7-Gq4XFOJJbGSFyvv57AOsUnvt9nuAII,4677
|
|
5
|
+
indicate/logging.py,sha256=AGzjUiPusLS7QJnMdjHDSWRkxC_WoL4jKv0_b2YuhJ0,96
|
|
6
|
+
indicate/transliterate.py,sha256=jrVfz7VaR2p-FthcMXFSDym-UmU9WeFu30ZrXzAzR34,667
|
|
7
|
+
indicate/utils.py,sha256=gD7WjTlBBsh9tPXi8i8wO3d6JDYCtp-rnUNyOnZN-04,3168
|
|
8
|
+
indicate/data/model/hindi_to_english/english_tokens.json,sha256=g_pssuCQyXnE05DRVqing852wHMcxZ-eT5753feCTJs,14480
|
|
9
|
+
indicate/data/model/hindi_to_english/hindi_tokens.json,sha256=mMZYNM8HkKz7eCHFgHGe8KEKbrXMGz-Mq3v-zQ2lJvU,22545
|
|
10
|
+
indicate/data/model/hindi_to_english/saved_weights/decoder.weights.h5,sha256=e-An6CE6yVi2J8Y8cGn7adogLhCD3UVPqNxf8kbX75o,39543312
|
|
11
|
+
indicate/data/model/hindi_to_english/saved_weights/encoder.weights.h5,sha256=6sd66LpwQlCXbK6od_hd2tR_KOzGHG0Q5PqLk7-h7gE,21208200
|
|
12
|
+
indicate/notebooks/Data_Preparation.ipynb,sha256=O65ytgmQRxujLS5LShxFtTnXr4ZTockDkHEPub0cGiw,37341
|
|
13
|
+
indicate/notebooks/Inference.ipynb,sha256=Y2k7oHAfQP579XTB-KL3ahiQTGdFygidBCqNNr6LqEI,72057
|
|
14
|
+
indicate/notebooks/Inference_latest.ipynb,sha256=2cGLK-KHWg3gKAJcABhZbEEuo6uoP2rzWAQt-Q8U9Jc,95628
|
|
15
|
+
indicate/notebooks/Train.ipynb,sha256=ZmG5iZ_n79FSBbyJBNOkTdmz_7JkNnWkng9A6wQ8U-w,58879
|
|
16
|
+
indicate/notebooks/Train_latest.ipynb,sha256=m91MAI2zxCKvts_QVgjc_mWXYi3Wb9a5Dur8EymCIRc,69089
|
|
17
|
+
indicate/notebooks/Transliterate.ipynb,sha256=u4MZ-9za3S0Dvz87QtKZnh1CIo8uYGR2RYYCj7hlA0I,308328
|
|
18
|
+
indicate/notebooks/transliterate_cricket.ipynb,sha256=Dr0iu3r0XwIF0a_DwQqQZ7qauKk4AfLkFEWjglRDb58,13072
|
|
19
|
+
indicate/tests/__init__.py,sha256=ezvdIE58d5GooM9wrJ51pNk9U3pb-LWMWE2nKYqiQP0,275
|
|
20
|
+
indicate/tests/test_010_hindi_translate.py,sha256=69XHcpDlRtOw9gMYv11cSBBOSsG31q_pfqi0mT9K1EE,590
|
|
21
|
+
indicate-0.2.0.dist-info/LICENSE,sha256=iEky6BOuLfXUmOQ67jpWOyqapJVj37qeXqUIHIhVHYM,1106
|
|
22
|
+
indicate-0.2.0.dist-info/METADATA,sha256=wxXC9ZmUTgWUZ6ysC17hCIlzHnT5W8_ph7b5MXnirv8,5269
|
|
23
|
+
indicate-0.2.0.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
|
|
24
|
+
indicate-0.2.0.dist-info/entry_points.txt,sha256=LrGK3DYYXfR5vWPG5RvW9wj7R6ZoSfa0iuxvXIkmZec,62
|
|
25
|
+
indicate-0.2.0.dist-info/top_level.txt,sha256=MH_uxn6O8-g_WCij25wOlx2cNuv9-mScUuZDBPSb2AA,9
|
|
26
|
+
indicate-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
indicate
|