indicate 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indicate/__init__.py +3 -0
- indicate/data/model/hindi_to_english/english_tokens.json +1 -0
- indicate/data/model/hindi_to_english/hindi_tokens.json +1 -0
- indicate/data/model/hindi_to_english/saved_weights/decoder.weights.h5 +0 -0
- indicate/data/model/hindi_to_english/saved_weights/encoder.weights.h5 +0 -0
- indicate/decoder.py +62 -0
- indicate/encoder.py +24 -0
- indicate/hindi2english.py +138 -0
- indicate/logging.py +6 -0
- indicate/notebooks/Data_Preparation.ipynb +1 -0
- indicate/notebooks/Inference.ipynb +1 -0
- indicate/notebooks/Inference_latest.ipynb +2331 -0
- indicate/notebooks/Train.ipynb +1 -0
- indicate/notebooks/Train_latest.ipynb +1746 -0
- indicate/notebooks/Transliterate.ipynb +2501 -0
- indicate/notebooks/transliterate_cricket.ipynb +305 -0
- indicate/tests/__init__.py +12 -0
- indicate/tests/test_010_hindi_translate.py +20 -0
- indicate/transliterate.py +28 -0
- indicate/utils.py +86 -0
- indicate-0.2.0.dist-info/LICENSE +21 -0
- indicate-0.2.0.dist-info/METADATA +139 -0
- indicate-0.2.0.dist-info/RECORD +26 -0
- indicate-0.2.0.dist-info/WHEEL +6 -0
- indicate-0.2.0.dist-info/entry_points.txt +2 -0
- indicate-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2331 @@
|
|
|
1
|
+
{
|
|
2
|
+
"nbformat": 4,
|
|
3
|
+
"nbformat_minor": 0,
|
|
4
|
+
"metadata": {
|
|
5
|
+
"colab": {
|
|
6
|
+
"provenance": [],
|
|
7
|
+
"machine_shape": "hm"
|
|
8
|
+
},
|
|
9
|
+
"kernelspec": {
|
|
10
|
+
"name": "python3",
|
|
11
|
+
"display_name": "Python 3"
|
|
12
|
+
},
|
|
13
|
+
"language_info": {
|
|
14
|
+
"name": "python"
|
|
15
|
+
},
|
|
16
|
+
"accelerator": "GPU"
|
|
17
|
+
},
|
|
18
|
+
"cells": [
|
|
19
|
+
{
|
|
20
|
+
"cell_type": "code",
|
|
21
|
+
"source": [
|
|
22
|
+
"%cd /content/drive/MyDrive/Colab/indicate/"
|
|
23
|
+
],
|
|
24
|
+
"metadata": {
|
|
25
|
+
"colab": {
|
|
26
|
+
"base_uri": "https://localhost:8080/"
|
|
27
|
+
},
|
|
28
|
+
"id": "k8cdWuWzeR_6",
|
|
29
|
+
"outputId": "3ebba234-6591-4d63-a405-6d7c3cfe7583"
|
|
30
|
+
},
|
|
31
|
+
"execution_count": 1,
|
|
32
|
+
"outputs": [
|
|
33
|
+
{
|
|
34
|
+
"output_type": "stream",
|
|
35
|
+
"name": "stdout",
|
|
36
|
+
"text": [
|
|
37
|
+
"/content/drive/MyDrive/Colab/indicate\n"
|
|
38
|
+
]
|
|
39
|
+
}
|
|
40
|
+
]
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"cell_type": "code",
|
|
44
|
+
"execution_count": 2,
|
|
45
|
+
"metadata": {
|
|
46
|
+
"id": "K0BNndbWw0Rd"
|
|
47
|
+
},
|
|
48
|
+
"outputs": [],
|
|
49
|
+
"source": [
|
|
50
|
+
"import os\n",
|
|
51
|
+
"import json\n",
|
|
52
|
+
"\n",
|
|
53
|
+
"import pandas as pd\n",
|
|
54
|
+
"import tensorflow as tf"
|
|
55
|
+
]
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"cell_type": "code",
|
|
59
|
+
"source": [
|
|
60
|
+
"class Encoder(tf.keras.Model):\n",
|
|
61
|
+
" def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
|
|
62
|
+
" super(Encoder, self).__init__()\n",
|
|
63
|
+
" self.batch_sz = batch_sz\n",
|
|
64
|
+
" self.enc_units = enc_units\n",
|
|
65
|
+
" self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
|
|
66
|
+
"\n",
|
|
67
|
+
" ##-------- LSTM layer in Encoder ------- ##\n",
|
|
68
|
+
" self.lstm_layer = tf.keras.layers.LSTM(self.enc_units,\n",
|
|
69
|
+
" return_sequences=True,\n",
|
|
70
|
+
" return_state=True,\n",
|
|
71
|
+
" recurrent_initializer='glorot_uniform')\n",
|
|
72
|
+
"\n",
|
|
73
|
+
"\n",
|
|
74
|
+
"\n",
|
|
75
|
+
" def call(self, x, hidden, training=None):\n",
|
|
76
|
+
" x = self.embedding(x)\n",
|
|
77
|
+
" output, h, c = self.lstm_layer(x, initial_state = hidden)\n",
|
|
78
|
+
" return output, h, c\n",
|
|
79
|
+
"\n",
|
|
80
|
+
" def initialize_hidden_state(self):\n",
|
|
81
|
+
" return [tf.zeros((self.batch_sz, self.enc_units)), tf.zeros((self.batch_sz, self.enc_units))]"
|
|
82
|
+
],
|
|
83
|
+
"metadata": {
|
|
84
|
+
"id": "I8kzDy_SPRWH"
|
|
85
|
+
},
|
|
86
|
+
"execution_count": 3,
|
|
87
|
+
"outputs": []
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"cell_type": "code",
|
|
91
|
+
"source": [
|
|
92
|
+
"class Decoder(tf.keras.Model):\n",
|
|
93
|
+
" def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz,\n",
|
|
94
|
+
" max_length_input, max_length_output, attention_type='luong'):\n",
|
|
95
|
+
" super(Decoder, self).__init__()\n",
|
|
96
|
+
" self.batch_sz = batch_sz\n",
|
|
97
|
+
" self.dec_units = dec_units\n",
|
|
98
|
+
" self.attention_type = attention_type\n",
|
|
99
|
+
" self.max_length_input = max_length_input\n",
|
|
100
|
+
" self.max_length_output = max_length_output\n",
|
|
101
|
+
"\n",
|
|
102
|
+
" # Embedding Layer\n",
|
|
103
|
+
" self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
|
|
104
|
+
"\n",
|
|
105
|
+
" # Final Dense layer on which softmax will be applied\n",
|
|
106
|
+
" self.fc = tf.keras.layers.Dense(vocab_size)\n",
|
|
107
|
+
"\n",
|
|
108
|
+
" # Define the fundamental cell for decoder recurrent structure\n",
|
|
109
|
+
" self.decoder_rnn_cell = tf.keras.layers.LSTMCell(self.dec_units)\n",
|
|
110
|
+
" self.decoder_rnn = tf.keras.layers.RNN(self.decoder_rnn_cell, return_sequences=True, return_state=True)\n",
|
|
111
|
+
"\n",
|
|
112
|
+
" # Create attention mechanism\n",
|
|
113
|
+
" # For Luong attention: project the query to match the encoder output's dimension (dec_units)\n",
|
|
114
|
+
" if self.attention_type == 'luong':\n",
|
|
115
|
+
" self.query_layer = tf.keras.layers.Dense(dec_units)\n",
|
|
116
|
+
"\n",
|
|
117
|
+
" self.attention_mechanism = self.build_attention_mechanism(None)\n",
|
|
118
|
+
"\n",
|
|
119
|
+
" def build_attention_mechanism(self, memory):\n",
|
|
120
|
+
" if self.attention_type == 'bahdanau':\n",
|
|
121
|
+
" return tf.keras.layers.AdditiveAttention()\n",
|
|
122
|
+
" else:\n",
|
|
123
|
+
" return tf.keras.layers.Attention()\n",
|
|
124
|
+
"\n",
|
|
125
|
+
" def call(self, inputs, initial_state, encoder_outputs):\n",
|
|
126
|
+
" # Get the embeddings of the inputs\n",
|
|
127
|
+
" x = self.embedding(inputs)\n",
|
|
128
|
+
"\n",
|
|
129
|
+
" if self.attention_type == 'luong':\n",
|
|
130
|
+
" query = self.query_layer(x)\n",
|
|
131
|
+
" else:\n",
|
|
132
|
+
" query = x\n",
|
|
133
|
+
"\n",
|
|
134
|
+
" # Feature dimensions to the attention layer\n",
|
|
135
|
+
" attention_output, attention_weights = self.attention_mechanism([query, encoder_outputs], return_attention_scores=True)\n",
|
|
136
|
+
"\n",
|
|
137
|
+
" # Concatenate the attention output with the LSTM Cell output\n",
|
|
138
|
+
" lstm_input = tf.concat([attention_output, x], axis=-1)\n",
|
|
139
|
+
"\n",
|
|
140
|
+
" # Process through the LSTM cell\n",
|
|
141
|
+
" outputs, state_h, state_c = self.decoder_rnn(lstm_input, initial_state)\n",
|
|
142
|
+
"\n",
|
|
143
|
+
" # Pass through the final dense layer\n",
|
|
144
|
+
" outputs = self.fc(outputs)\n",
|
|
145
|
+
"\n",
|
|
146
|
+
" return outputs, (state_h, state_c), attention_weights\n",
|
|
147
|
+
"\n",
|
|
148
|
+
" def build_initial_state(self, batch_sz, encoder_state):\n",
|
|
149
|
+
" hidden_state, cell_state = encoder_state\n",
|
|
150
|
+
" return [hidden_state, cell_state]"
|
|
151
|
+
],
|
|
152
|
+
"metadata": {
|
|
153
|
+
"id": "1o5NB8_QPtHL"
|
|
154
|
+
},
|
|
155
|
+
"execution_count": 4,
|
|
156
|
+
"outputs": []
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
"cell_type": "markdown",
|
|
160
|
+
"source": [
|
|
161
|
+
"# Hyperparameters"
|
|
162
|
+
],
|
|
163
|
+
"metadata": {
|
|
164
|
+
"id": "x34MMPFkgw16"
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
"cell_type": "code",
|
|
169
|
+
"source": [
|
|
170
|
+
"embedding_dim = 256\n",
|
|
171
|
+
"units = 1024\n",
|
|
172
|
+
"BATCH_SIZE = 64\n",
|
|
173
|
+
"BUFFER_SIZE = 120000\n",
|
|
174
|
+
"\n",
|
|
175
|
+
"max_length_input = 47\n",
|
|
176
|
+
"max_length_output = 173"
|
|
177
|
+
],
|
|
178
|
+
"metadata": {
|
|
179
|
+
"id": "7xTucF3FgwKK"
|
|
180
|
+
},
|
|
181
|
+
"execution_count": 35,
|
|
182
|
+
"outputs": []
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
"cell_type": "markdown",
|
|
186
|
+
"source": [
|
|
187
|
+
"# Load Vocab"
|
|
188
|
+
],
|
|
189
|
+
"metadata": {
|
|
190
|
+
"id": "F0lrvAYvhZcW"
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
"cell_type": "code",
|
|
195
|
+
"source": [
|
|
196
|
+
"%ls"
|
|
197
|
+
],
|
|
198
|
+
"metadata": {
|
|
199
|
+
"colab": {
|
|
200
|
+
"base_uri": "https://localhost:8080/"
|
|
201
|
+
},
|
|
202
|
+
"id": "fHfOd7s4Ss7A",
|
|
203
|
+
"outputId": "de373d25-2842-4c1f-d5ec-e6ca2d9f7c6a"
|
|
204
|
+
},
|
|
205
|
+
"execution_count": 6,
|
|
206
|
+
"outputs": [
|
|
207
|
+
{
|
|
208
|
+
"output_type": "stream",
|
|
209
|
+
"name": "stdout",
|
|
210
|
+
"text": [
|
|
211
|
+
"\u001b[0m\u001b[01;34mdata\u001b[0m/ encoder.pt Inference_checkpoints_latest.ipynb Train_tf_latest.ipynb\n",
|
|
212
|
+
"data.ipynb english_tokens.json train.ipynb train_w_lstm.ipynb\n",
|
|
213
|
+
"decoder.pt hindi_tokens.json train_old_data.ipynb train_w_transformers.ipynb\n"
|
|
214
|
+
]
|
|
215
|
+
}
|
|
216
|
+
]
|
|
217
|
+
},
|
|
218
|
+
{
|
|
219
|
+
"cell_type": "code",
|
|
220
|
+
"source": [
|
|
221
|
+
"with open('hindi_tokens.json') as f:\n",
|
|
222
|
+
" data = json.load(f)\n",
|
|
223
|
+
" input_lang_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(data)\n",
|
|
224
|
+
"\n",
|
|
225
|
+
"with open('english_tokens.json') as f:\n",
|
|
226
|
+
" data = json.load(f)\n",
|
|
227
|
+
" target_lang_tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(data)\n",
|
|
228
|
+
"\n",
|
|
229
|
+
"vocab_inp_size = len(input_lang_tokenizer.word_index)+1\n",
|
|
230
|
+
"vocab_tar_size = len(target_lang_tokenizer.word_index)+1"
|
|
231
|
+
],
|
|
232
|
+
"metadata": {
|
|
233
|
+
"id": "WRdqxqINhY_M"
|
|
234
|
+
},
|
|
235
|
+
"execution_count": 7,
|
|
236
|
+
"outputs": []
|
|
237
|
+
},
|
|
238
|
+
{
|
|
239
|
+
"cell_type": "code",
|
|
240
|
+
"source": [
|
|
241
|
+
"print(f\"input vocab size - {vocab_inp_size}\")\n",
|
|
242
|
+
"print(f\"target vocab size - {vocab_tar_size}\")\n",
|
|
243
|
+
"\n",
|
|
244
|
+
"print(f\"max length of input - {max_length_input}\")\n",
|
|
245
|
+
"print(f\"max length of target - {max_length_output}\")"
|
|
246
|
+
],
|
|
247
|
+
"metadata": {
|
|
248
|
+
"colab": {
|
|
249
|
+
"base_uri": "https://localhost:8080/"
|
|
250
|
+
},
|
|
251
|
+
"id": "Jc0zHxL9FV1i",
|
|
252
|
+
"outputId": "0518923a-d7f6-4ea5-a7bb-4c11b0a86919"
|
|
253
|
+
},
|
|
254
|
+
"execution_count": 36,
|
|
255
|
+
"outputs": [
|
|
256
|
+
{
|
|
257
|
+
"output_type": "stream",
|
|
258
|
+
"name": "stdout",
|
|
259
|
+
"text": [
|
|
260
|
+
"input vocab size - 200\n",
|
|
261
|
+
"target vocab size - 137\n",
|
|
262
|
+
"max length of input - 47\n",
|
|
263
|
+
"max length of target - 173\n"
|
|
264
|
+
]
|
|
265
|
+
}
|
|
266
|
+
]
|
|
267
|
+
},
|
|
268
|
+
{
|
|
269
|
+
"cell_type": "markdown",
|
|
270
|
+
"source": [
|
|
271
|
+
"# Encoder, Decoder, optimizer"
|
|
272
|
+
],
|
|
273
|
+
"metadata": {
|
|
274
|
+
"id": "YA24vPP0khOK"
|
|
275
|
+
}
|
|
276
|
+
},
|
|
277
|
+
{
|
|
278
|
+
"cell_type": "code",
|
|
279
|
+
"source": [
|
|
280
|
+
"encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
|
|
281
|
+
"decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE,\n",
|
|
282
|
+
" max_length_input, max_length_output, 'luong')\n",
|
|
283
|
+
"optimizer = tf.keras.optimizers.Adam()"
|
|
284
|
+
],
|
|
285
|
+
"metadata": {
|
|
286
|
+
"id": "5UtLbxPegpsM"
|
|
287
|
+
},
|
|
288
|
+
"execution_count": 71,
|
|
289
|
+
"outputs": []
|
|
290
|
+
},
|
|
291
|
+
{
|
|
292
|
+
"cell_type": "code",
|
|
293
|
+
"source": [
|
|
294
|
+
"encoder.embedding.variables"
|
|
295
|
+
],
|
|
296
|
+
"metadata": {
|
|
297
|
+
"colab": {
|
|
298
|
+
"base_uri": "https://localhost:8080/"
|
|
299
|
+
},
|
|
300
|
+
"id": "LkpffmEAABYD",
|
|
301
|
+
"outputId": "d6089465-ec54-43f7-d99b-47cfe46ba6d2"
|
|
302
|
+
},
|
|
303
|
+
"execution_count": 72,
|
|
304
|
+
"outputs": [
|
|
305
|
+
{
|
|
306
|
+
"output_type": "execute_result",
|
|
307
|
+
"data": {
|
|
308
|
+
"text/plain": [
|
|
309
|
+
"[]"
|
|
310
|
+
]
|
|
311
|
+
},
|
|
312
|
+
"metadata": {},
|
|
313
|
+
"execution_count": 72
|
|
314
|
+
}
|
|
315
|
+
]
|
|
316
|
+
},
|
|
317
|
+
{
|
|
318
|
+
"cell_type": "code",
|
|
319
|
+
"source": [
|
|
320
|
+
"decoder.embedding.variables"
|
|
321
|
+
],
|
|
322
|
+
"metadata": {
|
|
323
|
+
"colab": {
|
|
324
|
+
"base_uri": "https://localhost:8080/"
|
|
325
|
+
},
|
|
326
|
+
"id": "SJ6LSOhLADJ9",
|
|
327
|
+
"outputId": "4d023564-3f54-4e4d-fa4c-ce56a0eb17ba"
|
|
328
|
+
},
|
|
329
|
+
"execution_count": 73,
|
|
330
|
+
"outputs": [
|
|
331
|
+
{
|
|
332
|
+
"output_type": "execute_result",
|
|
333
|
+
"data": {
|
|
334
|
+
"text/plain": [
|
|
335
|
+
"[]"
|
|
336
|
+
]
|
|
337
|
+
},
|
|
338
|
+
"metadata": {},
|
|
339
|
+
"execution_count": 73
|
|
340
|
+
}
|
|
341
|
+
]
|
|
342
|
+
},
|
|
343
|
+
{
|
|
344
|
+
"cell_type": "code",
|
|
345
|
+
"source": [
|
|
346
|
+
"# sample input\n",
|
|
347
|
+
"example_input_batch = tf.random.uniform(shape=[64,47])\n",
|
|
348
|
+
"sample_hidden = encoder.initialize_hidden_state()\n",
|
|
349
|
+
"sample_output, sample_h, sample_c = encoder(example_input_batch, sample_hidden)\n",
|
|
350
|
+
"print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
|
|
351
|
+
"print ('Encoder h vecotr shape: (batch size, units) {}'.format(sample_h.shape))\n",
|
|
352
|
+
"print ('Encoder c vector shape: (batch size, units) {}'.format(sample_c.shape))"
|
|
353
|
+
],
|
|
354
|
+
"metadata": {
|
|
355
|
+
"colab": {
|
|
356
|
+
"base_uri": "https://localhost:8080/"
|
|
357
|
+
},
|
|
358
|
+
"id": "KtjT9vaXssne",
|
|
359
|
+
"outputId": "169d40f5-467e-40ff-9838-6ade7ea43130"
|
|
360
|
+
},
|
|
361
|
+
"execution_count": 74,
|
|
362
|
+
"outputs": [
|
|
363
|
+
{
|
|
364
|
+
"output_type": "stream",
|
|
365
|
+
"name": "stdout",
|
|
366
|
+
"text": [
|
|
367
|
+
"Encoder output shape: (batch size, sequence length, units) (64, 47, 1024)\n",
|
|
368
|
+
"Encoder h vecotr shape: (batch size, units) (64, 1024)\n",
|
|
369
|
+
"Encoder c vector shape: (batch size, units) (64, 1024)\n"
|
|
370
|
+
]
|
|
371
|
+
}
|
|
372
|
+
]
|
|
373
|
+
},
|
|
374
|
+
{
|
|
375
|
+
"cell_type": "code",
|
|
376
|
+
"source": [
|
|
377
|
+
"encoder.embedding.variables"
|
|
378
|
+
],
|
|
379
|
+
"metadata": {
|
|
380
|
+
"colab": {
|
|
381
|
+
"base_uri": "https://localhost:8080/"
|
|
382
|
+
},
|
|
383
|
+
"id": "l7N2B7otAJox",
|
|
384
|
+
"outputId": "d3ddfced-b16d-4757-9443-29d34039c0db"
|
|
385
|
+
},
|
|
386
|
+
"execution_count": 75,
|
|
387
|
+
"outputs": [
|
|
388
|
+
{
|
|
389
|
+
"output_type": "execute_result",
|
|
390
|
+
"data": {
|
|
391
|
+
"text/plain": [
|
|
392
|
+
"[<Variable path=encoder_3/embedding_6/embeddings, shape=(200, 256), dtype=float32, value=[[-0.00686686 -0.0010526 0.03595546 ... 0.04294099 -0.01272945\n",
|
|
393
|
+
" 0.00456731]\n",
|
|
394
|
+
" [ 0.00348008 -0.03233007 -0.03155342 ... 0.02613237 -0.00144179\n",
|
|
395
|
+
" 0.02792479]\n",
|
|
396
|
+
" [-0.00382755 -0.03357872 -0.0355198 ... 0.02060625 0.04350809\n",
|
|
397
|
+
" -0.03944601]\n",
|
|
398
|
+
" ...\n",
|
|
399
|
+
" [ 0.02380589 0.04296638 0.01664678 ... -0.03105517 0.02877896\n",
|
|
400
|
+
" 0.03039264]\n",
|
|
401
|
+
" [ 0.03506743 -0.00562978 -0.04282135 ... -0.01053991 0.0236184\n",
|
|
402
|
+
" 0.04497857]\n",
|
|
403
|
+
" [ 0.0213498 0.01751263 0.02058559 ... -0.03325355 0.03340849\n",
|
|
404
|
+
" -0.04312873]]>]"
|
|
405
|
+
]
|
|
406
|
+
},
|
|
407
|
+
"metadata": {},
|
|
408
|
+
"execution_count": 75
|
|
409
|
+
}
|
|
410
|
+
]
|
|
411
|
+
},
|
|
412
|
+
{
|
|
413
|
+
"cell_type": "code",
|
|
414
|
+
"source": [
|
|
415
|
+
"# Test decoder stack\n",
|
|
416
|
+
"sample_x = tf.random.uniform((BATCH_SIZE, max_length_output))\n",
|
|
417
|
+
"initial_state = decoder.build_initial_state(BATCH_SIZE, [sample_h, sample_c])\n",
|
|
418
|
+
"\n",
|
|
419
|
+
"\n",
|
|
420
|
+
"sample_decoder_outputs = decoder(sample_x, initial_state, sample_output)\n",
|
|
421
|
+
"logits, new_state_h, new_state_c = sample_decoder_outputs\n",
|
|
422
|
+
"\n",
|
|
423
|
+
"print(\"Decoder Outputs Shape: \", sample_decoder_outputs[0].shape)"
|
|
424
|
+
],
|
|
425
|
+
"metadata": {
|
|
426
|
+
"colab": {
|
|
427
|
+
"base_uri": "https://localhost:8080/"
|
|
428
|
+
},
|
|
429
|
+
"id": "1o-WETR7OdLp",
|
|
430
|
+
"outputId": "4026884a-4395-40cf-a248-0f7a16455c3c"
|
|
431
|
+
},
|
|
432
|
+
"execution_count": 76,
|
|
433
|
+
"outputs": [
|
|
434
|
+
{
|
|
435
|
+
"output_type": "stream",
|
|
436
|
+
"name": "stdout",
|
|
437
|
+
"text": [
|
|
438
|
+
"Decoder Outputs Shape: (64, 173, 137)\n"
|
|
439
|
+
]
|
|
440
|
+
}
|
|
441
|
+
]
|
|
442
|
+
},
|
|
443
|
+
{
|
|
444
|
+
"cell_type": "code",
|
|
445
|
+
"source": [
|
|
446
|
+
"decoder.embedding.variables"
|
|
447
|
+
],
|
|
448
|
+
"metadata": {
|
|
449
|
+
"colab": {
|
|
450
|
+
"base_uri": "https://localhost:8080/"
|
|
451
|
+
},
|
|
452
|
+
"id": "sz-LXyVxAM4S",
|
|
453
|
+
"outputId": "24f480c9-243f-47de-8a5a-ea91156817f8"
|
|
454
|
+
},
|
|
455
|
+
"execution_count": 77,
|
|
456
|
+
"outputs": [
|
|
457
|
+
{
|
|
458
|
+
"output_type": "execute_result",
|
|
459
|
+
"data": {
|
|
460
|
+
"text/plain": [
|
|
461
|
+
"[<Variable path=decoder_3/embedding_7/embeddings, shape=(137, 256), dtype=float32, value=[[-0.00802499 0.03800768 -0.04571027 ... -0.04793583 0.01777229\n",
|
|
462
|
+
" 0.00134502]\n",
|
|
463
|
+
" [ 0.02571804 -0.0299448 -0.00607223 ... -0.02471362 0.00100492\n",
|
|
464
|
+
" -0.0391422 ]\n",
|
|
465
|
+
" [-0.04521506 -0.015232 -0.01456056 ... 0.02043522 0.04228887\n",
|
|
466
|
+
" -0.01479077]\n",
|
|
467
|
+
" ...\n",
|
|
468
|
+
" [-0.00173284 0.00543803 -0.02407183 ... -0.04090531 -0.0238791\n",
|
|
469
|
+
" -0.00550655]\n",
|
|
470
|
+
" [ 0.0098604 -0.01415495 0.00713523 ... 0.03206344 0.04017801\n",
|
|
471
|
+
" -0.04943197]\n",
|
|
472
|
+
" [-0.0247746 0.01139496 -0.0258055 ... -0.02702575 0.03122982\n",
|
|
473
|
+
" 0.03643204]]>]"
|
|
474
|
+
]
|
|
475
|
+
},
|
|
476
|
+
"metadata": {},
|
|
477
|
+
"execution_count": 77
|
|
478
|
+
}
|
|
479
|
+
]
|
|
480
|
+
},
|
|
481
|
+
{
|
|
482
|
+
"cell_type": "markdown",
|
|
483
|
+
"source": [
|
|
484
|
+
"# Restore the latest checkpoint"
|
|
485
|
+
],
|
|
486
|
+
"metadata": {
|
|
487
|
+
"id": "68fvuRtPd_D-"
|
|
488
|
+
}
|
|
489
|
+
},
|
|
490
|
+
{
|
|
491
|
+
"cell_type": "code",
|
|
492
|
+
"source": [
|
|
493
|
+
"checkpoint_dir = './data/training_checkpoints'\n",
|
|
494
|
+
"checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
|
|
495
|
+
" encoder=encoder,\n",
|
|
496
|
+
" decoder=decoder)"
|
|
497
|
+
],
|
|
498
|
+
"metadata": {
|
|
499
|
+
"id": "kou79vmeeNt9"
|
|
500
|
+
},
|
|
501
|
+
"execution_count": 78,
|
|
502
|
+
"outputs": []
|
|
503
|
+
},
|
|
504
|
+
{
|
|
505
|
+
"cell_type": "code",
|
|
506
|
+
"source": [
|
|
507
|
+
"tf.train.latest_checkpoint(checkpoint_dir)"
|
|
508
|
+
],
|
|
509
|
+
"metadata": {
|
|
510
|
+
"colab": {
|
|
511
|
+
"base_uri": "https://localhost:8080/",
|
|
512
|
+
"height": 35
|
|
513
|
+
},
|
|
514
|
+
"id": "Si8q4-lBB4-G",
|
|
515
|
+
"outputId": "c2eaddca-f228-4514-99c8-84dc875003df"
|
|
516
|
+
},
|
|
517
|
+
"execution_count": 79,
|
|
518
|
+
"outputs": [
|
|
519
|
+
{
|
|
520
|
+
"output_type": "execute_result",
|
|
521
|
+
"data": {
|
|
522
|
+
"text/plain": [
|
|
523
|
+
"'./data/training_checkpoints/ckpt-5'"
|
|
524
|
+
],
|
|
525
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
526
|
+
"type": "string"
|
|
527
|
+
}
|
|
528
|
+
},
|
|
529
|
+
"metadata": {},
|
|
530
|
+
"execution_count": 79
|
|
531
|
+
}
|
|
532
|
+
]
|
|
533
|
+
},
|
|
534
|
+
{
|
|
535
|
+
"cell_type": "code",
|
|
536
|
+
"source": [
|
|
537
|
+
"checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
|
|
538
|
+
],
|
|
539
|
+
"metadata": {
|
|
540
|
+
"colab": {
|
|
541
|
+
"base_uri": "https://localhost:8080/"
|
|
542
|
+
},
|
|
543
|
+
"id": "zg4vWb7jDTo3",
|
|
544
|
+
"outputId": "f6c4ba41-79f8-4ad1-bfd2-1e891e5a06fe"
|
|
545
|
+
},
|
|
546
|
+
"execution_count": 80,
|
|
547
|
+
"outputs": [
|
|
548
|
+
{
|
|
549
|
+
"output_type": "execute_result",
|
|
550
|
+
"data": {
|
|
551
|
+
"text/plain": [
|
|
552
|
+
"<tensorflow.python.checkpoint.checkpoint.CheckpointLoadStatus at 0x7f18bef3dfd0>"
|
|
553
|
+
]
|
|
554
|
+
},
|
|
555
|
+
"metadata": {},
|
|
556
|
+
"execution_count": 80
|
|
557
|
+
}
|
|
558
|
+
]
|
|
559
|
+
},
|
|
560
|
+
{
|
|
561
|
+
"cell_type": "code",
|
|
562
|
+
"source": [
|
|
563
|
+
"encoder.embedding.variables"
|
|
564
|
+
],
|
|
565
|
+
"metadata": {
|
|
566
|
+
"colab": {
|
|
567
|
+
"base_uri": "https://localhost:8080/"
|
|
568
|
+
},
|
|
569
|
+
"id": "3XnDYSVy_kzK",
|
|
570
|
+
"outputId": "79649d0c-92ef-49e6-b336-6cfa79746568"
|
|
571
|
+
},
|
|
572
|
+
"execution_count": 81,
|
|
573
|
+
"outputs": [
|
|
574
|
+
{
|
|
575
|
+
"output_type": "execute_result",
|
|
576
|
+
"data": {
|
|
577
|
+
"text/plain": [
|
|
578
|
+
"[<Variable path=encoder_3/embedding_6/embeddings, shape=(200, 256), dtype=float32, value=[[ 0.26681638 0.2754885 -0.03031236 ... 0.22623836 0.27917588\n",
|
|
579
|
+
" 0.297866 ]\n",
|
|
580
|
+
" [-0.04418039 0.02793723 0.11009125 ... 0.00912727 -0.23427705\n",
|
|
581
|
+
" 0.12682076]\n",
|
|
582
|
+
" [ 0.0630974 -0.01531512 0.04576264 ... 0.12889968 0.08963458\n",
|
|
583
|
+
" 0.15908036]\n",
|
|
584
|
+
" ...\n",
|
|
585
|
+
" [ 0.06958606 0.07699304 0.01560043 ... -0.01147923 -0.00868758\n",
|
|
586
|
+
" 0.04236265]\n",
|
|
587
|
+
" [ 0.05035917 -0.08790015 -0.06968533 ... 0.0825654 0.11627398\n",
|
|
588
|
+
" -0.05767357]\n",
|
|
589
|
+
" [-0.0197444 -0.02649814 0.03845655 ... 0.03345997 0.03370145\n",
|
|
590
|
+
" -0.03743879]]>]"
|
|
591
|
+
]
|
|
592
|
+
},
|
|
593
|
+
"metadata": {},
|
|
594
|
+
"execution_count": 81
|
|
595
|
+
}
|
|
596
|
+
]
|
|
597
|
+
},
|
|
598
|
+
{
|
|
599
|
+
"cell_type": "code",
|
|
600
|
+
"source": [
|
|
601
|
+
"decoder.embedding.variables"
|
|
602
|
+
],
|
|
603
|
+
"metadata": {
|
|
604
|
+
"colab": {
|
|
605
|
+
"base_uri": "https://localhost:8080/"
|
|
606
|
+
},
|
|
607
|
+
"id": "8EcQxPd4uEsg",
|
|
608
|
+
"outputId": "0d5acf0d-3e18-48d4-ef23-0561d76761a3"
|
|
609
|
+
},
|
|
610
|
+
"execution_count": 82,
|
|
611
|
+
"outputs": [
|
|
612
|
+
{
|
|
613
|
+
"output_type": "execute_result",
|
|
614
|
+
"data": {
|
|
615
|
+
"text/plain": [
|
|
616
|
+
"[<Variable path=decoder_3/embedding_7/embeddings, shape=(137, 256), dtype=float32, value=[[-0.03412235 -0.00510821 0.04777941 ... 0.03236498 0.01061462\n",
|
|
617
|
+
" 0.00176196]\n",
|
|
618
|
+
" [ 0.14992759 -0.07939371 -0.24454711 ... -0.29890257 0.04763342\n",
|
|
619
|
+
" 0.10742956]\n",
|
|
620
|
+
" [ 0.0089919 -0.0043212 0.0303094 ... 0.00179068 0.00056501\n",
|
|
621
|
+
" -0.04161235]\n",
|
|
622
|
+
" ...\n",
|
|
623
|
+
" [ 0.03066176 -0.05368591 0.0289726 ... -0.04912771 0.06342489\n",
|
|
624
|
+
" -0.02452039]\n",
|
|
625
|
+
" [-0.10702807 0.1304991 -0.04743847 ... -0.00367014 -0.04885023\n",
|
|
626
|
+
" 0.02426663]\n",
|
|
627
|
+
" [ 0.07773516 -0.00087182 -0.04214528 ... -0.05326953 -0.01027754\n",
|
|
628
|
+
" 0.0061043 ]]>]"
|
|
629
|
+
]
|
|
630
|
+
},
|
|
631
|
+
"metadata": {},
|
|
632
|
+
"execution_count": 82
|
|
633
|
+
}
|
|
634
|
+
]
|
|
635
|
+
},
|
|
636
|
+
{
|
|
637
|
+
"cell_type": "code",
|
|
638
|
+
"source": [
|
|
639
|
+
"encoder.summary()"
|
|
640
|
+
],
|
|
641
|
+
"metadata": {
|
|
642
|
+
"colab": {
|
|
643
|
+
"base_uri": "https://localhost:8080/",
|
|
644
|
+
"height": 215
|
|
645
|
+
},
|
|
646
|
+
"id": "f2B59spYXlZt",
|
|
647
|
+
"outputId": "7efa2fbd-7779-4e75-bf19-f4d29e17f724"
|
|
648
|
+
},
|
|
649
|
+
"execution_count": 49,
|
|
650
|
+
"outputs": [
|
|
651
|
+
{
|
|
652
|
+
"output_type": "display_data",
|
|
653
|
+
"data": {
|
|
654
|
+
"text/plain": [
|
|
655
|
+
"\u001b[1mModel: \"encoder_1\"\u001b[0m\n"
|
|
656
|
+
],
|
|
657
|
+
"text/html": [
|
|
658
|
+
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"encoder_1\"</span>\n",
|
|
659
|
+
"</pre>\n"
|
|
660
|
+
]
|
|
661
|
+
},
|
|
662
|
+
"metadata": {}
|
|
663
|
+
},
|
|
664
|
+
{
|
|
665
|
+
"output_type": "display_data",
|
|
666
|
+
"data": {
|
|
667
|
+
"text/plain": [
|
|
668
|
+
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
|
|
669
|
+
"┃\u001b[1m \u001b[0m\u001b[1mLayer (type) \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
|
|
670
|
+
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
|
|
671
|
+
"│ embedding_2 (\u001b[38;5;33mEmbedding\u001b[0m) │ (\u001b[38;5;34m64\u001b[0m, \u001b[38;5;34m47\u001b[0m, \u001b[38;5;34m256\u001b[0m) │ \u001b[38;5;34m51,200\u001b[0m │\n",
|
|
672
|
+
"├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
|
|
673
|
+
"│ lstm_1 (\u001b[38;5;33mLSTM\u001b[0m) │ ((\u001b[38;5;34m64\u001b[0m, \u001b[38;5;34m47\u001b[0m, \u001b[38;5;34m1024\u001b[0m), (\u001b[38;5;34m64\u001b[0m, │ \u001b[38;5;34m5,246,976\u001b[0m │\n",
|
|
674
|
+
"│ │ \u001b[38;5;34m1024\u001b[0m), (\u001b[38;5;34m64\u001b[0m, \u001b[38;5;34m1024\u001b[0m)) │ │\n",
|
|
675
|
+
"└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n"
|
|
676
|
+
],
|
|
677
|
+
"text/html": [
|
|
678
|
+
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
|
|
679
|
+
"┃<span style=\"font-weight: bold\"> Layer (type) </span>┃<span style=\"font-weight: bold\"> Output Shape </span>┃<span style=\"font-weight: bold\"> Param # </span>┃\n",
|
|
680
|
+
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
|
|
681
|
+
"│ embedding_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>) │ (<span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">47</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>) │ <span style=\"color: #00af00; text-decoration-color: #00af00\">51,200</span> │\n",
|
|
682
|
+
"├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
|
|
683
|
+
"│ lstm_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">LSTM</span>) │ ((<span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">47</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1024</span>), (<span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>, │ <span style=\"color: #00af00; text-decoration-color: #00af00\">5,246,976</span> │\n",
|
|
684
|
+
"│ │ <span style=\"color: #00af00; text-decoration-color: #00af00\">1024</span>), (<span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1024</span>)) │ │\n",
|
|
685
|
+
"└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n",
|
|
686
|
+
"</pre>\n"
|
|
687
|
+
]
|
|
688
|
+
},
|
|
689
|
+
"metadata": {}
|
|
690
|
+
},
|
|
691
|
+
{
|
|
692
|
+
"output_type": "display_data",
|
|
693
|
+
"data": {
|
|
694
|
+
"text/plain": [
|
|
695
|
+
"\u001b[1m Total params: \u001b[0m\u001b[38;5;34m5,298,176\u001b[0m (20.21 MB)\n"
|
|
696
|
+
],
|
|
697
|
+
"text/html": [
|
|
698
|
+
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">5,298,176</span> (20.21 MB)\n",
|
|
699
|
+
"</pre>\n"
|
|
700
|
+
]
|
|
701
|
+
},
|
|
702
|
+
"metadata": {}
|
|
703
|
+
},
|
|
704
|
+
{
|
|
705
|
+
"output_type": "display_data",
|
|
706
|
+
"data": {
|
|
707
|
+
"text/plain": [
|
|
708
|
+
"\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m5,298,176\u001b[0m (20.21 MB)\n"
|
|
709
|
+
],
|
|
710
|
+
"text/html": [
|
|
711
|
+
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">5,298,176</span> (20.21 MB)\n",
|
|
712
|
+
"</pre>\n"
|
|
713
|
+
]
|
|
714
|
+
},
|
|
715
|
+
"metadata": {}
|
|
716
|
+
},
|
|
717
|
+
{
|
|
718
|
+
"output_type": "display_data",
|
|
719
|
+
"data": {
|
|
720
|
+
"text/plain": [
|
|
721
|
+
"\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
|
|
722
|
+
],
|
|
723
|
+
"text/html": [
|
|
724
|
+
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
|
|
725
|
+
"</pre>\n"
|
|
726
|
+
]
|
|
727
|
+
},
|
|
728
|
+
"metadata": {}
|
|
729
|
+
}
|
|
730
|
+
]
|
|
731
|
+
},
|
|
732
|
+
{
|
|
733
|
+
"cell_type": "code",
|
|
734
|
+
"source": [
|
|
735
|
+
"def sequence_to_chars(tokenizer, sequence):\n",
|
|
736
|
+
" \"\"\"Convert a sequence of indices back to characters.\"\"\"\n",
|
|
737
|
+
" word_index = tokenizer.word_index\n",
|
|
738
|
+
" reverse_map = {val: key for key, val in word_index.items()}\n",
|
|
739
|
+
" retext = ''\n",
|
|
740
|
+
"\n",
|
|
741
|
+
" # Convert tensor to numpy array if it's a tensor\n",
|
|
742
|
+
" if tf.is_tensor(sequence):\n",
|
|
743
|
+
" sequence = sequence.numpy()\n",
|
|
744
|
+
"\n",
|
|
745
|
+
" for q in sequence:\n",
|
|
746
|
+
" q_int = int(q) # Convert to integer\n",
|
|
747
|
+
" if q_int != 0: # Skip padding token\n",
|
|
748
|
+
" retext += reverse_map[q_int]\n",
|
|
749
|
+
" return retext\n",
|
|
750
|
+
"\n",
|
|
751
|
+
"def evaluate_sentence(sentence, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
752
|
+
" encoder, decoder, max_length_input):\n",
|
|
753
|
+
" \"\"\"Evaluate/translate a single sentence.\"\"\"\n",
|
|
754
|
+
" # Convert input sentence to token indices\n",
|
|
755
|
+
" inputs = [input_lang_tokenizer.word_index[i] for i in sentence]\n",
|
|
756
|
+
" inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n",
|
|
757
|
+
" maxlen=max_length_input,\n",
|
|
758
|
+
" padding='post')\n",
|
|
759
|
+
" inputs = tf.convert_to_tensor(inputs)\n",
|
|
760
|
+
"\n",
|
|
761
|
+
" # Use the same batch size as decoder.batch_sz for consistency\n",
|
|
762
|
+
" inference_batch_size = inputs.shape[0]\n",
|
|
763
|
+
"\n",
|
|
764
|
+
" inputs = tf.tile(inputs, [inference_batch_size, 1]) # Replicate input to match batch size\n",
|
|
765
|
+
"\n",
|
|
766
|
+
" # Initialize encoder state\n",
|
|
767
|
+
" enc_start_state = [tf.zeros((inference_batch_size, units)),\n",
|
|
768
|
+
" tf.zeros((inference_batch_size, units))]\n",
|
|
769
|
+
"\n",
|
|
770
|
+
" # Get encoder output\n",
|
|
771
|
+
" enc_out, enc_h, enc_c = encoder(inputs, enc_start_state)\n",
|
|
772
|
+
"\n",
|
|
773
|
+
" # Initialize decoder with encoder's final state\n",
|
|
774
|
+
" dec_state = [enc_h, enc_c]\n",
|
|
775
|
+
"\n",
|
|
776
|
+
" # Prepare decoder input with start token\n",
|
|
777
|
+
" dec_input = tf.fill([inference_batch_size, 1],\n",
|
|
778
|
+
" target_lang_tokenizer.word_index['^'])\n",
|
|
779
|
+
"\n",
|
|
780
|
+
" outputs = []\n",
|
|
781
|
+
"\n",
|
|
782
|
+
" # Decoding loop\n",
|
|
783
|
+
" for t in range(decoder.max_length_output):\n",
|
|
784
|
+
" # Call decoder\n",
|
|
785
|
+
" predictions, dec_state, _ = decoder(dec_input, dec_state, enc_out)\n",
|
|
786
|
+
"\n",
|
|
787
|
+
" # Get predicted token (use first batch item since all are the same)\n",
|
|
788
|
+
" predicted_id = tf.argmax(predictions, axis=-1).numpy()[0, -1]\n",
|
|
789
|
+
"\n",
|
|
790
|
+
" # Append prediction\n",
|
|
791
|
+
" outputs.append(predicted_id)\n",
|
|
792
|
+
"\n",
|
|
793
|
+
" # Break if end token is predicted\n",
|
|
794
|
+
" if predicted_id == target_lang_tokenizer.word_index['$']:\n",
|
|
795
|
+
" break\n",
|
|
796
|
+
"\n",
|
|
797
|
+
" # Update decoder input\n",
|
|
798
|
+
" dec_input = tf.fill([inference_batch_size, 1], predicted_id)\n",
|
|
799
|
+
"\n",
|
|
800
|
+
" return tf.convert_to_tensor(outputs)\n",
|
|
801
|
+
"\n",
|
|
802
|
+
"\n",
|
|
803
|
+
"def translate(sentence, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
804
|
+
" encoder, decoder, max_length_input):\n",
|
|
805
|
+
" \"\"\"Translate a sentence from source to target language.\"\"\"\n",
|
|
806
|
+
" result = evaluate_sentence(sentence,\n",
|
|
807
|
+
" units,\n",
|
|
808
|
+
" input_lang_tokenizer,\n",
|
|
809
|
+
" target_lang_tokenizer,\n",
|
|
810
|
+
" encoder,\n",
|
|
811
|
+
" decoder,\n",
|
|
812
|
+
" max_length_input)\n",
|
|
813
|
+
"\n",
|
|
814
|
+
" # Convert the output tokens back to characters\n",
|
|
815
|
+
" translated_text = sequence_to_chars(target_lang_tokenizer, result)\n",
|
|
816
|
+
"\n",
|
|
817
|
+
" # Remove the end token and return\n",
|
|
818
|
+
" return translated_text.strip('$')"
|
|
819
|
+
],
|
|
820
|
+
"metadata": {
|
|
821
|
+
"id": "BQjcWrjbes7N"
|
|
822
|
+
},
|
|
823
|
+
"execution_count": 50,
|
|
824
|
+
"outputs": []
|
|
825
|
+
},
|
|
826
|
+
{
|
|
827
|
+
"cell_type": "code",
|
|
828
|
+
"source": [
|
|
829
|
+
"%pwd"
|
|
830
|
+
],
|
|
831
|
+
"metadata": {
|
|
832
|
+
"colab": {
|
|
833
|
+
"base_uri": "https://localhost:8080/",
|
|
834
|
+
"height": 35
|
|
835
|
+
},
|
|
836
|
+
"id": "kcLNEYJyADXF",
|
|
837
|
+
"outputId": "1d37bfd7-09e3-4fb4-c310-d0a6a509739f"
|
|
838
|
+
},
|
|
839
|
+
"execution_count": 51,
|
|
840
|
+
"outputs": [
|
|
841
|
+
{
|
|
842
|
+
"output_type": "execute_result",
|
|
843
|
+
"data": {
|
|
844
|
+
"text/plain": [
|
|
845
|
+
"'/content/drive/MyDrive/Colab/indicate'"
|
|
846
|
+
],
|
|
847
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
848
|
+
"type": "string"
|
|
849
|
+
}
|
|
850
|
+
},
|
|
851
|
+
"metadata": {},
|
|
852
|
+
"execution_count": 51
|
|
853
|
+
}
|
|
854
|
+
]
|
|
855
|
+
},
|
|
856
|
+
{
|
|
857
|
+
"cell_type": "code",
|
|
858
|
+
"source": [
|
|
859
|
+
"encoder.save_weights('./data/final_model/encoder.weights.h5')"
|
|
860
|
+
],
|
|
861
|
+
"metadata": {
|
|
862
|
+
"id": "e7cLv1PCaB_K"
|
|
863
|
+
},
|
|
864
|
+
"execution_count": 52,
|
|
865
|
+
"outputs": []
|
|
866
|
+
},
|
|
867
|
+
{
|
|
868
|
+
"cell_type": "code",
|
|
869
|
+
"source": [
|
|
870
|
+
"decoder.save_weights('./data/final_model/decoder.weights.h5')"
|
|
871
|
+
],
|
|
872
|
+
"metadata": {
|
|
873
|
+
"id": "AnYLAiRDdc-5"
|
|
874
|
+
},
|
|
875
|
+
"execution_count": 53,
|
|
876
|
+
"outputs": []
|
|
877
|
+
},
|
|
878
|
+
{
|
|
879
|
+
"cell_type": "code",
|
|
880
|
+
"source": [
|
|
881
|
+
"!du -sh ./data/final_model/"
|
|
882
|
+
],
|
|
883
|
+
"metadata": {
|
|
884
|
+
"colab": {
|
|
885
|
+
"base_uri": "https://localhost:8080/"
|
|
886
|
+
},
|
|
887
|
+
"id": "Tk1uzGHzhOyy",
|
|
888
|
+
"outputId": "1c00ce1a-31ea-46ce-ba0a-39439304dcbd"
|
|
889
|
+
},
|
|
890
|
+
"execution_count": 54,
|
|
891
|
+
"outputs": [
|
|
892
|
+
{
|
|
893
|
+
"output_type": "stream",
|
|
894
|
+
"name": "stdout",
|
|
895
|
+
"text": [
|
|
896
|
+
"58M\t./data/final_model/\n"
|
|
897
|
+
]
|
|
898
|
+
}
|
|
899
|
+
]
|
|
900
|
+
},
|
|
901
|
+
{
|
|
902
|
+
"cell_type": "markdown",
|
|
903
|
+
"source": [
|
|
904
|
+
"# Basic Decoder"
|
|
905
|
+
],
|
|
906
|
+
"metadata": {
|
|
907
|
+
"id": "ZUk_JuKyoPLf"
|
|
908
|
+
}
|
|
909
|
+
},
|
|
910
|
+
{
|
|
911
|
+
"cell_type": "code",
|
|
912
|
+
"source": [
|
|
913
|
+
"test_input = 'राजशेखर'\n",
|
|
914
|
+
"translate(test_input, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
915
|
+
" encoder, decoder, max_length_input)"
|
|
916
|
+
],
|
|
917
|
+
"metadata": {
|
|
918
|
+
"colab": {
|
|
919
|
+
"base_uri": "https://localhost:8080/",
|
|
920
|
+
"height": 35
|
|
921
|
+
},
|
|
922
|
+
"id": "uuaT1icFoSiI",
|
|
923
|
+
"outputId": "e16d9925-1044-47bd-d576-60fc8af86d31"
|
|
924
|
+
},
|
|
925
|
+
"execution_count": 83,
|
|
926
|
+
"outputs": [
|
|
927
|
+
{
|
|
928
|
+
"output_type": "execute_result",
|
|
929
|
+
"data": {
|
|
930
|
+
"text/plain": [
|
|
931
|
+
"'rajshekhar'"
|
|
932
|
+
],
|
|
933
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
934
|
+
"type": "string"
|
|
935
|
+
}
|
|
936
|
+
},
|
|
937
|
+
"metadata": {},
|
|
938
|
+
"execution_count": 83
|
|
939
|
+
}
|
|
940
|
+
]
|
|
941
|
+
},
|
|
942
|
+
{
|
|
943
|
+
"cell_type": "code",
|
|
944
|
+
"source": [
|
|
945
|
+
"test_input = 'चिंतालपति'\n",
|
|
946
|
+
"translate(test_input, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
947
|
+
" encoder, decoder, max_length_input)"
|
|
948
|
+
],
|
|
949
|
+
"metadata": {
|
|
950
|
+
"colab": {
|
|
951
|
+
"base_uri": "https://localhost:8080/",
|
|
952
|
+
"height": 35
|
|
953
|
+
},
|
|
954
|
+
"id": "C_2BuWpfGPri",
|
|
955
|
+
"outputId": "ccab0419-8fa9-4509-c0b2-2adba0e7f2c1"
|
|
956
|
+
},
|
|
957
|
+
"execution_count": 84,
|
|
958
|
+
"outputs": [
|
|
959
|
+
{
|
|
960
|
+
"output_type": "execute_result",
|
|
961
|
+
"data": {
|
|
962
|
+
"text/plain": [
|
|
963
|
+
"'chintalpati'"
|
|
964
|
+
],
|
|
965
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
966
|
+
"type": "string"
|
|
967
|
+
}
|
|
968
|
+
},
|
|
969
|
+
"metadata": {},
|
|
970
|
+
"execution_count": 84
|
|
971
|
+
}
|
|
972
|
+
]
|
|
973
|
+
},
|
|
974
|
+
{
|
|
975
|
+
"cell_type": "code",
|
|
976
|
+
"source": [
|
|
977
|
+
"test_input = 'गौरव'\n",
|
|
978
|
+
"translate(test_input, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
979
|
+
" encoder, decoder, max_length_input)"
|
|
980
|
+
],
|
|
981
|
+
"metadata": {
|
|
982
|
+
"colab": {
|
|
983
|
+
"base_uri": "https://localhost:8080/",
|
|
984
|
+
"height": 35
|
|
985
|
+
},
|
|
986
|
+
"id": "MesVovlwGHKQ",
|
|
987
|
+
"outputId": "8e9593d2-eca2-4c17-a079-0ed1a0e3a079"
|
|
988
|
+
},
|
|
989
|
+
"execution_count": 85,
|
|
990
|
+
"outputs": [
|
|
991
|
+
{
|
|
992
|
+
"output_type": "execute_result",
|
|
993
|
+
"data": {
|
|
994
|
+
"text/plain": [
|
|
995
|
+
"'gaurav'"
|
|
996
|
+
],
|
|
997
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
998
|
+
"type": "string"
|
|
999
|
+
}
|
|
1000
|
+
},
|
|
1001
|
+
"metadata": {},
|
|
1002
|
+
"execution_count": 85
|
|
1003
|
+
}
|
|
1004
|
+
]
|
|
1005
|
+
},
|
|
1006
|
+
{
|
|
1007
|
+
"cell_type": "code",
|
|
1008
|
+
"source": [
|
|
1009
|
+
"test_input = 'सूद'\n",
|
|
1010
|
+
"translate(test_input, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
1011
|
+
" encoder, decoder, max_length_input)"
|
|
1012
|
+
],
|
|
1013
|
+
"metadata": {
|
|
1014
|
+
"colab": {
|
|
1015
|
+
"base_uri": "https://localhost:8080/",
|
|
1016
|
+
"height": 35
|
|
1017
|
+
},
|
|
1018
|
+
"id": "YJw64aW6quw5",
|
|
1019
|
+
"outputId": "6f51db87-fad2-4624-c6e0-2a547c0e353e"
|
|
1020
|
+
},
|
|
1021
|
+
"execution_count": 86,
|
|
1022
|
+
"outputs": [
|
|
1023
|
+
{
|
|
1024
|
+
"output_type": "execute_result",
|
|
1025
|
+
"data": {
|
|
1026
|
+
"text/plain": [
|
|
1027
|
+
"'sood'"
|
|
1028
|
+
],
|
|
1029
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
1030
|
+
"type": "string"
|
|
1031
|
+
}
|
|
1032
|
+
},
|
|
1033
|
+
"metadata": {},
|
|
1034
|
+
"execution_count": 86
|
|
1035
|
+
}
|
|
1036
|
+
]
|
|
1037
|
+
},
|
|
1038
|
+
{
|
|
1039
|
+
"cell_type": "code",
|
|
1040
|
+
"source": [
|
|
1041
|
+
"# load h5 weights\n",
|
|
1042
|
+
"encoder_h = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
|
|
1043
|
+
"decoder_h = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE,\n",
|
|
1044
|
+
" max_length_input, max_length_output, 'luong')"
|
|
1045
|
+
],
|
|
1046
|
+
"metadata": {
|
|
1047
|
+
"id": "heSUIOPYJakL"
|
|
1048
|
+
},
|
|
1049
|
+
"execution_count": 88,
|
|
1050
|
+
"outputs": []
|
|
1051
|
+
},
|
|
1052
|
+
{
|
|
1053
|
+
"cell_type": "code",
|
|
1054
|
+
"source": [
|
|
1055
|
+
"example_input_batch = tf.random.uniform(shape=[64,47])\n",
|
|
1056
|
+
"sample_hidden = encoder_h.initialize_hidden_state()\n",
|
|
1057
|
+
"sample_output, sample_h, sample_c = encoder_h(example_input_batch, sample_hidden)\n",
|
|
1058
|
+
"\n",
|
|
1059
|
+
"encoder_h.load_weights('./data/final_model/encoder.weights.h5')"
|
|
1060
|
+
],
|
|
1061
|
+
"metadata": {
|
|
1062
|
+
"id": "0EV3GYIhKQzw"
|
|
1063
|
+
},
|
|
1064
|
+
"execution_count": 89,
|
|
1065
|
+
"outputs": []
|
|
1066
|
+
},
|
|
1067
|
+
{
|
|
1068
|
+
"cell_type": "code",
|
|
1069
|
+
"source": [
|
|
1070
|
+
"sample_x = tf.random.uniform((BATCH_SIZE, max_length_output))\n",
|
|
1071
|
+
"initial_state = decoder_h.build_initial_state(BATCH_SIZE, [sample_h, sample_c])\n",
|
|
1072
|
+
"\n",
|
|
1073
|
+
"sample_decoder_outputs = decoder_h(sample_x, initial_state, sample_output)\n",
|
|
1074
|
+
"logits, new_state_h, new_state_c = sample_decoder_outputs\n",
|
|
1075
|
+
"\n",
|
|
1076
|
+
"decoder_h.load_weights('./data/final_model/decoder.weights.h5')"
|
|
1077
|
+
],
|
|
1078
|
+
"metadata": {
|
|
1079
|
+
"id": "qxad6IsxKooz"
|
|
1080
|
+
},
|
|
1081
|
+
"execution_count": 90,
|
|
1082
|
+
"outputs": []
|
|
1083
|
+
},
|
|
1084
|
+
{
|
|
1085
|
+
"cell_type": "code",
|
|
1086
|
+
"source": [
|
|
1087
|
+
"test_inputs = [\"राजशेखर\",\"चिंतालपति\",\"गौरव\",\"सूद\"]\n",
|
|
1088
|
+
"for test_input in test_inputs:\n",
|
|
1089
|
+
" print(translate(test_input, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
1090
|
+
" encoder_h, decoder_h, max_length_input))"
|
|
1091
|
+
],
|
|
1092
|
+
"metadata": {
|
|
1093
|
+
"colab": {
|
|
1094
|
+
"base_uri": "https://localhost:8080/"
|
|
1095
|
+
},
|
|
1096
|
+
"id": "We6nBG4-K8Ax",
|
|
1097
|
+
"outputId": "0327f288-5823-4b89-b74f-c9995a7849e2"
|
|
1098
|
+
},
|
|
1099
|
+
"execution_count": 91,
|
|
1100
|
+
"outputs": [
|
|
1101
|
+
{
|
|
1102
|
+
"output_type": "stream",
|
|
1103
|
+
"name": "stdout",
|
|
1104
|
+
"text": [
|
|
1105
|
+
"rajshekhar\n",
|
|
1106
|
+
"chintalpati\n",
|
|
1107
|
+
"gaurav\n",
|
|
1108
|
+
"sood\n"
|
|
1109
|
+
]
|
|
1110
|
+
}
|
|
1111
|
+
]
|
|
1112
|
+
},
|
|
1113
|
+
{
|
|
1114
|
+
"cell_type": "code",
|
|
1115
|
+
"source": [
|
|
1116
|
+
"# lets test above with df with columns as names from test inputs\n",
|
|
1117
|
+
"test_df = pd.DataFrame(test_inputs, columns=['hindi'])\n",
|
|
1118
|
+
"test_df"
|
|
1119
|
+
],
|
|
1120
|
+
"metadata": {
|
|
1121
|
+
"colab": {
|
|
1122
|
+
"base_uri": "https://localhost:8080/",
|
|
1123
|
+
"height": 175
|
|
1124
|
+
},
|
|
1125
|
+
"id": "CjJ1oEN7OU75",
|
|
1126
|
+
"outputId": "06510ef4-22b8-439b-f1ec-a31eae593f1c"
|
|
1127
|
+
},
|
|
1128
|
+
"execution_count": 92,
|
|
1129
|
+
"outputs": [
|
|
1130
|
+
{
|
|
1131
|
+
"output_type": "execute_result",
|
|
1132
|
+
"data": {
|
|
1133
|
+
"text/plain": [
|
|
1134
|
+
" hindi\n",
|
|
1135
|
+
"0 राजशेखर\n",
|
|
1136
|
+
"1 चिंतालपति\n",
|
|
1137
|
+
"2 गौरव\n",
|
|
1138
|
+
"3 सूद"
|
|
1139
|
+
],
|
|
1140
|
+
"text/html": [
|
|
1141
|
+
"\n",
|
|
1142
|
+
" <div id=\"df-49c970ab-5339-4ce3-83bb-a7a63bd5bcbe\" class=\"colab-df-container\">\n",
|
|
1143
|
+
" <div>\n",
|
|
1144
|
+
"<style scoped>\n",
|
|
1145
|
+
" .dataframe tbody tr th:only-of-type {\n",
|
|
1146
|
+
" vertical-align: middle;\n",
|
|
1147
|
+
" }\n",
|
|
1148
|
+
"\n",
|
|
1149
|
+
" .dataframe tbody tr th {\n",
|
|
1150
|
+
" vertical-align: top;\n",
|
|
1151
|
+
" }\n",
|
|
1152
|
+
"\n",
|
|
1153
|
+
" .dataframe thead th {\n",
|
|
1154
|
+
" text-align: right;\n",
|
|
1155
|
+
" }\n",
|
|
1156
|
+
"</style>\n",
|
|
1157
|
+
"<table border=\"1\" class=\"dataframe\">\n",
|
|
1158
|
+
" <thead>\n",
|
|
1159
|
+
" <tr style=\"text-align: right;\">\n",
|
|
1160
|
+
" <th></th>\n",
|
|
1161
|
+
" <th>hindi</th>\n",
|
|
1162
|
+
" </tr>\n",
|
|
1163
|
+
" </thead>\n",
|
|
1164
|
+
" <tbody>\n",
|
|
1165
|
+
" <tr>\n",
|
|
1166
|
+
" <th>0</th>\n",
|
|
1167
|
+
" <td>राजशेखर</td>\n",
|
|
1168
|
+
" </tr>\n",
|
|
1169
|
+
" <tr>\n",
|
|
1170
|
+
" <th>1</th>\n",
|
|
1171
|
+
" <td>चिंतालपति</td>\n",
|
|
1172
|
+
" </tr>\n",
|
|
1173
|
+
" <tr>\n",
|
|
1174
|
+
" <th>2</th>\n",
|
|
1175
|
+
" <td>गौरव</td>\n",
|
|
1176
|
+
" </tr>\n",
|
|
1177
|
+
" <tr>\n",
|
|
1178
|
+
" <th>3</th>\n",
|
|
1179
|
+
" <td>सूद</td>\n",
|
|
1180
|
+
" </tr>\n",
|
|
1181
|
+
" </tbody>\n",
|
|
1182
|
+
"</table>\n",
|
|
1183
|
+
"</div>\n",
|
|
1184
|
+
" <div class=\"colab-df-buttons\">\n",
|
|
1185
|
+
"\n",
|
|
1186
|
+
" <div class=\"colab-df-container\">\n",
|
|
1187
|
+
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-49c970ab-5339-4ce3-83bb-a7a63bd5bcbe')\"\n",
|
|
1188
|
+
" title=\"Convert this dataframe to an interactive table.\"\n",
|
|
1189
|
+
" style=\"display:none;\">\n",
|
|
1190
|
+
"\n",
|
|
1191
|
+
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
|
|
1192
|
+
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
|
|
1193
|
+
" </svg>\n",
|
|
1194
|
+
" </button>\n",
|
|
1195
|
+
"\n",
|
|
1196
|
+
" <style>\n",
|
|
1197
|
+
" .colab-df-container {\n",
|
|
1198
|
+
" display:flex;\n",
|
|
1199
|
+
" gap: 12px;\n",
|
|
1200
|
+
" }\n",
|
|
1201
|
+
"\n",
|
|
1202
|
+
" .colab-df-convert {\n",
|
|
1203
|
+
" background-color: #E8F0FE;\n",
|
|
1204
|
+
" border: none;\n",
|
|
1205
|
+
" border-radius: 50%;\n",
|
|
1206
|
+
" cursor: pointer;\n",
|
|
1207
|
+
" display: none;\n",
|
|
1208
|
+
" fill: #1967D2;\n",
|
|
1209
|
+
" height: 32px;\n",
|
|
1210
|
+
" padding: 0 0 0 0;\n",
|
|
1211
|
+
" width: 32px;\n",
|
|
1212
|
+
" }\n",
|
|
1213
|
+
"\n",
|
|
1214
|
+
" .colab-df-convert:hover {\n",
|
|
1215
|
+
" background-color: #E2EBFA;\n",
|
|
1216
|
+
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
1217
|
+
" fill: #174EA6;\n",
|
|
1218
|
+
" }\n",
|
|
1219
|
+
"\n",
|
|
1220
|
+
" .colab-df-buttons div {\n",
|
|
1221
|
+
" margin-bottom: 4px;\n",
|
|
1222
|
+
" }\n",
|
|
1223
|
+
"\n",
|
|
1224
|
+
" [theme=dark] .colab-df-convert {\n",
|
|
1225
|
+
" background-color: #3B4455;\n",
|
|
1226
|
+
" fill: #D2E3FC;\n",
|
|
1227
|
+
" }\n",
|
|
1228
|
+
"\n",
|
|
1229
|
+
" [theme=dark] .colab-df-convert:hover {\n",
|
|
1230
|
+
" background-color: #434B5C;\n",
|
|
1231
|
+
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
|
|
1232
|
+
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
|
|
1233
|
+
" fill: #FFFFFF;\n",
|
|
1234
|
+
" }\n",
|
|
1235
|
+
" </style>\n",
|
|
1236
|
+
"\n",
|
|
1237
|
+
" <script>\n",
|
|
1238
|
+
" const buttonEl =\n",
|
|
1239
|
+
" document.querySelector('#df-49c970ab-5339-4ce3-83bb-a7a63bd5bcbe button.colab-df-convert');\n",
|
|
1240
|
+
" buttonEl.style.display =\n",
|
|
1241
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
1242
|
+
"\n",
|
|
1243
|
+
" async function convertToInteractive(key) {\n",
|
|
1244
|
+
" const element = document.querySelector('#df-49c970ab-5339-4ce3-83bb-a7a63bd5bcbe');\n",
|
|
1245
|
+
" const dataTable =\n",
|
|
1246
|
+
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
|
|
1247
|
+
" [key], {});\n",
|
|
1248
|
+
" if (!dataTable) return;\n",
|
|
1249
|
+
"\n",
|
|
1250
|
+
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
|
|
1251
|
+
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
|
|
1252
|
+
" + ' to learn more about interactive tables.';\n",
|
|
1253
|
+
" element.innerHTML = '';\n",
|
|
1254
|
+
" dataTable['output_type'] = 'display_data';\n",
|
|
1255
|
+
" await google.colab.output.renderOutput(dataTable, element);\n",
|
|
1256
|
+
" const docLink = document.createElement('div');\n",
|
|
1257
|
+
" docLink.innerHTML = docLinkHtml;\n",
|
|
1258
|
+
" element.appendChild(docLink);\n",
|
|
1259
|
+
" }\n",
|
|
1260
|
+
" </script>\n",
|
|
1261
|
+
" </div>\n",
|
|
1262
|
+
"\n",
|
|
1263
|
+
"\n",
|
|
1264
|
+
"<div id=\"df-803131aa-bbb2-42f3-afb5-475ad224fafa\">\n",
|
|
1265
|
+
" <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-803131aa-bbb2-42f3-afb5-475ad224fafa')\"\n",
|
|
1266
|
+
" title=\"Suggest charts\"\n",
|
|
1267
|
+
" style=\"display:none;\">\n",
|
|
1268
|
+
"\n",
|
|
1269
|
+
"<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
|
|
1270
|
+
" width=\"24px\">\n",
|
|
1271
|
+
" <g>\n",
|
|
1272
|
+
" <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
|
|
1273
|
+
" </g>\n",
|
|
1274
|
+
"</svg>\n",
|
|
1275
|
+
" </button>\n",
|
|
1276
|
+
"\n",
|
|
1277
|
+
"<style>\n",
|
|
1278
|
+
" .colab-df-quickchart {\n",
|
|
1279
|
+
" --bg-color: #E8F0FE;\n",
|
|
1280
|
+
" --fill-color: #1967D2;\n",
|
|
1281
|
+
" --hover-bg-color: #E2EBFA;\n",
|
|
1282
|
+
" --hover-fill-color: #174EA6;\n",
|
|
1283
|
+
" --disabled-fill-color: #AAA;\n",
|
|
1284
|
+
" --disabled-bg-color: #DDD;\n",
|
|
1285
|
+
" }\n",
|
|
1286
|
+
"\n",
|
|
1287
|
+
" [theme=dark] .colab-df-quickchart {\n",
|
|
1288
|
+
" --bg-color: #3B4455;\n",
|
|
1289
|
+
" --fill-color: #D2E3FC;\n",
|
|
1290
|
+
" --hover-bg-color: #434B5C;\n",
|
|
1291
|
+
" --hover-fill-color: #FFFFFF;\n",
|
|
1292
|
+
" --disabled-bg-color: #3B4455;\n",
|
|
1293
|
+
" --disabled-fill-color: #666;\n",
|
|
1294
|
+
" }\n",
|
|
1295
|
+
"\n",
|
|
1296
|
+
" .colab-df-quickchart {\n",
|
|
1297
|
+
" background-color: var(--bg-color);\n",
|
|
1298
|
+
" border: none;\n",
|
|
1299
|
+
" border-radius: 50%;\n",
|
|
1300
|
+
" cursor: pointer;\n",
|
|
1301
|
+
" display: none;\n",
|
|
1302
|
+
" fill: var(--fill-color);\n",
|
|
1303
|
+
" height: 32px;\n",
|
|
1304
|
+
" padding: 0;\n",
|
|
1305
|
+
" width: 32px;\n",
|
|
1306
|
+
" }\n",
|
|
1307
|
+
"\n",
|
|
1308
|
+
" .colab-df-quickchart:hover {\n",
|
|
1309
|
+
" background-color: var(--hover-bg-color);\n",
|
|
1310
|
+
" box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
1311
|
+
" fill: var(--button-hover-fill-color);\n",
|
|
1312
|
+
" }\n",
|
|
1313
|
+
"\n",
|
|
1314
|
+
" .colab-df-quickchart-complete:disabled,\n",
|
|
1315
|
+
" .colab-df-quickchart-complete:disabled:hover {\n",
|
|
1316
|
+
" background-color: var(--disabled-bg-color);\n",
|
|
1317
|
+
" fill: var(--disabled-fill-color);\n",
|
|
1318
|
+
" box-shadow: none;\n",
|
|
1319
|
+
" }\n",
|
|
1320
|
+
"\n",
|
|
1321
|
+
" .colab-df-spinner {\n",
|
|
1322
|
+
" border: 2px solid var(--fill-color);\n",
|
|
1323
|
+
" border-color: transparent;\n",
|
|
1324
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
1325
|
+
" animation:\n",
|
|
1326
|
+
" spin 1s steps(1) infinite;\n",
|
|
1327
|
+
" }\n",
|
|
1328
|
+
"\n",
|
|
1329
|
+
" @keyframes spin {\n",
|
|
1330
|
+
" 0% {\n",
|
|
1331
|
+
" border-color: transparent;\n",
|
|
1332
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
1333
|
+
" border-left-color: var(--fill-color);\n",
|
|
1334
|
+
" }\n",
|
|
1335
|
+
" 20% {\n",
|
|
1336
|
+
" border-color: transparent;\n",
|
|
1337
|
+
" border-left-color: var(--fill-color);\n",
|
|
1338
|
+
" border-top-color: var(--fill-color);\n",
|
|
1339
|
+
" }\n",
|
|
1340
|
+
" 30% {\n",
|
|
1341
|
+
" border-color: transparent;\n",
|
|
1342
|
+
" border-left-color: var(--fill-color);\n",
|
|
1343
|
+
" border-top-color: var(--fill-color);\n",
|
|
1344
|
+
" border-right-color: var(--fill-color);\n",
|
|
1345
|
+
" }\n",
|
|
1346
|
+
" 40% {\n",
|
|
1347
|
+
" border-color: transparent;\n",
|
|
1348
|
+
" border-right-color: var(--fill-color);\n",
|
|
1349
|
+
" border-top-color: var(--fill-color);\n",
|
|
1350
|
+
" }\n",
|
|
1351
|
+
" 60% {\n",
|
|
1352
|
+
" border-color: transparent;\n",
|
|
1353
|
+
" border-right-color: var(--fill-color);\n",
|
|
1354
|
+
" }\n",
|
|
1355
|
+
" 80% {\n",
|
|
1356
|
+
" border-color: transparent;\n",
|
|
1357
|
+
" border-right-color: var(--fill-color);\n",
|
|
1358
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
1359
|
+
" }\n",
|
|
1360
|
+
" 90% {\n",
|
|
1361
|
+
" border-color: transparent;\n",
|
|
1362
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
1363
|
+
" }\n",
|
|
1364
|
+
" }\n",
|
|
1365
|
+
"</style>\n",
|
|
1366
|
+
"\n",
|
|
1367
|
+
" <script>\n",
|
|
1368
|
+
" async function quickchart(key) {\n",
|
|
1369
|
+
" const quickchartButtonEl =\n",
|
|
1370
|
+
" document.querySelector('#' + key + ' button');\n",
|
|
1371
|
+
" quickchartButtonEl.disabled = true; // To prevent multiple clicks.\n",
|
|
1372
|
+
" quickchartButtonEl.classList.add('colab-df-spinner');\n",
|
|
1373
|
+
" try {\n",
|
|
1374
|
+
" const charts = await google.colab.kernel.invokeFunction(\n",
|
|
1375
|
+
" 'suggestCharts', [key], {});\n",
|
|
1376
|
+
" } catch (error) {\n",
|
|
1377
|
+
" console.error('Error during call to suggestCharts:', error);\n",
|
|
1378
|
+
" }\n",
|
|
1379
|
+
" quickchartButtonEl.classList.remove('colab-df-spinner');\n",
|
|
1380
|
+
" quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
|
|
1381
|
+
" }\n",
|
|
1382
|
+
" (() => {\n",
|
|
1383
|
+
" let quickchartButtonEl =\n",
|
|
1384
|
+
" document.querySelector('#df-803131aa-bbb2-42f3-afb5-475ad224fafa button');\n",
|
|
1385
|
+
" quickchartButtonEl.style.display =\n",
|
|
1386
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
1387
|
+
" })();\n",
|
|
1388
|
+
" </script>\n",
|
|
1389
|
+
"</div>\n",
|
|
1390
|
+
"\n",
|
|
1391
|
+
" <div id=\"id_7c9c962f-6ed0-4b76-924c-931df3377baa\">\n",
|
|
1392
|
+
" <style>\n",
|
|
1393
|
+
" .colab-df-generate {\n",
|
|
1394
|
+
" background-color: #E8F0FE;\n",
|
|
1395
|
+
" border: none;\n",
|
|
1396
|
+
" border-radius: 50%;\n",
|
|
1397
|
+
" cursor: pointer;\n",
|
|
1398
|
+
" display: none;\n",
|
|
1399
|
+
" fill: #1967D2;\n",
|
|
1400
|
+
" height: 32px;\n",
|
|
1401
|
+
" padding: 0 0 0 0;\n",
|
|
1402
|
+
" width: 32px;\n",
|
|
1403
|
+
" }\n",
|
|
1404
|
+
"\n",
|
|
1405
|
+
" .colab-df-generate:hover {\n",
|
|
1406
|
+
" background-color: #E2EBFA;\n",
|
|
1407
|
+
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
1408
|
+
" fill: #174EA6;\n",
|
|
1409
|
+
" }\n",
|
|
1410
|
+
"\n",
|
|
1411
|
+
" [theme=dark] .colab-df-generate {\n",
|
|
1412
|
+
" background-color: #3B4455;\n",
|
|
1413
|
+
" fill: #D2E3FC;\n",
|
|
1414
|
+
" }\n",
|
|
1415
|
+
"\n",
|
|
1416
|
+
" [theme=dark] .colab-df-generate:hover {\n",
|
|
1417
|
+
" background-color: #434B5C;\n",
|
|
1418
|
+
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
|
|
1419
|
+
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
|
|
1420
|
+
" fill: #FFFFFF;\n",
|
|
1421
|
+
" }\n",
|
|
1422
|
+
" </style>\n",
|
|
1423
|
+
" <button class=\"colab-df-generate\" onclick=\"generateWithVariable('test_df')\"\n",
|
|
1424
|
+
" title=\"Generate code using this dataframe.\"\n",
|
|
1425
|
+
" style=\"display:none;\">\n",
|
|
1426
|
+
"\n",
|
|
1427
|
+
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
|
|
1428
|
+
" width=\"24px\">\n",
|
|
1429
|
+
" <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
|
|
1430
|
+
" </svg>\n",
|
|
1431
|
+
" </button>\n",
|
|
1432
|
+
" <script>\n",
|
|
1433
|
+
" (() => {\n",
|
|
1434
|
+
" const buttonEl =\n",
|
|
1435
|
+
" document.querySelector('#id_7c9c962f-6ed0-4b76-924c-931df3377baa button.colab-df-generate');\n",
|
|
1436
|
+
" buttonEl.style.display =\n",
|
|
1437
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
1438
|
+
"\n",
|
|
1439
|
+
" buttonEl.onclick = () => {\n",
|
|
1440
|
+
" google.colab.notebook.generateWithVariable('test_df');\n",
|
|
1441
|
+
" }\n",
|
|
1442
|
+
" })();\n",
|
|
1443
|
+
" </script>\n",
|
|
1444
|
+
" </div>\n",
|
|
1445
|
+
"\n",
|
|
1446
|
+
" </div>\n",
|
|
1447
|
+
" </div>\n"
|
|
1448
|
+
],
|
|
1449
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
1450
|
+
"type": "dataframe",
|
|
1451
|
+
"variable_name": "test_df",
|
|
1452
|
+
"summary": "{\n \"name\": \"test_df\",\n \"rows\": 4,\n \"fields\": [\n {\n \"column\": \"hindi\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"\\u091a\\u093f\\u0902\\u0924\\u093e\\u0932\\u092a\\u0924\\u093f\",\n \"\\u0938\\u0942\\u0926\",\n \"\\u0930\\u093e\\u091c\\u0936\\u0947\\u0916\\u0930\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
|
|
1453
|
+
}
|
|
1454
|
+
},
|
|
1455
|
+
"metadata": {},
|
|
1456
|
+
"execution_count": 92
|
|
1457
|
+
}
|
|
1458
|
+
]
|
|
1459
|
+
},
|
|
1460
|
+
{
|
|
1461
|
+
"cell_type": "code",
|
|
1462
|
+
"source": [
|
|
1463
|
+
"# add new column english to test_df\n",
|
|
1464
|
+
"test_df['english'] = test_df['hindi'].apply(lambda x: translate(x, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
1465
|
+
" encoder_h, decoder_h, max_length_input))"
|
|
1466
|
+
],
|
|
1467
|
+
"metadata": {
|
|
1468
|
+
"id": "yZ7IdfngOe_8"
|
|
1469
|
+
},
|
|
1470
|
+
"execution_count": 93,
|
|
1471
|
+
"outputs": []
|
|
1472
|
+
},
|
|
1473
|
+
{
|
|
1474
|
+
"cell_type": "code",
|
|
1475
|
+
"source": [
|
|
1476
|
+
"test_df"
|
|
1477
|
+
],
|
|
1478
|
+
"metadata": {
|
|
1479
|
+
"colab": {
|
|
1480
|
+
"base_uri": "https://localhost:8080/",
|
|
1481
|
+
"height": 175
|
|
1482
|
+
},
|
|
1483
|
+
"id": "PM_c4kSfOvYk",
|
|
1484
|
+
"outputId": "2933d1b4-732c-44d8-c85b-8bf61d359c68"
|
|
1485
|
+
},
|
|
1486
|
+
"execution_count": 94,
|
|
1487
|
+
"outputs": [
|
|
1488
|
+
{
|
|
1489
|
+
"output_type": "execute_result",
|
|
1490
|
+
"data": {
|
|
1491
|
+
"text/plain": [
|
|
1492
|
+
" hindi english\n",
|
|
1493
|
+
"0 राजशेखर rajshekhar\n",
|
|
1494
|
+
"1 चिंतालपति chintalpati\n",
|
|
1495
|
+
"2 गौरव gaurav\n",
|
|
1496
|
+
"3 सूद sood"
|
|
1497
|
+
],
|
|
1498
|
+
"text/html": [
|
|
1499
|
+
"\n",
|
|
1500
|
+
" <div id=\"df-83607790-a0cf-4606-874f-d949c590f4dd\" class=\"colab-df-container\">\n",
|
|
1501
|
+
" <div>\n",
|
|
1502
|
+
"<style scoped>\n",
|
|
1503
|
+
" .dataframe tbody tr th:only-of-type {\n",
|
|
1504
|
+
" vertical-align: middle;\n",
|
|
1505
|
+
" }\n",
|
|
1506
|
+
"\n",
|
|
1507
|
+
" .dataframe tbody tr th {\n",
|
|
1508
|
+
" vertical-align: top;\n",
|
|
1509
|
+
" }\n",
|
|
1510
|
+
"\n",
|
|
1511
|
+
" .dataframe thead th {\n",
|
|
1512
|
+
" text-align: right;\n",
|
|
1513
|
+
" }\n",
|
|
1514
|
+
"</style>\n",
|
|
1515
|
+
"<table border=\"1\" class=\"dataframe\">\n",
|
|
1516
|
+
" <thead>\n",
|
|
1517
|
+
" <tr style=\"text-align: right;\">\n",
|
|
1518
|
+
" <th></th>\n",
|
|
1519
|
+
" <th>hindi</th>\n",
|
|
1520
|
+
" <th>english</th>\n",
|
|
1521
|
+
" </tr>\n",
|
|
1522
|
+
" </thead>\n",
|
|
1523
|
+
" <tbody>\n",
|
|
1524
|
+
" <tr>\n",
|
|
1525
|
+
" <th>0</th>\n",
|
|
1526
|
+
" <td>राजशेखर</td>\n",
|
|
1527
|
+
" <td>rajshekhar</td>\n",
|
|
1528
|
+
" </tr>\n",
|
|
1529
|
+
" <tr>\n",
|
|
1530
|
+
" <th>1</th>\n",
|
|
1531
|
+
" <td>चिंतालपति</td>\n",
|
|
1532
|
+
" <td>chintalpati</td>\n",
|
|
1533
|
+
" </tr>\n",
|
|
1534
|
+
" <tr>\n",
|
|
1535
|
+
" <th>2</th>\n",
|
|
1536
|
+
" <td>गौरव</td>\n",
|
|
1537
|
+
" <td>gaurav</td>\n",
|
|
1538
|
+
" </tr>\n",
|
|
1539
|
+
" <tr>\n",
|
|
1540
|
+
" <th>3</th>\n",
|
|
1541
|
+
" <td>सूद</td>\n",
|
|
1542
|
+
" <td>sood</td>\n",
|
|
1543
|
+
" </tr>\n",
|
|
1544
|
+
" </tbody>\n",
|
|
1545
|
+
"</table>\n",
|
|
1546
|
+
"</div>\n",
|
|
1547
|
+
" <div class=\"colab-df-buttons\">\n",
|
|
1548
|
+
"\n",
|
|
1549
|
+
" <div class=\"colab-df-container\">\n",
|
|
1550
|
+
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-83607790-a0cf-4606-874f-d949c590f4dd')\"\n",
|
|
1551
|
+
" title=\"Convert this dataframe to an interactive table.\"\n",
|
|
1552
|
+
" style=\"display:none;\">\n",
|
|
1553
|
+
"\n",
|
|
1554
|
+
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
|
|
1555
|
+
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
|
|
1556
|
+
" </svg>\n",
|
|
1557
|
+
" </button>\n",
|
|
1558
|
+
"\n",
|
|
1559
|
+
" <style>\n",
|
|
1560
|
+
" .colab-df-container {\n",
|
|
1561
|
+
" display:flex;\n",
|
|
1562
|
+
" gap: 12px;\n",
|
|
1563
|
+
" }\n",
|
|
1564
|
+
"\n",
|
|
1565
|
+
" .colab-df-convert {\n",
|
|
1566
|
+
" background-color: #E8F0FE;\n",
|
|
1567
|
+
" border: none;\n",
|
|
1568
|
+
" border-radius: 50%;\n",
|
|
1569
|
+
" cursor: pointer;\n",
|
|
1570
|
+
" display: none;\n",
|
|
1571
|
+
" fill: #1967D2;\n",
|
|
1572
|
+
" height: 32px;\n",
|
|
1573
|
+
" padding: 0 0 0 0;\n",
|
|
1574
|
+
" width: 32px;\n",
|
|
1575
|
+
" }\n",
|
|
1576
|
+
"\n",
|
|
1577
|
+
" .colab-df-convert:hover {\n",
|
|
1578
|
+
" background-color: #E2EBFA;\n",
|
|
1579
|
+
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
1580
|
+
" fill: #174EA6;\n",
|
|
1581
|
+
" }\n",
|
|
1582
|
+
"\n",
|
|
1583
|
+
" .colab-df-buttons div {\n",
|
|
1584
|
+
" margin-bottom: 4px;\n",
|
|
1585
|
+
" }\n",
|
|
1586
|
+
"\n",
|
|
1587
|
+
" [theme=dark] .colab-df-convert {\n",
|
|
1588
|
+
" background-color: #3B4455;\n",
|
|
1589
|
+
" fill: #D2E3FC;\n",
|
|
1590
|
+
" }\n",
|
|
1591
|
+
"\n",
|
|
1592
|
+
" [theme=dark] .colab-df-convert:hover {\n",
|
|
1593
|
+
" background-color: #434B5C;\n",
|
|
1594
|
+
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
|
|
1595
|
+
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
|
|
1596
|
+
" fill: #FFFFFF;\n",
|
|
1597
|
+
" }\n",
|
|
1598
|
+
" </style>\n",
|
|
1599
|
+
"\n",
|
|
1600
|
+
" <script>\n",
|
|
1601
|
+
" const buttonEl =\n",
|
|
1602
|
+
" document.querySelector('#df-83607790-a0cf-4606-874f-d949c590f4dd button.colab-df-convert');\n",
|
|
1603
|
+
" buttonEl.style.display =\n",
|
|
1604
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
1605
|
+
"\n",
|
|
1606
|
+
" async function convertToInteractive(key) {\n",
|
|
1607
|
+
" const element = document.querySelector('#df-83607790-a0cf-4606-874f-d949c590f4dd');\n",
|
|
1608
|
+
" const dataTable =\n",
|
|
1609
|
+
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
|
|
1610
|
+
" [key], {});\n",
|
|
1611
|
+
" if (!dataTable) return;\n",
|
|
1612
|
+
"\n",
|
|
1613
|
+
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
|
|
1614
|
+
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
|
|
1615
|
+
" + ' to learn more about interactive tables.';\n",
|
|
1616
|
+
" element.innerHTML = '';\n",
|
|
1617
|
+
" dataTable['output_type'] = 'display_data';\n",
|
|
1618
|
+
" await google.colab.output.renderOutput(dataTable, element);\n",
|
|
1619
|
+
" const docLink = document.createElement('div');\n",
|
|
1620
|
+
" docLink.innerHTML = docLinkHtml;\n",
|
|
1621
|
+
" element.appendChild(docLink);\n",
|
|
1622
|
+
" }\n",
|
|
1623
|
+
" </script>\n",
|
|
1624
|
+
" </div>\n",
|
|
1625
|
+
"\n",
|
|
1626
|
+
"\n",
|
|
1627
|
+
"<div id=\"df-54b66c33-97af-4c06-9735-a41863f2c8d8\">\n",
|
|
1628
|
+
" <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-54b66c33-97af-4c06-9735-a41863f2c8d8')\"\n",
|
|
1629
|
+
" title=\"Suggest charts\"\n",
|
|
1630
|
+
" style=\"display:none;\">\n",
|
|
1631
|
+
"\n",
|
|
1632
|
+
"<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
|
|
1633
|
+
" width=\"24px\">\n",
|
|
1634
|
+
" <g>\n",
|
|
1635
|
+
" <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
|
|
1636
|
+
" </g>\n",
|
|
1637
|
+
"</svg>\n",
|
|
1638
|
+
" </button>\n",
|
|
1639
|
+
"\n",
|
|
1640
|
+
"<style>\n",
|
|
1641
|
+
" .colab-df-quickchart {\n",
|
|
1642
|
+
" --bg-color: #E8F0FE;\n",
|
|
1643
|
+
" --fill-color: #1967D2;\n",
|
|
1644
|
+
" --hover-bg-color: #E2EBFA;\n",
|
|
1645
|
+
" --hover-fill-color: #174EA6;\n",
|
|
1646
|
+
" --disabled-fill-color: #AAA;\n",
|
|
1647
|
+
" --disabled-bg-color: #DDD;\n",
|
|
1648
|
+
" }\n",
|
|
1649
|
+
"\n",
|
|
1650
|
+
" [theme=dark] .colab-df-quickchart {\n",
|
|
1651
|
+
" --bg-color: #3B4455;\n",
|
|
1652
|
+
" --fill-color: #D2E3FC;\n",
|
|
1653
|
+
" --hover-bg-color: #434B5C;\n",
|
|
1654
|
+
" --hover-fill-color: #FFFFFF;\n",
|
|
1655
|
+
" --disabled-bg-color: #3B4455;\n",
|
|
1656
|
+
" --disabled-fill-color: #666;\n",
|
|
1657
|
+
" }\n",
|
|
1658
|
+
"\n",
|
|
1659
|
+
" .colab-df-quickchart {\n",
|
|
1660
|
+
" background-color: var(--bg-color);\n",
|
|
1661
|
+
" border: none;\n",
|
|
1662
|
+
" border-radius: 50%;\n",
|
|
1663
|
+
" cursor: pointer;\n",
|
|
1664
|
+
" display: none;\n",
|
|
1665
|
+
" fill: var(--fill-color);\n",
|
|
1666
|
+
" height: 32px;\n",
|
|
1667
|
+
" padding: 0;\n",
|
|
1668
|
+
" width: 32px;\n",
|
|
1669
|
+
" }\n",
|
|
1670
|
+
"\n",
|
|
1671
|
+
" .colab-df-quickchart:hover {\n",
|
|
1672
|
+
" background-color: var(--hover-bg-color);\n",
|
|
1673
|
+
" box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
1674
|
+
" fill: var(--button-hover-fill-color);\n",
|
|
1675
|
+
" }\n",
|
|
1676
|
+
"\n",
|
|
1677
|
+
" .colab-df-quickchart-complete:disabled,\n",
|
|
1678
|
+
" .colab-df-quickchart-complete:disabled:hover {\n",
|
|
1679
|
+
" background-color: var(--disabled-bg-color);\n",
|
|
1680
|
+
" fill: var(--disabled-fill-color);\n",
|
|
1681
|
+
" box-shadow: none;\n",
|
|
1682
|
+
" }\n",
|
|
1683
|
+
"\n",
|
|
1684
|
+
" .colab-df-spinner {\n",
|
|
1685
|
+
" border: 2px solid var(--fill-color);\n",
|
|
1686
|
+
" border-color: transparent;\n",
|
|
1687
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
1688
|
+
" animation:\n",
|
|
1689
|
+
" spin 1s steps(1) infinite;\n",
|
|
1690
|
+
" }\n",
|
|
1691
|
+
"\n",
|
|
1692
|
+
" @keyframes spin {\n",
|
|
1693
|
+
" 0% {\n",
|
|
1694
|
+
" border-color: transparent;\n",
|
|
1695
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
1696
|
+
" border-left-color: var(--fill-color);\n",
|
|
1697
|
+
" }\n",
|
|
1698
|
+
" 20% {\n",
|
|
1699
|
+
" border-color: transparent;\n",
|
|
1700
|
+
" border-left-color: var(--fill-color);\n",
|
|
1701
|
+
" border-top-color: var(--fill-color);\n",
|
|
1702
|
+
" }\n",
|
|
1703
|
+
" 30% {\n",
|
|
1704
|
+
" border-color: transparent;\n",
|
|
1705
|
+
" border-left-color: var(--fill-color);\n",
|
|
1706
|
+
" border-top-color: var(--fill-color);\n",
|
|
1707
|
+
" border-right-color: var(--fill-color);\n",
|
|
1708
|
+
" }\n",
|
|
1709
|
+
" 40% {\n",
|
|
1710
|
+
" border-color: transparent;\n",
|
|
1711
|
+
" border-right-color: var(--fill-color);\n",
|
|
1712
|
+
" border-top-color: var(--fill-color);\n",
|
|
1713
|
+
" }\n",
|
|
1714
|
+
" 60% {\n",
|
|
1715
|
+
" border-color: transparent;\n",
|
|
1716
|
+
" border-right-color: var(--fill-color);\n",
|
|
1717
|
+
" }\n",
|
|
1718
|
+
" 80% {\n",
|
|
1719
|
+
" border-color: transparent;\n",
|
|
1720
|
+
" border-right-color: var(--fill-color);\n",
|
|
1721
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
1722
|
+
" }\n",
|
|
1723
|
+
" 90% {\n",
|
|
1724
|
+
" border-color: transparent;\n",
|
|
1725
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
1726
|
+
" }\n",
|
|
1727
|
+
" }\n",
|
|
1728
|
+
"</style>\n",
|
|
1729
|
+
"\n",
|
|
1730
|
+
" <script>\n",
|
|
1731
|
+
" async function quickchart(key) {\n",
|
|
1732
|
+
" const quickchartButtonEl =\n",
|
|
1733
|
+
" document.querySelector('#' + key + ' button');\n",
|
|
1734
|
+
" quickchartButtonEl.disabled = true; // To prevent multiple clicks.\n",
|
|
1735
|
+
" quickchartButtonEl.classList.add('colab-df-spinner');\n",
|
|
1736
|
+
" try {\n",
|
|
1737
|
+
" const charts = await google.colab.kernel.invokeFunction(\n",
|
|
1738
|
+
" 'suggestCharts', [key], {});\n",
|
|
1739
|
+
" } catch (error) {\n",
|
|
1740
|
+
" console.error('Error during call to suggestCharts:', error);\n",
|
|
1741
|
+
" }\n",
|
|
1742
|
+
" quickchartButtonEl.classList.remove('colab-df-spinner');\n",
|
|
1743
|
+
" quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
|
|
1744
|
+
" }\n",
|
|
1745
|
+
" (() => {\n",
|
|
1746
|
+
" let quickchartButtonEl =\n",
|
|
1747
|
+
" document.querySelector('#df-54b66c33-97af-4c06-9735-a41863f2c8d8 button');\n",
|
|
1748
|
+
" quickchartButtonEl.style.display =\n",
|
|
1749
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
1750
|
+
" })();\n",
|
|
1751
|
+
" </script>\n",
|
|
1752
|
+
"</div>\n",
|
|
1753
|
+
"\n",
|
|
1754
|
+
" <div id=\"id_403dff92-e8ae-4697-9e35-20c5f5742a01\">\n",
|
|
1755
|
+
" <style>\n",
|
|
1756
|
+
" .colab-df-generate {\n",
|
|
1757
|
+
" background-color: #E8F0FE;\n",
|
|
1758
|
+
" border: none;\n",
|
|
1759
|
+
" border-radius: 50%;\n",
|
|
1760
|
+
" cursor: pointer;\n",
|
|
1761
|
+
" display: none;\n",
|
|
1762
|
+
" fill: #1967D2;\n",
|
|
1763
|
+
" height: 32px;\n",
|
|
1764
|
+
" padding: 0 0 0 0;\n",
|
|
1765
|
+
" width: 32px;\n",
|
|
1766
|
+
" }\n",
|
|
1767
|
+
"\n",
|
|
1768
|
+
" .colab-df-generate:hover {\n",
|
|
1769
|
+
" background-color: #E2EBFA;\n",
|
|
1770
|
+
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
1771
|
+
" fill: #174EA6;\n",
|
|
1772
|
+
" }\n",
|
|
1773
|
+
"\n",
|
|
1774
|
+
" [theme=dark] .colab-df-generate {\n",
|
|
1775
|
+
" background-color: #3B4455;\n",
|
|
1776
|
+
" fill: #D2E3FC;\n",
|
|
1777
|
+
" }\n",
|
|
1778
|
+
"\n",
|
|
1779
|
+
" [theme=dark] .colab-df-generate:hover {\n",
|
|
1780
|
+
" background-color: #434B5C;\n",
|
|
1781
|
+
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
|
|
1782
|
+
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
|
|
1783
|
+
" fill: #FFFFFF;\n",
|
|
1784
|
+
" }\n",
|
|
1785
|
+
" </style>\n",
|
|
1786
|
+
" <button class=\"colab-df-generate\" onclick=\"generateWithVariable('test_df')\"\n",
|
|
1787
|
+
" title=\"Generate code using this dataframe.\"\n",
|
|
1788
|
+
" style=\"display:none;\">\n",
|
|
1789
|
+
"\n",
|
|
1790
|
+
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
|
|
1791
|
+
" width=\"24px\">\n",
|
|
1792
|
+
" <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
|
|
1793
|
+
" </svg>\n",
|
|
1794
|
+
" </button>\n",
|
|
1795
|
+
" <script>\n",
|
|
1796
|
+
" (() => {\n",
|
|
1797
|
+
" const buttonEl =\n",
|
|
1798
|
+
" document.querySelector('#id_403dff92-e8ae-4697-9e35-20c5f5742a01 button.colab-df-generate');\n",
|
|
1799
|
+
" buttonEl.style.display =\n",
|
|
1800
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
1801
|
+
"\n",
|
|
1802
|
+
" buttonEl.onclick = () => {\n",
|
|
1803
|
+
" google.colab.notebook.generateWithVariable('test_df');\n",
|
|
1804
|
+
" }\n",
|
|
1805
|
+
" })();\n",
|
|
1806
|
+
" </script>\n",
|
|
1807
|
+
" </div>\n",
|
|
1808
|
+
"\n",
|
|
1809
|
+
" </div>\n",
|
|
1810
|
+
" </div>\n"
|
|
1811
|
+
],
|
|
1812
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
1813
|
+
"type": "dataframe",
|
|
1814
|
+
"variable_name": "test_df",
|
|
1815
|
+
"summary": "{\n \"name\": \"test_df\",\n \"rows\": 4,\n \"fields\": [\n {\n \"column\": \"hindi\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"\\u091a\\u093f\\u0902\\u0924\\u093e\\u0932\\u092a\\u0924\\u093f\",\n \"\\u0938\\u0942\\u0926\",\n \"\\u0930\\u093e\\u091c\\u0936\\u0947\\u0916\\u0930\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"english\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"chintalpati\",\n \"sood\",\n \"rajshekhar\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
|
|
1816
|
+
}
|
|
1817
|
+
},
|
|
1818
|
+
"metadata": {},
|
|
1819
|
+
"execution_count": 94
|
|
1820
|
+
}
|
|
1821
|
+
]
|
|
1822
|
+
},
|
|
1823
|
+
{
|
|
1824
|
+
"cell_type": "markdown",
|
|
1825
|
+
"source": [
|
|
1826
|
+
"# Validation on Dakshina Test Dataset"
|
|
1827
|
+
],
|
|
1828
|
+
"metadata": {
|
|
1829
|
+
"id": "IZQDxlAS5tIe"
|
|
1830
|
+
}
|
|
1831
|
+
},
|
|
1832
|
+
{
|
|
1833
|
+
"cell_type": "code",
|
|
1834
|
+
"source": [
|
|
1835
|
+
"dakshina_test_df = pd.read_csv('./data/hi.translit.sampled.test.tsv', sep='\\t', names=[\"hindi\", \"english\", \"freq\"])"
|
|
1836
|
+
],
|
|
1837
|
+
"metadata": {
|
|
1838
|
+
"id": "wPkXVPU25oCU"
|
|
1839
|
+
},
|
|
1840
|
+
"execution_count": null,
|
|
1841
|
+
"outputs": []
|
|
1842
|
+
},
|
|
1843
|
+
{
|
|
1844
|
+
"cell_type": "code",
|
|
1845
|
+
"source": [
|
|
1846
|
+
"from tqdm import tqdm\n",
|
|
1847
|
+
"tqdm.pandas()\n",
|
|
1848
|
+
"\n",
|
|
1849
|
+
"predicted = []\n",
|
|
1850
|
+
"\n",
|
|
1851
|
+
"def predict(hin):\n",
|
|
1852
|
+
" predicted.append(translate(hin, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
1853
|
+
" encoder, decoder, max_length_input))\n",
|
|
1854
|
+
"\n",
|
|
1855
|
+
"dakshina_test_df.progress_apply(lambda x: predict(x['hindi']), axis=1)"
|
|
1856
|
+
],
|
|
1857
|
+
"metadata": {
|
|
1858
|
+
"colab": {
|
|
1859
|
+
"base_uri": "https://localhost:8080/",
|
|
1860
|
+
"height": 476
|
|
1861
|
+
},
|
|
1862
|
+
"id": "jM-zkNGsKtlH",
|
|
1863
|
+
"outputId": "a29a198c-5a7c-4925-a2b3-852bd03157f8"
|
|
1864
|
+
},
|
|
1865
|
+
"execution_count": null,
|
|
1866
|
+
"outputs": [
|
|
1867
|
+
{
|
|
1868
|
+
"output_type": "stream",
|
|
1869
|
+
"name": "stderr",
|
|
1870
|
+
"text": [
|
|
1871
|
+
"100%|██████████| 4502/4502 [09:04<00:00, 8.27it/s]\n"
|
|
1872
|
+
]
|
|
1873
|
+
},
|
|
1874
|
+
{
|
|
1875
|
+
"output_type": "execute_result",
|
|
1876
|
+
"data": {
|
|
1877
|
+
"text/plain": [
|
|
1878
|
+
"0 None\n",
|
|
1879
|
+
"1 None\n",
|
|
1880
|
+
"2 None\n",
|
|
1881
|
+
"3 None\n",
|
|
1882
|
+
"4 None\n",
|
|
1883
|
+
" ... \n",
|
|
1884
|
+
"4497 None\n",
|
|
1885
|
+
"4498 None\n",
|
|
1886
|
+
"4499 None\n",
|
|
1887
|
+
"4500 None\n",
|
|
1888
|
+
"4501 None\n",
|
|
1889
|
+
"Length: 4502, dtype: object"
|
|
1890
|
+
],
|
|
1891
|
+
"text/html": [
|
|
1892
|
+
"<div>\n",
|
|
1893
|
+
"<style scoped>\n",
|
|
1894
|
+
" .dataframe tbody tr th:only-of-type {\n",
|
|
1895
|
+
" vertical-align: middle;\n",
|
|
1896
|
+
" }\n",
|
|
1897
|
+
"\n",
|
|
1898
|
+
" .dataframe tbody tr th {\n",
|
|
1899
|
+
" vertical-align: top;\n",
|
|
1900
|
+
" }\n",
|
|
1901
|
+
"\n",
|
|
1902
|
+
" .dataframe thead th {\n",
|
|
1903
|
+
" text-align: right;\n",
|
|
1904
|
+
" }\n",
|
|
1905
|
+
"</style>\n",
|
|
1906
|
+
"<table border=\"1\" class=\"dataframe\">\n",
|
|
1907
|
+
" <thead>\n",
|
|
1908
|
+
" <tr style=\"text-align: right;\">\n",
|
|
1909
|
+
" <th></th>\n",
|
|
1910
|
+
" <th>0</th>\n",
|
|
1911
|
+
" </tr>\n",
|
|
1912
|
+
" </thead>\n",
|
|
1913
|
+
" <tbody>\n",
|
|
1914
|
+
" <tr>\n",
|
|
1915
|
+
" <th>0</th>\n",
|
|
1916
|
+
" <td>None</td>\n",
|
|
1917
|
+
" </tr>\n",
|
|
1918
|
+
" <tr>\n",
|
|
1919
|
+
" <th>1</th>\n",
|
|
1920
|
+
" <td>None</td>\n",
|
|
1921
|
+
" </tr>\n",
|
|
1922
|
+
" <tr>\n",
|
|
1923
|
+
" <th>2</th>\n",
|
|
1924
|
+
" <td>None</td>\n",
|
|
1925
|
+
" </tr>\n",
|
|
1926
|
+
" <tr>\n",
|
|
1927
|
+
" <th>3</th>\n",
|
|
1928
|
+
" <td>None</td>\n",
|
|
1929
|
+
" </tr>\n",
|
|
1930
|
+
" <tr>\n",
|
|
1931
|
+
" <th>4</th>\n",
|
|
1932
|
+
" <td>None</td>\n",
|
|
1933
|
+
" </tr>\n",
|
|
1934
|
+
" <tr>\n",
|
|
1935
|
+
" <th>...</th>\n",
|
|
1936
|
+
" <td>...</td>\n",
|
|
1937
|
+
" </tr>\n",
|
|
1938
|
+
" <tr>\n",
|
|
1939
|
+
" <th>4497</th>\n",
|
|
1940
|
+
" <td>None</td>\n",
|
|
1941
|
+
" </tr>\n",
|
|
1942
|
+
" <tr>\n",
|
|
1943
|
+
" <th>4498</th>\n",
|
|
1944
|
+
" <td>None</td>\n",
|
|
1945
|
+
" </tr>\n",
|
|
1946
|
+
" <tr>\n",
|
|
1947
|
+
" <th>4499</th>\n",
|
|
1948
|
+
" <td>None</td>\n",
|
|
1949
|
+
" </tr>\n",
|
|
1950
|
+
" <tr>\n",
|
|
1951
|
+
" <th>4500</th>\n",
|
|
1952
|
+
" <td>None</td>\n",
|
|
1953
|
+
" </tr>\n",
|
|
1954
|
+
" <tr>\n",
|
|
1955
|
+
" <th>4501</th>\n",
|
|
1956
|
+
" <td>None</td>\n",
|
|
1957
|
+
" </tr>\n",
|
|
1958
|
+
" </tbody>\n",
|
|
1959
|
+
"</table>\n",
|
|
1960
|
+
"<p>4502 rows × 1 columns</p>\n",
|
|
1961
|
+
"</div><br><label><b>dtype:</b> object</label>"
|
|
1962
|
+
]
|
|
1963
|
+
},
|
|
1964
|
+
"metadata": {},
|
|
1965
|
+
"execution_count": 27
|
|
1966
|
+
}
|
|
1967
|
+
]
|
|
1968
|
+
},
|
|
1969
|
+
{
|
|
1970
|
+
"cell_type": "code",
|
|
1971
|
+
"source": [
|
|
1972
|
+
"dakshina_test_df.head()"
|
|
1973
|
+
],
|
|
1974
|
+
"metadata": {
|
|
1975
|
+
"colab": {
|
|
1976
|
+
"base_uri": "https://localhost:8080/",
|
|
1977
|
+
"height": 206
|
|
1978
|
+
},
|
|
1979
|
+
"id": "dpMHM_uiOyuA",
|
|
1980
|
+
"outputId": "72d2f83b-94af-4b59-b3d2-fe2631a21218"
|
|
1981
|
+
},
|
|
1982
|
+
"execution_count": null,
|
|
1983
|
+
"outputs": [
|
|
1984
|
+
{
|
|
1985
|
+
"output_type": "execute_result",
|
|
1986
|
+
"data": {
|
|
1987
|
+
"text/plain": [
|
|
1988
|
+
" hindi english freq\n",
|
|
1989
|
+
"0 अंक ank 5\n",
|
|
1990
|
+
"1 अंक anka 1\n",
|
|
1991
|
+
"2 अंकित ankit 3\n",
|
|
1992
|
+
"3 अंकों anakon 1\n",
|
|
1993
|
+
"4 अंकों ankhon 1"
|
|
1994
|
+
],
|
|
1995
|
+
"text/html": [
|
|
1996
|
+
"\n",
|
|
1997
|
+
" <div id=\"df-a1d39375-28f7-4212-84a9-aae8a61bb0a5\" class=\"colab-df-container\">\n",
|
|
1998
|
+
" <div>\n",
|
|
1999
|
+
"<style scoped>\n",
|
|
2000
|
+
" .dataframe tbody tr th:only-of-type {\n",
|
|
2001
|
+
" vertical-align: middle;\n",
|
|
2002
|
+
" }\n",
|
|
2003
|
+
"\n",
|
|
2004
|
+
" .dataframe tbody tr th {\n",
|
|
2005
|
+
" vertical-align: top;\n",
|
|
2006
|
+
" }\n",
|
|
2007
|
+
"\n",
|
|
2008
|
+
" .dataframe thead th {\n",
|
|
2009
|
+
" text-align: right;\n",
|
|
2010
|
+
" }\n",
|
|
2011
|
+
"</style>\n",
|
|
2012
|
+
"<table border=\"1\" class=\"dataframe\">\n",
|
|
2013
|
+
" <thead>\n",
|
|
2014
|
+
" <tr style=\"text-align: right;\">\n",
|
|
2015
|
+
" <th></th>\n",
|
|
2016
|
+
" <th>hindi</th>\n",
|
|
2017
|
+
" <th>english</th>\n",
|
|
2018
|
+
" <th>freq</th>\n",
|
|
2019
|
+
" </tr>\n",
|
|
2020
|
+
" </thead>\n",
|
|
2021
|
+
" <tbody>\n",
|
|
2022
|
+
" <tr>\n",
|
|
2023
|
+
" <th>0</th>\n",
|
|
2024
|
+
" <td>अंक</td>\n",
|
|
2025
|
+
" <td>ank</td>\n",
|
|
2026
|
+
" <td>5</td>\n",
|
|
2027
|
+
" </tr>\n",
|
|
2028
|
+
" <tr>\n",
|
|
2029
|
+
" <th>1</th>\n",
|
|
2030
|
+
" <td>अंक</td>\n",
|
|
2031
|
+
" <td>anka</td>\n",
|
|
2032
|
+
" <td>1</td>\n",
|
|
2033
|
+
" </tr>\n",
|
|
2034
|
+
" <tr>\n",
|
|
2035
|
+
" <th>2</th>\n",
|
|
2036
|
+
" <td>अंकित</td>\n",
|
|
2037
|
+
" <td>ankit</td>\n",
|
|
2038
|
+
" <td>3</td>\n",
|
|
2039
|
+
" </tr>\n",
|
|
2040
|
+
" <tr>\n",
|
|
2041
|
+
" <th>3</th>\n",
|
|
2042
|
+
" <td>अंकों</td>\n",
|
|
2043
|
+
" <td>anakon</td>\n",
|
|
2044
|
+
" <td>1</td>\n",
|
|
2045
|
+
" </tr>\n",
|
|
2046
|
+
" <tr>\n",
|
|
2047
|
+
" <th>4</th>\n",
|
|
2048
|
+
" <td>अंकों</td>\n",
|
|
2049
|
+
" <td>ankhon</td>\n",
|
|
2050
|
+
" <td>1</td>\n",
|
|
2051
|
+
" </tr>\n",
|
|
2052
|
+
" </tbody>\n",
|
|
2053
|
+
"</table>\n",
|
|
2054
|
+
"</div>\n",
|
|
2055
|
+
" <div class=\"colab-df-buttons\">\n",
|
|
2056
|
+
"\n",
|
|
2057
|
+
" <div class=\"colab-df-container\">\n",
|
|
2058
|
+
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a1d39375-28f7-4212-84a9-aae8a61bb0a5')\"\n",
|
|
2059
|
+
" title=\"Convert this dataframe to an interactive table.\"\n",
|
|
2060
|
+
" style=\"display:none;\">\n",
|
|
2061
|
+
"\n",
|
|
2062
|
+
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
|
|
2063
|
+
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
|
|
2064
|
+
" </svg>\n",
|
|
2065
|
+
" </button>\n",
|
|
2066
|
+
"\n",
|
|
2067
|
+
" <style>\n",
|
|
2068
|
+
" .colab-df-container {\n",
|
|
2069
|
+
" display:flex;\n",
|
|
2070
|
+
" gap: 12px;\n",
|
|
2071
|
+
" }\n",
|
|
2072
|
+
"\n",
|
|
2073
|
+
" .colab-df-convert {\n",
|
|
2074
|
+
" background-color: #E8F0FE;\n",
|
|
2075
|
+
" border: none;\n",
|
|
2076
|
+
" border-radius: 50%;\n",
|
|
2077
|
+
" cursor: pointer;\n",
|
|
2078
|
+
" display: none;\n",
|
|
2079
|
+
" fill: #1967D2;\n",
|
|
2080
|
+
" height: 32px;\n",
|
|
2081
|
+
" padding: 0 0 0 0;\n",
|
|
2082
|
+
" width: 32px;\n",
|
|
2083
|
+
" }\n",
|
|
2084
|
+
"\n",
|
|
2085
|
+
" .colab-df-convert:hover {\n",
|
|
2086
|
+
" background-color: #E2EBFA;\n",
|
|
2087
|
+
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
2088
|
+
" fill: #174EA6;\n",
|
|
2089
|
+
" }\n",
|
|
2090
|
+
"\n",
|
|
2091
|
+
" .colab-df-buttons div {\n",
|
|
2092
|
+
" margin-bottom: 4px;\n",
|
|
2093
|
+
" }\n",
|
|
2094
|
+
"\n",
|
|
2095
|
+
" [theme=dark] .colab-df-convert {\n",
|
|
2096
|
+
" background-color: #3B4455;\n",
|
|
2097
|
+
" fill: #D2E3FC;\n",
|
|
2098
|
+
" }\n",
|
|
2099
|
+
"\n",
|
|
2100
|
+
" [theme=dark] .colab-df-convert:hover {\n",
|
|
2101
|
+
" background-color: #434B5C;\n",
|
|
2102
|
+
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
|
|
2103
|
+
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
|
|
2104
|
+
" fill: #FFFFFF;\n",
|
|
2105
|
+
" }\n",
|
|
2106
|
+
" </style>\n",
|
|
2107
|
+
"\n",
|
|
2108
|
+
" <script>\n",
|
|
2109
|
+
" const buttonEl =\n",
|
|
2110
|
+
" document.querySelector('#df-a1d39375-28f7-4212-84a9-aae8a61bb0a5 button.colab-df-convert');\n",
|
|
2111
|
+
" buttonEl.style.display =\n",
|
|
2112
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
2113
|
+
"\n",
|
|
2114
|
+
" async function convertToInteractive(key) {\n",
|
|
2115
|
+
" const element = document.querySelector('#df-a1d39375-28f7-4212-84a9-aae8a61bb0a5');\n",
|
|
2116
|
+
" const dataTable =\n",
|
|
2117
|
+
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
|
|
2118
|
+
" [key], {});\n",
|
|
2119
|
+
" if (!dataTable) return;\n",
|
|
2120
|
+
"\n",
|
|
2121
|
+
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
|
|
2122
|
+
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
|
|
2123
|
+
" + ' to learn more about interactive tables.';\n",
|
|
2124
|
+
" element.innerHTML = '';\n",
|
|
2125
|
+
" dataTable['output_type'] = 'display_data';\n",
|
|
2126
|
+
" await google.colab.output.renderOutput(dataTable, element);\n",
|
|
2127
|
+
" const docLink = document.createElement('div');\n",
|
|
2128
|
+
" docLink.innerHTML = docLinkHtml;\n",
|
|
2129
|
+
" element.appendChild(docLink);\n",
|
|
2130
|
+
" }\n",
|
|
2131
|
+
" </script>\n",
|
|
2132
|
+
" </div>\n",
|
|
2133
|
+
"\n",
|
|
2134
|
+
"\n",
|
|
2135
|
+
"<div id=\"df-c7d47cd1-8b6c-4080-9ea6-5ef75f4cd0b6\">\n",
|
|
2136
|
+
" <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c7d47cd1-8b6c-4080-9ea6-5ef75f4cd0b6')\"\n",
|
|
2137
|
+
" title=\"Suggest charts\"\n",
|
|
2138
|
+
" style=\"display:none;\">\n",
|
|
2139
|
+
"\n",
|
|
2140
|
+
"<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
|
|
2141
|
+
" width=\"24px\">\n",
|
|
2142
|
+
" <g>\n",
|
|
2143
|
+
" <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
|
|
2144
|
+
" </g>\n",
|
|
2145
|
+
"</svg>\n",
|
|
2146
|
+
" </button>\n",
|
|
2147
|
+
"\n",
|
|
2148
|
+
"<style>\n",
|
|
2149
|
+
" .colab-df-quickchart {\n",
|
|
2150
|
+
" --bg-color: #E8F0FE;\n",
|
|
2151
|
+
" --fill-color: #1967D2;\n",
|
|
2152
|
+
" --hover-bg-color: #E2EBFA;\n",
|
|
2153
|
+
" --hover-fill-color: #174EA6;\n",
|
|
2154
|
+
" --disabled-fill-color: #AAA;\n",
|
|
2155
|
+
" --disabled-bg-color: #DDD;\n",
|
|
2156
|
+
" }\n",
|
|
2157
|
+
"\n",
|
|
2158
|
+
" [theme=dark] .colab-df-quickchart {\n",
|
|
2159
|
+
" --bg-color: #3B4455;\n",
|
|
2160
|
+
" --fill-color: #D2E3FC;\n",
|
|
2161
|
+
" --hover-bg-color: #434B5C;\n",
|
|
2162
|
+
" --hover-fill-color: #FFFFFF;\n",
|
|
2163
|
+
" --disabled-bg-color: #3B4455;\n",
|
|
2164
|
+
" --disabled-fill-color: #666;\n",
|
|
2165
|
+
" }\n",
|
|
2166
|
+
"\n",
|
|
2167
|
+
" .colab-df-quickchart {\n",
|
|
2168
|
+
" background-color: var(--bg-color);\n",
|
|
2169
|
+
" border: none;\n",
|
|
2170
|
+
" border-radius: 50%;\n",
|
|
2171
|
+
" cursor: pointer;\n",
|
|
2172
|
+
" display: none;\n",
|
|
2173
|
+
" fill: var(--fill-color);\n",
|
|
2174
|
+
" height: 32px;\n",
|
|
2175
|
+
" padding: 0;\n",
|
|
2176
|
+
" width: 32px;\n",
|
|
2177
|
+
" }\n",
|
|
2178
|
+
"\n",
|
|
2179
|
+
" .colab-df-quickchart:hover {\n",
|
|
2180
|
+
" background-color: var(--hover-bg-color);\n",
|
|
2181
|
+
" box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
2182
|
+
" fill: var(--button-hover-fill-color);\n",
|
|
2183
|
+
" }\n",
|
|
2184
|
+
"\n",
|
|
2185
|
+
" .colab-df-quickchart-complete:disabled,\n",
|
|
2186
|
+
" .colab-df-quickchart-complete:disabled:hover {\n",
|
|
2187
|
+
" background-color: var(--disabled-bg-color);\n",
|
|
2188
|
+
" fill: var(--disabled-fill-color);\n",
|
|
2189
|
+
" box-shadow: none;\n",
|
|
2190
|
+
" }\n",
|
|
2191
|
+
"\n",
|
|
2192
|
+
" .colab-df-spinner {\n",
|
|
2193
|
+
" border: 2px solid var(--fill-color);\n",
|
|
2194
|
+
" border-color: transparent;\n",
|
|
2195
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
2196
|
+
" animation:\n",
|
|
2197
|
+
" spin 1s steps(1) infinite;\n",
|
|
2198
|
+
" }\n",
|
|
2199
|
+
"\n",
|
|
2200
|
+
" @keyframes spin {\n",
|
|
2201
|
+
" 0% {\n",
|
|
2202
|
+
" border-color: transparent;\n",
|
|
2203
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
2204
|
+
" border-left-color: var(--fill-color);\n",
|
|
2205
|
+
" }\n",
|
|
2206
|
+
" 20% {\n",
|
|
2207
|
+
" border-color: transparent;\n",
|
|
2208
|
+
" border-left-color: var(--fill-color);\n",
|
|
2209
|
+
" border-top-color: var(--fill-color);\n",
|
|
2210
|
+
" }\n",
|
|
2211
|
+
" 30% {\n",
|
|
2212
|
+
" border-color: transparent;\n",
|
|
2213
|
+
" border-left-color: var(--fill-color);\n",
|
|
2214
|
+
" border-top-color: var(--fill-color);\n",
|
|
2215
|
+
" border-right-color: var(--fill-color);\n",
|
|
2216
|
+
" }\n",
|
|
2217
|
+
" 40% {\n",
|
|
2218
|
+
" border-color: transparent;\n",
|
|
2219
|
+
" border-right-color: var(--fill-color);\n",
|
|
2220
|
+
" border-top-color: var(--fill-color);\n",
|
|
2221
|
+
" }\n",
|
|
2222
|
+
" 60% {\n",
|
|
2223
|
+
" border-color: transparent;\n",
|
|
2224
|
+
" border-right-color: var(--fill-color);\n",
|
|
2225
|
+
" }\n",
|
|
2226
|
+
" 80% {\n",
|
|
2227
|
+
" border-color: transparent;\n",
|
|
2228
|
+
" border-right-color: var(--fill-color);\n",
|
|
2229
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
2230
|
+
" }\n",
|
|
2231
|
+
" 90% {\n",
|
|
2232
|
+
" border-color: transparent;\n",
|
|
2233
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
2234
|
+
" }\n",
|
|
2235
|
+
" }\n",
|
|
2236
|
+
"</style>\n",
|
|
2237
|
+
"\n",
|
|
2238
|
+
" <script>\n",
|
|
2239
|
+
" async function quickchart(key) {\n",
|
|
2240
|
+
" const quickchartButtonEl =\n",
|
|
2241
|
+
" document.querySelector('#' + key + ' button');\n",
|
|
2242
|
+
" quickchartButtonEl.disabled = true; // To prevent multiple clicks.\n",
|
|
2243
|
+
" quickchartButtonEl.classList.add('colab-df-spinner');\n",
|
|
2244
|
+
" try {\n",
|
|
2245
|
+
" const charts = await google.colab.kernel.invokeFunction(\n",
|
|
2246
|
+
" 'suggestCharts', [key], {});\n",
|
|
2247
|
+
" } catch (error) {\n",
|
|
2248
|
+
" console.error('Error during call to suggestCharts:', error);\n",
|
|
2249
|
+
" }\n",
|
|
2250
|
+
" quickchartButtonEl.classList.remove('colab-df-spinner');\n",
|
|
2251
|
+
" quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
|
|
2252
|
+
" }\n",
|
|
2253
|
+
" (() => {\n",
|
|
2254
|
+
" let quickchartButtonEl =\n",
|
|
2255
|
+
" document.querySelector('#df-c7d47cd1-8b6c-4080-9ea6-5ef75f4cd0b6 button');\n",
|
|
2256
|
+
" quickchartButtonEl.style.display =\n",
|
|
2257
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
2258
|
+
" })();\n",
|
|
2259
|
+
" </script>\n",
|
|
2260
|
+
"</div>\n",
|
|
2261
|
+
"\n",
|
|
2262
|
+
" </div>\n",
|
|
2263
|
+
" </div>\n"
|
|
2264
|
+
],
|
|
2265
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
2266
|
+
"type": "dataframe",
|
|
2267
|
+
"variable_name": "dakshina_test_df",
|
|
2268
|
+
"summary": "{\n \"name\": \"dakshina_test_df\",\n \"rows\": 4502,\n \"fields\": [\n {\n \"column\": \"hindi\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 2500,\n \"samples\": [\n \"\\u092b\\u094d\\u092f\\u0942\\u091c\\u0928\",\n \"\\u0928\\u0935\\u091c\\u093e\\u0917\\u0930\\u0923\",\n \"\\u0927\\u0928\\u0927\\u093e\\u0928\\u094d\\u092f\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"english\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4442,\n \"samples\": [\n \"pythogaras\",\n \"navya\",\n \"dhokebaaj\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"freq\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 1,\n \"min\": 1,\n \"max\": 10,\n \"num_unique_values\": 10,\n \"samples\": [\n 9,\n 1,\n 4\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
|
|
2269
|
+
}
|
|
2270
|
+
},
|
|
2271
|
+
"metadata": {},
|
|
2272
|
+
"execution_count": 28
|
|
2273
|
+
}
|
|
2274
|
+
]
|
|
2275
|
+
},
|
|
2276
|
+
{
|
|
2277
|
+
"cell_type": "code",
|
|
2278
|
+
"source": [
|
|
2279
|
+
"def edit_distance(s1, s2):\n",
|
|
2280
|
+
" t1 = tf.sparse.from_dense(tf.expand_dims(tf.strings.bytes_split(s1), axis=0))\n",
|
|
2281
|
+
" t2 = tf.sparse.from_dense(tf.expand_dims(tf.strings.bytes_split(s2), axis=0))\n",
|
|
2282
|
+
" distance = tf.edit_distance(t1, t2)\n",
|
|
2283
|
+
" return distance[0].numpy()\n",
|
|
2284
|
+
"\n",
|
|
2285
|
+
"model_edit_dist = []\n",
|
|
2286
|
+
"\n",
|
|
2287
|
+
"def edit_dist(x):\n",
|
|
2288
|
+
" model_edit_dist.append(edit_distance(x['model_predicted'], x['english']))\n",
|
|
2289
|
+
"\n",
|
|
2290
|
+
"dakshina_test_df['model_predicted'] = predicted\n",
|
|
2291
|
+
"dakshina_test_df.progress_apply(lambda x: edit_dist(x), axis=1)\n",
|
|
2292
|
+
"dakshina_test_df['model_edit_distance'] = model_edit_dist\n",
|
|
2293
|
+
"\n",
|
|
2294
|
+
"model_edit_dedup = dakshina_test_df.groupby(['hindi'])['model_edit_distance'].min().reset_index()\n",
|
|
2295
|
+
"model_acc_percent = len(model_edit_dedup[model_edit_dedup['model_edit_distance'] == 0])/len(model_edit_dedup)\n",
|
|
2296
|
+
"print(f\"Out of {len(model_edit_dedup)}, model predicted {model_acc_percent * 100}% correctly\")"
|
|
2297
|
+
],
|
|
2298
|
+
"metadata": {
|
|
2299
|
+
"colab": {
|
|
2300
|
+
"base_uri": "https://localhost:8080/"
|
|
2301
|
+
},
|
|
2302
|
+
"id": "vLtgblD6LDoj",
|
|
2303
|
+
"outputId": "262f85db-1208-425e-fac2-ebe0910a2ec4"
|
|
2304
|
+
},
|
|
2305
|
+
"execution_count": null,
|
|
2306
|
+
"outputs": [
|
|
2307
|
+
{
|
|
2308
|
+
"output_type": "stream",
|
|
2309
|
+
"name": "stderr",
|
|
2310
|
+
"text": [
|
|
2311
|
+
"100%|██████████| 4502/4502 [01:07<00:00, 66.45it/s]"
|
|
2312
|
+
]
|
|
2313
|
+
},
|
|
2314
|
+
{
|
|
2315
|
+
"output_type": "stream",
|
|
2316
|
+
"name": "stdout",
|
|
2317
|
+
"text": [
|
|
2318
|
+
"Out of 2500, model predicted 78.44% correctly\n"
|
|
2319
|
+
]
|
|
2320
|
+
},
|
|
2321
|
+
{
|
|
2322
|
+
"output_type": "stream",
|
|
2323
|
+
"name": "stderr",
|
|
2324
|
+
"text": [
|
|
2325
|
+
"\n"
|
|
2326
|
+
]
|
|
2327
|
+
}
|
|
2328
|
+
]
|
|
2329
|
+
}
|
|
2330
|
+
]
|
|
2331
|
+
}
|