indicate 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indicate/__init__.py +3 -0
- indicate/data/model/hindi_to_english/english_tokens.json +1 -0
- indicate/data/model/hindi_to_english/hindi_tokens.json +1 -0
- indicate/data/model/hindi_to_english/saved_weights/decoder.weights.h5 +0 -0
- indicate/data/model/hindi_to_english/saved_weights/encoder.weights.h5 +0 -0
- indicate/decoder.py +62 -0
- indicate/encoder.py +24 -0
- indicate/hindi2english.py +138 -0
- indicate/logging.py +6 -0
- indicate/notebooks/Data_Preparation.ipynb +1 -0
- indicate/notebooks/Inference.ipynb +1 -0
- indicate/notebooks/Inference_latest.ipynb +2331 -0
- indicate/notebooks/Train.ipynb +1 -0
- indicate/notebooks/Train_latest.ipynb +1746 -0
- indicate/notebooks/Transliterate.ipynb +2501 -0
- indicate/notebooks/transliterate_cricket.ipynb +305 -0
- indicate/tests/__init__.py +12 -0
- indicate/tests/test_010_hindi_translate.py +20 -0
- indicate/transliterate.py +28 -0
- indicate/utils.py +86 -0
- indicate-0.2.0.dist-info/LICENSE +21 -0
- indicate-0.2.0.dist-info/METADATA +139 -0
- indicate-0.2.0.dist-info/RECORD +26 -0
- indicate-0.2.0.dist-info/WHEEL +6 -0
- indicate-0.2.0.dist-info/entry_points.txt +2 -0
- indicate-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1746 @@
|
|
|
1
|
+
{
|
|
2
|
+
"nbformat": 4,
|
|
3
|
+
"nbformat_minor": 0,
|
|
4
|
+
"metadata": {
|
|
5
|
+
"colab": {
|
|
6
|
+
"provenance": [],
|
|
7
|
+
"machine_shape": "hm",
|
|
8
|
+
"gpuType": "A100"
|
|
9
|
+
},
|
|
10
|
+
"kernelspec": {
|
|
11
|
+
"name": "python3",
|
|
12
|
+
"display_name": "Python 3"
|
|
13
|
+
},
|
|
14
|
+
"language_info": {
|
|
15
|
+
"name": "python"
|
|
16
|
+
},
|
|
17
|
+
"accelerator": "GPU"
|
|
18
|
+
},
|
|
19
|
+
"cells": [
|
|
20
|
+
{
|
|
21
|
+
"cell_type": "code",
|
|
22
|
+
"source": [
|
|
23
|
+
"%cd /content/drive/MyDrive/Colab/indicate/"
|
|
24
|
+
],
|
|
25
|
+
"metadata": {
|
|
26
|
+
"colab": {
|
|
27
|
+
"base_uri": "https://localhost:8080/"
|
|
28
|
+
},
|
|
29
|
+
"id": "X3542qmUrVqO",
|
|
30
|
+
"outputId": "8bae4e69-e5fe-425f-df03-dbd29656493d"
|
|
31
|
+
},
|
|
32
|
+
"execution_count": null,
|
|
33
|
+
"outputs": [
|
|
34
|
+
{
|
|
35
|
+
"output_type": "stream",
|
|
36
|
+
"name": "stdout",
|
|
37
|
+
"text": [
|
|
38
|
+
"/content/drive/MyDrive/Colab/indicate\n"
|
|
39
|
+
]
|
|
40
|
+
}
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"cell_type": "code",
|
|
45
|
+
"source": [
|
|
46
|
+
"import json\n",
|
|
47
|
+
"import os\n",
|
|
48
|
+
"import time\n",
|
|
49
|
+
"\n",
|
|
50
|
+
"import pandas as pd\n",
|
|
51
|
+
"import tensorflow as tf\n",
|
|
52
|
+
"\n",
|
|
53
|
+
"from sklearn.model_selection import train_test_split"
|
|
54
|
+
],
|
|
55
|
+
"metadata": {
|
|
56
|
+
"id": "1wHPGoNrrd2s"
|
|
57
|
+
},
|
|
58
|
+
"execution_count": null,
|
|
59
|
+
"outputs": []
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"cell_type": "code",
|
|
63
|
+
"source": [
|
|
64
|
+
"class Encoder(tf.keras.Model):\n",
|
|
65
|
+
" def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
|
|
66
|
+
" super(Encoder, self).__init__()\n",
|
|
67
|
+
" self.batch_sz = batch_sz\n",
|
|
68
|
+
" self.enc_units = enc_units\n",
|
|
69
|
+
" self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
|
|
70
|
+
"\n",
|
|
71
|
+
" ##-------- LSTM layer in Encoder ------- ##\n",
|
|
72
|
+
" self.lstm_layer = tf.keras.layers.LSTM(self.enc_units,\n",
|
|
73
|
+
" return_sequences=True,\n",
|
|
74
|
+
" return_state=True,\n",
|
|
75
|
+
" recurrent_initializer='glorot_uniform')\n",
|
|
76
|
+
"\n",
|
|
77
|
+
"\n",
|
|
78
|
+
"\n",
|
|
79
|
+
" def call(self, x, hidden, training=None):\n",
|
|
80
|
+
" x = self.embedding(x)\n",
|
|
81
|
+
" output, h, c = self.lstm_layer(x, initial_state = hidden)\n",
|
|
82
|
+
" return output, h, c\n",
|
|
83
|
+
"\n",
|
|
84
|
+
" def initialize_hidden_state(self):\n",
|
|
85
|
+
" return [tf.zeros((self.batch_sz, self.enc_units)), tf.zeros((self.batch_sz, self.enc_units))]"
|
|
86
|
+
],
|
|
87
|
+
"metadata": {
|
|
88
|
+
"id": "JfjRUNuuniGV"
|
|
89
|
+
},
|
|
90
|
+
"execution_count": null,
|
|
91
|
+
"outputs": []
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"cell_type": "code",
|
|
95
|
+
"source": [
|
|
96
|
+
"class Decoder(tf.keras.Model):\n",
|
|
97
|
+
" def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz,\n",
|
|
98
|
+
" max_length_input, max_length_output, attention_type='luong'):\n",
|
|
99
|
+
" super(Decoder, self).__init__()\n",
|
|
100
|
+
" self.batch_sz = batch_sz\n",
|
|
101
|
+
" self.dec_units = dec_units\n",
|
|
102
|
+
" self.attention_type = attention_type\n",
|
|
103
|
+
" self.max_length_input = max_length_input\n",
|
|
104
|
+
" self.max_length_output = max_length_output\n",
|
|
105
|
+
"\n",
|
|
106
|
+
" # Embedding Layer\n",
|
|
107
|
+
" self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
|
|
108
|
+
"\n",
|
|
109
|
+
" # Final Dense layer on which softmax will be applied\n",
|
|
110
|
+
" self.fc = tf.keras.layers.Dense(vocab_size)\n",
|
|
111
|
+
"\n",
|
|
112
|
+
" # Define the fundamental cell for decoder recurrent structure\n",
|
|
113
|
+
" self.decoder_rnn_cell = tf.keras.layers.LSTMCell(self.dec_units)\n",
|
|
114
|
+
" self.decoder_rnn = tf.keras.layers.RNN(self.decoder_rnn_cell, return_sequences=True, return_state=True)\n",
|
|
115
|
+
"\n",
|
|
116
|
+
" # Create attention mechanism\n",
|
|
117
|
+
" # For Luong attention: project the query to match the encoder output's dimension (dec_units)\n",
|
|
118
|
+
" if self.attention_type == 'luong':\n",
|
|
119
|
+
" self.query_layer = tf.keras.layers.Dense(dec_units)\n",
|
|
120
|
+
"\n",
|
|
121
|
+
" self.attention_mechanism = self.build_attention_mechanism(None)\n",
|
|
122
|
+
"\n",
|
|
123
|
+
" def build_attention_mechanism(self, memory):\n",
|
|
124
|
+
" if self.attention_type == 'bahdanau':\n",
|
|
125
|
+
" return tf.keras.layers.AdditiveAttention()\n",
|
|
126
|
+
" else:\n",
|
|
127
|
+
" return tf.keras.layers.Attention()\n",
|
|
128
|
+
"\n",
|
|
129
|
+
" def call(self, inputs, initial_state, encoder_outputs):\n",
|
|
130
|
+
" # Get the embeddings of the inputs\n",
|
|
131
|
+
" x = self.embedding(inputs)\n",
|
|
132
|
+
"\n",
|
|
133
|
+
" if self.attention_type == 'luong':\n",
|
|
134
|
+
" query = self.query_layer(x)\n",
|
|
135
|
+
" else:\n",
|
|
136
|
+
" query = x\n",
|
|
137
|
+
"\n",
|
|
138
|
+
" # Feature dimensions to the attention layer\n",
|
|
139
|
+
" attention_output, attention_weights = self.attention_mechanism([query, encoder_outputs], return_attention_scores=True)\n",
|
|
140
|
+
"\n",
|
|
141
|
+
" # Concatenate the attention output with the LSTM Cell output\n",
|
|
142
|
+
" lstm_input = tf.concat([attention_output, x], axis=-1)\n",
|
|
143
|
+
"\n",
|
|
144
|
+
" # Process through the LSTM cell\n",
|
|
145
|
+
" outputs, state_h, state_c = self.decoder_rnn(lstm_input, initial_state)\n",
|
|
146
|
+
"\n",
|
|
147
|
+
" # Pass through the final dense layer\n",
|
|
148
|
+
" outputs = self.fc(outputs)\n",
|
|
149
|
+
"\n",
|
|
150
|
+
" return outputs, (state_h, state_c), attention_weights\n",
|
|
151
|
+
"\n",
|
|
152
|
+
" def build_initial_state(self, batch_sz, encoder_state):\n",
|
|
153
|
+
" hidden_state, cell_state = encoder_state\n",
|
|
154
|
+
" return [hidden_state, cell_state]"
|
|
155
|
+
],
|
|
156
|
+
"metadata": {
|
|
157
|
+
"id": "oC_ggIudwc5Y"
|
|
158
|
+
},
|
|
159
|
+
"execution_count": null,
|
|
160
|
+
"outputs": []
|
|
161
|
+
},
|
|
162
|
+
{
|
|
163
|
+
"cell_type": "code",
|
|
164
|
+
"source": [
|
|
165
|
+
"# Generate text from sequence (mapping back)\n",
|
|
166
|
+
"def sequence_to_chars(tokenizer, sequence):\n",
|
|
167
|
+
" word_index = tokenizer.word_index\n",
|
|
168
|
+
" reverse_map = {val: key for key, val in word_index.items()}\n",
|
|
169
|
+
" retext = ''\n",
|
|
170
|
+
" for q in sequence:\n",
|
|
171
|
+
" if q != 0:\n",
|
|
172
|
+
" retext += reverse_map[q]\n",
|
|
173
|
+
" return retext\n",
|
|
174
|
+
"\n",
|
|
175
|
+
"def evaluate_sentence(sentence, units, input_lang_tokenizer, target_lang_tokenizer,\n",
|
|
176
|
+
" encoder, decoder, max_length_input):\n",
|
|
177
|
+
" # sentence = '^' + sentence.strip() + '$'\n",
|
|
178
|
+
" inputs = [input_lang_tokenizer.word_index[i] for i in sentence]\n",
|
|
179
|
+
" inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n",
|
|
180
|
+
" maxlen=max_length_input,\n",
|
|
181
|
+
" padding='post')\n",
|
|
182
|
+
" inputs = tf.convert_to_tensor(inputs)\n",
|
|
183
|
+
" inference_batch_size = inputs.shape[0]\n",
|
|
184
|
+
" result = ''\n",
|
|
185
|
+
"\n",
|
|
186
|
+
" enc_start_state = [tf.zeros((inference_batch_size, units)), tf.zeros((inference_batch_size, units))]\n",
|
|
187
|
+
" enc_out, enc_h, enc_c = encoder(inputs, enc_start_state)\n",
|
|
188
|
+
"\n",
|
|
189
|
+
" dec_h = enc_h\n",
|
|
190
|
+
" dec_c = enc_c\n",
|
|
191
|
+
"\n",
|
|
192
|
+
" start_tokens = tf.fill([inference_batch_size], target_lang_tokenizer.word_index['^'])\n",
|
|
193
|
+
" end_token = target_lang_tokenizer.word_index['$']\n",
|
|
194
|
+
"\n",
|
|
195
|
+
" # Use a simple greedy decoding method\n",
|
|
196
|
+
" outputs = []\n",
|
|
197
|
+
" for t in range(decoder.max_length_output):\n",
|
|
198
|
+
" # The current input is the last output from previous time step\n",
|
|
199
|
+
" if t == 0:\n",
|
|
200
|
+
" input_dec = start_tokens\n",
|
|
201
|
+
" else:\n",
|
|
202
|
+
" input_dec = tf.expand_dims(sample_ids, 1)\n",
|
|
203
|
+
"\n",
|
|
204
|
+
" # Call the decoder\n",
|
|
205
|
+
" predictions, (dec_h, dec_c), _ = decoder(input_dec, (dec_h, dec_c), enc_out)\n",
|
|
206
|
+
"\n",
|
|
207
|
+
" # Sample the next token\n",
|
|
208
|
+
" sample_ids = tf.argmax(predictions, axis=-1).numpy()[:, -1]\n",
|
|
209
|
+
" outputs.append(sample_ids)\n",
|
|
210
|
+
"\n",
|
|
211
|
+
" return tf.concat(outputs, axis=0)\n",
|
|
212
|
+
"\n",
|
|
213
|
+
"def translate(sentence, units, input_lang_tokenizer, target_lang_tokenizer, encoder, decoder, max_length_input):\n",
|
|
214
|
+
" result = evaluate_sentence(sentence, units, input_lang_tokenizer, target_lang_tokenizer, encoder, decoder, max_length_input)\n",
|
|
215
|
+
" result = sequence_to_chars(target_lang_tokenizer, result[0])\n",
|
|
216
|
+
" return result.strip('$')"
|
|
217
|
+
],
|
|
218
|
+
"metadata": {
|
|
219
|
+
"id": "pnvLyemcnozh"
|
|
220
|
+
},
|
|
221
|
+
"execution_count": null,
|
|
222
|
+
"outputs": []
|
|
223
|
+
},
|
|
224
|
+
{
|
|
225
|
+
"cell_type": "code",
|
|
226
|
+
"source": [
|
|
227
|
+
"print(tf.__version__)"
|
|
228
|
+
],
|
|
229
|
+
"metadata": {
|
|
230
|
+
"colab": {
|
|
231
|
+
"base_uri": "https://localhost:8080/"
|
|
232
|
+
},
|
|
233
|
+
"id": "CKh1Ecloa9Pz",
|
|
234
|
+
"outputId": "a4e02f65-cc04-4585-9651-9db9a739515b"
|
|
235
|
+
},
|
|
236
|
+
"execution_count": null,
|
|
237
|
+
"outputs": [
|
|
238
|
+
{
|
|
239
|
+
"output_type": "stream",
|
|
240
|
+
"name": "stdout",
|
|
241
|
+
"text": [
|
|
242
|
+
"2.18.0\n"
|
|
243
|
+
]
|
|
244
|
+
}
|
|
245
|
+
]
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
"cell_type": "markdown",
|
|
249
|
+
"source": [
|
|
250
|
+
"# Load Dataset"
|
|
251
|
+
],
|
|
252
|
+
"metadata": {
|
|
253
|
+
"id": "a1wzzI6zrsQa"
|
|
254
|
+
}
|
|
255
|
+
},
|
|
256
|
+
{
|
|
257
|
+
"cell_type": "code",
|
|
258
|
+
"source": [
|
|
259
|
+
"input_target_df = pd.read_csv('./data/hindi.csv')"
|
|
260
|
+
],
|
|
261
|
+
"metadata": {
|
|
262
|
+
"id": "63PtIy_-rGtN"
|
|
263
|
+
},
|
|
264
|
+
"execution_count": null,
|
|
265
|
+
"outputs": []
|
|
266
|
+
},
|
|
267
|
+
{
|
|
268
|
+
"cell_type": "markdown",
|
|
269
|
+
"source": [
|
|
270
|
+
"# Prepare Data for training"
|
|
271
|
+
],
|
|
272
|
+
"metadata": {
|
|
273
|
+
"id": "jLfWz0d0ryHu"
|
|
274
|
+
}
|
|
275
|
+
},
|
|
276
|
+
{
|
|
277
|
+
"cell_type": "markdown",
|
|
278
|
+
"source": [
|
|
279
|
+
"## Append start and end of sequence for target"
|
|
280
|
+
],
|
|
281
|
+
"metadata": {
|
|
282
|
+
"id": "RbXARDGdsByb"
|
|
283
|
+
}
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
"cell_type": "code",
|
|
287
|
+
"source": [
|
|
288
|
+
"# append start and end of sequence for target\n",
|
|
289
|
+
"sos = '^'\n",
|
|
290
|
+
"eos = '$'\n",
|
|
291
|
+
"\n",
|
|
292
|
+
"input_target_df['english'] = sos + input_target_df['english'].astype(str) + eos"
|
|
293
|
+
],
|
|
294
|
+
"metadata": {
|
|
295
|
+
"id": "lqEt-Smdrxas"
|
|
296
|
+
},
|
|
297
|
+
"execution_count": null,
|
|
298
|
+
"outputs": []
|
|
299
|
+
},
|
|
300
|
+
{
|
|
301
|
+
"cell_type": "code",
|
|
302
|
+
"source": [
|
|
303
|
+
"input_target_df.head()"
|
|
304
|
+
],
|
|
305
|
+
"metadata": {
|
|
306
|
+
"colab": {
|
|
307
|
+
"base_uri": "https://localhost:8080/",
|
|
308
|
+
"height": 206
|
|
309
|
+
},
|
|
310
|
+
"id": "7SH3SP1Xsbp1",
|
|
311
|
+
"outputId": "f5bd5bcb-c53e-44ee-9b21-77d058a12c31"
|
|
312
|
+
},
|
|
313
|
+
"execution_count": null,
|
|
314
|
+
"outputs": [
|
|
315
|
+
{
|
|
316
|
+
"output_type": "execute_result",
|
|
317
|
+
"data": {
|
|
318
|
+
"text/plain": [
|
|
319
|
+
" hindi english\n",
|
|
320
|
+
"0 प्रयात सुबोध चन्द्र दास ^late subodh chandra das$\n",
|
|
321
|
+
"1 स्वर्गीय पिनाकि कुंडू ^late pinaki kundu$\n",
|
|
322
|
+
"2 मृत नूर महम्मद बिश्वास ^late nur mohammad biswas$\n",
|
|
323
|
+
"3 नारायण चंद्र डॉं ^narayan chandra dawn$\n",
|
|
324
|
+
"4 आब्दुल मन्नान ^abdul mannan$"
|
|
325
|
+
],
|
|
326
|
+
"text/html": [
|
|
327
|
+
"\n",
|
|
328
|
+
" <div id=\"df-6cc869bf-2284-4e79-9db2-710c93dd4b8d\" class=\"colab-df-container\">\n",
|
|
329
|
+
" <div>\n",
|
|
330
|
+
"<style scoped>\n",
|
|
331
|
+
" .dataframe tbody tr th:only-of-type {\n",
|
|
332
|
+
" vertical-align: middle;\n",
|
|
333
|
+
" }\n",
|
|
334
|
+
"\n",
|
|
335
|
+
" .dataframe tbody tr th {\n",
|
|
336
|
+
" vertical-align: top;\n",
|
|
337
|
+
" }\n",
|
|
338
|
+
"\n",
|
|
339
|
+
" .dataframe thead th {\n",
|
|
340
|
+
" text-align: right;\n",
|
|
341
|
+
" }\n",
|
|
342
|
+
"</style>\n",
|
|
343
|
+
"<table border=\"1\" class=\"dataframe\">\n",
|
|
344
|
+
" <thead>\n",
|
|
345
|
+
" <tr style=\"text-align: right;\">\n",
|
|
346
|
+
" <th></th>\n",
|
|
347
|
+
" <th>hindi</th>\n",
|
|
348
|
+
" <th>english</th>\n",
|
|
349
|
+
" </tr>\n",
|
|
350
|
+
" </thead>\n",
|
|
351
|
+
" <tbody>\n",
|
|
352
|
+
" <tr>\n",
|
|
353
|
+
" <th>0</th>\n",
|
|
354
|
+
" <td>प्रयात सुबोध चन्द्र दास</td>\n",
|
|
355
|
+
" <td>^late subodh chandra das$</td>\n",
|
|
356
|
+
" </tr>\n",
|
|
357
|
+
" <tr>\n",
|
|
358
|
+
" <th>1</th>\n",
|
|
359
|
+
" <td>स्वर्गीय पिनाकि कुंडू</td>\n",
|
|
360
|
+
" <td>^late pinaki kundu$</td>\n",
|
|
361
|
+
" </tr>\n",
|
|
362
|
+
" <tr>\n",
|
|
363
|
+
" <th>2</th>\n",
|
|
364
|
+
" <td>मृत नूर महम्मद बिश्वास</td>\n",
|
|
365
|
+
" <td>^late nur mohammad biswas$</td>\n",
|
|
366
|
+
" </tr>\n",
|
|
367
|
+
" <tr>\n",
|
|
368
|
+
" <th>3</th>\n",
|
|
369
|
+
" <td>नारायण चंद्र डॉं</td>\n",
|
|
370
|
+
" <td>^narayan chandra dawn$</td>\n",
|
|
371
|
+
" </tr>\n",
|
|
372
|
+
" <tr>\n",
|
|
373
|
+
" <th>4</th>\n",
|
|
374
|
+
" <td>आब्दुल मन्नान</td>\n",
|
|
375
|
+
" <td>^abdul mannan$</td>\n",
|
|
376
|
+
" </tr>\n",
|
|
377
|
+
" </tbody>\n",
|
|
378
|
+
"</table>\n",
|
|
379
|
+
"</div>\n",
|
|
380
|
+
" <div class=\"colab-df-buttons\">\n",
|
|
381
|
+
"\n",
|
|
382
|
+
" <div class=\"colab-df-container\">\n",
|
|
383
|
+
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6cc869bf-2284-4e79-9db2-710c93dd4b8d')\"\n",
|
|
384
|
+
" title=\"Convert this dataframe to an interactive table.\"\n",
|
|
385
|
+
" style=\"display:none;\">\n",
|
|
386
|
+
"\n",
|
|
387
|
+
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
|
|
388
|
+
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
|
|
389
|
+
" </svg>\n",
|
|
390
|
+
" </button>\n",
|
|
391
|
+
"\n",
|
|
392
|
+
" <style>\n",
|
|
393
|
+
" .colab-df-container {\n",
|
|
394
|
+
" display:flex;\n",
|
|
395
|
+
" gap: 12px;\n",
|
|
396
|
+
" }\n",
|
|
397
|
+
"\n",
|
|
398
|
+
" .colab-df-convert {\n",
|
|
399
|
+
" background-color: #E8F0FE;\n",
|
|
400
|
+
" border: none;\n",
|
|
401
|
+
" border-radius: 50%;\n",
|
|
402
|
+
" cursor: pointer;\n",
|
|
403
|
+
" display: none;\n",
|
|
404
|
+
" fill: #1967D2;\n",
|
|
405
|
+
" height: 32px;\n",
|
|
406
|
+
" padding: 0 0 0 0;\n",
|
|
407
|
+
" width: 32px;\n",
|
|
408
|
+
" }\n",
|
|
409
|
+
"\n",
|
|
410
|
+
" .colab-df-convert:hover {\n",
|
|
411
|
+
" background-color: #E2EBFA;\n",
|
|
412
|
+
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
413
|
+
" fill: #174EA6;\n",
|
|
414
|
+
" }\n",
|
|
415
|
+
"\n",
|
|
416
|
+
" .colab-df-buttons div {\n",
|
|
417
|
+
" margin-bottom: 4px;\n",
|
|
418
|
+
" }\n",
|
|
419
|
+
"\n",
|
|
420
|
+
" [theme=dark] .colab-df-convert {\n",
|
|
421
|
+
" background-color: #3B4455;\n",
|
|
422
|
+
" fill: #D2E3FC;\n",
|
|
423
|
+
" }\n",
|
|
424
|
+
"\n",
|
|
425
|
+
" [theme=dark] .colab-df-convert:hover {\n",
|
|
426
|
+
" background-color: #434B5C;\n",
|
|
427
|
+
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
|
|
428
|
+
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
|
|
429
|
+
" fill: #FFFFFF;\n",
|
|
430
|
+
" }\n",
|
|
431
|
+
" </style>\n",
|
|
432
|
+
"\n",
|
|
433
|
+
" <script>\n",
|
|
434
|
+
" const buttonEl =\n",
|
|
435
|
+
" document.querySelector('#df-6cc869bf-2284-4e79-9db2-710c93dd4b8d button.colab-df-convert');\n",
|
|
436
|
+
" buttonEl.style.display =\n",
|
|
437
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
438
|
+
"\n",
|
|
439
|
+
" async function convertToInteractive(key) {\n",
|
|
440
|
+
" const element = document.querySelector('#df-6cc869bf-2284-4e79-9db2-710c93dd4b8d');\n",
|
|
441
|
+
" const dataTable =\n",
|
|
442
|
+
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
|
|
443
|
+
" [key], {});\n",
|
|
444
|
+
" if (!dataTable) return;\n",
|
|
445
|
+
"\n",
|
|
446
|
+
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
|
|
447
|
+
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
|
|
448
|
+
" + ' to learn more about interactive tables.';\n",
|
|
449
|
+
" element.innerHTML = '';\n",
|
|
450
|
+
" dataTable['output_type'] = 'display_data';\n",
|
|
451
|
+
" await google.colab.output.renderOutput(dataTable, element);\n",
|
|
452
|
+
" const docLink = document.createElement('div');\n",
|
|
453
|
+
" docLink.innerHTML = docLinkHtml;\n",
|
|
454
|
+
" element.appendChild(docLink);\n",
|
|
455
|
+
" }\n",
|
|
456
|
+
" </script>\n",
|
|
457
|
+
" </div>\n",
|
|
458
|
+
"\n",
|
|
459
|
+
"\n",
|
|
460
|
+
"<div id=\"df-f3ad6501-6696-466b-99ae-6373b70f380e\">\n",
|
|
461
|
+
" <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f3ad6501-6696-466b-99ae-6373b70f380e')\"\n",
|
|
462
|
+
" title=\"Suggest charts\"\n",
|
|
463
|
+
" style=\"display:none;\">\n",
|
|
464
|
+
"\n",
|
|
465
|
+
"<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
|
|
466
|
+
" width=\"24px\">\n",
|
|
467
|
+
" <g>\n",
|
|
468
|
+
" <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
|
|
469
|
+
" </g>\n",
|
|
470
|
+
"</svg>\n",
|
|
471
|
+
" </button>\n",
|
|
472
|
+
"\n",
|
|
473
|
+
"<style>\n",
|
|
474
|
+
" .colab-df-quickchart {\n",
|
|
475
|
+
" --bg-color: #E8F0FE;\n",
|
|
476
|
+
" --fill-color: #1967D2;\n",
|
|
477
|
+
" --hover-bg-color: #E2EBFA;\n",
|
|
478
|
+
" --hover-fill-color: #174EA6;\n",
|
|
479
|
+
" --disabled-fill-color: #AAA;\n",
|
|
480
|
+
" --disabled-bg-color: #DDD;\n",
|
|
481
|
+
" }\n",
|
|
482
|
+
"\n",
|
|
483
|
+
" [theme=dark] .colab-df-quickchart {\n",
|
|
484
|
+
" --bg-color: #3B4455;\n",
|
|
485
|
+
" --fill-color: #D2E3FC;\n",
|
|
486
|
+
" --hover-bg-color: #434B5C;\n",
|
|
487
|
+
" --hover-fill-color: #FFFFFF;\n",
|
|
488
|
+
" --disabled-bg-color: #3B4455;\n",
|
|
489
|
+
" --disabled-fill-color: #666;\n",
|
|
490
|
+
" }\n",
|
|
491
|
+
"\n",
|
|
492
|
+
" .colab-df-quickchart {\n",
|
|
493
|
+
" background-color: var(--bg-color);\n",
|
|
494
|
+
" border: none;\n",
|
|
495
|
+
" border-radius: 50%;\n",
|
|
496
|
+
" cursor: pointer;\n",
|
|
497
|
+
" display: none;\n",
|
|
498
|
+
" fill: var(--fill-color);\n",
|
|
499
|
+
" height: 32px;\n",
|
|
500
|
+
" padding: 0;\n",
|
|
501
|
+
" width: 32px;\n",
|
|
502
|
+
" }\n",
|
|
503
|
+
"\n",
|
|
504
|
+
" .colab-df-quickchart:hover {\n",
|
|
505
|
+
" background-color: var(--hover-bg-color);\n",
|
|
506
|
+
" box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
|
|
507
|
+
" fill: var(--button-hover-fill-color);\n",
|
|
508
|
+
" }\n",
|
|
509
|
+
"\n",
|
|
510
|
+
" .colab-df-quickchart-complete:disabled,\n",
|
|
511
|
+
" .colab-df-quickchart-complete:disabled:hover {\n",
|
|
512
|
+
" background-color: var(--disabled-bg-color);\n",
|
|
513
|
+
" fill: var(--disabled-fill-color);\n",
|
|
514
|
+
" box-shadow: none;\n",
|
|
515
|
+
" }\n",
|
|
516
|
+
"\n",
|
|
517
|
+
" .colab-df-spinner {\n",
|
|
518
|
+
" border: 2px solid var(--fill-color);\n",
|
|
519
|
+
" border-color: transparent;\n",
|
|
520
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
521
|
+
" animation:\n",
|
|
522
|
+
" spin 1s steps(1) infinite;\n",
|
|
523
|
+
" }\n",
|
|
524
|
+
"\n",
|
|
525
|
+
" @keyframes spin {\n",
|
|
526
|
+
" 0% {\n",
|
|
527
|
+
" border-color: transparent;\n",
|
|
528
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
529
|
+
" border-left-color: var(--fill-color);\n",
|
|
530
|
+
" }\n",
|
|
531
|
+
" 20% {\n",
|
|
532
|
+
" border-color: transparent;\n",
|
|
533
|
+
" border-left-color: var(--fill-color);\n",
|
|
534
|
+
" border-top-color: var(--fill-color);\n",
|
|
535
|
+
" }\n",
|
|
536
|
+
" 30% {\n",
|
|
537
|
+
" border-color: transparent;\n",
|
|
538
|
+
" border-left-color: var(--fill-color);\n",
|
|
539
|
+
" border-top-color: var(--fill-color);\n",
|
|
540
|
+
" border-right-color: var(--fill-color);\n",
|
|
541
|
+
" }\n",
|
|
542
|
+
" 40% {\n",
|
|
543
|
+
" border-color: transparent;\n",
|
|
544
|
+
" border-right-color: var(--fill-color);\n",
|
|
545
|
+
" border-top-color: var(--fill-color);\n",
|
|
546
|
+
" }\n",
|
|
547
|
+
" 60% {\n",
|
|
548
|
+
" border-color: transparent;\n",
|
|
549
|
+
" border-right-color: var(--fill-color);\n",
|
|
550
|
+
" }\n",
|
|
551
|
+
" 80% {\n",
|
|
552
|
+
" border-color: transparent;\n",
|
|
553
|
+
" border-right-color: var(--fill-color);\n",
|
|
554
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
555
|
+
" }\n",
|
|
556
|
+
" 90% {\n",
|
|
557
|
+
" border-color: transparent;\n",
|
|
558
|
+
" border-bottom-color: var(--fill-color);\n",
|
|
559
|
+
" }\n",
|
|
560
|
+
" }\n",
|
|
561
|
+
"</style>\n",
|
|
562
|
+
"\n",
|
|
563
|
+
" <script>\n",
|
|
564
|
+
" async function quickchart(key) {\n",
|
|
565
|
+
" const quickchartButtonEl =\n",
|
|
566
|
+
" document.querySelector('#' + key + ' button');\n",
|
|
567
|
+
" quickchartButtonEl.disabled = true; // To prevent multiple clicks.\n",
|
|
568
|
+
" quickchartButtonEl.classList.add('colab-df-spinner');\n",
|
|
569
|
+
" try {\n",
|
|
570
|
+
" const charts = await google.colab.kernel.invokeFunction(\n",
|
|
571
|
+
" 'suggestCharts', [key], {});\n",
|
|
572
|
+
" } catch (error) {\n",
|
|
573
|
+
" console.error('Error during call to suggestCharts:', error);\n",
|
|
574
|
+
" }\n",
|
|
575
|
+
" quickchartButtonEl.classList.remove('colab-df-spinner');\n",
|
|
576
|
+
" quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
|
|
577
|
+
" }\n",
|
|
578
|
+
" (() => {\n",
|
|
579
|
+
" let quickchartButtonEl =\n",
|
|
580
|
+
" document.querySelector('#df-f3ad6501-6696-466b-99ae-6373b70f380e button');\n",
|
|
581
|
+
" quickchartButtonEl.style.display =\n",
|
|
582
|
+
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
|
|
583
|
+
" })();\n",
|
|
584
|
+
" </script>\n",
|
|
585
|
+
"</div>\n",
|
|
586
|
+
"\n",
|
|
587
|
+
" </div>\n",
|
|
588
|
+
" </div>\n"
|
|
589
|
+
],
|
|
590
|
+
"application/vnd.google.colaboratory.intrinsic+json": {
|
|
591
|
+
"type": "dataframe",
|
|
592
|
+
"variable_name": "input_target_df"
|
|
593
|
+
}
|
|
594
|
+
},
|
|
595
|
+
"metadata": {},
|
|
596
|
+
"execution_count": 9
|
|
597
|
+
}
|
|
598
|
+
]
|
|
599
|
+
},
|
|
600
|
+
{
|
|
601
|
+
"cell_type": "markdown",
|
|
602
|
+
"source": [
|
|
603
|
+
"## Construct Vocab for Source and Target"
|
|
604
|
+
],
|
|
605
|
+
"metadata": {
|
|
606
|
+
"id": "LobqH0_ysdXj"
|
|
607
|
+
}
|
|
608
|
+
},
|
|
609
|
+
{
|
|
610
|
+
"cell_type": "code",
|
|
611
|
+
"source": [
|
|
612
|
+
"input_words = input_target_df['hindi'].tolist()\n",
|
|
613
|
+
"target_words = input_target_df['english'].tolist()"
|
|
614
|
+
],
|
|
615
|
+
"metadata": {
|
|
616
|
+
"id": "-dRUCb3dsLn_"
|
|
617
|
+
},
|
|
618
|
+
"execution_count": null,
|
|
619
|
+
"outputs": []
|
|
620
|
+
},
|
|
621
|
+
{
|
|
622
|
+
"cell_type": "code",
|
|
623
|
+
"source": [
|
|
624
|
+
"print(f\"Total number of input words {len(input_words)}\")\n",
|
|
625
|
+
"print(f\"Total number of target words {len(target_words)}\")"
|
|
626
|
+
],
|
|
627
|
+
"metadata": {
|
|
628
|
+
"colab": {
|
|
629
|
+
"base_uri": "https://localhost:8080/"
|
|
630
|
+
},
|
|
631
|
+
"id": "u0_y53xksY4Y",
|
|
632
|
+
"outputId": "51f88d98-a55a-4781-a9fa-92e0bb7d1202"
|
|
633
|
+
},
|
|
634
|
+
"execution_count": null,
|
|
635
|
+
"outputs": [
|
|
636
|
+
{
|
|
637
|
+
"output_type": "stream",
|
|
638
|
+
"name": "stdout",
|
|
639
|
+
"text": [
|
|
640
|
+
"Total number of input words 371035\n",
|
|
641
|
+
"Total number of target words 371035\n"
|
|
642
|
+
]
|
|
643
|
+
}
|
|
644
|
+
]
|
|
645
|
+
},
|
|
646
|
+
{
|
|
647
|
+
"cell_type": "code",
|
|
648
|
+
"source": [
|
|
649
|
+
"input_lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='', char_level=True)\n",
|
|
650
|
+
"input_lang_tokenizer.fit_on_texts(input_words)\n",
|
|
651
|
+
"input_tensor = input_lang_tokenizer.texts_to_sequences(input_words)\n",
|
|
652
|
+
"input_tensor = tf.keras.preprocessing.sequence.pad_sequences(input_tensor, padding='post')"
|
|
653
|
+
],
|
|
654
|
+
"metadata": {
|
|
655
|
+
"id": "zKIjiRn_suXm"
|
|
656
|
+
},
|
|
657
|
+
"execution_count": null,
|
|
658
|
+
"outputs": []
|
|
659
|
+
},
|
|
660
|
+
{
|
|
661
|
+
"cell_type": "code",
|
|
662
|
+
"source": [
|
|
663
|
+
"target_lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='', char_level=True)\n",
|
|
664
|
+
"target_lang_tokenizer.fit_on_texts(target_words)\n",
|
|
665
|
+
"target_tensor = target_lang_tokenizer.texts_to_sequences(target_words)\n",
|
|
666
|
+
"target_tensor = tf.keras.preprocessing.sequence.pad_sequences(target_tensor, padding='post')"
|
|
667
|
+
],
|
|
668
|
+
"metadata": {
|
|
669
|
+
"id": "GAUAkBKZs_B8"
|
|
670
|
+
},
|
|
671
|
+
"execution_count": null,
|
|
672
|
+
"outputs": []
|
|
673
|
+
},
|
|
674
|
+
{
|
|
675
|
+
"cell_type": "markdown",
|
|
676
|
+
"source": [
|
|
677
|
+
"## Save Vocab for Source and Target"
|
|
678
|
+
],
|
|
679
|
+
"metadata": {
|
|
680
|
+
"id": "xbx21xm5tKG0"
|
|
681
|
+
}
|
|
682
|
+
},
|
|
683
|
+
{
|
|
684
|
+
"cell_type": "code",
|
|
685
|
+
"source": [
|
|
686
|
+
"# save tokenizers\n",
|
|
687
|
+
"hindi_tokenizer_json = input_lang_tokenizer.to_json()\n",
|
|
688
|
+
"with open('hindi_tokens.json', 'w') as f:\n",
|
|
689
|
+
" json.dump(hindi_tokenizer_json, f)\n",
|
|
690
|
+
"\n",
|
|
691
|
+
"english_tokenizer_json = target_lang_tokenizer.to_json()\n",
|
|
692
|
+
"with open('english_tokens.json', 'w') as f:\n",
|
|
693
|
+
" json.dump(english_tokenizer_json, f)"
|
|
694
|
+
],
|
|
695
|
+
"metadata": {
|
|
696
|
+
"id": "N_tEU-mctE64"
|
|
697
|
+
},
|
|
698
|
+
"execution_count": null,
|
|
699
|
+
"outputs": []
|
|
700
|
+
},
|
|
701
|
+
{
|
|
702
|
+
"cell_type": "markdown",
|
|
703
|
+
"source": [
|
|
704
|
+
"## Split data into Train and Validation"
|
|
705
|
+
],
|
|
706
|
+
"metadata": {
|
|
707
|
+
"id": "YP6tOVP_tglo"
|
|
708
|
+
}
|
|
709
|
+
},
|
|
710
|
+
{
|
|
711
|
+
"cell_type": "code",
|
|
712
|
+
"source": [
|
|
713
|
+
"# split train and validation\n",
|
|
714
|
+
"input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)"
|
|
715
|
+
],
|
|
716
|
+
"metadata": {
|
|
717
|
+
"id": "XuednRVRtVPW"
|
|
718
|
+
},
|
|
719
|
+
"execution_count": null,
|
|
720
|
+
"outputs": []
|
|
721
|
+
},
|
|
722
|
+
{
|
|
723
|
+
"cell_type": "code",
|
|
724
|
+
"source": [
|
|
725
|
+
"BATCH_SIZE = 64\n",
|
|
726
|
+
"BUFFER_SIZE = 120000\n",
|
|
727
|
+
"\n",
|
|
728
|
+
"train_dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train))\n",
|
|
729
|
+
"train_dataset = train_dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n",
|
|
730
|
+
"\n",
|
|
731
|
+
"val_dataset = tf.data.Dataset.from_tensor_slices((input_tensor_val, target_tensor_val))\n",
|
|
732
|
+
"val_dataset = val_dataset.batch(BATCH_SIZE, drop_remainder=True)"
|
|
733
|
+
],
|
|
734
|
+
"metadata": {
|
|
735
|
+
"id": "8euCEPBKtxrI"
|
|
736
|
+
},
|
|
737
|
+
"execution_count": null,
|
|
738
|
+
"outputs": []
|
|
739
|
+
},
|
|
740
|
+
{
|
|
741
|
+
"cell_type": "code",
|
|
742
|
+
"source": [
|
|
743
|
+
"example_input_batch, example_target_batch = next(iter(train_dataset))\n",
|
|
744
|
+
"example_input_batch.shape, example_target_batch.shape"
|
|
745
|
+
],
|
|
746
|
+
"metadata": {
|
|
747
|
+
"colab": {
|
|
748
|
+
"base_uri": "https://localhost:8080/"
|
|
749
|
+
},
|
|
750
|
+
"id": "ecn7UTUAt5dX",
|
|
751
|
+
"outputId": "69054e78-1b57-4c2d-bf63-51cf39efed77"
|
|
752
|
+
},
|
|
753
|
+
"execution_count": null,
|
|
754
|
+
"outputs": [
|
|
755
|
+
{
|
|
756
|
+
"output_type": "execute_result",
|
|
757
|
+
"data": {
|
|
758
|
+
"text/plain": [
|
|
759
|
+
"(TensorShape([64, 47]), TensorShape([64, 173]))"
|
|
760
|
+
]
|
|
761
|
+
},
|
|
762
|
+
"metadata": {},
|
|
763
|
+
"execution_count": 17
|
|
764
|
+
}
|
|
765
|
+
]
|
|
766
|
+
},
|
|
767
|
+
{
|
|
768
|
+
"cell_type": "markdown",
|
|
769
|
+
"source": [
|
|
770
|
+
"## Summary of prepared dataset"
|
|
771
|
+
],
|
|
772
|
+
"metadata": {
|
|
773
|
+
"id": "BncLpNAEuWP7"
|
|
774
|
+
}
|
|
775
|
+
},
|
|
776
|
+
{
|
|
777
|
+
"cell_type": "code",
|
|
778
|
+
"source": [
|
|
779
|
+
"vocab_inp_size = len(input_lang_tokenizer.word_index)+1\n",
|
|
780
|
+
"vocab_tar_size = len(target_lang_tokenizer.word_index)+1\n",
|
|
781
|
+
"\n",
|
|
782
|
+
"max_length_input = input_tensor.shape[1]\n",
|
|
783
|
+
"max_length_output = target_tensor.shape[1]\n",
|
|
784
|
+
"\n",
|
|
785
|
+
"embedding_dim = 256\n",
|
|
786
|
+
"units = 1024\n",
|
|
787
|
+
"steps_per_epoch = input_tensor.shape[0]//BATCH_SIZE"
|
|
788
|
+
],
|
|
789
|
+
"metadata": {
|
|
790
|
+
"id": "7UzIiLwTuG3h"
|
|
791
|
+
},
|
|
792
|
+
"execution_count": null,
|
|
793
|
+
"outputs": []
|
|
794
|
+
},
|
|
795
|
+
{
|
|
796
|
+
"cell_type": "code",
|
|
797
|
+
"source": [
|
|
798
|
+
"print('Number of samples:', len(input_words))\n",
|
|
799
|
+
"\n",
|
|
800
|
+
"print('Input Vocab size:', vocab_inp_size)\n",
|
|
801
|
+
"print('Target Vocab size:', vocab_tar_size)\n",
|
|
802
|
+
"\n",
|
|
803
|
+
"print('Max sequence length for inputs:', max_length_input)\n",
|
|
804
|
+
"print('Max sequence length for outputs:', max_length_output)"
|
|
805
|
+
],
|
|
806
|
+
"metadata": {
|
|
807
|
+
"colab": {
|
|
808
|
+
"base_uri": "https://localhost:8080/"
|
|
809
|
+
},
|
|
810
|
+
"id": "W3NgN0bRuhWQ",
|
|
811
|
+
"outputId": "6c29c81a-3162-4f94-adbb-48e255c772f9"
|
|
812
|
+
},
|
|
813
|
+
"execution_count": null,
|
|
814
|
+
"outputs": [
|
|
815
|
+
{
|
|
816
|
+
"output_type": "stream",
|
|
817
|
+
"name": "stdout",
|
|
818
|
+
"text": [
|
|
819
|
+
"Number of samples: 371035\n",
|
|
820
|
+
"Input Vocab size: 200\n",
|
|
821
|
+
"Target Vocab size: 137\n",
|
|
822
|
+
"Max sequence length for inputs: 47\n",
|
|
823
|
+
"Max sequence length for outputs: 173\n"
|
|
824
|
+
]
|
|
825
|
+
}
|
|
826
|
+
]
|
|
827
|
+
},
|
|
828
|
+
{
|
|
829
|
+
"cell_type": "markdown",
|
|
830
|
+
"source": [
|
|
831
|
+
"# Encoder"
|
|
832
|
+
],
|
|
833
|
+
"metadata": {
|
|
834
|
+
"id": "Qr9di8mBupTC"
|
|
835
|
+
}
|
|
836
|
+
},
|
|
837
|
+
{
|
|
838
|
+
"cell_type": "code",
|
|
839
|
+
"source": [
|
|
840
|
+
"encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)"
|
|
841
|
+
],
|
|
842
|
+
"metadata": {
|
|
843
|
+
"id": "_jOmoDVEutZJ"
|
|
844
|
+
},
|
|
845
|
+
"execution_count": null,
|
|
846
|
+
"outputs": []
|
|
847
|
+
},
|
|
848
|
+
{
|
|
849
|
+
"cell_type": "markdown",
|
|
850
|
+
"source": [
|
|
851
|
+
"## Test Encoder"
|
|
852
|
+
],
|
|
853
|
+
"metadata": {
|
|
854
|
+
"id": "zxMmclrzuyJw"
|
|
855
|
+
}
|
|
856
|
+
},
|
|
857
|
+
{
|
|
858
|
+
"cell_type": "code",
|
|
859
|
+
"source": [
|
|
860
|
+
"## Test Encoder Stack\n",
|
|
861
|
+
"# sample input\n",
|
|
862
|
+
"sample_hidden = encoder.initialize_hidden_state()\n",
|
|
863
|
+
"sample_output, sample_h, sample_c = encoder(example_input_batch, sample_hidden)\n",
|
|
864
|
+
"print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
|
|
865
|
+
"print ('Encoder h vecotr shape: (batch size, units) {}'.format(sample_h.shape))\n",
|
|
866
|
+
"print ('Encoder c vector shape: (batch size, units) {}'.format(sample_c.shape))\n",
|
|
867
|
+
"print('Encoder output vector shape: (batch size, sequence length, units) {}'.format(sample_output.shape))"
|
|
868
|
+
],
|
|
869
|
+
"metadata": {
|
|
870
|
+
"colab": {
|
|
871
|
+
"base_uri": "https://localhost:8080/"
|
|
872
|
+
},
|
|
873
|
+
"id": "qlNRs98cuxdp",
|
|
874
|
+
"outputId": "d71bd178-ff67-4721-c43c-d6abe3100454"
|
|
875
|
+
},
|
|
876
|
+
"execution_count": null,
|
|
877
|
+
"outputs": [
|
|
878
|
+
{
|
|
879
|
+
"output_type": "stream",
|
|
880
|
+
"name": "stdout",
|
|
881
|
+
"text": [
|
|
882
|
+
"Encoder output shape: (batch size, sequence length, units) (64, 47, 1024)\n",
|
|
883
|
+
"Encoder h vecotr shape: (batch size, units) (64, 1024)\n",
|
|
884
|
+
"Encoder c vector shape: (batch size, units) (64, 1024)\n",
|
|
885
|
+
"Encoder output vector shape: (batch size, sequence length, units) (64, 47, 1024)\n"
|
|
886
|
+
]
|
|
887
|
+
}
|
|
888
|
+
]
|
|
889
|
+
},
|
|
890
|
+
{
|
|
891
|
+
"cell_type": "markdown",
|
|
892
|
+
"source": [
|
|
893
|
+
"# Decoder"
|
|
894
|
+
],
|
|
895
|
+
"metadata": {
|
|
896
|
+
"id": "Qw4ebubBu7MW"
|
|
897
|
+
}
|
|
898
|
+
},
|
|
899
|
+
{
|
|
900
|
+
"cell_type": "code",
|
|
901
|
+
"source": [
|
|
902
|
+
"decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE,\n",
|
|
903
|
+
" max_length_input, max_length_output, 'luong')"
|
|
904
|
+
],
|
|
905
|
+
"metadata": {
|
|
906
|
+
"id": "EFLPrhRVu9hK"
|
|
907
|
+
},
|
|
908
|
+
"execution_count": null,
|
|
909
|
+
"outputs": []
|
|
910
|
+
},
|
|
911
|
+
{
|
|
912
|
+
"cell_type": "markdown",
|
|
913
|
+
"source": [
|
|
914
|
+
"## Test Decoder"
|
|
915
|
+
],
|
|
916
|
+
"metadata": {
|
|
917
|
+
"id": "-bDN2S0TvEIT"
|
|
918
|
+
}
|
|
919
|
+
},
|
|
920
|
+
{
|
|
921
|
+
"cell_type": "code",
|
|
922
|
+
"source": [
|
|
923
|
+
"# Test decoder stack\n",
|
|
924
|
+
"sample_x = tf.random.uniform((BATCH_SIZE, max_length_output))\n",
|
|
925
|
+
"initial_state = decoder.build_initial_state(BATCH_SIZE, [sample_h, sample_c])\n",
|
|
926
|
+
"\n",
|
|
927
|
+
"\n",
|
|
928
|
+
"sample_decoder_outputs = decoder(sample_x, initial_state, sample_output)\n",
|
|
929
|
+
"\n",
|
|
930
|
+
"print(\"Decoder Outputs Shape: \", sample_decoder_outputs[0].shape)"
|
|
931
|
+
],
|
|
932
|
+
"metadata": {
|
|
933
|
+
"colab": {
|
|
934
|
+
"base_uri": "https://localhost:8080/"
|
|
935
|
+
},
|
|
936
|
+
"id": "pTNciGVdvCzs",
|
|
937
|
+
"outputId": "365a06cb-ff2b-43e7-a4f9-6012c0fb764f"
|
|
938
|
+
},
|
|
939
|
+
"execution_count": null,
|
|
940
|
+
"outputs": [
|
|
941
|
+
{
|
|
942
|
+
"output_type": "stream",
|
|
943
|
+
"name": "stdout",
|
|
944
|
+
"text": [
|
|
945
|
+
"Decoder Outputs Shape: (64, 173, 137)\n"
|
|
946
|
+
]
|
|
947
|
+
}
|
|
948
|
+
]
|
|
949
|
+
},
|
|
950
|
+
{
|
|
951
|
+
"cell_type": "markdown",
|
|
952
|
+
"source": [
|
|
953
|
+
"# Optimizer and Loss Function"
|
|
954
|
+
],
|
|
955
|
+
"metadata": {
|
|
956
|
+
"id": "Xh07QJ3AwEkV"
|
|
957
|
+
}
|
|
958
|
+
},
|
|
959
|
+
{
|
|
960
|
+
"cell_type": "code",
|
|
961
|
+
"source": [
|
|
962
|
+
"optimizer = tf.keras.optimizers.Adam()\n",
|
|
963
|
+
"\n",
|
|
964
|
+
"def loss_function(real, pred):\n",
|
|
965
|
+
" # real shape = (BATCH_SIZE, max_length_output)\n",
|
|
966
|
+
" # pred shape = (BATCH_SIZE, max_length_output, tar_vocab_size )\n",
|
|
967
|
+
" cross_entropy = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
|
|
968
|
+
" loss = cross_entropy(y_true=real, y_pred=pred)\n",
|
|
969
|
+
" mask = tf.logical_not(tf.math.equal(real,0)) #output 0 for y=0 else output 1\n",
|
|
970
|
+
" mask = tf.cast(mask, dtype=loss.dtype)\n",
|
|
971
|
+
" loss = mask* loss\n",
|
|
972
|
+
" loss = tf.reduce_mean(loss)\n",
|
|
973
|
+
" return loss"
|
|
974
|
+
],
|
|
975
|
+
"metadata": {
|
|
976
|
+
"id": "GAN_YevuwH9X"
|
|
977
|
+
},
|
|
978
|
+
"execution_count": null,
|
|
979
|
+
"outputs": []
|
|
980
|
+
},
|
|
981
|
+
{
|
|
982
|
+
"cell_type": "markdown",
|
|
983
|
+
"source": [
|
|
984
|
+
"# Checkpoints"
|
|
985
|
+
],
|
|
986
|
+
"metadata": {
|
|
987
|
+
"id": "n04SWU2lwMAo"
|
|
988
|
+
}
|
|
989
|
+
},
|
|
990
|
+
{
|
|
991
|
+
"cell_type": "code",
|
|
992
|
+
"source": [
|
|
993
|
+
"checkpoint_dir = './data/training_checkpoints'\n",
|
|
994
|
+
"checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
|
|
995
|
+
"checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
|
|
996
|
+
" encoder=encoder,\n",
|
|
997
|
+
" decoder=decoder)"
|
|
998
|
+
],
|
|
999
|
+
"metadata": {
|
|
1000
|
+
"id": "-RchUEMMwLTt"
|
|
1001
|
+
},
|
|
1002
|
+
"execution_count": null,
|
|
1003
|
+
"outputs": []
|
|
1004
|
+
},
|
|
1005
|
+
{
|
|
1006
|
+
"cell_type": "markdown",
|
|
1007
|
+
"source": [
|
|
1008
|
+
"# One train_step operations"
|
|
1009
|
+
],
|
|
1010
|
+
"metadata": {
|
|
1011
|
+
"id": "kn92tVRdcvb1"
|
|
1012
|
+
}
|
|
1013
|
+
},
|
|
1014
|
+
{
|
|
1015
|
+
"cell_type": "code",
|
|
1016
|
+
"source": [
|
|
1017
|
+
"@tf.function\n",
|
|
1018
|
+
"def train_step(inp, targ, enc_hidden):\n",
|
|
1019
|
+
" loss = 0\n",
|
|
1020
|
+
"\n",
|
|
1021
|
+
" with tf.GradientTape() as tape:\n",
|
|
1022
|
+
" enc_output, enc_h, enc_c = encoder(inp, enc_hidden)\n",
|
|
1023
|
+
"\n",
|
|
1024
|
+
" # For the decoder, the input is the target sequence excluding the <end> token;\n",
|
|
1025
|
+
" # and the “real” sequence is target shifted by one (excluding the <start> token).\n",
|
|
1026
|
+
" dec_input = targ[ : , :-1 ] # Ignore <end> token\n",
|
|
1027
|
+
" real = targ[ : , 1: ] # ignore <start> token\n",
|
|
1028
|
+
"\n",
|
|
1029
|
+
" # Build the initial state for the decoder.\n",
|
|
1030
|
+
" decoder_initial_state = decoder.build_initial_state(BATCH_SIZE, [enc_h, enc_c])\n",
|
|
1031
|
+
"\n",
|
|
1032
|
+
" # Unpack the outputs from the decoder.\n",
|
|
1033
|
+
" logits, decoder_state, attention_weights = decoder(dec_input, decoder_initial_state, enc_output)\n",
|
|
1034
|
+
" # Compute the loss from logits and the true labels.\n",
|
|
1035
|
+
" loss = loss_function(real, logits)\n",
|
|
1036
|
+
"\n",
|
|
1037
|
+
" variables = encoder.trainable_variables + decoder.trainable_variables\n",
|
|
1038
|
+
" gradients = tape.gradient(loss, variables)\n",
|
|
1039
|
+
" optimizer.apply_gradients(zip(gradients, variables))\n",
|
|
1040
|
+
"\n",
|
|
1041
|
+
" return loss"
|
|
1042
|
+
],
|
|
1043
|
+
"metadata": {
|
|
1044
|
+
"id": "Z4wTBov2c1Sr"
|
|
1045
|
+
},
|
|
1046
|
+
"execution_count": null,
|
|
1047
|
+
"outputs": []
|
|
1048
|
+
},
|
|
1049
|
+
{
|
|
1050
|
+
"cell_type": "code",
|
|
1051
|
+
"source": [
|
|
1052
|
+
"# Validation Step (no gradient tape needed)\n",
|
|
1053
|
+
"@tf.function\n",
|
|
1054
|
+
"def val_step(inp, targ, enc_hidden):\n",
|
|
1055
|
+
" # Run the encoder on the validation input.\n",
|
|
1056
|
+
" enc_output, enc_h, enc_c = encoder(inp, enc_hidden)\n",
|
|
1057
|
+
" # Prepare the decoder input and targets as in training.\n",
|
|
1058
|
+
" dec_input = targ[:, :-1]\n",
|
|
1059
|
+
" real = targ[:, 1:]\n",
|
|
1060
|
+
" # Build the initial state for the decoder.\n",
|
|
1061
|
+
" decoder_initial_state = decoder.build_initial_state(BATCH_SIZE, [enc_h, enc_c])\n",
|
|
1062
|
+
" # Get the logits from the decoder.\n",
|
|
1063
|
+
" logits, _, _ = decoder(dec_input, decoder_initial_state, enc_output)\n",
|
|
1064
|
+
" # Compute and return the loss.\n",
|
|
1065
|
+
" loss = loss_function(real, logits)\n",
|
|
1066
|
+
" return loss"
|
|
1067
|
+
],
|
|
1068
|
+
"metadata": {
|
|
1069
|
+
"id": "3eZgjJnxOMTm"
|
|
1070
|
+
},
|
|
1071
|
+
"execution_count": null,
|
|
1072
|
+
"outputs": []
|
|
1073
|
+
},
|
|
1074
|
+
{
|
|
1075
|
+
"cell_type": "markdown",
|
|
1076
|
+
"source": [
|
|
1077
|
+
"# Train"
|
|
1078
|
+
],
|
|
1079
|
+
"metadata": {
|
|
1080
|
+
"id": "Jv13BMMfc5rB"
|
|
1081
|
+
}
|
|
1082
|
+
},
|
|
1083
|
+
{
|
|
1084
|
+
"cell_type": "code",
|
|
1085
|
+
"source": [
|
|
1086
|
+
"EPOCHS = 25\n",
|
|
1087
|
+
"patience = 5\n",
|
|
1088
|
+
"epochs_without_improvement = 0\n",
|
|
1089
|
+
"best_val_loss = float('inf')\n",
|
|
1090
|
+
"\n",
|
|
1091
|
+
"best_loss = 1\n",
|
|
1092
|
+
"for epoch in range(EPOCHS):\n",
|
|
1093
|
+
" start = time.time()\n",
|
|
1094
|
+
"\n",
|
|
1095
|
+
" # -- Training phase --\n",
|
|
1096
|
+
" enc_hidden = encoder.initialize_hidden_state()\n",
|
|
1097
|
+
" total_train_loss = 0\n",
|
|
1098
|
+
" # loop over the training batches\n",
|
|
1099
|
+
" for (batch, (inp, targ)) in enumerate(train_dataset.take(steps_per_epoch)):\n",
|
|
1100
|
+
" batch_loss = train_step(inp, targ, enc_hidden)\n",
|
|
1101
|
+
" total_train_loss += batch_loss\n",
|
|
1102
|
+
"\n",
|
|
1103
|
+
" if batch % 100 == 0:\n",
|
|
1104
|
+
" print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
|
|
1105
|
+
" batch,\n",
|
|
1106
|
+
" batch_loss.numpy()))\n",
|
|
1107
|
+
"\n",
|
|
1108
|
+
" loss = total_train_loss / steps_per_epoch\n",
|
|
1109
|
+
"\n",
|
|
1110
|
+
"\n",
|
|
1111
|
+
" # -- validation phase --\n",
|
|
1112
|
+
" total_val_loss = 0\n",
|
|
1113
|
+
" for (batch, (inp, targ)) in enumerate(val_dataset):\n",
|
|
1114
|
+
" enc_hidden = encoder.initialize_hidden_state()\n",
|
|
1115
|
+
" batch_val_loss = val_step(inp, targ, enc_hidden)\n",
|
|
1116
|
+
" total_val_loss += batch_val_loss\n",
|
|
1117
|
+
"\n",
|
|
1118
|
+
" val_loss = total_val_loss / len(val_dataset)\n",
|
|
1119
|
+
" print('Epoch {} Loss {:.4f} Val Loss {:.4f}'.format(epoch + 1,\n",
|
|
1120
|
+
" loss,\n",
|
|
1121
|
+
" val_loss))\n",
|
|
1122
|
+
"\n",
|
|
1123
|
+
" # -- checkpoint & early stopping --\n",
|
|
1124
|
+
" if val_loss < best_val_loss:\n",
|
|
1125
|
+
" best_val_loss = val_loss\n",
|
|
1126
|
+
" print('Saving checkpoint for epoch {} Loss {:.4f}'.format(epoch + 1, val_loss))\n",
|
|
1127
|
+
" checkpoint.save(file_prefix = checkpoint_prefix)\n",
|
|
1128
|
+
" epochs_without_improvement = 0\n",
|
|
1129
|
+
" else:\n",
|
|
1130
|
+
" epochs_without_improvement += 1\n",
|
|
1131
|
+
"\n",
|
|
1132
|
+
" # break the loop if there is no improvement\n",
|
|
1133
|
+
" if epochs_without_improvement >= patience:\n",
|
|
1134
|
+
" print(f\"Early stopping triggered after {patience} consecutive epochs w/o improvement\")\n",
|
|
1135
|
+
" break\n",
|
|
1136
|
+
"\n",
|
|
1137
|
+
" print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
|
|
1138
|
+
],
|
|
1139
|
+
"metadata": {
|
|
1140
|
+
"colab": {
|
|
1141
|
+
"base_uri": "https://localhost:8080/"
|
|
1142
|
+
},
|
|
1143
|
+
"id": "_ZheDY-jc7P7",
|
|
1144
|
+
"outputId": "4b1bd2dd-4bdf-426b-a062-bb4ae0dc433e"
|
|
1145
|
+
},
|
|
1146
|
+
"execution_count": null,
|
|
1147
|
+
"outputs": [
|
|
1148
|
+
{
|
|
1149
|
+
"output_type": "stream",
|
|
1150
|
+
"name": "stdout",
|
|
1151
|
+
"text": [
|
|
1152
|
+
"Epoch 1 Batch 0 Loss 0.2329\n",
|
|
1153
|
+
"Epoch 1 Batch 100 Loss 0.1237\n",
|
|
1154
|
+
"Epoch 1 Batch 200 Loss 0.0989\n",
|
|
1155
|
+
"Epoch 1 Batch 300 Loss 0.0988\n",
|
|
1156
|
+
"Epoch 1 Batch 400 Loss 0.0887\n",
|
|
1157
|
+
"Epoch 1 Batch 500 Loss 0.0944\n",
|
|
1158
|
+
"Epoch 1 Batch 600 Loss 0.0800\n",
|
|
1159
|
+
"Epoch 1 Batch 700 Loss 0.0663\n",
|
|
1160
|
+
"Epoch 1 Batch 800 Loss 0.0466\n",
|
|
1161
|
+
"Epoch 1 Batch 900 Loss 0.0326\n",
|
|
1162
|
+
"Epoch 1 Batch 1000 Loss 0.0208\n",
|
|
1163
|
+
"Epoch 1 Batch 1100 Loss 0.0239\n",
|
|
1164
|
+
"Epoch 1 Batch 1200 Loss 0.0202\n",
|
|
1165
|
+
"Epoch 1 Batch 1300 Loss 0.0175\n",
|
|
1166
|
+
"Epoch 1 Batch 1400 Loss 0.0211\n",
|
|
1167
|
+
"Epoch 1 Batch 1500 Loss 0.0181\n",
|
|
1168
|
+
"Epoch 1 Batch 1600 Loss 0.0248\n",
|
|
1169
|
+
"Epoch 1 Batch 1700 Loss 0.0215\n",
|
|
1170
|
+
"Epoch 1 Batch 1800 Loss 0.0208\n",
|
|
1171
|
+
"Epoch 1 Batch 1900 Loss 0.0168\n",
|
|
1172
|
+
"Epoch 1 Batch 2000 Loss 0.0163\n",
|
|
1173
|
+
"Epoch 1 Batch 2100 Loss 0.0212\n",
|
|
1174
|
+
"Epoch 1 Batch 2200 Loss 0.0224\n",
|
|
1175
|
+
"Epoch 1 Batch 2300 Loss 0.0178\n",
|
|
1176
|
+
"Epoch 1 Batch 2400 Loss 0.0144\n",
|
|
1177
|
+
"Epoch 1 Batch 2500 Loss 0.0170\n",
|
|
1178
|
+
"Epoch 1 Batch 2600 Loss 0.0156\n",
|
|
1179
|
+
"Epoch 1 Batch 2700 Loss 0.0208\n",
|
|
1180
|
+
"Epoch 1 Batch 2800 Loss 0.0179\n",
|
|
1181
|
+
"Epoch 1 Batch 2900 Loss 0.0169\n",
|
|
1182
|
+
"Epoch 1 Batch 3000 Loss 0.0161\n",
|
|
1183
|
+
"Epoch 1 Batch 3100 Loss 0.0189\n",
|
|
1184
|
+
"Epoch 1 Batch 3200 Loss 0.0132\n",
|
|
1185
|
+
"Epoch 1 Batch 3300 Loss 0.0180\n",
|
|
1186
|
+
"Epoch 1 Batch 3400 Loss 0.0127\n",
|
|
1187
|
+
"Epoch 1 Batch 3500 Loss 0.0159\n",
|
|
1188
|
+
"Epoch 1 Batch 3600 Loss 0.0147\n",
|
|
1189
|
+
"Epoch 1 Batch 3700 Loss 0.0175\n",
|
|
1190
|
+
"Epoch 1 Batch 3800 Loss 0.0137\n",
|
|
1191
|
+
"Epoch 1 Batch 3900 Loss 0.0175\n",
|
|
1192
|
+
"Epoch 1 Batch 4000 Loss 0.0162\n",
|
|
1193
|
+
"Epoch 1 Batch 4100 Loss 0.0151\n",
|
|
1194
|
+
"Epoch 1 Batch 4200 Loss 0.0243\n",
|
|
1195
|
+
"Epoch 1 Batch 4300 Loss 0.0127\n",
|
|
1196
|
+
"Epoch 1 Batch 4400 Loss 0.0201\n",
|
|
1197
|
+
"Epoch 1 Batch 4500 Loss 0.0172\n",
|
|
1198
|
+
"Epoch 1 Batch 4600 Loss 0.0129\n",
|
|
1199
|
+
"Epoch 1 Loss 0.0257 Val Loss 0.0154\n",
|
|
1200
|
+
"Saving checkpoint for epoch 1 Loss 0.0154\n",
|
|
1201
|
+
"Time taken for 1 epoch 738.509771823883 sec\n",
|
|
1202
|
+
"\n",
|
|
1203
|
+
"Epoch 2 Batch 0 Loss 0.0162\n",
|
|
1204
|
+
"Epoch 2 Batch 100 Loss 0.0158\n",
|
|
1205
|
+
"Epoch 2 Batch 200 Loss 0.0142\n",
|
|
1206
|
+
"Epoch 2 Batch 300 Loss 0.0158\n",
|
|
1207
|
+
"Epoch 2 Batch 400 Loss 0.0136\n",
|
|
1208
|
+
"Epoch 2 Batch 500 Loss 0.0154\n",
|
|
1209
|
+
"Epoch 2 Batch 600 Loss 0.0178\n",
|
|
1210
|
+
"Epoch 2 Batch 700 Loss 0.0231\n",
|
|
1211
|
+
"Epoch 2 Batch 800 Loss 0.0108\n",
|
|
1212
|
+
"Epoch 2 Batch 900 Loss 0.0191\n",
|
|
1213
|
+
"Epoch 2 Batch 1000 Loss 0.0166\n",
|
|
1214
|
+
"Epoch 2 Batch 1100 Loss 0.0155\n",
|
|
1215
|
+
"Epoch 2 Batch 1200 Loss 0.0133\n",
|
|
1216
|
+
"Epoch 2 Batch 1300 Loss 0.0135\n",
|
|
1217
|
+
"Epoch 2 Batch 1400 Loss 0.0152\n",
|
|
1218
|
+
"Epoch 2 Batch 1500 Loss 0.0171\n",
|
|
1219
|
+
"Epoch 2 Batch 1600 Loss 0.0114\n",
|
|
1220
|
+
"Epoch 2 Batch 1700 Loss 0.0130\n",
|
|
1221
|
+
"Epoch 2 Batch 1800 Loss 0.0176\n",
|
|
1222
|
+
"Epoch 2 Batch 1900 Loss 0.0146\n",
|
|
1223
|
+
"Epoch 2 Batch 2000 Loss 0.0141\n",
|
|
1224
|
+
"Epoch 2 Batch 2100 Loss 0.0084\n",
|
|
1225
|
+
"Epoch 2 Batch 2200 Loss 0.0137\n",
|
|
1226
|
+
"Epoch 2 Batch 2300 Loss 0.0157\n",
|
|
1227
|
+
"Epoch 2 Batch 2400 Loss 0.0119\n",
|
|
1228
|
+
"Epoch 2 Batch 2500 Loss 0.0161\n",
|
|
1229
|
+
"Epoch 2 Batch 2600 Loss 0.0184\n",
|
|
1230
|
+
"Epoch 2 Batch 2700 Loss 0.0114\n",
|
|
1231
|
+
"Epoch 2 Batch 2800 Loss 0.0123\n",
|
|
1232
|
+
"Epoch 2 Batch 2900 Loss 0.0166\n",
|
|
1233
|
+
"Epoch 2 Batch 3000 Loss 0.0149\n",
|
|
1234
|
+
"Epoch 2 Batch 3100 Loss 0.0179\n",
|
|
1235
|
+
"Epoch 2 Batch 3200 Loss 0.0171\n",
|
|
1236
|
+
"Epoch 2 Batch 3300 Loss 0.0137\n",
|
|
1237
|
+
"Epoch 2 Batch 3400 Loss 0.0194\n",
|
|
1238
|
+
"Epoch 2 Batch 3500 Loss 0.0135\n",
|
|
1239
|
+
"Epoch 2 Batch 3600 Loss 0.0117\n",
|
|
1240
|
+
"Epoch 2 Batch 3700 Loss 0.0157\n",
|
|
1241
|
+
"Epoch 2 Batch 3800 Loss 0.0109\n",
|
|
1242
|
+
"Epoch 2 Batch 3900 Loss 0.0160\n",
|
|
1243
|
+
"Epoch 2 Batch 4000 Loss 0.0122\n",
|
|
1244
|
+
"Epoch 2 Batch 4100 Loss 0.0119\n",
|
|
1245
|
+
"Epoch 2 Batch 4200 Loss 0.0159\n",
|
|
1246
|
+
"Epoch 2 Batch 4300 Loss 0.0130\n",
|
|
1247
|
+
"Epoch 2 Batch 4400 Loss 0.0117\n",
|
|
1248
|
+
"Epoch 2 Batch 4500 Loss 0.0095\n",
|
|
1249
|
+
"Epoch 2 Batch 4600 Loss 0.0123\n",
|
|
1250
|
+
"Epoch 2 Loss 0.0112 Val Loss 0.0137\n",
|
|
1251
|
+
"Saving checkpoint for epoch 2 Loss 0.0137\n",
|
|
1252
|
+
"Time taken for 1 epoch 733.146365404129 sec\n",
|
|
1253
|
+
"\n",
|
|
1254
|
+
"Epoch 3 Batch 0 Loss 0.0163\n",
|
|
1255
|
+
"Epoch 3 Batch 100 Loss 0.0084\n",
|
|
1256
|
+
"Epoch 3 Batch 200 Loss 0.0094\n",
|
|
1257
|
+
"Epoch 3 Batch 300 Loss 0.0172\n",
|
|
1258
|
+
"Epoch 3 Batch 400 Loss 0.0141\n",
|
|
1259
|
+
"Epoch 3 Batch 500 Loss 0.0119\n",
|
|
1260
|
+
"Epoch 3 Batch 600 Loss 0.0108\n",
|
|
1261
|
+
"Epoch 3 Batch 700 Loss 0.0114\n",
|
|
1262
|
+
"Epoch 3 Batch 800 Loss 0.0096\n",
|
|
1263
|
+
"Epoch 3 Batch 900 Loss 0.0154\n",
|
|
1264
|
+
"Epoch 3 Batch 1000 Loss 0.0098\n",
|
|
1265
|
+
"Epoch 3 Batch 1100 Loss 0.0109\n",
|
|
1266
|
+
"Epoch 3 Batch 1200 Loss 0.0156\n",
|
|
1267
|
+
"Epoch 3 Batch 1300 Loss 0.0125\n",
|
|
1268
|
+
"Epoch 3 Batch 1400 Loss 0.0154\n",
|
|
1269
|
+
"Epoch 3 Batch 1500 Loss 0.0140\n",
|
|
1270
|
+
"Epoch 3 Batch 1600 Loss 0.0116\n",
|
|
1271
|
+
"Epoch 3 Batch 1700 Loss 0.0091\n",
|
|
1272
|
+
"Epoch 3 Batch 1800 Loss 0.0131\n",
|
|
1273
|
+
"Epoch 3 Batch 1900 Loss 0.0123\n",
|
|
1274
|
+
"Epoch 3 Batch 2000 Loss 0.0103\n",
|
|
1275
|
+
"Epoch 3 Batch 2100 Loss 0.0129\n",
|
|
1276
|
+
"Epoch 3 Batch 2200 Loss 0.0098\n",
|
|
1277
|
+
"Epoch 3 Batch 2300 Loss 0.0124\n",
|
|
1278
|
+
"Epoch 3 Batch 2400 Loss 0.0107\n",
|
|
1279
|
+
"Epoch 3 Batch 2500 Loss 0.0152\n",
|
|
1280
|
+
"Epoch 3 Batch 2600 Loss 0.0131\n",
|
|
1281
|
+
"Epoch 3 Batch 2700 Loss 0.0114\n",
|
|
1282
|
+
"Epoch 3 Batch 2800 Loss 0.0116\n",
|
|
1283
|
+
"Epoch 3 Batch 2900 Loss 0.0119\n",
|
|
1284
|
+
"Epoch 3 Batch 3000 Loss 0.0151\n",
|
|
1285
|
+
"Epoch 3 Batch 3100 Loss 0.0115\n",
|
|
1286
|
+
"Epoch 3 Batch 3200 Loss 0.0115\n",
|
|
1287
|
+
"Epoch 3 Batch 3300 Loss 0.0208\n",
|
|
1288
|
+
"Epoch 3 Batch 3400 Loss 0.0084\n",
|
|
1289
|
+
"Epoch 3 Batch 3500 Loss 0.0082\n",
|
|
1290
|
+
"Epoch 3 Batch 3600 Loss 0.0158\n",
|
|
1291
|
+
"Epoch 3 Batch 3700 Loss 0.0125\n",
|
|
1292
|
+
"Epoch 3 Batch 3800 Loss 0.0130\n",
|
|
1293
|
+
"Epoch 3 Batch 3900 Loss 0.0103\n",
|
|
1294
|
+
"Epoch 3 Batch 4000 Loss 0.0152\n",
|
|
1295
|
+
"Epoch 3 Batch 4100 Loss 0.0118\n",
|
|
1296
|
+
"Epoch 3 Batch 4200 Loss 0.0122\n",
|
|
1297
|
+
"Epoch 3 Batch 4300 Loss 0.0097\n",
|
|
1298
|
+
"Epoch 3 Batch 4400 Loss 0.0147\n",
|
|
1299
|
+
"Epoch 3 Batch 4500 Loss 0.0107\n",
|
|
1300
|
+
"Epoch 3 Batch 4600 Loss 0.0094\n",
|
|
1301
|
+
"Epoch 3 Loss 0.0097 Val Loss 0.0130\n",
|
|
1302
|
+
"Saving checkpoint for epoch 3 Loss 0.0130\n",
|
|
1303
|
+
"Time taken for 1 epoch 733.5029106140137 sec\n",
|
|
1304
|
+
"\n",
|
|
1305
|
+
"Epoch 4 Batch 0 Loss 0.0120\n",
|
|
1306
|
+
"Epoch 4 Batch 100 Loss 0.0107\n",
|
|
1307
|
+
"Epoch 4 Batch 200 Loss 0.0120\n",
|
|
1308
|
+
"Epoch 4 Batch 300 Loss 0.0105\n",
|
|
1309
|
+
"Epoch 4 Batch 400 Loss 0.0117\n",
|
|
1310
|
+
"Epoch 4 Batch 500 Loss 0.0090\n",
|
|
1311
|
+
"Epoch 4 Batch 600 Loss 0.0118\n",
|
|
1312
|
+
"Epoch 4 Batch 700 Loss 0.0096\n",
|
|
1313
|
+
"Epoch 4 Batch 800 Loss 0.0085\n",
|
|
1314
|
+
"Epoch 4 Batch 900 Loss 0.0101\n",
|
|
1315
|
+
"Epoch 4 Batch 1000 Loss 0.0104\n",
|
|
1316
|
+
"Epoch 4 Batch 1100 Loss 0.0163\n",
|
|
1317
|
+
"Epoch 4 Batch 1200 Loss 0.0126\n",
|
|
1318
|
+
"Epoch 4 Batch 1300 Loss 0.0115\n",
|
|
1319
|
+
"Epoch 4 Batch 1400 Loss 0.0114\n",
|
|
1320
|
+
"Epoch 4 Batch 1500 Loss 0.0138\n",
|
|
1321
|
+
"Epoch 4 Batch 1600 Loss 0.0092\n",
|
|
1322
|
+
"Epoch 4 Batch 1700 Loss 0.0076\n",
|
|
1323
|
+
"Epoch 4 Batch 1800 Loss 0.0109\n",
|
|
1324
|
+
"Epoch 4 Batch 1900 Loss 0.0059\n",
|
|
1325
|
+
"Epoch 4 Batch 2000 Loss 0.0131\n",
|
|
1326
|
+
"Epoch 4 Batch 2100 Loss 0.0106\n",
|
|
1327
|
+
"Epoch 4 Batch 2200 Loss 0.0101\n",
|
|
1328
|
+
"Epoch 4 Batch 2300 Loss 0.0084\n",
|
|
1329
|
+
"Epoch 4 Batch 2400 Loss 0.0104\n",
|
|
1330
|
+
"Epoch 4 Batch 2500 Loss 0.0152\n",
|
|
1331
|
+
"Epoch 4 Batch 2600 Loss 0.0105\n",
|
|
1332
|
+
"Epoch 4 Batch 2700 Loss 0.0127\n",
|
|
1333
|
+
"Epoch 4 Batch 2800 Loss 0.0113\n",
|
|
1334
|
+
"Epoch 4 Batch 2900 Loss 0.0082\n",
|
|
1335
|
+
"Epoch 4 Batch 3000 Loss 0.0187\n",
|
|
1336
|
+
"Epoch 4 Batch 3100 Loss 0.0087\n",
|
|
1337
|
+
"Epoch 4 Batch 3200 Loss 0.0091\n",
|
|
1338
|
+
"Epoch 4 Batch 3300 Loss 0.0106\n",
|
|
1339
|
+
"Epoch 4 Batch 3400 Loss 0.0085\n",
|
|
1340
|
+
"Epoch 4 Batch 3500 Loss 0.0134\n",
|
|
1341
|
+
"Epoch 4 Batch 3600 Loss 0.0098\n",
|
|
1342
|
+
"Epoch 4 Batch 3700 Loss 0.0118\n",
|
|
1343
|
+
"Epoch 4 Batch 3800 Loss 0.0115\n",
|
|
1344
|
+
"Epoch 4 Batch 3900 Loss 0.0092\n",
|
|
1345
|
+
"Epoch 4 Batch 4000 Loss 0.0122\n",
|
|
1346
|
+
"Epoch 4 Batch 4100 Loss 0.0087\n",
|
|
1347
|
+
"Epoch 4 Batch 4200 Loss 0.0114\n",
|
|
1348
|
+
"Epoch 4 Batch 4300 Loss 0.0128\n",
|
|
1349
|
+
"Epoch 4 Batch 4400 Loss 0.0097\n",
|
|
1350
|
+
"Epoch 4 Batch 4500 Loss 0.0119\n",
|
|
1351
|
+
"Epoch 4 Batch 4600 Loss 0.0118\n",
|
|
1352
|
+
"Epoch 4 Loss 0.0086 Val Loss 0.0128\n",
|
|
1353
|
+
"Saving checkpoint for epoch 4 Loss 0.0128\n",
|
|
1354
|
+
"Time taken for 1 epoch 731.1751847267151 sec\n",
|
|
1355
|
+
"\n",
|
|
1356
|
+
"Epoch 5 Batch 0 Loss 0.0082\n",
|
|
1357
|
+
"Epoch 5 Batch 100 Loss 0.0091\n",
|
|
1358
|
+
"Epoch 5 Batch 200 Loss 0.0114\n",
|
|
1359
|
+
"Epoch 5 Batch 300 Loss 0.0095\n",
|
|
1360
|
+
"Epoch 5 Batch 400 Loss 0.0071\n",
|
|
1361
|
+
"Epoch 5 Batch 500 Loss 0.0109\n",
|
|
1362
|
+
"Epoch 5 Batch 600 Loss 0.0102\n",
|
|
1363
|
+
"Epoch 5 Batch 700 Loss 0.0097\n",
|
|
1364
|
+
"Epoch 5 Batch 800 Loss 0.0060\n",
|
|
1365
|
+
"Epoch 5 Batch 900 Loss 0.0086\n",
|
|
1366
|
+
"Epoch 5 Batch 1000 Loss 0.0097\n",
|
|
1367
|
+
"Epoch 5 Batch 1100 Loss 0.0083\n",
|
|
1368
|
+
"Epoch 5 Batch 1200 Loss 0.0115\n",
|
|
1369
|
+
"Epoch 5 Batch 1300 Loss 0.0075\n",
|
|
1370
|
+
"Epoch 5 Batch 1400 Loss 0.0068\n",
|
|
1371
|
+
"Epoch 5 Batch 1500 Loss 0.0091\n",
|
|
1372
|
+
"Epoch 5 Batch 1600 Loss 0.0093\n",
|
|
1373
|
+
"Epoch 5 Batch 1700 Loss 0.0092\n",
|
|
1374
|
+
"Epoch 5 Batch 1800 Loss 0.0078\n",
|
|
1375
|
+
"Epoch 5 Batch 1900 Loss 0.0076\n",
|
|
1376
|
+
"Epoch 5 Batch 2000 Loss 0.0076\n",
|
|
1377
|
+
"Epoch 5 Batch 2100 Loss 0.0096\n",
|
|
1378
|
+
"Epoch 5 Batch 2200 Loss 0.0087\n",
|
|
1379
|
+
"Epoch 5 Batch 2300 Loss 0.0080\n",
|
|
1380
|
+
"Epoch 5 Batch 2400 Loss 0.0097\n",
|
|
1381
|
+
"Epoch 5 Batch 2500 Loss 0.0082\n",
|
|
1382
|
+
"Epoch 5 Batch 2600 Loss 0.0111\n",
|
|
1383
|
+
"Epoch 5 Batch 2700 Loss 0.0106\n",
|
|
1384
|
+
"Epoch 5 Batch 2800 Loss 0.0113\n",
|
|
1385
|
+
"Epoch 5 Batch 2900 Loss 0.0162\n",
|
|
1386
|
+
"Epoch 5 Batch 3000 Loss 0.0081\n",
|
|
1387
|
+
"Epoch 5 Batch 3100 Loss 0.0067\n",
|
|
1388
|
+
"Epoch 5 Batch 3200 Loss 0.0107\n",
|
|
1389
|
+
"Epoch 5 Batch 3300 Loss 0.0091\n",
|
|
1390
|
+
"Epoch 5 Batch 3400 Loss 0.0113\n",
|
|
1391
|
+
"Epoch 5 Batch 3500 Loss 0.0120\n",
|
|
1392
|
+
"Epoch 5 Batch 3600 Loss 0.0099\n",
|
|
1393
|
+
"Epoch 5 Batch 3700 Loss 0.0094\n",
|
|
1394
|
+
"Epoch 5 Batch 3800 Loss 0.0079\n",
|
|
1395
|
+
"Epoch 5 Batch 3900 Loss 0.0115\n",
|
|
1396
|
+
"Epoch 5 Batch 4000 Loss 0.0099\n",
|
|
1397
|
+
"Epoch 5 Batch 4100 Loss 0.0118\n",
|
|
1398
|
+
"Epoch 5 Batch 4200 Loss 0.0086\n",
|
|
1399
|
+
"Epoch 5 Batch 4300 Loss 0.0104\n",
|
|
1400
|
+
"Epoch 5 Batch 4400 Loss 0.0069\n",
|
|
1401
|
+
"Epoch 5 Batch 4500 Loss 0.0106\n",
|
|
1402
|
+
"Epoch 5 Batch 4600 Loss 0.0087\n",
|
|
1403
|
+
"Epoch 5 Loss 0.0078 Val Loss 0.0128\n",
|
|
1404
|
+
"Saving checkpoint for epoch 5 Loss 0.0128\n",
|
|
1405
|
+
"Time taken for 1 epoch 730.5528981685638 sec\n",
|
|
1406
|
+
"\n",
|
|
1407
|
+
"Epoch 6 Batch 0 Loss 0.0103\n",
|
|
1408
|
+
"Epoch 6 Batch 100 Loss 0.0072\n",
|
|
1409
|
+
"Epoch 6 Batch 200 Loss 0.0085\n",
|
|
1410
|
+
"Epoch 6 Batch 300 Loss 0.0106\n",
|
|
1411
|
+
"Epoch 6 Batch 400 Loss 0.0118\n",
|
|
1412
|
+
"Epoch 6 Batch 500 Loss 0.0084\n",
|
|
1413
|
+
"Epoch 6 Batch 600 Loss 0.0081\n",
|
|
1414
|
+
"Epoch 6 Batch 700 Loss 0.0099\n",
|
|
1415
|
+
"Epoch 6 Batch 800 Loss 0.0110\n",
|
|
1416
|
+
"Epoch 6 Batch 900 Loss 0.0089\n",
|
|
1417
|
+
"Epoch 6 Batch 1000 Loss 0.0082\n",
|
|
1418
|
+
"Epoch 6 Batch 1100 Loss 0.0108\n",
|
|
1419
|
+
"Epoch 6 Batch 1200 Loss 0.0076\n",
|
|
1420
|
+
"Epoch 6 Batch 1300 Loss 0.0099\n",
|
|
1421
|
+
"Epoch 6 Batch 1400 Loss 0.0101\n",
|
|
1422
|
+
"Epoch 6 Batch 1500 Loss 0.0097\n",
|
|
1423
|
+
"Epoch 6 Batch 1600 Loss 0.0076\n",
|
|
1424
|
+
"Epoch 6 Batch 1700 Loss 0.0062\n",
|
|
1425
|
+
"Epoch 6 Batch 1800 Loss 0.0090\n",
|
|
1426
|
+
"Epoch 6 Batch 1900 Loss 0.0123\n",
|
|
1427
|
+
"Epoch 6 Batch 2000 Loss 0.0083\n",
|
|
1428
|
+
"Epoch 6 Batch 2100 Loss 0.0074\n",
|
|
1429
|
+
"Epoch 6 Batch 2200 Loss 0.0075\n",
|
|
1430
|
+
"Epoch 6 Batch 2300 Loss 0.0099\n",
|
|
1431
|
+
"Epoch 6 Batch 2400 Loss 0.0066\n",
|
|
1432
|
+
"Epoch 6 Batch 2500 Loss 0.0111\n",
|
|
1433
|
+
"Epoch 6 Batch 2600 Loss 0.0074\n",
|
|
1434
|
+
"Epoch 6 Batch 2700 Loss 0.0082\n",
|
|
1435
|
+
"Epoch 6 Batch 2800 Loss 0.0113\n",
|
|
1436
|
+
"Epoch 6 Batch 2900 Loss 0.0095\n",
|
|
1437
|
+
"Epoch 6 Batch 3000 Loss 0.0058\n",
|
|
1438
|
+
"Epoch 6 Batch 3100 Loss 0.0077\n",
|
|
1439
|
+
"Epoch 6 Batch 3200 Loss 0.0078\n",
|
|
1440
|
+
"Epoch 6 Batch 3300 Loss 0.0114\n",
|
|
1441
|
+
"Epoch 6 Batch 3400 Loss 0.0086\n",
|
|
1442
|
+
"Epoch 6 Batch 3500 Loss 0.0070\n",
|
|
1443
|
+
"Epoch 6 Batch 3600 Loss 0.0082\n",
|
|
1444
|
+
"Epoch 6 Batch 3700 Loss 0.0081\n",
|
|
1445
|
+
"Epoch 6 Batch 3800 Loss 0.0077\n",
|
|
1446
|
+
"Epoch 6 Batch 3900 Loss 0.0105\n",
|
|
1447
|
+
"Epoch 6 Batch 4000 Loss 0.0098\n",
|
|
1448
|
+
"Epoch 6 Batch 4100 Loss 0.0104\n",
|
|
1449
|
+
"Epoch 6 Batch 4200 Loss 0.0095\n",
|
|
1450
|
+
"Epoch 6 Batch 4300 Loss 0.0084\n",
|
|
1451
|
+
"Epoch 6 Batch 4400 Loss 0.0097\n",
|
|
1452
|
+
"Epoch 6 Batch 4500 Loss 0.0083\n",
|
|
1453
|
+
"Epoch 6 Batch 4600 Loss 0.0049\n",
|
|
1454
|
+
"Epoch 6 Loss 0.0070 Val Loss 0.0131\n",
|
|
1455
|
+
"Time taken for 1 epoch 731.5087742805481 sec\n",
|
|
1456
|
+
"\n",
|
|
1457
|
+
"Epoch 7 Batch 0 Loss 0.0111\n",
|
|
1458
|
+
"Epoch 7 Batch 100 Loss 0.0057\n",
|
|
1459
|
+
"Epoch 7 Batch 200 Loss 0.0063\n",
|
|
1460
|
+
"Epoch 7 Batch 300 Loss 0.0078\n",
|
|
1461
|
+
"Epoch 7 Batch 400 Loss 0.0109\n",
|
|
1462
|
+
"Epoch 7 Batch 500 Loss 0.0087\n",
|
|
1463
|
+
"Epoch 7 Batch 600 Loss 0.0088\n",
|
|
1464
|
+
"Epoch 7 Batch 700 Loss 0.0054\n",
|
|
1465
|
+
"Epoch 7 Batch 800 Loss 0.0075\n",
|
|
1466
|
+
"Epoch 7 Batch 900 Loss 0.0118\n",
|
|
1467
|
+
"Epoch 7 Batch 1000 Loss 0.0082\n",
|
|
1468
|
+
"Epoch 7 Batch 1100 Loss 0.0073\n",
|
|
1469
|
+
"Epoch 7 Batch 1200 Loss 0.0075\n",
|
|
1470
|
+
"Epoch 7 Batch 1300 Loss 0.0089\n",
|
|
1471
|
+
"Epoch 7 Batch 1400 Loss 0.0099\n",
|
|
1472
|
+
"Epoch 7 Batch 1500 Loss 0.0066\n",
|
|
1473
|
+
"Epoch 7 Batch 1600 Loss 0.0061\n",
|
|
1474
|
+
"Epoch 7 Batch 1700 Loss 0.0083\n",
|
|
1475
|
+
"Epoch 7 Batch 1800 Loss 0.0092\n",
|
|
1476
|
+
"Epoch 7 Batch 1900 Loss 0.0071\n",
|
|
1477
|
+
"Epoch 7 Batch 2000 Loss 0.0103\n",
|
|
1478
|
+
"Epoch 7 Batch 2100 Loss 0.0052\n",
|
|
1479
|
+
"Epoch 7 Batch 2200 Loss 0.0095\n",
|
|
1480
|
+
"Epoch 7 Batch 2300 Loss 0.0063\n",
|
|
1481
|
+
"Epoch 7 Batch 2400 Loss 0.0081\n",
|
|
1482
|
+
"Epoch 7 Batch 2500 Loss 0.0071\n",
|
|
1483
|
+
"Epoch 7 Batch 2600 Loss 0.0077\n",
|
|
1484
|
+
"Epoch 7 Batch 2700 Loss 0.0091\n",
|
|
1485
|
+
"Epoch 7 Batch 2800 Loss 0.0073\n",
|
|
1486
|
+
"Epoch 7 Batch 2900 Loss 0.0070\n",
|
|
1487
|
+
"Epoch 7 Batch 3000 Loss 0.0073\n",
|
|
1488
|
+
"Epoch 7 Batch 3100 Loss 0.0057\n",
|
|
1489
|
+
"Epoch 7 Batch 3200 Loss 0.0074\n",
|
|
1490
|
+
"Epoch 7 Batch 3300 Loss 0.0071\n",
|
|
1491
|
+
"Epoch 7 Batch 3400 Loss 0.0074\n",
|
|
1492
|
+
"Epoch 7 Batch 3500 Loss 0.0079\n",
|
|
1493
|
+
"Epoch 7 Batch 3600 Loss 0.0089\n",
|
|
1494
|
+
"Epoch 7 Batch 3700 Loss 0.0070\n",
|
|
1495
|
+
"Epoch 7 Batch 3800 Loss 0.0072\n",
|
|
1496
|
+
"Epoch 7 Batch 3900 Loss 0.0077\n",
|
|
1497
|
+
"Epoch 7 Batch 4000 Loss 0.0080\n",
|
|
1498
|
+
"Epoch 7 Batch 4100 Loss 0.0083\n",
|
|
1499
|
+
"Epoch 7 Batch 4200 Loss 0.0081\n",
|
|
1500
|
+
"Epoch 7 Batch 4300 Loss 0.0090\n",
|
|
1501
|
+
"Epoch 7 Batch 4400 Loss 0.0077\n",
|
|
1502
|
+
"Epoch 7 Batch 4500 Loss 0.0072\n",
|
|
1503
|
+
"Epoch 7 Batch 4600 Loss 0.0119\n",
|
|
1504
|
+
"Epoch 7 Loss 0.0064 Val Loss 0.0133\n",
|
|
1505
|
+
"Time taken for 1 epoch 729.9929449558258 sec\n",
|
|
1506
|
+
"\n",
|
|
1507
|
+
"Epoch 8 Batch 0 Loss 0.0066\n",
|
|
1508
|
+
"Epoch 8 Batch 100 Loss 0.0049\n",
|
|
1509
|
+
"Epoch 8 Batch 200 Loss 0.0059\n",
|
|
1510
|
+
"Epoch 8 Batch 300 Loss 0.0073\n",
|
|
1511
|
+
"Epoch 8 Batch 400 Loss 0.0081\n",
|
|
1512
|
+
"Epoch 8 Batch 500 Loss 0.0062\n",
|
|
1513
|
+
"Epoch 8 Batch 600 Loss 0.0069\n",
|
|
1514
|
+
"Epoch 8 Batch 700 Loss 0.0062\n",
|
|
1515
|
+
"Epoch 8 Batch 800 Loss 0.0086\n",
|
|
1516
|
+
"Epoch 8 Batch 900 Loss 0.0062\n",
|
|
1517
|
+
"Epoch 8 Batch 1000 Loss 0.0051\n",
|
|
1518
|
+
"Epoch 8 Batch 1100 Loss 0.0068\n",
|
|
1519
|
+
"Epoch 8 Batch 1200 Loss 0.0051\n",
|
|
1520
|
+
"Epoch 8 Batch 1300 Loss 0.0084\n",
|
|
1521
|
+
"Epoch 8 Batch 1400 Loss 0.0056\n",
|
|
1522
|
+
"Epoch 8 Batch 1500 Loss 0.0078\n",
|
|
1523
|
+
"Epoch 8 Batch 1600 Loss 0.0055\n",
|
|
1524
|
+
"Epoch 8 Batch 1700 Loss 0.0055\n",
|
|
1525
|
+
"Epoch 8 Batch 1800 Loss 0.0055\n",
|
|
1526
|
+
"Epoch 8 Batch 1900 Loss 0.0081\n",
|
|
1527
|
+
"Epoch 8 Batch 2000 Loss 0.0078\n",
|
|
1528
|
+
"Epoch 8 Batch 2100 Loss 0.0095\n",
|
|
1529
|
+
"Epoch 8 Batch 2200 Loss 0.0050\n",
|
|
1530
|
+
"Epoch 8 Batch 2300 Loss 0.0086\n",
|
|
1531
|
+
"Epoch 8 Batch 2400 Loss 0.0054\n",
|
|
1532
|
+
"Epoch 8 Batch 2500 Loss 0.0100\n",
|
|
1533
|
+
"Epoch 8 Batch 2600 Loss 0.0062\n",
|
|
1534
|
+
"Epoch 8 Batch 2700 Loss 0.0090\n",
|
|
1535
|
+
"Epoch 8 Batch 2800 Loss 0.0085\n",
|
|
1536
|
+
"Epoch 8 Batch 2900 Loss 0.0075\n",
|
|
1537
|
+
"Epoch 8 Batch 3000 Loss 0.0071\n",
|
|
1538
|
+
"Epoch 8 Batch 3100 Loss 0.0066\n",
|
|
1539
|
+
"Epoch 8 Batch 3200 Loss 0.0098\n",
|
|
1540
|
+
"Epoch 8 Batch 3300 Loss 0.0076\n",
|
|
1541
|
+
"Epoch 8 Batch 3400 Loss 0.0084\n",
|
|
1542
|
+
"Epoch 8 Batch 3500 Loss 0.0081\n",
|
|
1543
|
+
"Epoch 8 Batch 3600 Loss 0.0077\n",
|
|
1544
|
+
"Epoch 8 Batch 3700 Loss 0.0092\n",
|
|
1545
|
+
"Epoch 8 Batch 3800 Loss 0.0061\n",
|
|
1546
|
+
"Epoch 8 Batch 3900 Loss 0.0081\n",
|
|
1547
|
+
"Epoch 8 Batch 4000 Loss 0.0087\n",
|
|
1548
|
+
"Epoch 8 Batch 4100 Loss 0.0102\n",
|
|
1549
|
+
"Epoch 8 Batch 4200 Loss 0.0071\n",
|
|
1550
|
+
"Epoch 8 Batch 4300 Loss 0.0078\n",
|
|
1551
|
+
"Epoch 8 Batch 4400 Loss 0.0081\n",
|
|
1552
|
+
"Epoch 8 Batch 4500 Loss 0.0066\n",
|
|
1553
|
+
"Epoch 8 Batch 4600 Loss 0.0068\n",
|
|
1554
|
+
"Epoch 8 Loss 0.0059 Val Loss 0.0137\n",
|
|
1555
|
+
"Time taken for 1 epoch 730.8820261955261 sec\n",
|
|
1556
|
+
"\n",
|
|
1557
|
+
"Epoch 9 Batch 0 Loss 0.0069\n",
|
|
1558
|
+
"Epoch 9 Batch 100 Loss 0.0094\n",
|
|
1559
|
+
"Epoch 9 Batch 200 Loss 0.0072\n",
|
|
1560
|
+
"Epoch 9 Batch 300 Loss 0.0038\n",
|
|
1561
|
+
"Epoch 9 Batch 400 Loss 0.0074\n",
|
|
1562
|
+
"Epoch 9 Batch 500 Loss 0.0064\n",
|
|
1563
|
+
"Epoch 9 Batch 600 Loss 0.0090\n",
|
|
1564
|
+
"Epoch 9 Batch 700 Loss 0.0078\n",
|
|
1565
|
+
"Epoch 9 Batch 800 Loss 0.0056\n",
|
|
1566
|
+
"Epoch 9 Batch 900 Loss 0.0052\n",
|
|
1567
|
+
"Epoch 9 Batch 1000 Loss 0.0065\n",
|
|
1568
|
+
"Epoch 9 Batch 1100 Loss 0.0057\n",
|
|
1569
|
+
"Epoch 9 Batch 1200 Loss 0.0061\n",
|
|
1570
|
+
"Epoch 9 Batch 1300 Loss 0.0057\n",
|
|
1571
|
+
"Epoch 9 Batch 1400 Loss 0.0071\n",
|
|
1572
|
+
"Epoch 9 Batch 1500 Loss 0.0067\n",
|
|
1573
|
+
"Epoch 9 Batch 1600 Loss 0.0075\n",
|
|
1574
|
+
"Epoch 9 Batch 1700 Loss 0.0073\n",
|
|
1575
|
+
"Epoch 9 Batch 1800 Loss 0.0053\n",
|
|
1576
|
+
"Epoch 9 Batch 1900 Loss 0.0064\n",
|
|
1577
|
+
"Epoch 9 Batch 2000 Loss 0.0056\n",
|
|
1578
|
+
"Epoch 9 Batch 2100 Loss 0.0076\n",
|
|
1579
|
+
"Epoch 9 Batch 2200 Loss 0.0061\n",
|
|
1580
|
+
"Epoch 9 Batch 2300 Loss 0.0086\n",
|
|
1581
|
+
"Epoch 9 Batch 2400 Loss 0.0069\n",
|
|
1582
|
+
"Epoch 9 Batch 2500 Loss 0.0083\n",
|
|
1583
|
+
"Epoch 9 Batch 2600 Loss 0.0075\n",
|
|
1584
|
+
"Epoch 9 Batch 2700 Loss 0.0086\n",
|
|
1585
|
+
"Epoch 9 Batch 2800 Loss 0.0088\n",
|
|
1586
|
+
"Epoch 9 Batch 2900 Loss 0.0065\n",
|
|
1587
|
+
"Epoch 9 Batch 3000 Loss 0.0069\n",
|
|
1588
|
+
"Epoch 9 Batch 3100 Loss 0.0061\n",
|
|
1589
|
+
"Epoch 9 Batch 3200 Loss 0.0076\n",
|
|
1590
|
+
"Epoch 9 Batch 3300 Loss 0.0049\n",
|
|
1591
|
+
"Epoch 9 Batch 3400 Loss 0.0094\n",
|
|
1592
|
+
"Epoch 9 Batch 3500 Loss 0.0070\n",
|
|
1593
|
+
"Epoch 9 Batch 3600 Loss 0.0081\n",
|
|
1594
|
+
"Epoch 9 Batch 3700 Loss 0.0072\n",
|
|
1595
|
+
"Epoch 9 Batch 3800 Loss 0.0085\n",
|
|
1596
|
+
"Epoch 9 Batch 3900 Loss 0.0059\n",
|
|
1597
|
+
"Epoch 9 Batch 4000 Loss 0.0055\n",
|
|
1598
|
+
"Epoch 9 Batch 4100 Loss 0.0069\n",
|
|
1599
|
+
"Epoch 9 Batch 4200 Loss 0.0062\n",
|
|
1600
|
+
"Epoch 9 Batch 4300 Loss 0.0055\n",
|
|
1601
|
+
"Epoch 9 Batch 4400 Loss 0.0095\n",
|
|
1602
|
+
"Epoch 9 Batch 4500 Loss 0.0087\n",
|
|
1603
|
+
"Epoch 9 Batch 4600 Loss 0.0066\n",
|
|
1604
|
+
"Epoch 9 Loss 0.0055 Val Loss 0.0139\n",
|
|
1605
|
+
"Time taken for 1 epoch 734.5921323299408 sec\n",
|
|
1606
|
+
"\n",
|
|
1607
|
+
"Epoch 10 Batch 0 Loss 0.0077\n",
|
|
1608
|
+
"Epoch 10 Batch 100 Loss 0.0051\n",
|
|
1609
|
+
"Epoch 10 Batch 200 Loss 0.0051\n",
|
|
1610
|
+
"Epoch 10 Batch 300 Loss 0.0075\n",
|
|
1611
|
+
"Epoch 10 Batch 400 Loss 0.0063\n",
|
|
1612
|
+
"Epoch 10 Batch 500 Loss 0.0073\n",
|
|
1613
|
+
"Epoch 10 Batch 600 Loss 0.0057\n",
|
|
1614
|
+
"Epoch 10 Batch 700 Loss 0.0053\n",
|
|
1615
|
+
"Epoch 10 Batch 800 Loss 0.0063\n",
|
|
1616
|
+
"Epoch 10 Batch 900 Loss 0.0054\n",
|
|
1617
|
+
"Epoch 10 Batch 1000 Loss 0.0050\n",
|
|
1618
|
+
"Epoch 10 Batch 1100 Loss 0.0064\n",
|
|
1619
|
+
"Epoch 10 Batch 1200 Loss 0.0062\n",
|
|
1620
|
+
"Epoch 10 Batch 1300 Loss 0.0069\n",
|
|
1621
|
+
"Epoch 10 Batch 1400 Loss 0.0083\n",
|
|
1622
|
+
"Epoch 10 Batch 1500 Loss 0.0082\n",
|
|
1623
|
+
"Epoch 10 Batch 1600 Loss 0.0068\n",
|
|
1624
|
+
"Epoch 10 Batch 1700 Loss 0.0065\n",
|
|
1625
|
+
"Epoch 10 Batch 1800 Loss 0.0068\n",
|
|
1626
|
+
"Epoch 10 Batch 1900 Loss 0.0063\n",
|
|
1627
|
+
"Epoch 10 Batch 2000 Loss 0.0054\n",
|
|
1628
|
+
"Epoch 10 Batch 2100 Loss 0.0063\n",
|
|
1629
|
+
"Epoch 10 Batch 2200 Loss 0.0070\n",
|
|
1630
|
+
"Epoch 10 Batch 2300 Loss 0.0056\n",
|
|
1631
|
+
"Epoch 10 Batch 2400 Loss 0.0091\n",
|
|
1632
|
+
"Epoch 10 Batch 2500 Loss 0.0062\n",
|
|
1633
|
+
"Epoch 10 Batch 2600 Loss 0.0073\n",
|
|
1634
|
+
"Epoch 10 Batch 2700 Loss 0.0065\n",
|
|
1635
|
+
"Epoch 10 Batch 2800 Loss 0.0075\n",
|
|
1636
|
+
"Epoch 10 Batch 2900 Loss 0.0075\n",
|
|
1637
|
+
"Epoch 10 Batch 3000 Loss 0.0083\n",
|
|
1638
|
+
"Epoch 10 Batch 3100 Loss 0.0083\n",
|
|
1639
|
+
"Epoch 10 Batch 3200 Loss 0.0085\n",
|
|
1640
|
+
"Epoch 10 Batch 3300 Loss 0.0076\n",
|
|
1641
|
+
"Epoch 10 Batch 3400 Loss 0.0071\n",
|
|
1642
|
+
"Epoch 10 Batch 3500 Loss 0.0062\n",
|
|
1643
|
+
"Epoch 10 Batch 3600 Loss 0.0082\n",
|
|
1644
|
+
"Epoch 10 Batch 3700 Loss 0.0070\n",
|
|
1645
|
+
"Epoch 10 Batch 3800 Loss 0.0093\n",
|
|
1646
|
+
"Epoch 10 Batch 3900 Loss 0.0078\n",
|
|
1647
|
+
"Epoch 10 Batch 4000 Loss 0.0080\n",
|
|
1648
|
+
"Epoch 10 Batch 4100 Loss 0.0058\n",
|
|
1649
|
+
"Epoch 10 Batch 4200 Loss 0.0079\n",
|
|
1650
|
+
"Epoch 10 Batch 4300 Loss 0.0059\n",
|
|
1651
|
+
"Epoch 10 Batch 4400 Loss 0.0065\n",
|
|
1652
|
+
"Epoch 10 Batch 4500 Loss 0.0064\n",
|
|
1653
|
+
"Epoch 10 Batch 4600 Loss 0.0058\n",
|
|
1654
|
+
"Epoch 10 Loss 0.0052 Val Loss 0.0142\n",
|
|
1655
|
+
"Early stopping triggered after 5 consecutive epochs w/o improvement\n"
|
|
1656
|
+
]
|
|
1657
|
+
}
|
|
1658
|
+
]
|
|
1659
|
+
},
|
|
1660
|
+
{
|
|
1661
|
+
"cell_type": "code",
|
|
1662
|
+
"source": [
|
|
1663
|
+
"encoder.embedding.variables"
|
|
1664
|
+
],
|
|
1665
|
+
"metadata": {
|
|
1666
|
+
"colab": {
|
|
1667
|
+
"base_uri": "https://localhost:8080/"
|
|
1668
|
+
},
|
|
1669
|
+
"id": "Ron5dNu1nG7a",
|
|
1670
|
+
"outputId": "9c2f9ad8-216a-4f86-d4c5-0ed25adcfad1"
|
|
1671
|
+
},
|
|
1672
|
+
"execution_count": null,
|
|
1673
|
+
"outputs": [
|
|
1674
|
+
{
|
|
1675
|
+
"output_type": "execute_result",
|
|
1676
|
+
"data": {
|
|
1677
|
+
"text/plain": [
|
|
1678
|
+
"[<Variable path=encoder_2/embedding_8/embeddings, shape=(166, 256), dtype=float32, value=[[-0.0377021 -0.13381055 -0.11647186 ... 0.02190447 -0.18612249\n",
|
|
1679
|
+
" 0.3392803 ]\n",
|
|
1680
|
+
" [-0.03409722 0.14195894 -0.0854991 ... -0.07803918 -0.00742843\n",
|
|
1681
|
+
" 0.12786649]\n",
|
|
1682
|
+
" [ 0.02340595 -0.08440623 0.10921772 ... 0.11545631 -0.07786053\n",
|
|
1683
|
+
" 0.02020929]\n",
|
|
1684
|
+
" ...\n",
|
|
1685
|
+
" [ 0.01179477 0.05889428 0.01100738 ... 0.03654375 -0.00295061\n",
|
|
1686
|
+
" -0.02032111]\n",
|
|
1687
|
+
" [ 0.09261505 0.00799699 -0.10922566 ... 0.16155736 -0.04701557\n",
|
|
1688
|
+
" -0.1135793 ]\n",
|
|
1689
|
+
" [ 0.02345644 -0.14304264 0.14749458 ... -0.05203421 0.19353712\n",
|
|
1690
|
+
" 0.01092249]]>]"
|
|
1691
|
+
]
|
|
1692
|
+
},
|
|
1693
|
+
"metadata": {},
|
|
1694
|
+
"execution_count": 92
|
|
1695
|
+
}
|
|
1696
|
+
]
|
|
1697
|
+
},
|
|
1698
|
+
{
|
|
1699
|
+
"cell_type": "code",
|
|
1700
|
+
"source": [
|
|
1701
|
+
"decoder.embedding.variables"
|
|
1702
|
+
],
|
|
1703
|
+
"metadata": {
|
|
1704
|
+
"colab": {
|
|
1705
|
+
"base_uri": "https://localhost:8080/"
|
|
1706
|
+
},
|
|
1707
|
+
"id": "lv3y11v-nLK0",
|
|
1708
|
+
"outputId": "6e83ec22-9367-41e5-d997-00c207111ae3"
|
|
1709
|
+
},
|
|
1710
|
+
"execution_count": null,
|
|
1711
|
+
"outputs": [
|
|
1712
|
+
{
|
|
1713
|
+
"output_type": "execute_result",
|
|
1714
|
+
"data": {
|
|
1715
|
+
"text/plain": [
|
|
1716
|
+
"[<Variable path=decoder_6/embedding_9/embeddings, shape=(36, 256), dtype=float32, value=[[ 0.02532734 0.04688115 0.02036213 ... 0.02987245 -0.01590071\n",
|
|
1717
|
+
" 0.021944 ]\n",
|
|
1718
|
+
" [-0.24319564 0.12953956 -0.02794108 ... -0.14007922 0.00697453\n",
|
|
1719
|
+
" -0.16937147]\n",
|
|
1720
|
+
" [ 0.02966861 0.01044544 -0.01499049 ... -0.01741641 -0.00173\n",
|
|
1721
|
+
" 0.04175289]\n",
|
|
1722
|
+
" ...\n",
|
|
1723
|
+
" [ 0.13381338 -0.06536956 0.08742485 ... -0.03918008 -0.1341787\n",
|
|
1724
|
+
" 0.12816505]\n",
|
|
1725
|
+
" [ 0.10032396 -0.01028896 0.18520994 ... 0.02209266 -0.1323131\n",
|
|
1726
|
+
" 0.08644268]\n",
|
|
1727
|
+
" [ 0.00735058 0.06271109 0.0967354 ... -0.05375795 -0.10891018\n",
|
|
1728
|
+
" -0.0059194 ]]>]"
|
|
1729
|
+
]
|
|
1730
|
+
},
|
|
1731
|
+
"metadata": {},
|
|
1732
|
+
"execution_count": 93
|
|
1733
|
+
}
|
|
1734
|
+
]
|
|
1735
|
+
},
|
|
1736
|
+
{
|
|
1737
|
+
"cell_type": "code",
|
|
1738
|
+
"source": [],
|
|
1739
|
+
"metadata": {
|
|
1740
|
+
"id": "bL2ukuAgrCrN"
|
|
1741
|
+
},
|
|
1742
|
+
"execution_count": null,
|
|
1743
|
+
"outputs": []
|
|
1744
|
+
}
|
|
1745
|
+
]
|
|
1746
|
+
}
|