dotstat_io 1.0.3__tar.gz → 1.0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dotstat_io might be problematic. Click here for more details.
- {dotstat_io-1.0.3 → dotstat_io-1.0.5}/PKG-INFO +2 -2
- {dotstat_io-1.0.3 → dotstat_io-1.0.5}/dotstat_io/client.py +66 -55
- {dotstat_io-1.0.3 → dotstat_io-1.0.5}/pyproject.toml +1 -1
- {dotstat_io-1.0.3 → dotstat_io-1.0.5}/README.md +0 -0
- {dotstat_io-1.0.3 → dotstat_io-1.0.5}/dotstat_io/__init__.py +0 -0
- {dotstat_io-1.0.3 → dotstat_io-1.0.5}/dotstat_io/authentication.py +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from enum import IntEnum
|
|
1
2
|
import os
|
|
2
3
|
import requests
|
|
3
4
|
import chardet
|
|
@@ -8,6 +9,12 @@ import xml.etree.ElementTree as ET
|
|
|
8
9
|
from pathlib import Path
|
|
9
10
|
from time import sleep
|
|
10
11
|
|
|
12
|
+
from dotstat_io.authentication import Authentication
|
|
13
|
+
|
|
14
|
+
class ValidationType(IntEnum):
|
|
15
|
+
BASIC = 0
|
|
16
|
+
ADVANCED = 1
|
|
17
|
+
|
|
11
18
|
|
|
12
19
|
# class to download or upload data from/to .Stat Suite
|
|
13
20
|
class Client():
|
|
@@ -35,10 +42,10 @@ class Client():
|
|
|
35
42
|
|
|
36
43
|
# Initialise Client
|
|
37
44
|
def __init__(self,
|
|
38
|
-
access_token: str = None,
|
|
39
|
-
authentication_obj:
|
|
40
|
-
|
|
41
|
-
|
|
45
|
+
access_token: None | str = None,
|
|
46
|
+
authentication_obj: None | Authentication = None):
|
|
47
|
+
Client.__access_token = access_token
|
|
48
|
+
Client.__authentication_obj = authentication_obj
|
|
42
49
|
|
|
43
50
|
|
|
44
51
|
#
|
|
@@ -65,7 +72,7 @@ class Client():
|
|
|
65
72
|
@classmethod
|
|
66
73
|
def init_with_authentication_obj(
|
|
67
74
|
cls,
|
|
68
|
-
authentication_obj:
|
|
75
|
+
authentication_obj: Authentication
|
|
69
76
|
):
|
|
70
77
|
return cls(
|
|
71
78
|
authentication_obj=authentication_obj
|
|
@@ -78,18 +85,18 @@ class Client():
|
|
|
78
85
|
returned_result = ""
|
|
79
86
|
|
|
80
87
|
#
|
|
81
|
-
if
|
|
82
|
-
|
|
88
|
+
if Client.__authentication_obj is not None:
|
|
89
|
+
Client.__access_token = Client.__authentication_obj.get_token()
|
|
83
90
|
|
|
84
91
|
headers = {
|
|
85
92
|
'accept': content_format,
|
|
86
|
-
'authorization': 'Bearer '+
|
|
93
|
+
'authorization': 'Bearer '+Client.__access_token
|
|
87
94
|
}
|
|
88
95
|
|
|
89
96
|
#
|
|
90
97
|
response = requests.get(dotstat_url, verify=True, headers=headers)
|
|
91
98
|
except Exception as err:
|
|
92
|
-
returned_result =
|
|
99
|
+
returned_result = Client.__ERROR_OCCURRED + str(err) + os.linesep
|
|
93
100
|
|
|
94
101
|
# Write the result to the log
|
|
95
102
|
for line in returned_result.split(os.linesep):
|
|
@@ -97,7 +104,7 @@ class Client():
|
|
|
97
104
|
self.__log.info(' ' + line)
|
|
98
105
|
else:
|
|
99
106
|
if response.status_code != 200:
|
|
100
|
-
returned_result =
|
|
107
|
+
returned_result = Client.__ERROR_OCCURRED
|
|
101
108
|
if len(str(response.status_code)) > 0:
|
|
102
109
|
returned_result += 'Error code: ' + str(response.status_code) + os.linesep
|
|
103
110
|
if len(str(response.reason)) > 0:
|
|
@@ -111,7 +118,7 @@ class Client():
|
|
|
111
118
|
os.remove(file_path)
|
|
112
119
|
with open(file_path, "wb") as file:
|
|
113
120
|
file.write(response.content)
|
|
114
|
-
returned_result =
|
|
121
|
+
returned_result = Client.__DOWNLOAD_SUCCESS
|
|
115
122
|
|
|
116
123
|
# Write the result to the log
|
|
117
124
|
for line in returned_result.split(os.linesep):
|
|
@@ -127,19 +134,19 @@ class Client():
|
|
|
127
134
|
returned_result = ""
|
|
128
135
|
|
|
129
136
|
#
|
|
130
|
-
if
|
|
131
|
-
__access_token =
|
|
137
|
+
if Client.__authentication_obj is not None:
|
|
138
|
+
Client.__access_token = Client.__authentication_obj.get_token()
|
|
132
139
|
|
|
133
140
|
headers = {
|
|
134
141
|
'accept': content_format,
|
|
135
142
|
'Transfer-Encoding': 'chunked',
|
|
136
|
-
'authorization': 'Bearer '+
|
|
143
|
+
'authorization': 'Bearer '+Client.__access_token
|
|
137
144
|
}
|
|
138
145
|
|
|
139
146
|
#
|
|
140
147
|
return requests.get(dotstat_url, verify=True, headers=headers, stream=True)
|
|
141
148
|
except Exception as err:
|
|
142
|
-
returned_result =
|
|
149
|
+
returned_result = Client.__ERROR_OCCURRED + str(err) + os.linesep
|
|
143
150
|
return returned_result
|
|
144
151
|
|
|
145
152
|
|
|
@@ -149,41 +156,45 @@ class Client():
|
|
|
149
156
|
file_path: Path,
|
|
150
157
|
space: str,
|
|
151
158
|
validationType: int,
|
|
152
|
-
use_filepath: bool = False
|
|
159
|
+
use_filepath: bool = False,
|
|
160
|
+
optimize: bool = True):
|
|
153
161
|
try:
|
|
154
162
|
returned_result = ""
|
|
155
163
|
|
|
156
164
|
#
|
|
157
|
-
if
|
|
158
|
-
|
|
165
|
+
if Client.__authentication_obj is not None:
|
|
166
|
+
Client.__access_token = Client.__authentication_obj.get_token()
|
|
159
167
|
|
|
160
168
|
payload = {
|
|
161
169
|
'dataspace': space,
|
|
162
|
-
'validationType': validationType
|
|
170
|
+
'validationType': validationType,
|
|
171
|
+
'optimize': optimize
|
|
163
172
|
}
|
|
164
173
|
|
|
165
174
|
headers = {
|
|
166
175
|
'accept': 'application/json',
|
|
167
|
-
'authorization': "Bearer "+
|
|
176
|
+
'authorization': "Bearer "+Client.__access_token
|
|
168
177
|
}
|
|
169
178
|
|
|
170
179
|
if use_filepath:
|
|
171
180
|
files = {
|
|
172
181
|
'dataspace': (None, payload['dataspace']),
|
|
173
182
|
'validationType': (None, payload['validationType']),
|
|
183
|
+
'optimize': (None, payload['optimize']),
|
|
174
184
|
'filepath': (None, str(file_path))
|
|
175
185
|
}
|
|
176
186
|
else:
|
|
177
187
|
files = {
|
|
178
188
|
'dataspace': (None, payload['dataspace']),
|
|
179
189
|
'validationType': (None, payload['validationType']),
|
|
190
|
+
'optimize': (None, payload['optimize']),
|
|
180
191
|
'file': (os.path.realpath(file_path), open(os.path.realpath(file_path), 'rb'), 'text/csv', '')
|
|
181
192
|
}
|
|
182
193
|
|
|
183
194
|
#
|
|
184
195
|
response = requests.post(transfer_url, verify=True, headers=headers, files=files)
|
|
185
196
|
except Exception as err:
|
|
186
|
-
returned_result =
|
|
197
|
+
returned_result = Client.__ERROR_OCCURRED + str(err) + os.linesep
|
|
187
198
|
|
|
188
199
|
# Write the result to the log
|
|
189
200
|
for line in returned_result.split(os.linesep):
|
|
@@ -204,7 +215,7 @@ class Client():
|
|
|
204
215
|
returned_result = result + os.linesep
|
|
205
216
|
|
|
206
217
|
# Check the request status
|
|
207
|
-
if (result != "" and result.find(
|
|
218
|
+
if (result != "" and result.find(Client.__ERROR_OCCURRED ) == -1):
|
|
208
219
|
# Extract the request ID the returned message
|
|
209
220
|
start = 'with ID'
|
|
210
221
|
end = 'was successfully'
|
|
@@ -225,8 +236,8 @@ class Client():
|
|
|
225
236
|
sleep(3)
|
|
226
237
|
|
|
227
238
|
previous_result = result
|
|
228
|
-
while (result in [
|
|
229
|
-
or
|
|
239
|
+
while (result in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
|
|
240
|
+
or Client.__CONNECTION_ABORTED in result):
|
|
230
241
|
result = self.__check_request_status(transfer_url, requestId, space)
|
|
231
242
|
|
|
232
243
|
# Prevent loging again the same information such as "Queued" or "InProgress"
|
|
@@ -235,14 +246,14 @@ class Client():
|
|
|
235
246
|
|
|
236
247
|
# Write the result to the log
|
|
237
248
|
for line in previous_result.split(os.linesep):
|
|
238
|
-
if (len(line) > 0 and line not in [
|
|
239
|
-
and
|
|
249
|
+
if (len(line) > 0 and line not in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
|
|
250
|
+
and Client.__CONNECTION_ABORTED not in line):
|
|
240
251
|
self.__log.info(' ' + line)
|
|
241
252
|
sleep(3)
|
|
242
253
|
|
|
243
254
|
returned_result = returned_result + result + os.linesep
|
|
244
255
|
else:
|
|
245
|
-
returned_result =
|
|
256
|
+
returned_result = Client.__ERROR_OCCURRED
|
|
246
257
|
if len(str(response.status_code)) > 0:
|
|
247
258
|
returned_result += 'Error code: ' + str(response.status_code) + os.linesep
|
|
248
259
|
if len(str(response.reason)) > 0:
|
|
@@ -256,7 +267,7 @@ class Client():
|
|
|
256
267
|
if len(line) > 0:
|
|
257
268
|
self.__log.info(' ' + line)
|
|
258
269
|
except ValueError as err:
|
|
259
|
-
returned_result =
|
|
270
|
+
returned_result = Client.__ERROR_OCCURRED
|
|
260
271
|
if len(str(response.status_code)) > 0:
|
|
261
272
|
returned_result += 'Error code: ' + str(response.status_code) + os.linesep
|
|
262
273
|
if len(str(response.reason)) > 0:
|
|
@@ -282,8 +293,8 @@ class Client():
|
|
|
282
293
|
returned_result = ""
|
|
283
294
|
|
|
284
295
|
#
|
|
285
|
-
if
|
|
286
|
-
|
|
296
|
+
if Client.__authentication_obj is not None:
|
|
297
|
+
Client.__access_token = Client.__authentication_obj.get_token()
|
|
287
298
|
|
|
288
299
|
payload = {
|
|
289
300
|
'dataspace': space,
|
|
@@ -292,7 +303,7 @@ class Client():
|
|
|
292
303
|
|
|
293
304
|
headers = {
|
|
294
305
|
'accept': 'application/json',
|
|
295
|
-
'authorization': "Bearer "+
|
|
306
|
+
'authorization': "Bearer "+Client.__access_token
|
|
296
307
|
}
|
|
297
308
|
|
|
298
309
|
excel_file = open(os.path.realpath(excelfile_path), 'rb')
|
|
@@ -307,7 +318,7 @@ class Client():
|
|
|
307
318
|
#
|
|
308
319
|
response = requests.post(transfer_url, verify=True, headers=headers, files=files)
|
|
309
320
|
except Exception as err:
|
|
310
|
-
returned_result =
|
|
321
|
+
returned_result = Client.__ERROR_OCCURRED + str(err) + os.linesep
|
|
311
322
|
|
|
312
323
|
# Write the result to the log
|
|
313
324
|
for line in returned_result.split(os.linesep):
|
|
@@ -328,7 +339,7 @@ class Client():
|
|
|
328
339
|
returned_result = result + os.linesep
|
|
329
340
|
|
|
330
341
|
# Check the request status
|
|
331
|
-
if (result != "" and result.find(
|
|
342
|
+
if (result != "" and result.find(Client.__ERROR_OCCURRED ) == -1):
|
|
332
343
|
# Extract the request ID the returned message
|
|
333
344
|
start = 'with ID'
|
|
334
345
|
end = 'was successfully'
|
|
@@ -349,8 +360,8 @@ class Client():
|
|
|
349
360
|
sleep(3)
|
|
350
361
|
|
|
351
362
|
previous_result = result
|
|
352
|
-
while (result in [
|
|
353
|
-
or
|
|
363
|
+
while (result in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
|
|
364
|
+
or Client.__CONNECTION_ABORTED in result):
|
|
354
365
|
result = self.__check_request_status(transfer_url, requestId, space)
|
|
355
366
|
|
|
356
367
|
# Prevent loging again the same information such as "Queued" or "InProgress"
|
|
@@ -359,14 +370,14 @@ class Client():
|
|
|
359
370
|
|
|
360
371
|
# Write the result to the log
|
|
361
372
|
for line in previous_result.split(os.linesep):
|
|
362
|
-
if (len(line) > 0 and line not in [
|
|
363
|
-
and
|
|
373
|
+
if (len(line) > 0 and line not in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
|
|
374
|
+
and Client.__CONNECTION_ABORTED not in line):
|
|
364
375
|
self.__log.info(' ' + line)
|
|
365
376
|
sleep(3)
|
|
366
377
|
|
|
367
378
|
returned_result = returned_result + result + os.linesep
|
|
368
379
|
else:
|
|
369
|
-
returned_result =
|
|
380
|
+
returned_result = Client.__ERROR_OCCURRED
|
|
370
381
|
if len(str(response.status_code)) > 0:
|
|
371
382
|
returned_result += 'Error code: ' + str(response.status_code) + os.linesep
|
|
372
383
|
if len(str(response.reason)) > 0:
|
|
@@ -380,7 +391,7 @@ class Client():
|
|
|
380
391
|
if len(line) > 0:
|
|
381
392
|
self.__log.info(' ' + line)
|
|
382
393
|
except ValueError as err:
|
|
383
|
-
returned_result =
|
|
394
|
+
returned_result = Client.__ERROR_OCCURRED
|
|
384
395
|
if len(str(response.status_code)) > 0:
|
|
385
396
|
returned_result += 'Error code: ' + str(response.status_code) + os.linesep
|
|
386
397
|
if len(str(response.reason)) > 0:
|
|
@@ -401,8 +412,8 @@ class Client():
|
|
|
401
412
|
returned_result = ""
|
|
402
413
|
|
|
403
414
|
#
|
|
404
|
-
if
|
|
405
|
-
|
|
415
|
+
if Client.__authentication_obj is not None:
|
|
416
|
+
Client.__access_token = Client.__authentication_obj.get_token()
|
|
406
417
|
|
|
407
418
|
# Detect the encoding used in file
|
|
408
419
|
detected_encoding = self.__detect_encode(file_path)
|
|
@@ -417,13 +428,13 @@ class Client():
|
|
|
417
428
|
|
|
418
429
|
headers = {
|
|
419
430
|
'Content-Type': 'application/xml',
|
|
420
|
-
'authorization': "Bearer "+
|
|
431
|
+
'authorization': "Bearer "+Client.__access_token
|
|
421
432
|
}
|
|
422
433
|
|
|
423
434
|
#
|
|
424
435
|
response = requests.post(transfer_url, verify=True, headers=headers, data=xml_data)
|
|
425
436
|
except Exception as err:
|
|
426
|
-
returned_result =
|
|
437
|
+
returned_result = Client.__ERROR_OCCURRED + str(err) + os.linesep
|
|
427
438
|
|
|
428
439
|
# Write the result to the log
|
|
429
440
|
for line in returned_result.split(os.linesep):
|
|
@@ -433,7 +444,7 @@ class Client():
|
|
|
433
444
|
try:
|
|
434
445
|
response.raise_for_status()
|
|
435
446
|
except requests.exceptions.HTTPError as e:
|
|
436
|
-
returned_result = f'{
|
|
447
|
+
returned_result = f'{Client.__UPLOAD_FAILED}{response.status_code}: {e}'
|
|
437
448
|
|
|
438
449
|
# Write the result to the log
|
|
439
450
|
for line in returned_result.split(os.linesep):
|
|
@@ -441,11 +452,11 @@ class Client():
|
|
|
441
452
|
self.__log.info(' ' + line)
|
|
442
453
|
else:
|
|
443
454
|
response_tree = ET.XML(response.content)
|
|
444
|
-
for
|
|
445
|
-
text_element =
|
|
455
|
+
for element in response_tree.findall("./{0}ErrorMessage".format(Client.__NAMESPACE_MESSAGE)):
|
|
456
|
+
text_element = element.find("./{0}Text".format(Client.__NAMESPACE_COMMON))
|
|
446
457
|
if (text_element is not None):
|
|
447
458
|
if returned_result == "":
|
|
448
|
-
returned_result = f'{
|
|
459
|
+
returned_result = f'{Client.__UPLOAD_SUCCESS}with status code: {response.status_code}' + os.linesep
|
|
449
460
|
returned_result = returned_result + text_element.text + os.linesep
|
|
450
461
|
|
|
451
462
|
# Write the result to the log
|
|
@@ -479,12 +490,12 @@ class Client():
|
|
|
479
490
|
returned_result = ""
|
|
480
491
|
|
|
481
492
|
#
|
|
482
|
-
if
|
|
483
|
-
|
|
493
|
+
if Client.__authentication_obj is not None:
|
|
494
|
+
Client.__access_token = Client.__authentication_obj.get_token()
|
|
484
495
|
|
|
485
496
|
headers = {
|
|
486
497
|
'accept': 'application/json',
|
|
487
|
-
'authorization': "Bearer "+
|
|
498
|
+
'authorization': "Bearer "+Client.__access_token
|
|
488
499
|
}
|
|
489
500
|
|
|
490
501
|
payload = {
|
|
@@ -501,7 +512,7 @@ class Client():
|
|
|
501
512
|
#
|
|
502
513
|
response = requests.post(transfer_url, verify=True, headers=headers, data=payload)
|
|
503
514
|
except Exception as err:
|
|
504
|
-
returned_result =
|
|
515
|
+
returned_result = Client.__ERROR_OCCURRED + str(err)
|
|
505
516
|
else:
|
|
506
517
|
# If the response object cannot be converted to json, return an error
|
|
507
518
|
results_json = None
|
|
@@ -509,8 +520,8 @@ class Client():
|
|
|
509
520
|
results_json = json.loads(response.text)
|
|
510
521
|
if response.status_code == 200:
|
|
511
522
|
executionStatus = 'Execution status: ' + results_json['executionStatus']
|
|
512
|
-
if (results_json['executionStatus'] in [
|
|
513
|
-
or
|
|
523
|
+
if (results_json['executionStatus'] in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
|
|
524
|
+
or Client.__CONNECTION_ABORTED in results_json['executionStatus']):
|
|
514
525
|
returned_result = results_json['executionStatus']
|
|
515
526
|
else:
|
|
516
527
|
returned_result = executionStatus + os.linesep + 'Outcome: ' + results_json['outcome'] + os.linesep
|
|
@@ -519,7 +530,7 @@ class Client():
|
|
|
519
530
|
returned_result = returned_result + 'Log' + str(index) + ': ' + results_json['logs'][index]['message'] + os.linesep
|
|
520
531
|
index += 1
|
|
521
532
|
else:
|
|
522
|
-
returned_result =
|
|
533
|
+
returned_result = Client.__ERROR_OCCURRED
|
|
523
534
|
if len(str(response.status_code)) > 0:
|
|
524
535
|
returned_result += 'Error code: ' + str(response.status_code) + os.linesep
|
|
525
536
|
if len(str(response.reason)) > 0:
|
|
@@ -529,7 +540,7 @@ class Client():
|
|
|
529
540
|
|
|
530
541
|
returned_result += os.linesep
|
|
531
542
|
except ValueError as err:
|
|
532
|
-
returned_result =
|
|
543
|
+
returned_result = Client.__ERROR_OCCURRED
|
|
533
544
|
if len(str(response.status_code)) > 0:
|
|
534
545
|
returned_result += 'Error code: ' + str(response.status_code) + os.linesep
|
|
535
546
|
if len(str(response.reason)) > 0:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "dotstat_io"
|
|
3
|
-
version = "1.0.
|
|
3
|
+
version = "1.0.5"
|
|
4
4
|
description = "Utility to download or upload data from/to .Stat Suite using ADFS authentication to connect to it"
|
|
5
5
|
license = "MIT"
|
|
6
6
|
authors = ["Gyorgy Gyomai <gyorgy.gyomai@oecd.org>", "Abdel Aliaoui <abdel.aliaoui@oecd.org>"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|