dotstat_io 1.0.2__tar.gz → 1.0.4__tar.gz

{dotstat_io-1.0.2 → dotstat_io-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: dotstat_io
-Version: 1.0.2
+Version: 1.0.4
 Summary: Utility to download or upload data from/to .Stat Suite using ADFS authentication to connect to it
 License: MIT
 Author: Gyorgy Gyomai
@@ -147,12 +147,25 @@ returned_result = client_obj.upload_data_file(
         transfer_url, Path(file_path), space, validationType, use_filepath)
 ```
 * `transfer_url:` URL of the transfer service
-* `file_path:` The full path of the SDMX-CSV file, which will be uploaded to .Stat Suite
+* `file_path:` The full path of the SDMX-CSV file to be imported
 * `space:` Data space where the file will be uploaded
 * `validationType:` The type of validation to use during upload. Possible values: Basic Validation (0), Advanced Validation (1)
 * `use_filepath:` Use a file path of a shared folder accessible by the .Stat Suite data upload engine (for unlimited file sizes)
 
-#### 6. To upload a structure to .Stat Suite:
+#### 6. To upload an Excel data file to .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_excel_data_file(
+        transfer_url, Path(excelfile_path), Path(eddfile_path), space, validationType)
+```
+* `transfer_url:` URL of the transfer service
+* `excelfile_path:` The full path of the Excel file containing the data/referential metadata values to be imported
+* `eddfile_path:` The full path of the XML edd file containing the description of the Excel file to be imported
+* `space:` Data space where the file will be uploaded
+* `validationType:` The type of validation to use during upload. Possible values: Basic Validation (0), Advanced Validation (1)
+
+#### 7. To upload a structure to .Stat Suite:
 ```python
 from dotstat_io.client import Client
 client_obj = Client.init_with_authentication_obj(Authentication_obj)
@@ -160,7 +173,7 @@ returned_result = client_obj.upload_structure(
         transfer_url, Path(file_path))
 ```
 * `transfer_url:` URL of the transfer service
-* `file_path:` The full path of the SDMX-ML file, which will be uploaded to .Stat Suite
+* `file_path:` The full path of the SDMX-ML file to be imported
 
 ### For more information about how to use this package, all test cases can be accessed from this [`link`](https://gitlab.algobank.oecd.org/SD_ENGINEERING/dotstat_io/dotstat_io-package/-/blob/main/tests/test_cases.py)
 

{dotstat_io-1.0.2 → dotstat_io-1.0.4}/README.md

@@ -126,12 +126,25 @@ returned_result = client_obj.upload_data_file(
         transfer_url, Path(file_path), space, validationType, use_filepath)
 ```
 * `transfer_url:` URL of the transfer service
-* `file_path:` The full path of the SDMX-CSV file, which will be uploaded to .Stat Suite
+* `file_path:` The full path of the SDMX-CSV file to be imported
 * `space:` Data space where the file will be uploaded
 * `validationType:` The type of validation to use during upload. Possible values: Basic Validation (0), Advanced Validation (1)
 * `use_filepath:` Use a file path of a shared folder accessible by the .Stat Suite data upload engine (for unlimited file sizes)
 
-#### 6. To upload a structure to .Stat Suite:
+#### 6. To upload an Excel data file to .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_excel_data_file(
+        transfer_url, Path(excelfile_path), Path(eddfile_path), space, validationType)
+```
+* `transfer_url:` URL of the transfer service
+* `excelfile_path:` The full path of the Excel file containing the data/referential metadata values to be imported
+* `eddfile_path:` The full path of the XML edd file containing the description of the Excel file to be imported
+* `space:` Data space where the file will be uploaded
+* `validationType:` The type of validation to use during upload. Possible values: Basic Validation (0), Advanced Validation (1)
+
+#### 7. To upload a structure to .Stat Suite:
 ```python
 from dotstat_io.client import Client
 client_obj = Client.init_with_authentication_obj(Authentication_obj)
@@ -139,6 +152,6 @@ returned_result = client_obj.upload_structure(
         transfer_url, Path(file_path))
 ```
 * `transfer_url:` URL of the transfer service
-* `file_path:` The full path of the SDMX-ML file, which will be uploaded to .Stat Suite
+* `file_path:` The full path of the SDMX-ML file to be imported
 
 ### For more information about how to use this package, all test cases can be accessed from this [`link`](https://gitlab.algobank.oecd.org/SD_ENGINEERING/dotstat_io/dotstat_io-package/-/blob/main/tests/test_cases.py)

{dotstat_io-1.0.2 → dotstat_io-1.0.4}/dotstat_io/client.py

@@ -1,3 +1,4 @@
+from enum import IntEnum
 import os
 import requests
 import chardet
@@ -8,6 +9,12 @@ import xml.etree.ElementTree as ET
 from pathlib import Path
 from time import sleep
 
+from dotstat_io.authentication import Authentication
+
+class ValidationType(IntEnum):
+    BASIC = 0
+    ADVANCED = 1
+    
 
 # class to download or upload data from/to .Stat Suite
 class Client():
@@ -35,10 +42,10 @@ class Client():
 
     # Initialise Client
     def __init__(self,
-	    access_token: str = None,
-        authentication_obj: object = None):
-        self.__access_token = access_token
-        self.__authentication_obj = authentication_obj
+	    access_token: None | str = None,
+        authentication_obj: None | Authentication = None):
+        Client.__access_token = access_token
+        Client.__authentication_obj = authentication_obj
 
 
     # 
@@ -65,7 +72,7 @@ class Client():
     @classmethod
     def init_with_authentication_obj(
         cls, 
-        authentication_obj: object
+        authentication_obj: Authentication
     ):
         return cls(
             authentication_obj=authentication_obj
@@ -78,18 +85,18 @@ class Client():
             returned_result = ""
 
             # 
-            if self.__authentication_obj is not None:
-                self.__access_token = self.__authentication_obj.get_token()
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
 
             headers = {
                 'accept': content_format,
-                'authorization': 'Bearer '+self.__access_token
+                'authorization': 'Bearer '+Client.__access_token
             }
 
             #
             response = requests.get(dotstat_url, verify=True, headers=headers)
         except Exception as err:
-            returned_result = self.__ERROR_OCCURRED  + str(err) + os.linesep
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
 
             # Write the result to the log
             for line in returned_result.split(os.linesep):
@@ -97,7 +104,7 @@ class Client():
                     self.__log.info('   ' + line)
         else:
             if response.status_code != 200:
-                returned_result = self.__ERROR_OCCURRED
+                returned_result = Client.__ERROR_OCCURRED
                 if len(str(response.status_code)) > 0:
                     returned_result += 'Error code: ' + str(response.status_code) + os.linesep
                 if len(str(response.reason)) > 0:
@@ -111,7 +118,7 @@ class Client():
                     os.remove(file_path)
                 with open(file_path, "wb") as file:
                     file.write(response.content)
-                    returned_result = self.__DOWNLOAD_SUCCESS
+                    returned_result = Client.__DOWNLOAD_SUCCESS
             
             # Write the result to the log
             for line in returned_result.split(os.linesep):
@@ -127,19 +134,19 @@ class Client():
             returned_result = ""
 
             # 
-            if self.__authentication_obj is not None:
-                __access_token = self.__authentication_obj.get_token()
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
 
             headers = {
                 'accept': content_format,
                 'Transfer-Encoding': 'chunked',
-                'authorization': 'Bearer '+self.__access_token
+                'authorization': 'Bearer '+Client.__access_token
             }
 
             #
             return requests.get(dotstat_url, verify=True, headers=headers, stream=True)
         except Exception as err:
-            returned_result = self.__ERROR_OCCURRED  + str(err) + os.linesep
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
             return returned_result
 
 
@@ -154,8 +161,8 @@ class Client():
             returned_result = ""
 
             # 
-            if self.__authentication_obj is not None:
-                self.__access_token = self.__authentication_obj.get_token()
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
 
             payload = {
                 'dataspace': space,
@@ -164,7 +171,7 @@ class Client():
 
             headers = {
                 'accept': 'application/json',
-                'authorization': "Bearer "+self.__access_token
+                'authorization': "Bearer "+Client.__access_token
             }
 
             if  use_filepath:
@@ -183,7 +190,7 @@ class Client():
             #
             response = requests.post(transfer_url, verify=True, headers=headers, files=files)
         except Exception as err:
-            returned_result = self.__ERROR_OCCURRED  + str(err) + os.linesep
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
 
             # Write the result to the log
             for line in returned_result.split(os.linesep):
@@ -204,7 +211,7 @@ class Client():
                     returned_result = result + os.linesep
 
                     # Check the request status
-                    if (result != "" and result.find(self.__ERROR_OCCURRED ) == -1):
+                    if (result != "" and result.find(Client.__ERROR_OCCURRED ) == -1):
                         # Extract the request ID the returned message
                         start = 'with ID'
                         end = 'was successfully'
@@ -225,8 +232,8 @@ class Client():
                         sleep(3)
 
                         previous_result = result
-                        while (result in [self.__EXECUTION_IN_PROGRESS, self.__EXECUTION_IN_QUEUED]
-                                 or self.__CONNECTION_ABORTED in result):
+                        while (result in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                                 or Client.__CONNECTION_ABORTED in result):
                             result = self.__check_request_status(transfer_url, requestId, space)
 
                             # Prevent loging again the same information such as "Queued" or "InProgress" 
@@ -235,14 +242,14 @@ class Client():
 
                             # Write the result to the log
                             for line in previous_result.split(os.linesep):
-                              if (len(line) > 0 and line not in [self.__EXECUTION_IN_PROGRESS, self.__EXECUTION_IN_QUEUED]
-                                    and self.__CONNECTION_ABORTED not in line):
+                              if (len(line) > 0 and line not in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                                    and Client.__CONNECTION_ABORTED not in line):
                                  self.__log.info('   ' + line)
                             sleep(3)
 
                         returned_result = returned_result + result + os.linesep
                 else:
-                    returned_result = self.__ERROR_OCCURRED
+                    returned_result = Client.__ERROR_OCCURRED
                     if len(str(response.status_code)) > 0:
                         returned_result += 'Error code: ' + str(response.status_code) + os.linesep
                     if len(str(response.reason)) > 0:
@@ -256,7 +263,7 @@ class Client():
                         if len(line) > 0:
                             self.__log.info('   ' + line)
             except ValueError as err:
-                returned_result = self.__ERROR_OCCURRED
+                returned_result = Client.__ERROR_OCCURRED
                 if len(str(response.status_code)) > 0:
                     returned_result += 'Error code: ' + str(response.status_code) + os.linesep
                 if len(str(response.reason)) > 0:
@@ -271,14 +278,138 @@ class Client():
             return returned_result
 
 
+    # Upload an Excel data file to .Stat Suite
+    def upload_excel_data_file(self, 
+                    transfer_url: str, 
+                    excelfile_path: Path,
+                    eddfile_path: Path, 
+                    space: str, 
+                    validationType: int):
+        try:
+            returned_result = ""
+
+            # 
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
+
+            payload = {
+                'dataspace': space,
+                'validationType': validationType
+            }
+
+            headers = {
+                'accept': 'application/json',
+                'authorization': "Bearer "+Client.__access_token
+            }
+
+            excel_file = open(os.path.realpath(excelfile_path), 'rb')
+            eddfile_file = open(os.path.realpath(eddfile_path), 'rb')
+            files = {
+                'dataspace': (None, payload['dataspace']),
+                'validationType': (None, payload['validationType']),
+                'excelFile': (str(excelfile_path), excel_file, 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', ''),
+                'eddFile': (str(eddfile_path), eddfile_file, 'text/xml', '')
+            }
+
+            #
+            response = requests.post(transfer_url, verify=True, headers=headers, files=files)
+        except Exception as err:
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
+
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+        else:
+            # If the response object cannot be converted to json, return an error
+            results_json = None
+            try:
+                results_json = json.loads(response.text)
+                if response.status_code == 200:
+                    result = results_json['message']
+                    # Write the result to the log
+                    for line in result.split(os.linesep):
+                        if len(line) > 0:
+                            self.__log.info('   ' + line)
+
+                    returned_result = result + os.linesep
+
+                    # Check the request status
+                    if (result != "" and result.find(Client.__ERROR_OCCURRED ) == -1):
+                        # Extract the request ID the returned message
+                        start = 'with ID'
+                        end = 'was successfully'
+                        requestId = result[result.find(
+                            start)+len(start):result.rfind(end)]
+
+                        # Sleep a little bit before checking the request status
+                        sleep(3)
+
+                        # To avoid this error: maximum recursion depth exceeded while calling a Python object
+                        # replace the recursive calls with while loops.
+                        result = self.__check_request_status(transfer_url, requestId, space)
+
+                        # Write the result to the log
+                        for line in result.split(os.linesep):
+                            if len(line) > 0:
+                                self.__log.info('   ' + line)
+                        sleep(3)
+
+                        previous_result = result
+                        while (result in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                                 or Client.__CONNECTION_ABORTED in result):
+                            result = self.__check_request_status(transfer_url, requestId, space)
+
+                            # Prevent loging again the same information such as "Queued" or "InProgress" 
+                            if previous_result != result:
+                               previous_result = result
+
+                            # Write the result to the log
+                            for line in previous_result.split(os.linesep):
+                              if (len(line) > 0 and line not in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                                    and Client.__CONNECTION_ABORTED not in line):
+                                 self.__log.info('   ' + line)
+                            sleep(3)
+
+                        returned_result = returned_result + result + os.linesep
+                else:
+                    returned_result = Client.__ERROR_OCCURRED
+                    if len(str(response.status_code)) > 0:
+                        returned_result += 'Error code: ' + str(response.status_code) + os.linesep
+                    if len(str(response.reason)) > 0:
+                        returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                    if len(response.text) > 0:
+                        returned_result += 'Text: ' + response.text
+
+                    returned_result += os.linesep
+                    # Write the result to the log
+                    for line in returned_result.split(os.linesep):
+                        if len(line) > 0:
+                            self.__log.info('   ' + line)
+            except ValueError as err:
+                returned_result = Client.__ERROR_OCCURRED
+                if len(str(response.status_code)) > 0:
+                    returned_result += 'Error code: ' + str(response.status_code) + os.linesep
+                if len(str(response.reason)) > 0:
+                    returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                if len(response.text) > 0:
+                    returned_result += 'Text: ' + str(response.text)
+                else:
+                    returned_result += str(err)
+
+                returned_result += os.linesep
+        finally:
+            return returned_result
+        
+
     # Upload a structure file to .Stat Suite
     def upload_structure(self, transfer_url: str, file_path: Path):
         try:
             returned_result = ""
 
             # 
-            if self.__authentication_obj is not None:
-                self.__access_token = self.__authentication_obj.get_token()
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
 
             # Detect the encoding used in file 
             detected_encoding = self.__detect_encode(file_path)
@@ -293,13 +424,13 @@ class Client():
 
             headers = {
                 'Content-Type': 'application/xml',
-                'authorization': "Bearer "+self.__access_token
+                'authorization': "Bearer "+Client.__access_token
             }
 
             #
             response = requests.post(transfer_url, verify=True, headers=headers, data=xml_data)
         except Exception as err:
-            returned_result = self.__ERROR_OCCURRED  + str(err) + os.linesep
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
 
             # Write the result to the log
             for line in returned_result.split(os.linesep):
@@ -309,7 +440,7 @@ class Client():
             try:
                 response.raise_for_status()
             except requests.exceptions.HTTPError as e:
-                returned_result = f'{self.__UPLOAD_FAILED}{response.status_code}: {e}'
+                returned_result = f'{Client.__UPLOAD_FAILED}{response.status_code}: {e}'
 
                 # Write the result to the log
                 for line in returned_result.split(os.linesep):
@@ -317,11 +448,11 @@ class Client():
                         self.__log.info('   ' + line)
             else:
                 response_tree = ET.XML(response.content)
-                for self.__ERROR_OCCURRED in response_tree.findall("./{0}ErrorMessage".format(self.__NAMESPACE_MESSAGE)):
-                    text_element = self.__ERROR_OCCURRED.find("./{0}Text".format(self.__NAMESPACE_COMMON))
+                for element in response_tree.findall("./{0}ErrorMessage".format(Client.__NAMESPACE_MESSAGE)):
+                    text_element = element.find("./{0}Text".format(Client.__NAMESPACE_COMMON))
                     if (text_element is not None):
                         if returned_result == "":
-                            returned_result = f'{self.__UPLOAD_SUCCESS}with status code: {response.status_code}' + os.linesep
+                            returned_result = f'{Client.__UPLOAD_SUCCESS}with status code: {response.status_code}' + os.linesep
                         returned_result = returned_result + text_element.text + os.linesep
 
                 # Write the result to the log
@@ -355,12 +486,12 @@ class Client():
             returned_result = ""
 
             # 
-            if self.__authentication_obj is not None:
-                self.__access_token = self.__authentication_obj.get_token()
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
 
             headers = {
                 'accept': 'application/json',
-                'authorization': "Bearer "+self.__access_token
+                'authorization': "Bearer "+Client.__access_token
             }
 
             payload = {
@@ -369,12 +500,15 @@ class Client():
             }
 
             transfer_url = transfer_url.replace("import", "status")
-            transfer_url = transfer_url.replace("sdmxFile", "request")
-
+            if "sdmxFile" in transfer_url:
+                transfer_url = transfer_url.replace("sdmxFile", "request")
+            elif "excel" in transfer_url:
+                transfer_url = transfer_url.replace("excel", "request")
+            
             #
             response = requests.post(transfer_url, verify=True, headers=headers, data=payload)
         except Exception as err:
-            returned_result = self.__ERROR_OCCURRED  + str(err)
+            returned_result = Client.__ERROR_OCCURRED  + str(err)
         else:
             # If the response object cannot be converted to json, return an error
             results_json = None
@@ -382,8 +516,8 @@ class Client():
                 results_json = json.loads(response.text)
                 if response.status_code == 200:
                     executionStatus = 'Execution status: ' + results_json['executionStatus']
-                    if (results_json['executionStatus'] in [self.__EXECUTION_IN_PROGRESS, self.__EXECUTION_IN_QUEUED]
-                        or self.__CONNECTION_ABORTED in results_json['executionStatus']):
+                    if (results_json['executionStatus'] in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                        or Client.__CONNECTION_ABORTED in results_json['executionStatus']):
                         returned_result = results_json['executionStatus']
                     else:
                         returned_result = executionStatus + os.linesep + 'Outcome: ' + results_json['outcome'] + os.linesep
@@ -392,7 +526,7 @@ class Client():
                             returned_result = returned_result + 'Log' + str(index) + ': ' + results_json['logs'][index]['message'] + os.linesep
                             index += 1
                 else:
-                    returned_result = self.__ERROR_OCCURRED
+                    returned_result = Client.__ERROR_OCCURRED
                     if len(str(response.status_code)) > 0:
                         returned_result += 'Error code: ' + str(response.status_code) + os.linesep
                     if len(str(response.reason)) > 0:
@@ -402,7 +536,7 @@ class Client():
 
                     returned_result += os.linesep
             except ValueError as err:
-                returned_result = self.__ERROR_OCCURRED
+                returned_result = Client.__ERROR_OCCURRED
                 if len(str(response.status_code)) > 0:
                     returned_result += 'Error code: ' + str(response.status_code) + os.linesep
                 if len(str(response.reason)) > 0:

{dotstat_io-1.0.2 → dotstat_io-1.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dotstat_io"
-version = "1.0.2"
+version = "1.0.4"
 description = "Utility to download or upload data from/to .Stat Suite using ADFS authentication to connect to it"
 license = "MIT"
 authors = ["Gyorgy Gyomai <gyorgy.gyomai@oecd.org>", "Abdel Aliaoui <abdel.aliaoui@oecd.org>"]