dotstat_io 0.2.7__py3-none-any.whl → 1.0.0__py3-none-any.whl

dotstat_io/client.py

@@ -0,0 +1,392 @@
+import os
+import requests
+import chardet
+import json
+import logging
+import xml.etree.ElementTree as ET
+
+from pathlib import Path
+from time import sleep
+
+
+# class to download or upload data from/to .Stat Suite
+class Client():
+
+    # private constants
+    __ERROR_OCCURRED  = "An error occurred: "
+    __EXECUTION_IN_QUEUED = "Queued"
+    __EXECUTION_IN_PROGRESS = "InProgress"
+    __CONNECTION_ABORTED = "An existing connection was forcibly closed by the remote host"
+    __DOWNLOAD_SUCCESS = "Successful download"
+    __UPLOAD_SUCCESS = "The request was successfully processed "
+    __UPLOAD_FAILED = "The request failed with status code "
+
+    __NAMESPACE_MESSAGE = "{http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message}"
+    __NAMESPACE_COMMON = "{http://www.sdmx.org/resources/sdmxml/schemas/v2_1/common}"
+
+    # private variables
+    __access_token = None
+    __authentication_obj = None
+
+    # Prepare logging format
+    FORMAT = '%(message)s'
+    logging.basicConfig(format=FORMAT, level=logging.INFO)
+    __log = logging.getLogger(__name__)
+
+    # Initialise Client
+    def __init__(self,
+	    access_token: str = None,
+        authentication_obj: object = None):
+        self.__access_token = access_token
+        self.__authentication_obj = authentication_obj
+
+
+    # 
+    def __enter__(self):
+        return self
+
+    # 
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.__access_token = None
+        self.__authentication_obj = None
+
+
+    #           
+    @classmethod
+    def init_with_access_token(
+        cls, 
+        access_token: str
+    ):
+        return cls(
+            access_token=access_token
+        )
+    
+    #           
+    @classmethod
+    def init_with_authentication_obj(
+        cls, 
+        authentication_obj: object
+    ):
+        return cls(
+            authentication_obj=authentication_obj
+        )
+    
+
+    # Download a data file from .Stat Suite
+    def download_data_file(self, dotstat_url: str, content_format: str, file_path: Path):
+        try:
+            returned_result = ""
+
+            # 
+            if self.__authentication_obj is not None:
+                self.__access_token = self.__authentication_obj.get_token()
+
+            headers = {
+                'accept': content_format,
+                'authorization': 'Bearer '+self.__access_token
+            }
+
+            #
+            response = requests.get(dotstat_url, verify=True, headers=headers)
+        except Exception as err:
+            returned_result = self.__ERROR_OCCURRED  + str(err) + os.linesep
+
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+        else:
+            if response.status_code != 200:
+                returned_result = self.__ERROR_OCCURRED  + 'Error code: ' + \
+                    str(response.status_code) + ' Reason: ' + str(response.reason)
+                if len(response.text) > 0:
+                    returned_result += os.linesep + 'Text: ' + response.text
+            else:
+                if os.path.isfile(file_path):
+                    os.remove(file_path)
+                with open(file_path, "wb") as file:
+                    file.write(response.content)
+                    returned_result = self.__DOWNLOAD_SUCCESS
+            
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+        finally:
+            return returned_result
+
+
+    # Download streamed content from .Stat Suite
+    def download_data_stream(self, dotstat_url: str, content_format: str):
+        try:
+            returned_result = ""
+
+            # 
+            if self.__authentication_obj is not None:
+                __access_token = self.__authentication_obj.get_token()
+
+            headers = {
+                'accept': content_format,
+                'Transfer-Encoding': 'chunked',
+                'authorization': 'Bearer '+self.__access_token
+            }
+
+            #
+            return requests.get(dotstat_url, verify=True, headers=headers, stream=True)
+        except Exception as err:
+            returned_result = self.__ERROR_OCCURRED  + str(err) + os.linesep
+            return returned_result
+
+
+    # Upload a data file to .Stat Suite
+    def upload_data_file(self, 
+                    transfer_url: str, 
+                    file_path: Path, 
+                    space: str, 
+                    validationType: int, 
+                    use_filepath: bool = False):
+        try:
+            returned_result = ""
+
+            # 
+            if self.__authentication_obj is not None:
+                self.__access_token = self.__authentication_obj.get_token()
+
+            payload = {
+                'dataspace': space,
+                'validationType': validationType
+            }
+
+            headers = {
+                'accept': 'application/json',
+                'authorization': "Bearer "+self.__access_token
+            }
+
+            if  use_filepath:
+                files = {
+                    'dataspace': (None, payload['dataspace']),
+                    'validationType': (None, payload['validationType']),
+                    'filepath': (None, str(file_path))
+                }
+            else:
+                files = {
+                    'dataspace': (None, payload['dataspace']),
+                    'validationType': (None, payload['validationType']),
+                    'file': (os.path.realpath(file_path), open(os.path.realpath(file_path), 'rb'), 'text/csv', '')
+                }
+
+            #
+            response = requests.post(transfer_url, verify=True, headers=headers, files=files)
+        except Exception as err:
+            returned_result = self.__ERROR_OCCURRED  + str(err) + os.linesep
+
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+        else:
+            # If the response object cannot be converted to json, return an error
+            results_json = None
+            try:
+                results_json = json.loads(response.text)
+                if response.status_code == 200:
+                    result = results_json['message']
+                    # Write the result to the log
+                    for line in result.split(os.linesep):
+                        if len(line) > 0:
+                            self.__log.info('   ' + line)
+
+                    returned_result = result + os.linesep
+
+                    # Check the request status
+                    if (result != "" and result.find(self.__ERROR_OCCURRED ) == -1):
+                        # Extract the request ID the returned message
+                        start = 'with ID'
+                        end = 'was successfully'
+                        requestId = result[result.find(
+                            start)+len(start):result.rfind(end)]
+
+                        # Sleep a little bit before checking the request status
+                        sleep(3)
+
+                        # To avoid this error: maximum recursion depth exceeded while calling a Python object
+                        # replace the recursive calls with while loops.
+                        result = self.__check_request_status(transfer_url, requestId, space)
+
+                        # Write the result to the log
+                        for line in result.split(os.linesep):
+                            if len(line) > 0:
+                                self.__log.info('   ' + line)
+                        sleep(3)
+
+                        previous_result = result
+                        while (result in [self.__EXECUTION_IN_PROGRESS, self.__EXECUTION_IN_QUEUED]
+                                 or self.__CONNECTION_ABORTED in result):
+                            result = self.__check_request_status(transfer_url, requestId, space)
+
+                            # Prevent loging again the same information such as "Queued" or "InProgress" 
+                            if previous_result != result:
+                               previous_result = result
+
+                            # Write the result to the log
+                            for line in previous_result.split(os.linesep):
+                              if (len(line) > 0 and line not in [self.__EXECUTION_IN_PROGRESS, self.__EXECUTION_IN_QUEUED]
+                                    and self.__CONNECTION_ABORTED not in line):
+                                 self.__log.info('   ' + line)
+                            sleep(3)
+
+                        returned_result = returned_result + result + os.linesep
+                else:
+                    returned_result = self.__ERROR_OCCURRED  + 'Error code: ' + str(response.status_code) + os.linesep
+                    if len(str(response.reason)) > 0:
+                        returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                    if len(response.text) > 0:
+                        returned_result = returned_result + os.linesep + 'Text: ' + response.text
+
+                    returned_result = returned_result + os.linesep
+                    # Write the result to the log
+                    for line in returned_result.split(os.linesep):
+                        if len(line) > 0:
+                            self.__log.info('   ' + line)
+            except ValueError as err:
+                if len(response.text) > 0:
+                    self.init_status = self._ERROR + os.linesep + str(response.text)
+                else:
+                    self.init_status = self._ERROR + os.linesep + str(err)
+        finally:
+            return returned_result
+
+
+    # Upload a structure file to .Stat Suite
+    def upload_structure(self, transfer_url: str, file_path: Path):
+        try:
+            returned_result = ""
+
+            # 
+            if self.__authentication_obj is not None:
+                self.__access_token = self.__authentication_obj.get_token()
+
+            # Detect the encoding used in file 
+            detected_encoding = self.__detect_encode(file_path)
+
+            # Read file as a string "r+" with the detected encoding
+            with open(file=file_path, mode="r+", encoding=detected_encoding.get("encoding")) as file:
+                xml_data = file.read()
+
+            # Make sure the encoding is "utf-8"
+            tree = ET.fromstring(xml_data)
+            xml_data = ET.tostring(tree, encoding="utf-8", method='xml', xml_declaration=True)
+
+            headers = {
+                'Content-Type': 'application/xml',
+                'authorization': "Bearer "+self.__access_token
+            }
+
+            #
+            response = requests.post(transfer_url, verify=True, headers=headers, data=xml_data)
+        except Exception as err:
+            returned_result = self.__ERROR_OCCURRED  + str(err) + os.linesep
+
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+        else:
+            try:
+                response.raise_for_status()
+            except requests.exceptions.HTTPError as e:
+                returned_result = f'{self.__UPLOAD_FAILED}{response.status_code}: {e}'
+
+                # Write the result to the log
+                for line in returned_result.split(os.linesep):
+                    if len(line) > 0:
+                        self.__log.info('   ' + line)
+            else:
+                response_tree = ET.XML(response.content)
+                for self.__ERROR_OCCURRED in response_tree.findall("./{0}ErrorMessage".format(self.__NAMESPACE_MESSAGE)):
+                    text_element = self.__ERROR_OCCURRED.find("./{0}Text".format(self.__NAMESPACE_COMMON))
+                    if (text_element is not None):
+                        if returned_result == "":
+                            returned_result = f'{self.__UPLOAD_SUCCESS}with status code: {response.status_code}' + os.linesep
+                        returned_result = returned_result + text_element.text + os.linesep
+
+                # Write the result to the log
+                for line in returned_result.split(os.linesep):
+                    if len(line) > 0:
+                        self.__log.info('   ' + line)
+        finally:
+            return returned_result
+        
+
+    # Detect the encoding used in file
+    def __detect_encode(self, file_path):
+        detector = chardet.UniversalDetector()
+        detector.reset()
+        with open(file=file_path, mode="rb") as file:
+            for row in file:
+                detector.feed(row)
+                if detector.done: 
+                    break
+
+        detector.close()
+
+        return detector.result
+
+
+    # Check request sent to .Stat Suite status
+    # To avoid this error: maximum recursion depth exceeded while calling a Python object
+    # replace the recursive calls with while loops.
+    def __check_request_status(self, transfer_url, requestId, space):
+        try:
+            returned_result = ""
+
+            # 
+            if self.__authentication_obj is not None:
+                self.__access_token = self.__authentication_obj.get_token()
+
+            headers = {
+                'accept': 'application/json',
+                'authorization': "Bearer "+self.__access_token
+            }
+
+            payload = {
+                'dataspace': space,
+                'id': requestId
+            }
+
+            transfer_url = transfer_url.replace("import", "status")
+            transfer_url = transfer_url.replace("sdmxFile", "request")
+
+            #
+            response = requests.post(transfer_url, verify=True, headers=headers, data=payload)
+        except Exception as err:
+            returned_result = self.__ERROR_OCCURRED  + str(err)
+        else:
+            # If the response object cannot be converted to json, return an error
+            results_json = None
+            try:
+                results_json = json.loads(response.text)
+                if response.status_code == 200:
+                    executionStatus = 'Execution status: ' + results_json['executionStatus']
+                    if (results_json['executionStatus'] in [self.__EXECUTION_IN_PROGRESS, self.__EXECUTION_IN_QUEUED]
+                        or self.__CONNECTION_ABORTED in results_json['executionStatus']):
+                        returned_result = results_json['executionStatus']
+                    else:
+                        returned_result = executionStatus + os.linesep + 'Outcome: ' + results_json['outcome'] + os.linesep
+                        index = 0
+                        while index < len(results_json['logs']):
+                            returned_result = returned_result + 'Log' + str(index) + ': ' + results_json['logs'][index]['message'] + os.linesep
+                            index += 1
+                else:
+                    returned_result = self.__ERROR_OCCURRED  + 'Error code: ' + \
+                        str(response.status_code) + ' Reason: ' + str(response.reason)
+                    if len(response.text) > 0:
+                        returned_result = returned_result + os.linesep + 'Text: ' + response.text
+            except ValueError as err:
+                if len(response.text) > 0:
+                    self.init_status = self._ERROR + os.linesep + str(response.text)
+                else:
+                    self.init_status = self._ERROR + os.linesep + str(err)
+        finally:
+            return returned_result
+    

{dotstat_io-0.2.7.dist-info → dotstat_io-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dotstat_io
-Version: 0.2.7
+Version: 1.0.0
 Summary: Utility to download or upload data from/to .Stat Suite using ADFS authentication to connect to it
 License: MIT
 Author: Gyorgy Gyomai
@@ -11,13 +11,15 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: chardet (>=5.1.0,<6.0.0)
 Requires-Dist: msal (>=1.22.0,<2.0.0)
 Requires-Dist: requests (>=2.29.0,<3.0.0)
 Requires-Dist: requests-kerberos (>=0.14.0,<0.15.0)
+Requires-Dist: setuptools (>=75.6.0,<76.0.0)
 Description-Content-Type: text/markdown
 
-### DotStat_IO: 
+### DotStat_IO package: 
 A generic python package which could be integrated with other end-user applications and Gitlab runner to perform basic io with .Stat Suite. 
 Its role is to mask the complexities of authentication to connect to .Stat Suite.
 The user needs to provide a set of parameters and the package exposes a couple of methods which will download or upload data from/to .Stat Suite.
@@ -25,19 +27,19 @@ The user needs to provide a set of parameters and the package exposes a couple o
 ### This package contains three modules:
 - ADFSAuthentication module
 - KeycloakAuthentication module
-- Download_upload module
+- Client module
 
 ### In ADFSAuthentication module, four methods are available:
 #### 1. To initialise the module for interactive use:
 ```python
 from dotstat_io.authentication import AdfsAuthentication
-with AdfsAuthentication.interactive(
+interactive_obj = AdfsAuthentication.interactive(
         client_id=client_id,
         sdmx_resource_id=sdmx_resource_id,
         scopes=scopes,
         authority_url=authority_url,
-        redirect_port=redirect_port) as Interactive_obj:
-    access_token = Interactive_obj.get_token()
+        redirect_port=redirect_port)
+
 ```
 * `client_id:` The Application (client) ID that the ADFS assigned to your app
 * `sdmx_resource_id:` The ID of the application to be accessed such as .Stat Suite
@@ -48,13 +50,13 @@ with AdfsAuthentication.interactive(
 #### 2. To initialise the module for non-interactive use using a secret: 
 ```python
 from dotstat_io.authentication import AdfsAuthentication
-with AdfsAuthentication.nointeractive_with_secret(
+noninteractive_with_secret_obj = AdfsAuthentication.noninteractive_with_secret(
         client_id=client_id,
         sdmx_resource_id=sdmx_resource_id,
         scopes=scopes,
         authority_url=authority_url,
-        client_secret=client_secret) as Nointeractive_with_secret_obj:
-    access_token = Nointeractive_with_secret_obj.get_token()
+        client_secret=client_secret)
+
 ```
 * `client_id:` The Application (client) ID that the ADFS assigned to your app
 * `sdmx_resource_id:` The ID of the application to be accessed such as .Stat Suite
@@ -65,11 +67,11 @@ with AdfsAuthentication.nointeractive_with_secret(
 #### 3. To initialise the module for non-interactive use using windows client authentication:
 ```python
 from dotstat_io.authentication import AdfsAuthentication
-with AdfsAuthentication.nointeractive_with_adfs(
+noninteractive_with_adfs_obj = AdfsAuthentication.noninteractive_with_adfs(
         client_id=client_id,
         sdmx_resource_id=sdmx_resource_id,
-        token_url=token_url) as Nointeractive_with_adfs_obj:
-    access_token = Nointeractive_with_adfs_obj.get_token()
+        token_url=token_url)
+
 ```
 * `client_id:` The Application (client) ID that the ADFS assigned to your app
 * `sdmx_resource_id:` The ID of the application to be accessed such as .Stat Suite
@@ -84,12 +86,12 @@ access_token = [Authentication Object Name].get_token()
 #### 1. To initialise the module for Keycloak authentication:
 ```python
 from dotstat_io.authentication import KeycloakAuthentication
-with KeycloakAuthentication.nointeractive_with_secret(
+noninteractive_with_keycloak_obj = KeycloakAuthentication.noninteractive_with_secret(
         user=user,
         password=password, 
         token_url=token_url, 
-        proxy=proxy) as Nointeractive_with_keycloak_obj:
-    access_token = Nointeractive_with_keycloak_obj.get_token()
+        proxy=proxy)
+
 ```
 * `user:` User name for .Stat Suite authentication
 * `password:` User name's password
@@ -101,33 +103,47 @@ with KeycloakAuthentication.nointeractive_with_secret(
 access_token = [Authentication Object Name].get_token()
 ```
 
-### In Download_upload module, four methods are available:
-#### 1. To download a file from .Stat Suite:
+### In Client module, six methods are available:
+#### 1. To initialise the module using an Authentication object type AdfsAuthentication or KeycloakAuthentication:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+```
+* `Authentication_obj:` An initialized authentication object type AdfsAuthentication or KeycloakAuthentication
+
+#### 2. To initialise the module using an access token:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_access_token(access_token)
+```
+* `access_token:` An access token to make requests on .Stat Suite services (nsiws) using the authorisation service and underlying permission rules
+
+#### 3. To download a file from .Stat Suite:
 ```python
-from dotstat_io.download_upload import Download_upload
-with Download_upload(adfsAuthentication_obj, access_token) as Download_upload_obj:
-    Returned_Message = Download_upload_obj.download_file(
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.download_data_file(
         dotstat_url, content_format, Path(file_path))
 ```
 * `dotstat_url:` URL of data to be downloaded from .Stat Suite
 * `content_format:` Format of the downloaded content
 * `file_path:` The full path where the file will downloaded
 
-#### 2. To download streamed content from .Stat Suite:
+#### 4. To download streamed content from .Stat Suite:
 ```python
-from dotstat_io.download_upload import Download_upload
-with Download_upload(adfsAuthentication_obj, access_token) as Download_upload_obj:
-    Returned_Message = Download_upload_obj.download_stream(
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.download_data_stream(
         dotstat_url, content_format)
 ```
 * `dotstat_url:` URL of data to be downloaded from .Stat Suite
 * `content_format:` Format of the downloaded content
 
-#### 3. To upload a data file to .Stat Suite:
+#### 5. To upload a data file to .Stat Suite:
 ```python
-from dotstat_io.download_upload import Download_upload
-with Download_upload(adfsAuthentication_obj, access_token) as Download_upload_obj:
-    Returned_Message = Download_upload_obj.upload_file(
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_data_file(
         transfer_url, Path(file_path), space, validationType, use_filepath)
 ```
 * `transfer_url:` URL of the transfer service
@@ -136,15 +152,15 @@ with Download_upload(adfsAuthentication_obj, access_token) as Download_upload_ob
 * `validationType:` The type of validation to use during upload. Possible values: Basic Validation (0), Advanced Validation (1)
 * `use_filepath:` Use a file path of a shared folder accessible by the .Stat Suite data upload engine (for unlimited file sizes)
 
-#### 4. To upload a structure to .Stat Suite:
+#### 6. To upload a structure to .Stat Suite:
 ```python
-from dotstat_io.download_upload import Download_upload
-with Download_upload(adfsAuthentication_obj, access_token) as Download_upload_obj:
-    Returned_Message = Download_upload_obj.upload_structure(
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_structure(
         transfer_url, Path(file_path))
 ```
 * `transfer_url:` URL of the transfer service
 * `file_path:` The full path of the SDMX-ML file, which will be uploaded to .Stat Suite
 
-### For more information about how to use this package, all test cases can be accessed from this [`link`](https://gitlab.algobank.oecd.org/sdd-legacy/dotstat_io/-/blob/main/tests/test_cases.py)
+### For more information about how to use this package, all test cases can be accessed from this [`link`](https://gitlab.algobank.oecd.org/SD_ENGINEERING/dotstat_io/dotstat_io-package/-/blob/main/tests/test_cases.py)
 

dotstat_io-1.0.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+dotstat_io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dotstat_io/authentication.py,sha256=Ax8a2koJ5kjZ4jAvnN7S9U4GZfEfsurksI8bbw6JcqI,13230
+dotstat_io/client.py,sha256=Y4gIVD8uuBTQkeZ6bWiIMH8IZ723RAqMM1S14II6pxQ,16018
+dotstat_io-1.0.0.dist-info/METADATA,sha256=dC41wVc4qs94XP7520QGRWqN44pDJuE0atTWnddL474,7280
+dotstat_io-1.0.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+dotstat_io-1.0.0.dist-info/RECORD,,

{dotstat_io-0.2.7.dist-info → dotstat_io-1.0.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any