dotstat_io 1.0.7__tar.gz

dotstat_io-1.0.7/PKG-INFO

@@ -0,0 +1,181 @@
+Metadata-Version: 2.4
+Name: dotstat_io
+Version: 1.0.7
+Summary: Utility to download or upload data from/to .Stat Suite using ADFS authentication to connect to it
+License: MIT
+Author: Gyorgy Gyomai
+Author-email: gyorgy.gyomai@oecd.org
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: chardet (>=5.1.0,<6.0.0)
+Requires-Dist: msal (>=1.22.0,<2.0.0)
+Requires-Dist: requests (>=2.29.0,<3.0.0)
+Requires-Dist: requests-kerberos (>=0.14.0,<0.15.0)
+Requires-Dist: setuptools (>=75.6.0,<76.0.0)
+Description-Content-Type: text/markdown
+
+### DotStat_IO package: 
+A generic python package which could be integrated with other end-user applications and Gitlab runner to perform basic io with .Stat Suite. 
+Its role is to mask the complexities of authentication to connect to .Stat Suite.
+The user needs to provide a set of parameters and the package exposes a couple of methods which will download or upload data from/to .Stat Suite.
+
+### This package contains three modules:
+- ADFSAuthentication module
+- KeycloakAuthentication module
+- Client module
+
+### In ADFSAuthentication module, four methods are available:
+#### 1. To initialise the module for interactive use:
+```python
+from dotstat_io.authentication import AdfsAuthentication
+interactive_obj = AdfsAuthentication.interactive(
+        client_id=client_id,
+        sdmx_resource_id=sdmx_resource_id,
+        scopes=scopes,
+        authority_url=authority_url,
+        redirect_port=redirect_port)
+
+```
+* `client_id:` The Application (client) ID that the ADFS assigned to your app
+* `sdmx_resource_id:` The ID of the application to be accessed such as .Stat Suite
+* `scopes:` Scopes requested to access a protected API (a resource defined by sdmx_resource_id)
+* `authority_url:` URL that identifies a token authority
+* `redirect_port:` The port of the address to return to upon receiving a response from the authority
+
+#### 2. To initialise the module for non-interactive use using a secret: 
+```python
+from dotstat_io.authentication import AdfsAuthentication
+noninteractive_with_secret_obj = AdfsAuthentication.noninteractive_with_secret(
+        client_id=client_id,
+        sdmx_resource_id=sdmx_resource_id,
+        scopes=scopes,
+        authority_url=authority_url,
+        client_secret=client_secret)
+
+```
+* `client_id:` The Application (client) ID that the ADFS assigned to your app
+* `sdmx_resource_id:` The ID of the application to be accessed such as .Stat Suite
+* `scopes:` Scopes requested to access a protected API (a resource defined by sdmx_resource_id)
+* `authority_url:` URL that identifies a token authority
+* `client_secret:` The application secret that you created during app registration in ADFS
+
+#### 3. To initialise the module for non-interactive use using windows client authentication:
+```python
+from dotstat_io.authentication import AdfsAuthentication
+noninteractive_with_adfs_obj = AdfsAuthentication.noninteractive_with_adfs(
+        client_id=client_id,
+        sdmx_resource_id=sdmx_resource_id,
+        token_url=token_url)
+
+```
+* `client_id:` The Application (client) ID that the ADFS assigned to your app
+* `sdmx_resource_id:` The ID of the application to be accessed such as .Stat Suite
+* `token_url:` URL of the authentication service
+
+#### 4. To get a token after initialisation of "AdfsAuthentication" object as shown above: 
+```python
+access_token = [Authentication Object Name].get_token()
+```
+
+### In KeycloakAuthentication module, two methods are available:
+#### 1. To initialise the module for Keycloak authentication:
+```python
+from dotstat_io.authentication import KeycloakAuthentication
+noninteractive_with_keycloak_obj = KeycloakAuthentication.noninteractive_with_secret(
+        user=user,
+        password=password, 
+        token_url=token_url, 
+        proxy=proxy)
+
+```
+* `user:` User name for .Stat Suite authentication
+* `password:` User name's password
+* `token_url:` URL of the authentication service
+* `proxy:` URL of the SSL certificates
+
+#### 2. To get a token after initialisation of "KeycloakAuthentication" object as shown above: 
+```python
+access_token = [Authentication Object Name].get_token()
+```
+
+### In Client module, six methods are available:
+#### 1. To initialise the module using an Authentication object type AdfsAuthentication or KeycloakAuthentication:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+```
+* `Authentication_obj:` An initialized authentication object type AdfsAuthentication or KeycloakAuthentication
+
+#### 2. To initialise the module using an access token:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_access_token(access_token)
+```
+* `access_token:` An access token to make requests on .Stat Suite services (nsiws) using the authorisation service and underlying permission rules
+
+#### 3. To download a file from .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.download_data_file(
+        dotstat_url, content_format, Path(file_path))
+```
+* `dotstat_url:` URL of data to be downloaded from .Stat Suite
+* `content_format:` Format of the downloaded content
+* `file_path:` The full path where the file will downloaded
+
+#### 4. To download streamed content from .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.download_data_stream(
+        dotstat_url, content_format)
+```
+* `dotstat_url:` URL of data to be downloaded from .Stat Suite
+* `content_format:` Format of the downloaded content
+
+#### 5. To upload a data file to .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_data_file(
+        transfer_url, Path(file_path), space, validationType, use_filepath, optimize)
+```
+* `transfer_url:` URL of the transfer service
+* `file_path:` The full path of the SDMX-CSV file to be imported
+* `space:` Data space where the file will be uploaded
+* `validationType:` The type of validation to use during upload. Possible values: Basic Validation (0), Advanced Validation (1)
+* `use_filepath:` Use a file path of a shared folder accessible by the .Stat Suite data upload engine (for unlimited file sizes)
+* `optimize:` Controls whether optimization is applied after data changes
+
+#### 6. To upload an Excel data file to .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_excel_data_file(
+        transfer_url, Path(excelfile_path), Path(eddfile_path), space, validationType)
+```
+* `transfer_url:` URL of the transfer service
+* `excelfile_path:` The full path of the Excel file containing the data/referential metadata values to be imported
+* `eddfile_path:` The full path of the XML edd file containing the description of the Excel file to be imported
+* `space:` Data space where the file will be uploaded
+* `validationType:` The type of validation to use during upload. Possible values: Basic Validation (0), Advanced Validation (1)
+
+#### 7. To upload a structure to .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_structure(
+        transfer_url, Path(file_path))
+```
+* `transfer_url:` URL of the transfer service
+* `file_path:` The full path of the SDMX-ML file to be imported
+
+### For more information about how to use this package, all test cases can be accessed from this [`link`](https://gitlab.algobank.oecd.org/SD_ENGINEERING/dotstat_io/dotstat_io-package/-/blob/main/tests/test_cases.py)
+

dotstat_io-1.0.7/README.md

@@ -0,0 +1,158 @@
+### DotStat_IO package: 
+A generic python package which could be integrated with other end-user applications and Gitlab runner to perform basic io with .Stat Suite. 
+Its role is to mask the complexities of authentication to connect to .Stat Suite.
+The user needs to provide a set of parameters and the package exposes a couple of methods which will download or upload data from/to .Stat Suite.
+
+### This package contains three modules:
+- ADFSAuthentication module
+- KeycloakAuthentication module
+- Client module
+
+### In ADFSAuthentication module, four methods are available:
+#### 1. To initialise the module for interactive use:
+```python
+from dotstat_io.authentication import AdfsAuthentication
+interactive_obj = AdfsAuthentication.interactive(
+        client_id=client_id,
+        sdmx_resource_id=sdmx_resource_id,
+        scopes=scopes,
+        authority_url=authority_url,
+        redirect_port=redirect_port)
+
+```
+* `client_id:` The Application (client) ID that the ADFS assigned to your app
+* `sdmx_resource_id:` The ID of the application to be accessed such as .Stat Suite
+* `scopes:` Scopes requested to access a protected API (a resource defined by sdmx_resource_id)
+* `authority_url:` URL that identifies a token authority
+* `redirect_port:` The port of the address to return to upon receiving a response from the authority
+
+#### 2. To initialise the module for non-interactive use using a secret: 
+```python
+from dotstat_io.authentication import AdfsAuthentication
+noninteractive_with_secret_obj = AdfsAuthentication.noninteractive_with_secret(
+        client_id=client_id,
+        sdmx_resource_id=sdmx_resource_id,
+        scopes=scopes,
+        authority_url=authority_url,
+        client_secret=client_secret)
+
+```
+* `client_id:` The Application (client) ID that the ADFS assigned to your app
+* `sdmx_resource_id:` The ID of the application to be accessed such as .Stat Suite
+* `scopes:` Scopes requested to access a protected API (a resource defined by sdmx_resource_id)
+* `authority_url:` URL that identifies a token authority
+* `client_secret:` The application secret that you created during app registration in ADFS
+
+#### 3. To initialise the module for non-interactive use using windows client authentication:
+```python
+from dotstat_io.authentication import AdfsAuthentication
+noninteractive_with_adfs_obj = AdfsAuthentication.noninteractive_with_adfs(
+        client_id=client_id,
+        sdmx_resource_id=sdmx_resource_id,
+        token_url=token_url)
+
+```
+* `client_id:` The Application (client) ID that the ADFS assigned to your app
+* `sdmx_resource_id:` The ID of the application to be accessed such as .Stat Suite
+* `token_url:` URL of the authentication service
+
+#### 4. To get a token after initialisation of "AdfsAuthentication" object as shown above: 
+```python
+access_token = [Authentication Object Name].get_token()
+```
+
+### In KeycloakAuthentication module, two methods are available:
+#### 1. To initialise the module for Keycloak authentication:
+```python
+from dotstat_io.authentication import KeycloakAuthentication
+noninteractive_with_keycloak_obj = KeycloakAuthentication.noninteractive_with_secret(
+        user=user,
+        password=password, 
+        token_url=token_url, 
+        proxy=proxy)
+
+```
+* `user:` User name for .Stat Suite authentication
+* `password:` User name's password
+* `token_url:` URL of the authentication service
+* `proxy:` URL of the SSL certificates
+
+#### 2. To get a token after initialisation of "KeycloakAuthentication" object as shown above: 
+```python
+access_token = [Authentication Object Name].get_token()
+```
+
+### In Client module, six methods are available:
+#### 1. To initialise the module using an Authentication object type AdfsAuthentication or KeycloakAuthentication:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+```
+* `Authentication_obj:` An initialized authentication object type AdfsAuthentication or KeycloakAuthentication
+
+#### 2. To initialise the module using an access token:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_access_token(access_token)
+```
+* `access_token:` An access token to make requests on .Stat Suite services (nsiws) using the authorisation service and underlying permission rules
+
+#### 3. To download a file from .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.download_data_file(
+        dotstat_url, content_format, Path(file_path))
+```
+* `dotstat_url:` URL of data to be downloaded from .Stat Suite
+* `content_format:` Format of the downloaded content
+* `file_path:` The full path where the file will downloaded
+
+#### 4. To download streamed content from .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.download_data_stream(
+        dotstat_url, content_format)
+```
+* `dotstat_url:` URL of data to be downloaded from .Stat Suite
+* `content_format:` Format of the downloaded content
+
+#### 5. To upload a data file to .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_data_file(
+        transfer_url, Path(file_path), space, validationType, use_filepath, optimize)
+```
+* `transfer_url:` URL of the transfer service
+* `file_path:` The full path of the SDMX-CSV file to be imported
+* `space:` Data space where the file will be uploaded
+* `validationType:` The type of validation to use during upload. Possible values: Basic Validation (0), Advanced Validation (1)
+* `use_filepath:` Use a file path of a shared folder accessible by the .Stat Suite data upload engine (for unlimited file sizes)
+* `optimize:` Controls whether optimization is applied after data changes
+
+#### 6. To upload an Excel data file to .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_excel_data_file(
+        transfer_url, Path(excelfile_path), Path(eddfile_path), space, validationType)
+```
+* `transfer_url:` URL of the transfer service
+* `excelfile_path:` The full path of the Excel file containing the data/referential metadata values to be imported
+* `eddfile_path:` The full path of the XML edd file containing the description of the Excel file to be imported
+* `space:` Data space where the file will be uploaded
+* `validationType:` The type of validation to use during upload. Possible values: Basic Validation (0), Advanced Validation (1)
+
+#### 7. To upload a structure to .Stat Suite:
+```python
+from dotstat_io.client import Client
+client_obj = Client.init_with_authentication_obj(Authentication_obj)
+returned_result = client_obj.upload_structure(
+        transfer_url, Path(file_path))
+```
+* `transfer_url:` URL of the transfer service
+* `file_path:` The full path of the SDMX-ML file to be imported
+
+### For more information about how to use this package, all test cases can be accessed from this [`link`](https://gitlab.algobank.oecd.org/SD_ENGINEERING/dotstat_io/dotstat_io-package/-/blob/main/tests/test_cases.py)

dotstat_io-1.0.7/dotstat_io/authentication.py

@@ -0,0 +1,354 @@
+import requests
+import os
+import msal
+import requests_kerberos
+import json
+import time
+
+from abc import ABC
+from enum import IntEnum
+from datetime import datetime
+
+# 
+class AuthenticationMode(IntEnum):
+    INTERACTIVE = 1
+    NONINTERACTIVE_WITH_SECRET = 2
+    NONINTERACTIVE_WITH_ADFS = 3
+
+# super class to manage authentication
+class Authentication(ABC):
+
+    # protected variables
+    _access_token = None
+    _refresh_token = None
+    _creation_time = None
+    _expiration_time = None
+
+    # public variables
+    init_status = None
+
+    # Initialise authentication
+    def __init__(
+        self,
+	    client_id: str,
+        mode: AuthenticationMode,
+        client_secret: str = None,
+	    scopes: list[str] = [],
+	    token_url: str = None,
+        sdmx_resource_id: str = None,
+        authority_url: str = None,
+        redirect_port: int = 3000,
+	    user: str = None,
+        password: str = None,
+        proxy: str = None
+    ):
+        self._client_id = client_id
+        self._mode = mode
+        self._client_secret = client_secret
+        self._scopes = scopes     
+        self._token_url = token_url
+        self._sdmx_resource_id = sdmx_resource_id
+        self._authority_url = authority_url
+        self._redirect_port = redirect_port
+        self._user = user
+        self._password = password
+
+        if proxy:
+            self._proxies = {
+                "http": proxy,
+                "https": proxy
+            }
+        else:
+            self._proxies = None
+
+        #
+        self._initialize_token()
+
+    # 
+    def __enter__(self):
+        return self
+
+    # 
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._access_token = None
+        self._refresh_token = None
+        self._creation_time = None
+        self._expiration_time = None
+        self.init_status = None
+
+    # 
+    def _initialize_token(self):
+        pass
+
+    # 
+    def get_token(self):
+        if (self._access_token is None) \
+               or (datetime.fromtimestamp(self._expiration_time) is not None \
+                   and datetime.now() >= datetime.fromtimestamp(self._expiration_time)):
+            self._initialize_token()
+
+        return self._access_token
+
+
+# sub class to manage ADFS authentication using OIDC flows
+class AdfsAuthentication(Authentication):
+    # private constants
+    __ERROR_OCCURRED = "An error occurred: "
+    __SUCCESSFUL_AUTHENTICATION = "Successful authentication"
+
+    # 
+    def _initialize_token(self):
+        self._access_token = None
+        self._refresh_token = None
+        self._creation_time = None
+        self._expiration_time = None
+        self.init_status = None
+        match self._mode:
+            case AuthenticationMode.INTERACTIVE:
+                self._app = msal.PublicClientApplication(
+                    self._client_id, authority=self._authority_url)
+                self.__acquire_token_interactive()
+            case AuthenticationMode.NONINTERACTIVE_WITH_SECRET:
+                self._app = msal.ConfidentialClientApplication(
+                    self._client_id, authority=self._authority_url, client_credential=self._client_secret)
+                self.__acquire_token_noninteractive_with_secret()
+            case AuthenticationMode.NONINTERACTIVE_WITH_ADFS:
+                self.__acquire_token_noninteractive_with_adfs()
+
+    #           
+    @classmethod
+    def interactive(
+        cls, 
+        client_id: str, 
+        sdmx_resource_id: str, 
+        scopes: list[str], 
+        authority_url: str, 
+        redirect_port: int = 3000,
+        mode: AuthenticationMode = AuthenticationMode.INTERACTIVE
+    ):
+        return cls(
+            client_id=client_id, 
+            sdmx_resource_id=sdmx_resource_id, 
+            scopes=scopes, 
+            authority_url=authority_url,
+            redirect_port=redirect_port,
+            mode=mode
+        )
+
+    #
+    @classmethod
+    def noninteractive_with_secret(
+        cls, 
+        client_id: str, 
+        sdmx_resource_id: str,
+        scopes: list[str],
+        authority_url: str,
+        client_secret: str,
+        mode: AuthenticationMode = AuthenticationMode.NONINTERACTIVE_WITH_SECRET
+    ):
+        return cls(
+            client_id=client_id,
+            sdmx_resource_id=sdmx_resource_id,
+            scopes=scopes,
+            authority_url=authority_url,   
+            client_secret=client_secret,  
+            mode=mode
+            )
+
+    #
+    @classmethod
+    def noninteractive_with_adfs(
+        cls, 
+        client_id: str, 
+        sdmx_resource_id: str,
+        token_url: str,
+        mode: AuthenticationMode = AuthenticationMode.NONINTERACTIVE_WITH_ADFS
+    ):
+        return cls(
+            client_id=client_id, 
+            sdmx_resource_id=sdmx_resource_id,
+            token_url=token_url,
+            mode=mode)
+
+    # Authentication interactively - aka Authorization Code flow
+    def __acquire_token_interactive(self):
+        try:
+            # We now check the cache to see
+            # whether we already have some accounts that the end user already used to sign in before.
+            accounts = self._app.get_accounts()
+            if accounts:
+                account = accounts[0]
+            else:
+                account = None
+
+            # Firstly, looks up a access_token from cache, or using a refresh token
+            response_silent = self._app.acquire_token_silent(
+                self._scopes, account=account)
+            if not response_silent:
+                # Prompt the user to sign in interactively
+                response_interactive = self._app.acquire_token_interactive(
+                    scopes=self._scopes, port=self._redirect_port)
+                if "access_token" in response_interactive:
+                    self._access_token = response_interactive.get("access_token")
+                    self._refresh_token = response_interactive.get("refresh_token")
+                    self._creation_time = time.time()
+                    self._expiration_time = time.time() + int(response_interactive.get("expires_in")) -  60 # one minute margin
+                    self.init_status = self.__SUCCESSFUL_AUTHENTICATION
+                else:
+                    self.init_status = f'{self.__ERROR_OCCURRED}{response_interactive.get("error")} Error description: {response_interactive.get("error_description")}'
+            else:
+                if "access_token" in response_silent:
+                    self._access_token = response_silent.get("access_token")
+                    self._refresh_token = response_silent.get("refresh_token")
+                    self._creation_time = time.time()
+                    self._expiration_time = time.time() + int(response_silent.get("expires_in")) -  60 # one minute margin
+                    self.init_status = self.__SUCCESSFUL_AUTHENTICATION
+                else:
+                    self.init_status = f'{self.__ERROR_OCCURRED}{response_silent.get("error")} Error description: {response_silent.get("error_description")}'
+        except Exception as err:
+            self.init_status = f'{self.__ERROR_OCCURRED}{err}\n'
+
+    # Authentication non-interactively using any account - aka Client Credentials flow
+    def __acquire_token_noninteractive_with_secret(self):
+        try:
+            response = self._app.acquire_token_for_client(scopes=self._scopes)
+            if "access_token" in response:
+                self._access_token = response.get("access_token")
+                self._creation_time = time.time()
+                self._expiration_time = time.time() + int(response.get("expires_in")) -  60 # one minute margin
+                self.init_status = self.__SUCCESSFUL_AUTHENTICATION
+            else:
+                self.init_status = f'{self.__ERROR_OCCURRED}{response.get("error")} Error description: {response.get("error_description")}'
+
+        except Exception as err:
+            self.init_status = f'{self.__ERROR_OCCURRED}{err}\n'
+
+    # Authentication non-interactively using service account - aka Windows Client Authentication
+    def __acquire_token_noninteractive_with_adfs(self):
+        try:
+            headers = {
+                "Content-Type": "application/x-www-form-urlencoded"
+            }
+
+            payload = {
+                'client_id': self._client_id,
+                'use_windows_client_authentication': 'true',
+                'grant_type': 'client_credentials',
+                'scope': 'openid',
+                'resource': self._sdmx_resource_id
+            }
+
+            kerberos_auth = requests_kerberos.HTTPKerberosAuth(
+                mutual_authentication=requests_kerberos.OPTIONAL, force_preemptive=True)
+            response = requests.post(url=self._token_url, data=payload, auth=kerberos_auth)
+
+            # If the response object cannot be converted to json, return an error
+            results_json = None
+            try:
+                results_json = json.loads(response.text)
+                if response.status_code == 200:
+                    self._access_token = results_json['access_token']
+                    self._creation_time = time.time()
+                    self._expiration_time = time.time() + int(results_json['expires_in']) -  60 # one minute margin
+                    self.init_status = self.__SUCCESSFUL_AUTHENTICATION
+                else:
+                    message = f'{self.__ERROR_OCCURRED} Error code: {response.status_code}'
+                    if len(str(response.reason)) > 0:
+                        message += os.linesep + 'Reason: ' + str(response.reason) + os.linesep
+                    if len(response.text) > 0:
+                        message += f'{self.__ERROR_OCCURRED}{results_json.get("error")} Error description: {results_json.get("error_description")}\n'
+                        
+                    self.init_status = message
+            except ValueError as err:
+               self.init_status = self.__ERROR_OCCURRED + os.linesep
+               if len(str(response.status_code)) > 0:
+                  self.init_status += 'Error code: ' + str(response.status_code) + os.linesep
+               if len(str(response.reason)) > 0:
+                  self.init_status += 'Reason: ' + str(response.reason) + os.linesep
+               if len(response.text) > 0:
+                  self.init_status += str(response.text)
+               else:
+                  self.init_status += str(err)
+        except Exception as err:
+            self.init_status = f'{self.__ERROR_OCCURRED} {err}\n'
+
+
+# sub class to manage Keycloak authentication
+class KeycloakAuthentication(Authentication):
+    # private constants
+    __ERROR_OCCURRED = "An error occurred: "
+    __SUCCESSFUL_AUTHENTICATION = "Successful authentication"
+ 
+    #
+    def _initialize_token(self):
+        self._access_token = None
+        self._refresh_token = None
+        self._creation_time = None
+        self._expiration_time = None
+        self.init_status = None
+        match self._mode:
+            case AuthenticationMode.NONINTERACTIVE_WITH_SECRET:
+                self.__acquire_token_noninteractive_with_secret()
+
+    #
+    @classmethod
+    def noninteractive_with_secret(
+        cls, 
+        token_url: str,
+        user: str,
+        password: str,
+        client_id: str = "app",
+        client_secret: str = "",
+        proxy: str | None = None,
+        scopes: list[str] = [],
+        mode: AuthenticationMode = AuthenticationMode.NONINTERACTIVE_WITH_SECRET
+    ):
+        return cls(           
+            token_url=token_url,
+            user=user,
+            password=password,
+            client_id=client_id,   
+            client_secret=client_secret,
+            scopes=scopes,
+            proxy=proxy,  
+            mode=mode
+            )
+
+    # Authentication non-interactively using any account - aka Client Credentials flow
+    def __acquire_token_noninteractive_with_secret(self):
+        try:
+            payload = {
+            'grant_type': 'password',
+            'client_id': self._client_id,
+            'client_secret': self._client_secret,
+            'username': self._user,
+            'password': self._password
+            }
+
+            headers = {"Content-Type": "application/x-www-form-urlencoded"}
+
+            response = requests.post(self._token_url, proxies=self._proxies, headers=headers, data=payload)
+            if response.status_code not in {200, 201}:
+                self.init_status = f'{self.__ERROR_OCCURRED}{response}'
+            else:
+                # If the response object cannot be converted to json, return an error
+                results_json = None
+                try:
+                    results_json = json.loads(response.text)
+                    self._access_token = results_json['access_token']
+                    self._refresh_token = results_json['refresh_token']
+                    self._creation_time = time.time()
+                    self._expiration_time = time.time() + int(results_json['expires_in']) -  60 # one minute margin
+                    self.init_status = self.__SUCCESSFUL_AUTHENTICATION
+                except ValueError as err:
+                  self.init_status = self.__ERROR_OCCURRED + os.linesep
+                  if len(str(response.status_code)) > 0:
+                     self.init_status += 'Error code: ' + str(response.status_code) + os.linesep
+                  if len(str(response.reason)) > 0:
+                     self.init_status += 'Reason: ' + str(response.reason) + os.linesep
+                  if len(response.text) > 0:
+                     self.init_status += str(response.text)
+                  else:
+                     self.init_status += str(err)
+        except Exception as err:
+            self.init_status = f'{self.__ERROR_OCCURRED}{err}\n'

dotstat_io-1.0.7/dotstat_io/client.py

@@ -0,0 +1,552 @@
+from enum import IntEnum
+import os
+import requests
+import chardet
+import json
+import logging
+import xml.etree.ElementTree as ET
+
+from pathlib import Path
+from time import sleep
+
+from dotstat_io.authentication import Authentication
+
+class ValidationType(IntEnum):
+    BASIC = 0
+    ADVANCED = 1
+    
+
+# class to download or upload data from/to .Stat Suite
+class Client():
+
+    # private constants
+    __ERROR_OCCURRED  = "An error occurred: "
+    __EXECUTION_IN_QUEUED = "Queued"
+    __EXECUTION_IN_PROGRESS = "InProgress"
+    __CONNECTION_ABORTED = "An existing connection was forcibly closed by the remote host"
+    __DOWNLOAD_SUCCESS = "Successful download"
+    __UPLOAD_SUCCESS = "The request was successfully processed "
+    __UPLOAD_FAILED = "The request failed with status code "
+
+    __NAMESPACE_MESSAGE = "{http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message}"
+    __NAMESPACE_COMMON = "{http://www.sdmx.org/resources/sdmxml/schemas/v2_1/common}"
+
+    # private variables
+    __access_token = None
+    __authentication_obj = None
+
+    # Prepare logging format
+    FORMAT = '%(message)s'
+    logging.basicConfig(format=FORMAT, level=logging.INFO)
+    __log = logging.getLogger(__name__)
+
+    # Initialise Client
+    def __init__(self,
+	    access_token: None | str = None,
+        authentication_obj: None | Authentication = None):
+        Client.__access_token = access_token
+        Client.__authentication_obj = authentication_obj
+
+
+    # 
+    def __enter__(self):
+        return self
+
+    # 
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.__access_token = None
+        self.__authentication_obj = None
+
+
+    #           
+    @classmethod
+    def init_with_access_token(
+        cls, 
+        access_token: str
+    ):
+        return cls(
+            access_token=access_token
+        )
+    
+    #           
+    @classmethod
+    def init_with_authentication_obj(
+        cls, 
+        authentication_obj: Authentication
+    ):
+        return cls(
+            authentication_obj=authentication_obj
+        )
+    
+
+    # Download a data file from .Stat Suite
+    def download_data_file(self, dotstat_url: str, content_format: str, file_path: Path):
+        try:
+            returned_result = ""
+
+            # 
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
+
+            headers = {
+                'accept': content_format,
+                'authorization': 'Bearer '+Client.__access_token
+            }
+
+            #
+            response = requests.get(dotstat_url, verify=True, headers=headers)
+        except Exception as err:
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
+
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+            return returned_result
+        else:
+            if response.status_code != 200:
+                returned_result = Client.__ERROR_OCCURRED
+                if len(str(response.status_code)) > 0:
+                    returned_result += 'Error code: ' + str(response.status_code) + os.linesep
+                if len(str(response.reason)) > 0:
+                    returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                if len(response.text) > 0:
+                    returned_result += 'Text: ' + response.text
+
+                returned_result += os.linesep
+            else:
+                if os.path.isfile(file_path):
+                    os.remove(file_path)
+                with open(file_path, "wb") as file:
+                    file.write(response.content)
+                    returned_result = Client.__DOWNLOAD_SUCCESS
+            
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+            return returned_result
+
+
+    # Download streamed content from .Stat Suite
+    def download_data_stream(self, dotstat_url: str, content_format: str):
+        try:
+            returned_result = ""
+
+            # 
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
+
+            headers = {
+                'accept': content_format,
+                'Transfer-Encoding': 'chunked',
+                'authorization': 'Bearer '+Client.__access_token
+            }
+
+            #
+            return requests.get(dotstat_url, verify=True, headers=headers, stream=True)
+        except Exception as err:
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
+            return returned_result
+
+
+    # Upload a data file to .Stat Suite
+    def upload_data_file(self, 
+                    transfer_url: str, 
+                    file_path: Path, 
+                    space: str, 
+                    validationType: int, 
+                    use_filepath: bool = False,
+                    optimize: bool = True):
+        try:
+            returned_result = ""
+
+            # 
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
+
+            payload = {
+                'dataspace': space,
+                'validationType': validationType,
+                'optimize': optimize
+            }
+
+            headers = {
+                'accept': 'application/json',
+                'authorization': "Bearer "+Client.__access_token
+            }
+
+            if  use_filepath:
+                files = {
+                    'dataspace': (None, payload['dataspace']),
+                    'validationType': (None, payload['validationType']),
+                    'optimize': (None, payload['optimize']),
+                    'filepath': (None, str(file_path))
+                }
+            else:
+                files = {
+                    'dataspace': (None, payload['dataspace']),
+                    'validationType': (None, payload['validationType']),
+                    'optimize': (None, payload['optimize']),
+                    'file': (os.path.realpath(file_path), open(os.path.realpath(file_path), 'rb'), 'text/csv', '')
+                }
+
+            #
+            response = requests.post(transfer_url, verify=True, headers=headers, files=files)
+        except Exception as err:
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
+
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+            return returned_result
+        else:
+            # If the response object cannot be converted to json, return an error
+            results_json = None
+            try:
+                results_json = json.loads(response.text)
+                if response.status_code == 200:
+                    result = results_json['message']
+                    # Write the result to the log
+                    for line in result.split(os.linesep):
+                        if len(line) > 0:
+                            self.__log.info('   ' + line)
+
+                    returned_result = result + os.linesep
+
+                    # Check the request status
+                    if (result != "" and result.find(Client.__ERROR_OCCURRED ) == -1):
+                        # Extract the request ID the returned message
+                        start = 'with ID'
+                        end = 'was successfully'
+                        requestId = result[result.find(
+                            start)+len(start):result.rfind(end)]
+
+                        # Sleep a little bit before checking the request status
+                        sleep(3)
+
+                        # To avoid this error: maximum recursion depth exceeded while calling a Python object
+                        # replace the recursive calls with while loops.
+                        result = self.__check_request_status(transfer_url, requestId, space)
+
+                        # Write the result to the log
+                        for line in result.split(os.linesep):
+                            if len(line) > 0:
+                                self.__log.info('   ' + line)
+                        sleep(3)
+
+                        previous_result = result
+                        while (result in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                                 or Client.__CONNECTION_ABORTED in result):
+                            result = self.__check_request_status(transfer_url, requestId, space)
+
+                            # Prevent loging again the same information such as "Queued" or "InProgress" 
+                            if previous_result != result:
+                               previous_result = result
+
+                            # Write the result to the log
+                            for line in previous_result.split(os.linesep):
+                              if (len(line) > 0 and line not in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                                    and Client.__CONNECTION_ABORTED not in line):
+                                 self.__log.info('   ' + line)
+                            sleep(3)
+
+                        returned_result = returned_result + result + os.linesep
+                else:
+                    returned_result = Client.__ERROR_OCCURRED
+                    if len(str(response.status_code)) > 0:
+                        returned_result += 'Error code: ' + str(response.status_code) + os.linesep
+                    if len(str(response.reason)) > 0:
+                        returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                    if len(response.text) > 0:
+                        returned_result += 'Text: ' + response.text
+
+                    returned_result += os.linesep
+                    # Write the result to the log
+                    for line in returned_result.split(os.linesep):
+                        if len(line) > 0:
+                            self.__log.info('   ' + line)
+            except ValueError as err:
+                returned_result = Client.__ERROR_OCCURRED
+                if len(str(response.status_code)) > 0:
+                    returned_result += 'Error code: ' + str(response.status_code) + os.linesep
+                if len(str(response.reason)) > 0:
+                    returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                if len(response.text) > 0:
+                    returned_result += 'Text: ' + str(response.text)
+                else:
+                    returned_result += str(err)
+                returned_result += os.linesep
+            return returned_result
+
+
+    # Upload an Excel data file to .Stat Suite
+    def upload_excel_data_file(self, 
+                    transfer_url: str, 
+                    excelfile_path: Path,
+                    eddfile_path: Path, 
+                    space: str, 
+                    validationType: int):
+        try:
+            returned_result = ""
+
+            # 
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
+
+            payload = {
+                'dataspace': space,
+                'validationType': validationType
+            }
+
+            headers = {
+                'accept': 'application/json',
+                'authorization': "Bearer "+Client.__access_token
+            }
+
+            excel_file = open(os.path.realpath(excelfile_path), 'rb')
+            eddfile_file = open(os.path.realpath(eddfile_path), 'rb')
+            files = {
+                'dataspace': (None, payload['dataspace']),
+                'validationType': (None, payload['validationType']),
+                'excelFile': (str(excelfile_path), excel_file, 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', ''),
+                'eddFile': (str(eddfile_path), eddfile_file, 'text/xml', '')
+            }
+
+            #
+            response = requests.post(transfer_url, verify=True, headers=headers, files=files)
+        except Exception as err:
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
+
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+            return returned_result
+        else:
+            # If the response object cannot be converted to json, return an error
+            results_json = None
+            try:
+                results_json = json.loads(response.text)
+                if response.status_code == 200:
+                    result = results_json['message']
+                    # Write the result to the log
+                    for line in result.split(os.linesep):
+                        if len(line) > 0:
+                            self.__log.info('   ' + line)
+
+                    returned_result = result + os.linesep
+
+                    # Check the request status
+                    if (result != "" and result.find(Client.__ERROR_OCCURRED ) == -1):
+                        # Extract the request ID the returned message
+                        start = 'with ID'
+                        end = 'was successfully'
+                        requestId = result[result.find(
+                            start)+len(start):result.rfind(end)]
+
+                        # Sleep a little bit before checking the request status
+                        sleep(3)
+
+                        # To avoid this error: maximum recursion depth exceeded while calling a Python object
+                        # replace the recursive calls with while loops.
+                        result = self.__check_request_status(transfer_url, requestId, space)
+
+                        # Write the result to the log
+                        for line in result.split(os.linesep):
+                            if len(line) > 0:
+                                self.__log.info('   ' + line)
+                        sleep(3)
+
+                        previous_result = result
+                        while (result in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                                 or Client.__CONNECTION_ABORTED in result):
+                            result = self.__check_request_status(transfer_url, requestId, space)
+
+                            # Prevent loging again the same information such as "Queued" or "InProgress" 
+                            if previous_result != result:
+                               previous_result = result
+
+                            # Write the result to the log
+                            for line in previous_result.split(os.linesep):
+                              if (len(line) > 0 and line not in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                                    and Client.__CONNECTION_ABORTED not in line):
+                                 self.__log.info('   ' + line)
+                            sleep(3)
+
+                        returned_result = returned_result + result + os.linesep
+                else:
+                    returned_result = Client.__ERROR_OCCURRED
+                    if len(str(response.status_code)) > 0:
+                        returned_result += 'Error code: ' + str(response.status_code) + os.linesep
+                    if len(str(response.reason)) > 0:
+                        returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                    if len(response.text) > 0:
+                        returned_result += 'Text: ' + response.text
+
+                    returned_result += os.linesep
+                    # Write the result to the log
+                    for line in returned_result.split(os.linesep):
+                        if len(line) > 0:
+                            self.__log.info('   ' + line)
+            except ValueError as err:
+                returned_result = Client.__ERROR_OCCURRED
+                if len(str(response.status_code)) > 0:
+                    returned_result += 'Error code: ' + str(response.status_code) + os.linesep
+                if len(str(response.reason)) > 0:
+                    returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                if len(response.text) > 0:
+                    returned_result += 'Text: ' + str(response.text)
+                else:
+                    returned_result += str(err)
+                returned_result += os.linesep
+            return returned_result
+
+
+    # Upload a structure file to .Stat Suite
+    def upload_structure(self, transfer_url: str, file_path: Path):
+        try:
+            returned_result = ""
+
+            # 
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
+
+            # Detect the encoding used in file 
+            detected_encoding = self.__detect_encode(file_path)
+
+            # Read file as a string "r+" with the detected encoding
+            with open(file=file_path, mode="r+", encoding=detected_encoding.get("encoding")) as file:
+                xml_data = file.read()
+
+            # Make sure the encoding is "utf-8"
+            tree = ET.fromstring(xml_data)
+            xml_data = ET.tostring(tree, encoding="utf-8", method='xml', xml_declaration=True)
+
+            headers = {
+                'Content-Type': 'application/xml',
+                'authorization': "Bearer "+Client.__access_token
+            }
+
+            #
+            response = requests.post(transfer_url, verify=True, headers=headers, data=xml_data)
+        except Exception as err:
+            returned_result = Client.__ERROR_OCCURRED  + str(err) + os.linesep
+
+            # Write the result to the log
+            for line in returned_result.split(os.linesep):
+                if len(line) > 0:
+                    self.__log.info('   ' + line)
+            return returned_result
+        else:
+            try:
+                response.raise_for_status()
+            except requests.exceptions.HTTPError as e:
+                returned_result = f'{Client.__UPLOAD_FAILED}{response.status_code}: {e}'
+
+                # Write the result to the log
+                for line in returned_result.split(os.linesep):
+                    if len(line) > 0:
+                        self.__log.info('   ' + line)
+            else:
+                response_tree = ET.XML(response.content)
+                for element in response_tree.findall("./{0}ErrorMessage".format(Client.__NAMESPACE_MESSAGE)):
+                    text_element = element.find("./{0}Text".format(Client.__NAMESPACE_COMMON))
+                    if (text_element is not None):
+                        if returned_result == "":
+                            returned_result = f'{Client.__UPLOAD_SUCCESS}with status code: {response.status_code}' + os.linesep
+                        returned_result = returned_result + text_element.text + os.linesep
+
+                # Write the result to the log
+                for line in returned_result.split(os.linesep):
+                    if len(line) > 0:
+                        self.__log.info('   ' + line)
+            return returned_result
+
+
+    # Detect the encoding used in file
+    def __detect_encode(self, file_path):
+        detector = chardet.UniversalDetector()
+        detector.reset()
+        with open(file=file_path, mode="rb") as file:
+            for row in file:
+                detector.feed(row)
+                if detector.done: 
+                    break
+
+        detector.close()
+
+        return detector.result
+
+
+    # Check request sent to .Stat Suite status
+    # To avoid this error: maximum recursion depth exceeded while calling a Python object
+    # replace the recursive calls with while loops.
+    def __check_request_status(self, transfer_url, requestId, space):
+        try:
+            returned_result = ""
+
+            # 
+            if Client.__authentication_obj is not None:
+                Client.__access_token = Client.__authentication_obj.get_token()
+
+            headers = {
+                'accept': 'application/json',
+                'authorization': "Bearer "+Client.__access_token
+            }
+
+            payload = {
+                'dataspace': space,
+                'id': requestId
+            }
+
+            transfer_url = transfer_url.replace("import", "status")
+            if "sdmxFile" in transfer_url:
+                transfer_url = transfer_url.replace("sdmxFile", "request")
+            elif "excel" in transfer_url:
+                transfer_url = transfer_url.replace("excel", "request")
+            
+            #
+            response = requests.post(transfer_url, verify=True, headers=headers, data=payload)
+        except Exception as err:
+            returned_result = Client.__ERROR_OCCURRED  + str(err)
+            return returned_result
+        else:
+            # If the response object cannot be converted to json, return an error
+            results_json = None
+            try:
+                results_json = json.loads(response.text)
+                if response.status_code == 200:
+                    executionStatus = 'Execution status: ' + results_json['executionStatus']
+                    if (results_json['executionStatus'] in [Client.__EXECUTION_IN_PROGRESS, Client.__EXECUTION_IN_QUEUED]
+                        or Client.__CONNECTION_ABORTED in results_json['executionStatus']):
+                        returned_result = results_json['executionStatus']
+                    else:
+                        returned_result = executionStatus + os.linesep + 'Outcome: ' + results_json['outcome'] + os.linesep
+                        index = 0
+                        while index < len(results_json['logs']):
+                            returned_result = returned_result + 'Log' + str(index) + ': ' + results_json['logs'][index]['message'] + os.linesep
+                            index += 1
+                else:
+                    returned_result = Client.__ERROR_OCCURRED
+                    if len(str(response.status_code)) > 0:
+                        returned_result += 'Error code: ' + str(response.status_code) + os.linesep
+                    if len(str(response.reason)) > 0:
+                        returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                    if len(response.text) > 0:
+                        returned_result += 'Text: ' + str(response.text)
+
+                    returned_result += os.linesep
+            except ValueError as err:
+                returned_result = Client.__ERROR_OCCURRED
+                if len(str(response.status_code)) > 0:
+                    returned_result += 'Error code: ' + str(response.status_code) + os.linesep
+                if len(str(response.reason)) > 0:
+                    returned_result += 'Reason: ' + str(response.reason) + os.linesep
+                if len(response.text) > 0:
+                    returned_result += 'Text: ' + str(response.text)
+                else:
+                    returned_result += str(err)
+                returned_result += os.linesep
+            return returned_result

dotstat_io-1.0.7/pyproject.toml

@@ -0,0 +1,35 @@
+[tool.poetry]
+name = "dotstat_io"
+version = "1.0.7"
+description = "Utility to download or upload data from/to .Stat Suite using ADFS authentication to connect to it"
+license = "MIT"
+authors = ["Gyorgy Gyomai <gyorgy.gyomai@oecd.org>", "Abdel Aliaoui <abdel.aliaoui@oecd.org>"]
+readme = "README.md"
+packages = [{include = "dotstat_io"}]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+requests = "^2.29.0"
+requests-kerberos = "^0.14.0"
+msal = "^1.22.0"
+chardet = "^5.1.0"
+setuptools = "^75.6.0"
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^9.0.2"
+
+
+[[tool.poetry.source]]
+name = "fpho"
+url = "https://files.pythonhosted.org"
+priority = "primary"
+
+
+[[tool.poetry.source]]
+name = "PyPI"
+priority = "primary"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"