cosmotech-acceleration-library 1.0.1__tar.gz → 2.0.0__tar.gz

{cosmotech_acceleration_library-1.0.1/cosmotech_acceleration_library.egg-info → cosmotech_acceleration_library-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cosmotech_acceleration_library
-Version: 1.0.1
+Version: 2.0.0
 Summary: Acceleration libraries for CosmoTech cloud based solution development
 Author-email: Cosmo Tech <platform@cosmotech.com>
 Project-URL: Homepage, https://www.cosmotech.com
@@ -15,21 +15,21 @@ Requires-Dist: azure-kusto-data~=4.4.1
 Requires-Dist: azure-kusto-ingest~=4.4.1
 Requires-Dist: tenacity~=8.3.0
 Requires-Dist: python-keycloak~=4.7.3
-Requires-Dist: cosmotech-api~=3.2
+Requires-Dist: cosmotech-api~=5.0.0rc1
 Requires-Dist: boto3~=1.35.19
 Requires-Dist: requests~=2.32.3
 Requires-Dist: singlestoredb~=1.10.0
 Requires-Dist: cosmotech-run-orchestrator~=2.0.0
 Requires-Dist: pyarrow~=20.0.0
-Requires-Dist: adbc-driver-manager~=1.1.0
-Requires-Dist: adbc-driver-sqlite~=1.1.0
-Requires-Dist: adbc-driver-postgresql~=1.1.0
+Requires-Dist: adbc-driver-manager~=1.7.0
+Requires-Dist: adbc-driver-sqlite~=1.7.0
+Requires-Dist: adbc-driver-postgresql~=1.7.0
 Requires-Dist: click~=8.1.7
 Requires-Dist: rich-click~=1.7.3
 Requires-Dist: click-log~=0.4.0
 Requires-Dist: tqdm~=4.67.1
 Requires-Dist: openpyxl~=3.1
-Requires-Dist: pandas~=2.1
+Requires-Dist: pandas~=2.3
 Requires-Dist: python-dateutil~=2.8
 Requires-Dist: rich~=13.7
 Requires-Dist: setuptools
@@ -51,13 +51,12 @@ Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-docker; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
-Provides-Extra: extra
-Requires-Dist: pandas~=2.2.2; extra == "extra"
 Provides-Extra: dev
 Requires-Dist: black~=24.3.0; extra == "dev"
 Requires-Dist: pre-commit~=3.3.2; extra == "dev"
+Requires-Dist: isort~=7.0.0; extra == "dev"
 Provides-Extra: all
-Requires-Dist: CosmoTech_Acceleration_Library[dev,doc,extra,test]; extra == "all"
+Requires-Dist: CosmoTech_Acceleration_Library[dev,doc,test]; extra == "all"
 Dynamic: license-file
 
 # CosmoTech-Acceleration-Library (CoAL)

{cosmotech_acceleration_library-1.0.1 → cosmotech_acceleration_library-2.0.0}/cosmotech/coal/__init__.py

@@ -5,4 +5,4 @@
 # etc., to any person is prohibited unless it has been previously and
 # specifically authorized by written means by Cosmo Tech.
 
-__version__ = "1.0.1"
+__version__ = "2.0.0"

{cosmotech_acceleration_library-1.0.1 → cosmotech_acceleration_library-2.0.0}/cosmotech/coal/aws/__init__.py

@@ -12,12 +12,4 @@ This module provides functions for interacting with AWS services like S3.
 """
 
 # Re-export S3 functions for easier importing
-from cosmotech.coal.aws.s3 import (
-    create_s3_client,
-    create_s3_resource,
-    upload_file,
-    upload_folder,
-    download_files,
-    upload_data_stream,
-    delete_objects,
-)
+from cosmotech.coal.aws.s3 import S3

cosmotech_acceleration_library-2.0.0/cosmotech/coal/aws/s3.py

@@ -0,0 +1,202 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+"""
+s3 bucket operations module.
+
+this module provides functions for interacting with S3 buckets, including
+uploading, downloading, and deleting files.
+"""
+
+import pathlib
+from io import BytesIO
+
+import boto3
+from cosmotech.orchestrator.utils.translate import T
+
+from cosmotech.coal.utils.configuration import Configuration
+from cosmotech.coal.utils.logger import LOGGER
+
+
+class S3:
+
+    def __init__(self, configuration: Configuration):
+        self._configuration = configuration.s3
+
+    @property
+    def file_prefix(self):
+        if "bucket_prefix" in self._configuration:
+            return self._configuration.bucket_prefix
+        return ""
+
+    @property
+    def use_ssl(self):
+        if "use_ssl" in self._configuration:
+            return self._configuration.use_ssl
+        return True
+
+    @property
+    def ssl_cert_bundle(self):
+        if "ssl_cert_bundle" in self._configuration:
+            return self._configuration.ssl_cert_bundle
+        return None
+
+    @property
+    def access_key_id(self):
+        return self._configuration.access_key_id
+
+    @property
+    def endpoint_url(self):
+        return self._configuration.endpoint_url
+
+    @property
+    def bucket_name(self):
+        return self._configuration.bucket_name
+
+    @property
+    def secret_access_key(self):
+        return self._configuration.secret_access_key
+
+    @property
+    def output_type(self):
+        if "output_type" in self._configuration:
+            return self._configuration.output_type
+        return "csv"
+
+    @property
+    def client(self) -> boto3.client:
+        boto3_parameters = {
+            "use_ssl": self.use_ssl,
+            "endpoint_url": self.endpoint_url,
+            "aws_access_key_id": self.access_key_id,
+            "aws_secret_access_key": self.secret_access_key,
+        }
+        if self.ssl_cert_bundle:
+            boto3_parameters["verify"] = self.ssl_cert_bundle
+
+        return boto3.client("s3", **boto3_parameters)
+
+    @property
+    def resource(self) -> boto3.resource:
+        boto3_parameters = {
+            "use_ssl": self.use_ssl,
+            "endpoint_url": self.endpoint_url,
+            "aws_access_key_id": self.access_key_id,
+            "aws_secret_access_key": self.secret_access_key,
+        }
+        if self.ssl_cert_bundle:
+            boto3_parameters["verify"] = self.ssl_cert_bundle
+
+        return boto3.resource("s3", **boto3_parameters)
+
+    def upload_file(self, file_path: pathlib.Path) -> None:
+        """
+        Upload a single file to an S3 bucket.
+
+        Args:
+            file_path: Path to the file to upload
+        """
+        uploaded_file_name = self.file_prefix + file_path.name
+        LOGGER.info(
+            T("coal.common.data_transfer.file_sent").format(file_path=file_path, uploaded_name=uploaded_file_name)
+        )
+        self.resource.Bucket(self.bucket_name).upload_file(str(file_path), uploaded_file_name)
+
+    def upload_folder(self, source_folder: str, recursive: bool = False) -> None:
+        """
+        Upload files from a folder to an S3 bucket.
+
+        Args:
+            source_folder: Path to the folder containing files to upload
+            recursive: Whether to recursively upload files from subdirectories
+        """
+        source_path = pathlib.Path(source_folder)
+        if not source_path.exists():
+            LOGGER.error(T("coal.common.file_operations.not_found").format(source_folder=source_folder))
+            raise FileNotFoundError(T("coal.common.file_operations.not_found").format(source_folder=source_folder))
+
+        if source_path.is_dir():
+            _source_name = str(source_path)
+            for _file_path in source_path.glob("**/*" if recursive else "*"):
+                if _file_path.is_file():
+                    _file_name = str(_file_path).removeprefix(_source_name).removeprefix("/")
+                    uploaded_file_name = self.file_prefix + _file_name
+                    LOGGER.info(
+                        T("coal.common.data_transfer.file_sent").format(
+                            file_path=_file_path, uploaded_name=uploaded_file_name
+                        )
+                    )
+                    self.resource.Bucket(self.bucket_name).upload_file(str(_file_path), uploaded_file_name)
+        else:
+            self.upload_file(source_path)
+
+    def download_files(self, destination_folder: str) -> None:
+        """
+        Download files from an S3 bucket to a local folder.
+
+        Args:
+            destination_folder: Local folder to download files to
+        """
+        bucket = self.resource.Bucket(self.bucket_name)
+
+        pathlib.Path(destination_folder).mkdir(parents=True, exist_ok=True)
+        remove_prefix = False
+        if self.file_prefix:
+            bucket_files = bucket.objects.filter(Prefix=self.file_prefix)
+            if self.file_prefix.endswith("/"):
+                remove_prefix = True
+        else:
+            bucket_files = bucket.objects.all()
+        for _file in bucket_files:
+            if not (path_name := str(_file.key)).endswith("/"):
+                target_file = path_name
+                if remove_prefix:
+                    target_file = target_file.removeprefix(self.file_prefix)
+                output_file = f"{destination_folder}/{target_file}"
+                pathlib.Path(output_file).parent.mkdir(parents=True, exist_ok=True)
+                LOGGER.info(T("coal.services.azure_storage.downloading").format(path=path_name, output=output_file))
+                bucket.download_file(_file.key, output_file)
+
+    def upload_data_stream(self, data_stream: BytesIO, file_name: str) -> None:
+        """
+        Upload a data stream to an S3 bucket.
+
+        Args:
+            data_stream: BytesIO stream containing the data to upload
+            file_name: Name of the file to create in the bucket
+        """
+        uploaded_file_name = self.file_prefix + file_name
+        data_stream.seek(0)
+        size = len(data_stream.read())
+        data_stream.seek(0)
+
+        LOGGER.info(T("coal.common.data_transfer.sending_data").format(size=size))
+        self.client.upload_fileobj(data_stream, self.bucket_name, uploaded_file_name)
+
+    def delete_objects(self) -> None:
+        """
+        Delete objects from an S3 bucket, optionally filtered by prefix.
+
+        Args:
+            bucket_name: Name of the S3 bucket
+            s3_resource: S3 resource object
+            file_prefix: Optional prefix to filter objects to delete
+        """
+        bucket = self.resource.Bucket(self.bucket_name)
+
+        if self.file_prefix:
+            bucket_files = bucket.objects.filter(Prefix=self.file_prefix)
+        else:
+            bucket_files = bucket.objects.all()
+
+        boto_objects = [{"Key": _file.key} for _file in bucket_files if _file.key != self.file_prefix]
+        if boto_objects:
+            LOGGER.info(T("coal.services.azure_storage.deleting_objects").format(objects=boto_objects))
+            boto_delete_request = {"Objects": boto_objects}
+            bucket.delete_objects(Delete=boto_delete_request)
+        else:
+            LOGGER.info(T("coal.services.azure_storage.no_objects"))

{cosmotech_acceleration_library-1.0.1 → cosmotech_acceleration_library-2.0.0}/cosmotech/coal/azure/adx/auth.py

@@ -6,13 +6,13 @@
 # specifically authorized by written means by Cosmo Tech.
 
 import os
-from typing import Union, Optional, Tuple
+from typing import Optional, Tuple
 
 from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
 from azure.kusto.ingest import QueuedIngestClient
+from cosmotech.orchestrator.utils.translate import T
 
 from cosmotech.coal.utils.logger import LOGGER
-from cosmotech.orchestrator.utils.translate import T
 
 
 def create_kusto_client(

{cosmotech_acceleration_library-1.0.1 → cosmotech_acceleration_library-2.0.0}/cosmotech/coal/azure/adx/runner.py

@@ -13,25 +13,24 @@ This module provides functions for ingesting runner data into Azure Data Explore
 
 import pathlib
 import time
-from collections import defaultdict
-from typing import Dict, Any, List, Tuple, Optional
+from typing import Any, Dict
 
 from azure.kusto.data.response import KustoResponseDataSet
-from azure.kusto.ingest import ColumnMapping
-from azure.kusto.ingest import FileDescriptor
-from azure.kusto.ingest import IngestionMappingKind
-from azure.kusto.ingest import IngestionProperties
-from azure.kusto.ingest import IngestionResult
-from azure.kusto.ingest import ReportLevel
-
-from azure.kusto.data import KustoClient
-from azure.kusto.ingest import QueuedIngestClient
+from azure.kusto.ingest import (
+    ColumnMapping,
+    FileDescriptor,
+    IngestionMappingKind,
+    IngestionProperties,
+    IngestionResult,
+    QueuedIngestClient,
+    ReportLevel,
+)
+from cosmotech.orchestrator.utils.translate import T
 
 from cosmotech.coal.azure.adx.auth import initialize_clients
-from cosmotech.coal.azure.adx.query import run_query, run_command_query
-from cosmotech.coal.azure.adx.ingestion import check_ingestion_status, IngestionStatus
+from cosmotech.coal.azure.adx.ingestion import IngestionStatus, check_ingestion_status
+from cosmotech.coal.azure.adx.query import run_query
 from cosmotech.coal.utils.logger import LOGGER
-from cosmotech.orchestrator.utils.translate import T
 
 
 def prepare_csv_content(folder_path: str) -> Dict[str, Dict[str, Any]]:

{cosmotech_acceleration_library-1.0.1 → cosmotech_acceleration_library-2.0.0}/cosmotech/coal/azure/adx/store.py

@@ -7,26 +7,22 @@
 
 import os
 import tempfile
+import time
 import uuid
-from typing import Optional, List, Dict, Tuple, Union, Any
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import pyarrow
 import pyarrow.csv as pc
-import time
 from azure.kusto.data import KustoClient
 from azure.kusto.data.data_format import DataFormat
-from azure.kusto.ingest import IngestionProperties
-from azure.kusto.ingest import QueuedIngestClient
-from azure.kusto.ingest import ReportLevel
+from azure.kusto.ingest import IngestionProperties, QueuedIngestClient, ReportLevel
 from cosmotech.orchestrator.utils.translate import T
-from time import perf_counter
 
-from cosmotech.coal.azure.adx.tables import check_and_create_table, _drop_by_tag
 from cosmotech.coal.azure.adx.auth import initialize_clients
-from cosmotech.coal.azure.adx.ingestion import monitor_ingestion, handle_failures
+from cosmotech.coal.azure.adx.ingestion import handle_failures, monitor_ingestion
+from cosmotech.coal.azure.adx.tables import _drop_by_tag, check_and_create_table
 from cosmotech.coal.store.store import Store
 from cosmotech.coal.utils.logger import LOGGER
-from cosmotech.coal.utils.postgresql import send_pyarrow_table_to_postgresql
 
 
 def send_table_data(
@@ -176,80 +172,3 @@ def send_store_to_adx(
         LOGGER.warning(T("coal.services.adx.dropping_data").format(operation_tag=operation_tag))
         _drop_by_tag(kusto_client, database, operation_tag)
         raise e
-
-
-def dump_store_to_adx(
-    store_folder: str,
-    postgres_host: str,
-    postgres_port: int,
-    postgres_db: str,
-    postgres_schema: str,
-    postgres_user: str,
-    postgres_password: str,
-    table_prefix: str = "Cosmotech_",
-    replace: bool = True,
-) -> None:
-    """
-    Dump Store data to an Azure Data Explorer database.
-
-    Args:
-        store_folder: Folder containing the Store
-        postgres_host: PostgreSQL host
-        postgres_port: PostgreSQL port
-        postgres_db: PostgreSQL database name
-        postgres_schema: PostgreSQL schema
-        postgres_user: PostgreSQL username
-        postgres_password: PostgreSQL password
-        table_prefix: Table prefix
-        replace: Whether to replace existing tables
-    """
-    _s = Store(store_location=store_folder)
-
-    tables = list(_s.list_tables())
-    if len(tables):
-        LOGGER.info(T("coal.services.database.sending_data").format(table=f"{postgres_db}.{postgres_schema}"))
-        total_rows = 0
-        _process_start = perf_counter()
-        for table_name in tables:
-            _s_time = perf_counter()
-            target_table_name = f"{table_prefix}{table_name}"
-            LOGGER.info(T("coal.services.database.table_entry").format(table=target_table_name))
-            data = _s.get_table(table_name)
-            if not len(data):
-                LOGGER.info(T("coal.services.database.no_rows"))
-                continue
-            _dl_time = perf_counter()
-            rows = send_pyarrow_table_to_postgresql(
-                data,
-                target_table_name,
-                postgres_host,
-                postgres_port,
-                postgres_db,
-                postgres_schema,
-                postgres_user,
-                postgres_password,
-                replace,
-            )
-            total_rows += rows
-            _up_time = perf_counter()
-            LOGGER.info(T("coal.services.database.row_count").format(count=rows))
-            LOGGER.debug(
-                T("coal.common.timing.operation_completed").format(
-                    operation="Load from datastore", time=f"{_dl_time - _s_time:0.3}"
-                )
-            )
-            LOGGER.debug(
-                T("coal.common.timing.operation_completed").format(
-                    operation="Send to postgresql", time=f"{_up_time - _dl_time:0.3}"
-                )
-            )
-        _process_end = perf_counter()
-        LOGGER.info(
-            T("coal.services.database.rows_fetched").format(
-                table="all tables",
-                count=total_rows,
-                time=f"{_process_end - _process_start:0.3}",
-            )
-        )
-    else:
-        LOGGER.info(T("coal.services.database.store_empty"))

{cosmotech_acceleration_library-1.0.1 → cosmotech_acceleration_library-2.0.0}/cosmotech/coal/azure/adx/tables.py

@@ -5,13 +5,13 @@
 # etc., to any person is prohibited unless it has been previously and
 # specifically authorized by written means by Cosmo Tech.
 
-from typing import Dict, Any
+from typing import Dict
 
 import pyarrow
 from azure.kusto.data import KustoClient
+from cosmotech.orchestrator.utils.translate import T
 
 from cosmotech.coal.utils.logger import LOGGER
-from cosmotech.orchestrator.utils.translate import T
 
 
 def table_exists(client: KustoClient, database: str, table_name: str) -> bool:

{cosmotech_acceleration_library-1.0.1 → cosmotech_acceleration_library-2.0.0}/cosmotech/coal/azure/blob.py

@@ -12,19 +12,16 @@ This module provides functions for interacting with Azure Blob Storage,
 including uploading data from the Store.
 """
 
-import pathlib
 from io import BytesIO
-from typing import List, Optional
-
-from azure.identity import ClientSecretCredential
-from azure.storage.blob import BlobServiceClient
 
 import pyarrow.csv as pc
 import pyarrow.parquet as pq
+from azure.identity import ClientSecretCredential
+from azure.storage.blob import BlobServiceClient
+from cosmotech.orchestrator.utils.translate import T
 
 from cosmotech.coal.store.store import Store
 from cosmotech.coal.utils.logger import LOGGER
-from cosmotech.orchestrator.utils.translate import T
 
 VALID_TYPES = (
     "sqlite",
@@ -42,6 +39,7 @@ def dump_store_to_azure(
     client_secret: str,
     output_type: str = "sqlite",
     file_prefix: str = "",
+    selected_tables: list[str] = [],
 ) -> None:
     """
     Dump Store data to Azure Blob Storage.
@@ -90,6 +88,8 @@ def dump_store_to_azure(
             container_client.upload_blob(name=_uploaded_file_name, data=data, overwrite=True)
     else:
         tables = list(_s.list_tables())
+        if selected_tables:
+            tables = [t for t in tables if t in selected_tables]
         for table_name in tables:
             _data_stream = BytesIO()
             _file_name = None

{cosmotech_acceleration_library-1.0.1 → cosmotech_acceleration_library-2.0.0}/cosmotech/coal/azure/storage.py

@@ -15,9 +15,9 @@ uploading files to blob storage.
 import pathlib
 
 from azure.storage.blob import ContainerClient
+from cosmotech.orchestrator.utils.translate import T
 
 from cosmotech.coal.utils.logger import LOGGER
-from cosmotech.orchestrator.utils.translate import T
 
 
 def upload_file(
@@ -65,10 +65,10 @@ def upload_folder(
         raise FileNotFoundError(T("coal.common.file_operations.not_found").format(source_folder=source_folder))
 
     if source_path.is_dir():
-        _source_name = str(source_path)
+        # _source_name = str(source_path)
         for _file_path in source_path.glob("**/*" if recursive else "*"):
             if _file_path.is_file():
-                _file_name = str(_file_path).removeprefix(_source_name).removeprefix("/")
+                # _file_name = str(_file_path).removeprefix(_source_name).removeprefix("/")
                 upload_file(_file_path, blob_name, az_storage_sas_url, file_prefix)
     else:
         upload_file(source_path, blob_name, az_storage_sas_url, file_prefix)

cosmotech_acceleration_library-2.0.0/cosmotech/coal/cosmotech_api/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+"""
+Cosmotech API integration module.
+
+This module provides functions for interacting with the Cosmotech API.
+"""

cosmotech_acceleration_library-2.0.0/cosmotech/coal/cosmotech_api/apis/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.coal.cosmotech_api.apis.dataset import DatasetApi
+from cosmotech.coal.cosmotech_api.apis.meta import MetaApi
+from cosmotech.coal.cosmotech_api.apis.organization import OrganizationApi
+from cosmotech.coal.cosmotech_api.apis.run import RunApi
+from cosmotech.coal.cosmotech_api.apis.runner import RunnerApi
+from cosmotech.coal.cosmotech_api.apis.solution import SolutionApi
+from cosmotech.coal.cosmotech_api.apis.workspace import WorkspaceApi

cosmotech_acceleration_library-2.0.0/cosmotech/coal/cosmotech_api/apis/dataset.py

@@ -0,0 +1,103 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+from pathlib import Path
+from typing import Optional, Union
+
+from cosmotech.orchestrator.utils.translate import T
+from cosmotech_api import (
+    Dataset,
+)
+from cosmotech_api import DatasetApi as BaseDatasetApi
+from cosmotech_api import (
+    DatasetCreateRequest,
+    DatasetPartCreateRequest,
+    DatasetPartTypeEnum,
+)
+
+from cosmotech.coal.cosmotech_api.objects.connection import Connection
+from cosmotech.coal.utils.configuration import ENVIRONMENT_CONFIGURATION, Configuration
+from cosmotech.coal.utils.logger import LOGGER
+
+
+class DatasetApi(BaseDatasetApi, Connection):
+
+    def __init__(
+        self,
+        configuration: Configuration = ENVIRONMENT_CONFIGURATION,
+    ):
+        Connection.__init__(self, configuration)
+        BaseDatasetApi.__init__(self, self.api_client)
+
+        LOGGER.debug(T("coal.cosmotech_api.initialization.dataset_api_initialized"))
+
+    def download_dataset(self, dataset_id) -> Dataset:
+        dataset = self.get_dataset(
+            organization_id=self.configuration.cosmotech.organization_id,
+            workspace_id=self.configuration.cosmotech.workspace_id,
+            dataset_id=dataset_id,
+        )
+
+        dataset_dir = self.configuration.cosmotech.dataset_absolute_path
+        dataset_dir_path = Path(dataset_dir) / dataset_id
+        for part in dataset.parts:
+            part_file_path = dataset_dir_path / part.source_name
+            part_file_path.parent.mkdir(parents=True, exist_ok=True)
+            data_part = self.download_dataset_part(
+                organization_id=self.configuration.cosmotech.organization_id,
+                workspace_id=self.configuration.cosmotech.workspace_id,
+                dataset_id=dataset_id,
+                dataset_part_id=part.id,
+            )
+            with open(part_file_path, "wb") as binary_file:
+                binary_file.write(data_part)
+            LOGGER.debug(
+                T("coal.services.dataset.part_downloaded").format(part_name=part.source_name, file_path=part_file_path)
+            )
+        return dataset
+
+    @staticmethod
+    def path_to_parts(_path, part_type) -> list[tuple[str, Path, DatasetPartTypeEnum]]:
+        if (_path := Path(_path)).is_dir():
+            return list((str(_p.relative_to(_path)), _p, part_type) for _p in _path.rglob("*") if _p.is_file())
+        return list(((_path.name, _path, part_type),))
+
+    def upload_dataset(
+        self,
+        dataset_name: str,
+        as_files: Optional[list[Union[Path, str]]] = (),
+        as_db: Optional[list[Union[Path, str]]] = (),
+    ) -> Dataset:
+        _parts = list()
+
+        for _f in as_files:
+            _parts.extend(self.path_to_parts(_f, DatasetPartTypeEnum.FILE))
+
+        for _db in as_db:
+            _parts.extend(self.path_to_parts(_db, DatasetPartTypeEnum.DB))
+
+        d_request = DatasetCreateRequest(
+            name=dataset_name,
+            parts=list(
+                DatasetPartCreateRequest(
+                    name=_p_name,
+                    description=_p_name,
+                    sourceName=_p_name,
+                    type=_type,
+                )
+                for _p_name, _, _type in _parts
+            ),
+        )
+
+        d_ret = self.create_dataset(
+            self.configuration.cosmotech.organization_id,
+            self.configuration.cosmotech.workspace_id,
+            d_request,
+            files=list((_p[0], _p[1].open("rb").read()) for _p in _parts),
+        )
+
+        LOGGER.info(T("coal.services.dataset.dataset_created").format(dataset_id=d_ret.id))
+        return d_ret