cosmotech-acceleration-library 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

cosmotech/coal/__init__.py

@@ -5,4 +5,4 @@
 # etc., to any person is prohibited unless it has been previously and
 # specifically authorized by written means by Cosmo Tech.
 
-__version__ = "1.0.1"
+__version__ = "1.1.0"

cosmotech/coal/cosmotech_api/__init__.py

@@ -16,12 +16,16 @@ from cosmotech.coal.cosmotech_api.parameters import (
     write_parameters,
 )
 
-# Re-export functions from the twin_data_layer module
-from cosmotech.coal.cosmotech_api.twin_data_layer import (
-    get_dataset_id_from_runner,
-    send_files_to_tdl,
-    load_files_from_tdl,
-)
+from cosmotech.coal.utils.semver import semver_of
+
+csm_version = semver_of("cosmotech_api")
+if csm_version.major < 5:
+    # Re-export functions from the twin_data_layer module
+    from cosmotech.coal.cosmotech_api.twin_data_layer import (
+        get_dataset_id_from_runner,
+        send_files_to_tdl,
+        load_files_from_tdl,
+    )
 
 # Re-export functions from the run_data module
 from cosmotech.coal.cosmotech_api.run_data import (

cosmotech/coal/cosmotech_api/dataset/download/file.py

@@ -12,16 +12,130 @@ import os
 import tempfile
 import time
 from pathlib import Path
-from typing import Dict, List, Any, Optional, Union, Tuple
+from typing import Dict, Any, Optional, Union, Tuple
 
 from cosmotech_api import WorkspaceApi
 from openpyxl import load_workbook
 
+from cosmotech.coal.utils.decorator import timed
 from cosmotech.coal.utils.logger import LOGGER
 from cosmotech.orchestrator.utils.translate import T
 from cosmotech.coal.cosmotech_api.connection import get_api_client
 
 
+def process_xls(target_file) -> Dict[str, Any]:
+    content = {}
+
+    LOGGER.debug(T("coal.services.dataset.processing_excel").format(file_name=target_file))
+    wb = load_workbook(target_file, data_only=True)
+
+    for sheet_name in wb.sheetnames:
+        sheet = wb[sheet_name]
+        content[sheet_name] = list()
+        headers = next(sheet.iter_rows(max_row=1, values_only=True))
+
+        row_count = 0
+        for r in sheet.iter_rows(min_row=2, values_only=True):
+            row = {k: v for k, v in zip(headers, r)}
+            new_row = dict()
+
+            for key, value in row.items():
+                try:
+                    converted_value = json.load(io.StringIO(value))
+                except (json.decoder.JSONDecodeError, TypeError):
+                    converted_value = value
+
+                if converted_value is not None:
+                    new_row[key] = converted_value
+
+            if new_row:
+                content[sheet_name].append(new_row)
+                row_count += 1
+
+        LOGGER.debug(T("coal.services.dataset.sheet_processed").format(sheet_name=sheet_name, rows=row_count))
+    return content
+
+
+def process_csv(target_file) -> Dict[str, Any]:
+    content = {}
+
+    LOGGER.debug(T("coal.services.dataset.processing_csv").format(file_name=target_file))
+    with open(target_file, "r") as file:
+        current_filename = os.path.basename(target_file)[: -len(".csv")]
+        content[current_filename] = list()
+
+        row_count = 0
+        for csv_row in csv.DictReader(file):
+            csv_row: dict
+            new_row = dict()
+
+            for key, value in csv_row.items():
+                try:
+                    # Try to convert any json row to dict object
+                    converted_value = json.load(io.StringIO(value))
+                except json.decoder.JSONDecodeError:
+                    converted_value = value
+
+                if converted_value == "":
+                    converted_value = None
+
+                if converted_value is not None:
+                    new_row[key] = converted_value
+
+            content[current_filename].append(new_row)
+            row_count += 1
+
+        LOGGER.debug(T("coal.services.dataset.csv_processed").format(file_name=current_filename, rows=row_count))
+    return content
+
+
+def process_json(target_file) -> Dict[str, Any]:
+    content = {}
+    LOGGER.debug(T("coal.services.dataset.processing_json").format(file_name=target_file))
+    with open(target_file, "r") as _file:
+        current_filename = os.path.basename(target_file)
+        content[current_filename] = json.load(_file)
+
+        if isinstance(content[current_filename], dict):
+            item_count = len(content[current_filename])
+        elif isinstance(content[current_filename], list):
+            item_count = len(content[current_filename])
+        else:
+            item_count = 1
+
+        LOGGER.debug(T("coal.services.dataset.json_processed").format(file_name=current_filename, items=item_count))
+    return content
+
+
+def process_txt(target_file) -> Dict[str, Any]:
+    content = {}
+    LOGGER.debug(T("coal.services.dataset.processing_text").format(file_name=target_file))
+    with open(target_file, "r") as _file:
+        current_filename = os.path.basename(target_file)
+        content[current_filename] = _file.read()
+
+        line_count = content[current_filename].count("\n") + 1
+        LOGGER.debug(T("coal.services.dataset.text_processed").format(file_name=current_filename, lines=line_count))
+    return content
+
+
+def read_file(file_name, file):
+    @timed(f"process {file_name}", debug=True)
+    def timed_read_file(file_name, file):
+        content = {}
+        if ".xls" in file_name:
+            content.update(process_xls(file))
+        elif ".csv" in file_name:
+            content.update(process_csv(file))
+        elif ".json" in file_name:
+            content.update(process_json(file))
+        else:
+            content.update(process_txt(file))
+        return content
+
+    return timed_read_file(file_name, file)
+
+
 def download_file_dataset(
     organization_id: str,
     workspace_id: str,
@@ -105,109 +219,8 @@ def download_file_dataset(
                 )
             )
 
-            if not read_files:
-                continue
-
-            # Process file based on type
-            process_start = time.time()
-
-            if ".xls" in _file_name:
-                LOGGER.debug(T("coal.services.dataset.processing_excel").format(file_name=target_file))
-                wb = load_workbook(target_file, data_only=True)
-
-                for sheet_name in wb.sheetnames:
-                    sheet = wb[sheet_name]
-                    content[sheet_name] = list()
-                    headers = next(sheet.iter_rows(max_row=1, values_only=True))
-
-                    def item(_row: tuple) -> dict:
-                        return {k: v for k, v in zip(headers, _row)}
-
-                    row_count = 0
-                    for r in sheet.iter_rows(min_row=2, values_only=True):
-                        row = item(r)
-                        new_row = dict()
-
-                        for key, value in row.items():
-                            try:
-                                converted_value = json.load(io.StringIO(value))
-                            except (json.decoder.JSONDecodeError, TypeError):
-                                converted_value = value
-
-                            if converted_value is not None:
-                                new_row[key] = converted_value
-
-                        if new_row:
-                            content[sheet_name].append(new_row)
-                            row_count += 1
-
-                    LOGGER.debug(
-                        T("coal.services.dataset.sheet_processed").format(sheet_name=sheet_name, rows=row_count)
-                    )
-
-            elif ".csv" in _file_name:
-                LOGGER.debug(T("coal.services.dataset.processing_csv").format(file_name=target_file))
-                with open(target_file, "r") as file:
-                    current_filename = os.path.basename(target_file)[: -len(".csv")]
-                    content[current_filename] = list()
-
-                    row_count = 0
-                    for csv_row in csv.DictReader(file):
-                        csv_row: dict
-                        new_row = dict()
-
-                        for key, value in csv_row.items():
-                            try:
-                                # Try to convert any json row to dict object
-                                converted_value = json.load(io.StringIO(value))
-                            except json.decoder.JSONDecodeError:
-                                converted_value = value
-
-                            if converted_value == "":
-                                converted_value = None
-
-                            if converted_value is not None:
-                                new_row[key] = converted_value
-
-                        content[current_filename].append(new_row)
-                        row_count += 1
-
-                    LOGGER.debug(
-                        T("coal.services.dataset.csv_processed").format(file_name=current_filename, rows=row_count)
-                    )
-
-            elif ".json" in _file_name:
-                LOGGER.debug(T("coal.services.dataset.processing_json").format(file_name=target_file))
-                with open(target_file, "r") as _file:
-                    current_filename = os.path.basename(target_file)
-                    content[current_filename] = json.load(_file)
-
-                    if isinstance(content[current_filename], dict):
-                        item_count = len(content[current_filename])
-                    elif isinstance(content[current_filename], list):
-                        item_count = len(content[current_filename])
-                    else:
-                        item_count = 1
-
-                    LOGGER.debug(
-                        T("coal.services.dataset.json_processed").format(file_name=current_filename, items=item_count)
-                    )
-
-            else:
-                LOGGER.debug(T("coal.services.dataset.processing_text").format(file_name=target_file))
-                with open(target_file, "r") as _file:
-                    current_filename = os.path.basename(target_file)
-                    content[current_filename] = "\n".join(line for line in _file)
-
-                    line_count = content[current_filename].count("\n") + 1
-                    LOGGER.debug(
-                        T("coal.services.dataset.text_processed").format(file_name=current_filename, lines=line_count)
-                    )
-
-            process_time = time.time() - process_start
-            LOGGER.debug(
-                T("coal.common.timing.operation_completed").format(operation=f"process {_file_name}", time=process_time)
-            )
+            if read_files:
+                content.update(read_file(_file_name, target_file))
 
     elapsed_time = time.time() - start_time
     LOGGER.info(T("coal.common.timing.operation_completed").format(operation="File download", time=elapsed_time))

cosmotech/coal/cosmotech_api/dataset/download/twingraph.py

@@ -8,14 +8,9 @@
 import time
 import tempfile
 from pathlib import Path
-from typing import Dict, List, Any, Optional, Union, Tuple
+from typing import Dict, Any, Optional, Union, Tuple
 
-from cosmotech_api import (
-    DatasetApi,
-    DatasetTwinGraphQuery,
-    TwinGraphQuery,
-    TwingraphApi,
-)
+import cosmotech_api
 
 from cosmotech.coal.utils.logger import LOGGER
 from cosmotech.orchestrator.utils.translate import T
@@ -47,12 +42,12 @@ def download_twingraph_dataset(
     )
 
     with get_api_client()[0] as api_client:
-        dataset_api = DatasetApi(api_client)
+        dataset_api = cosmotech_api.DatasetApi(api_client)
 
         # Query nodes
         nodes_start = time.time()
         LOGGER.debug(T("coal.services.dataset.twingraph_querying_nodes").format(dataset_id=dataset_id))
-        nodes_query = DatasetTwinGraphQuery(query="MATCH(n) RETURN n")
+        nodes_query = cosmotech_api.DatasetTwinGraphQuery(query="MATCH(n) RETURN n")
 
         nodes = dataset_api.twingraph_query(
             organization_id=organization_id,
@@ -67,7 +62,9 @@ def download_twingraph_dataset(
         # Query edges
         edges_start = time.time()
         LOGGER.debug(T("coal.services.dataset.twingraph_querying_edges").format(dataset_id=dataset_id))
-        edges_query = DatasetTwinGraphQuery(query="MATCH(n)-[r]->(m) RETURN n as src, r as rel, m as dest")
+        edges_query = cosmotech_api.DatasetTwinGraphQuery(
+            query="MATCH(n)-[r]->(m) RETURN n as src, r as rel, m as dest"
+        )
 
         edges = dataset_api.twingraph_query(
             organization_id=organization_id,
@@ -129,12 +126,12 @@ def download_legacy_twingraph_dataset(
     )
 
     with get_api_client()[0] as api_client:
-        api_instance = TwingraphApi(api_client)
+        api_instance = cosmotech_api.TwingraphApi(api_client)
 
         # Query nodes
         nodes_start = time.time()
         LOGGER.debug(T("coal.services.dataset.legacy_twingraph_querying_nodes").format(cache_name=cache_name))
-        _query_nodes = TwinGraphQuery(query="MATCH(n) RETURN n")
+        _query_nodes = cosmotech_api.TwinGraphQuery(query="MATCH(n) RETURN n")
 
         nodes = api_instance.query(
             organization_id=organization_id,
@@ -149,7 +146,7 @@ def download_legacy_twingraph_dataset(
         # Query relationships
         rel_start = time.time()
         LOGGER.debug(T("coal.services.dataset.legacy_twingraph_querying_relations").format(cache_name=cache_name))
-        _query_rel = TwinGraphQuery(query="MATCH(n)-[r]->(m) RETURN n as src, r as rel, m as dest")
+        _query_rel = cosmotech_api.TwinGraphQuery(query="MATCH(n)-[r]->(m) RETURN n as src, r as rel, m as dest")
 
         rel = api_instance.query(
             organization_id=organization_id,

cosmotech/coal/cosmotech_api/dataset/upload.py

@@ -0,0 +1,41 @@
+import pathlib
+
+from cosmotech_api import Dataset
+from cosmotech_api import DatasetPartTypeEnum
+from cosmotech_api.api.dataset_api import DatasetApi
+from cosmotech_api.api.dataset_api import DatasetCreateRequest
+from cosmotech_api.api.dataset_api import DatasetPartCreateRequest
+import pprint
+
+from cosmotech.coal.cosmotech_api.connection import get_api_client
+from cosmotech.coal.utils.logger import LOGGER
+
+LOGGER.info("Generating dataset content")
+
+
+def upload_dataset(organization_id, workspace_id, dataset_name, dataset_dir) -> Dataset:
+    dataset_path = pathlib.Path(dataset_dir)
+
+    with get_api_client()[0] as client:
+        d_api = DatasetApi(client)
+        _files = list(_p for _p in dataset_path.rglob("*") if _p.is_file())
+        d_request = DatasetCreateRequest(
+            name=dataset_name,
+            parts=list(
+                DatasetPartCreateRequest(
+                    name=_p.name,
+                    description=str(_p.relative_to(dataset_path)),
+                    sourceName=str(_p.relative_to(dataset_path)),
+                    type=DatasetPartTypeEnum.FILE,
+                )
+                for _p in _files
+            ),
+        )
+        pprint.pprint(d_request.to_dict())
+        d_ret = d_api.create_dataset(
+            organization_id,
+            workspace_id,
+            d_request,
+            files=list((str(_p.relative_to(dataset_path)), _p.open("rb").read()) for _p in _files),
+        )
+    return d_ret

cosmotech/coal/cosmotech_api/runner/datasets.py

@@ -12,7 +12,7 @@ Dataset handling functions.
 import multiprocessing
 import tempfile
 from pathlib import Path
-from typing import Dict, List, Any, Optional, Union, Tuple
+from typing import Dict, List, Any, Optional, Union
 
 from azure.identity import DefaultAzureCredential
 from cosmotech_api.api.dataset_api import DatasetApi
@@ -25,6 +25,7 @@ from cosmotech.coal.cosmotech_api.dataset import (
     download_legacy_twingraph_dataset,
     download_file_dataset,
 )
+from cosmotech.coal.cosmotech_api.dataset.download import file
 from cosmotech.coal.utils.logger import LOGGER
 from cosmotech.orchestrator.utils.translate import T
 
@@ -54,7 +55,72 @@ def download_dataset(
     workspace_id: str,
     dataset_id: str,
     read_files: bool = True,
-    credentials: Optional[DefaultAzureCredential] = None,
+) -> Dict[str, Any]:
+    """
+    retro-compatibility to cosmo-api v4
+    """
+    from cosmotech.coal.utils.semver import semver_of
+
+    csm_version = semver_of("cosmotech_api")
+    if csm_version.major >= 5:
+        return download_dataset_v5(organization_id, workspace_id, dataset_id, read_files)
+    else:
+        return download_dataset_v4(organization_id, workspace_id, dataset_id, read_files)
+
+
+def download_dataset_v5(
+    organization_id: str,
+    workspace_id: str,
+    dataset_id: str,
+    read_files: bool = True,
+) -> Dict[str, Any]:
+    """
+    Download a single dataset by ID.
+
+    Args:
+        organization_id: Organization ID
+        workspace_id: Workspace ID
+        dataset_id: Dataset ID
+        read_files: Whether to read file contents
+
+    Returns:
+        Dataset information dictionary
+    """
+
+    # Get dataset information
+    with get_api_client()[0] as api_client:
+        dataset_api_instance = DatasetApi(api_client)
+        dataset = dataset_api_instance.get_dataset(
+            organization_id=organization_id, workspace_id=workspace_id, dataset_id=dataset_id
+        )
+
+        content = dict()
+        tmp_dataset_dir = tempfile.mkdtemp()
+        tmp_dataset_dir_path = Path(tmp_dataset_dir)
+        for part in dataset.parts:
+            part_file_path = tmp_dataset_dir_path / part.source_name
+            part_file_path.parent.mkdir(parents=True, exist_ok=True)
+            data_part = dataset_api_instance.download_dataset_part(organization_id, workspace_id, dataset_id, part.id)
+            with open(part_file_path, "wb") as binary_file:
+                binary_file.write(data_part)
+
+            if read_files:
+                content.update(file.read_file(part.source_name, part_file_path))
+
+        return {
+            "type": "csm_dataset",
+            "content": content,
+            "name": dataset.name,
+            "folder_path": tmp_dataset_dir,
+            "dataset_id": dataset_id,
+        }
+
+
+def download_dataset_v4(
+    organization_id: str,
+    workspace_id: str,
+    dataset_id: str,
+    read_files: bool = True,
 ) -> Dict[str, Any]:
     """
     Download a single dataset by ID.
@@ -64,7 +130,6 @@ def download_dataset(
         workspace_id: Workspace ID
         dataset_id: Dataset ID
         read_files: Whether to read file contents
-        credentials: Azure credentials (if None, uses DefaultAzureCredential if needed)
 
     Returns:
         Dataset information dictionary
@@ -91,7 +156,7 @@ def download_dataset(
         if is_adt:
             content, folder_path = download_adt_dataset(
                 adt_address=parameters["AZURE_DIGITAL_TWINS_URL"],
-                credentials=credentials,
+                credentials=DefaultAzureCredential(),
             )
             return {
                 "type": "adt",
@@ -159,9 +224,7 @@ def download_dataset(
             }
 
 
-def download_dataset_process(
-    _dataset_id, organization_id, workspace_id, read_files, credentials, _return_dict, _error_dict
-):
+def download_dataset_process(_dataset_id, organization_id, workspace_id, read_files, _return_dict, _error_dict):
     """
     Process function for downloading a dataset in a separate process.
 
@@ -174,7 +237,6 @@ def download_dataset_process(
         organization_id: Organization ID
         workspace_id: Workspace ID
         read_files: Whether to read file contents
-        credentials: Azure credentials (if None, uses DefaultAzureCredential if needed)
         _return_dict: Shared dictionary to store successful download results
         _error_dict: Shared dictionary to store error messages
 
@@ -187,7 +249,6 @@ def download_dataset_process(
             workspace_id=workspace_id,
             dataset_id=_dataset_id,
             read_files=read_files,
-            credentials=credentials,
         )
         _return_dict[_dataset_id] = _c
     except Exception as e:
@@ -200,7 +261,6 @@ def download_datasets_parallel(
     workspace_id: str,
     dataset_ids: List[str],
     read_files: bool = True,
-    credentials: Optional[DefaultAzureCredential] = None,
 ) -> Dict[str, Dict[str, Any]]:
     """
     Download multiple datasets in parallel.
@@ -210,7 +270,6 @@ def download_datasets_parallel(
         workspace_id: Workspace ID
         dataset_ids: List of dataset IDs
         read_files: Whether to read file contents
-        credentials: Azure credentials (if None, uses DefaultAzureCredential if needed)
 
     Returns:
         Dictionary mapping dataset IDs to dataset information
@@ -225,7 +284,7 @@ def download_datasets_parallel(
             dataset_id,
             multiprocessing.Process(
                 target=download_dataset_process,
-                args=(dataset_id, organization_id, workspace_id, read_files, credentials, return_dict, error_dict),
+                args=(dataset_id, organization_id, workspace_id, read_files, return_dict, error_dict),
             ),
         )
         for dataset_id in dataset_ids
@@ -251,7 +310,6 @@ def download_datasets_sequential(
     workspace_id: str,
     dataset_ids: List[str],
     read_files: bool = True,
-    credentials: Optional[DefaultAzureCredential] = None,
 ) -> Dict[str, Dict[str, Any]]:
     """
     Download multiple datasets sequentially.
@@ -261,7 +319,6 @@ def download_datasets_sequential(
         workspace_id: Workspace ID
         dataset_ids: List of dataset IDs
         read_files: Whether to read file contents
-        credentials: Azure credentials (if None, uses DefaultAzureCredential if needed)
 
     Returns:
         Dictionary mapping dataset IDs to dataset information
@@ -279,7 +336,6 @@ def download_datasets_sequential(
                 workspace_id=workspace_id,
                 dataset_id=dataset_id,
                 read_files=read_files,
-                credentials=credentials,
             )
         except Exception as e:
             error_dict[dataset_id] = f"{type(e).__name__}: {str(e)}"
@@ -294,7 +350,6 @@ def download_datasets(
     dataset_ids: List[str],
     read_files: bool = True,
     parallel: bool = True,
-    credentials: Optional[DefaultAzureCredential] = None,
 ) -> Dict[str, Dict[str, Any]]:
     """
     Download multiple datasets, either in parallel or sequentially.
@@ -305,7 +360,6 @@ def download_datasets(
         dataset_ids: List of dataset IDs
         read_files: Whether to read file contents
         parallel: Whether to download in parallel
-        credentials: Azure credentials (if None, uses DefaultAzureCredential if needed)
 
     Returns:
         Dictionary mapping dataset IDs to dataset information
@@ -319,7 +373,6 @@ def download_datasets(
             workspace_id=workspace_id,
             dataset_ids=dataset_ids,
             read_files=read_files,
-            credentials=credentials,
         )
     else:
         return download_datasets_sequential(
@@ -327,7 +380,6 @@ def download_datasets(
             workspace_id=workspace_id,
             dataset_ids=dataset_ids,
             read_files=read_files,
-            credentials=credentials,
         )
 
 

cosmotech/coal/cosmotech_api/runner/download.py

@@ -12,13 +12,8 @@ Orchestration functions for downloading runner and run data.
 import os
 import pathlib
 import shutil
-from typing import Dict, List, Any, Optional
+from typing import Dict, Any, Optional
 
-from azure.identity import DefaultAzureCredential
-from cosmotech_api.api.runner_api import RunnerApi
-from cosmotech_api.exceptions import ApiException
-
-from cosmotech.coal.cosmotech_api.connection import get_api_client
 from cosmotech.coal.cosmotech_api.runner.data import get_runner_data
 from cosmotech.coal.cosmotech_api.runner.parameters import (
     format_parameters_list,
@@ -65,11 +60,6 @@ def download_runner_data(
     """
     LOGGER.info(T("coal.cosmotech_api.runner.starting_download"))
 
-    # Get credentials if needed
-    credentials = None
-    if get_api_client()[1] == "Azure Entra Connection":
-        credentials = DefaultAzureCredential()
-
     # Get runner data
     runner_data = get_runner_data(organization_id, workspace_id, runner_id)
 
@@ -100,19 +90,18 @@ def download_runner_data(
                 dataset_ids=dataset_ids,
                 read_files=read_files,
                 parallel=parallel,
-                credentials=credentials,
             )
 
             result["datasets"] = datasets
 
-            # Process datasets
+            # List datasets set as parameter
             datasets_parameters_ids = {
                 param.value: param.parameter_id
                 for param in runner_data.parameters_values
                 if param.var_type == "%DATASETID%" and param.value
             }
 
-            # Save datasets to parameter folders
+            # Save parameter datasets to parameter folders
             for dataset_id, dataset_info in datasets.items():
                 # If dataset is referenced by a parameter, save to parameter folder
                 if dataset_id in datasets_parameters_ids:

cosmotech/coal/utils/decorator.py

@@ -0,0 +1,25 @@
+import time
+
+from functools import wraps
+
+from cosmotech.coal.utils.logger import LOGGER
+from cosmotech.orchestrator.utils.translate import T
+
+
+def timed(operation, debug=False):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            process_start = time.time()
+            r = func(*args, **kwargs)
+            process_time = time.time() - process_start
+            msg = T("coal.common.timing.operation_completed").format(operation=operation, time=process_time)
+            if debug:
+                LOGGER.debug(msg)
+            else:
+                LOGGER.info(msg)
+            return r
+
+        return wrapper
+
+    return decorator

cosmotech/coal/utils/semver.py

@@ -0,0 +1,6 @@
+from packaging.version import Version
+import importlib.metadata
+
+
+def semver_of(package: str) -> Version:
+    return Version(importlib.metadata.version(package))

cosmotech/csm_data/commands/api/wsf_load_file.py

@@ -8,9 +8,6 @@ import pathlib
 
 from cosmotech.csm_data.utils.click import click
 from cosmotech.csm_data.utils.decorators import web_help, translate_help
-from cosmotech.coal.cosmotech_api.connection import get_api_client
-from cosmotech.coal.cosmotech_api.workspace import download_workspace_file
-from cosmotech.coal.cosmotech_api.workspace import list_workspace_files
 from cosmotech.orchestrator.utils.translate import T
 
 
@@ -53,6 +50,11 @@ from cosmotech.orchestrator.utils.translate import T
 @web_help("csm-data/api/wsf-load-file")
 @translate_help("csm_data.commands.api.wsf_load_file.description")
 def wsf_load_file(organization_id, workspace_id, workspace_path: str, target_folder: str):
+
+    from cosmotech.coal.cosmotech_api.connection import get_api_client
+    from cosmotech.coal.cosmotech_api.workspace import download_workspace_file
+    from cosmotech.coal.cosmotech_api.workspace import list_workspace_files
+
     with get_api_client()[0] as api_client:
         target_list = list_workspace_files(api_client, organization_id, workspace_id, workspace_path)
 

cosmotech/csm_data/commands/api/wsf_send_file.py

@@ -7,8 +7,6 @@
 
 from cosmotech.csm_data.utils.click import click
 from cosmotech.csm_data.utils.decorators import web_help, translate_help
-from cosmotech.coal.cosmotech_api.connection import get_api_client
-from cosmotech.coal.cosmotech_api.workspace import upload_workspace_file
 from cosmotech.orchestrator.utils.translate import T
 
 
@@ -57,6 +55,10 @@ from cosmotech.orchestrator.utils.translate import T
 @web_help("csm-data/api/wsf-send-file")
 @translate_help("csm_data.commands.api.wsf_send_file.description")
 def wsf_send_file(organization_id, workspace_id, file_path, workspace_path: str, overwrite: bool):
+
+    from cosmotech.coal.cosmotech_api.connection import get_api_client
+    from cosmotech.coal.cosmotech_api.workspace import upload_workspace_file
+
     with get_api_client()[0] as api_client:
         upload_workspace_file(
             api_client,

cosmotech/csm_data/commands/store/store.py

@@ -17,7 +17,6 @@ from cosmotech.csm_data.commands.store.load_from_singlestore import (
 from cosmotech.csm_data.commands.store.reset import reset
 from cosmotech.csm_data.utils.click import click
 from cosmotech.csm_data.utils.decorators import web_help, translate_help
-from cosmotech.orchestrator.utils.translate import T
 
 
 @click.group()

{cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cosmotech_acceleration_library
-Version: 1.0.1
+Version: 1.1.0
 Summary: Acceleration libraries for CosmoTech cloud based solution development
 Author-email: Cosmo Tech <platform@cosmotech.com>
 Project-URL: Homepage, https://www.cosmotech.com
@@ -15,15 +15,15 @@ Requires-Dist: azure-kusto-data~=4.4.1
 Requires-Dist: azure-kusto-ingest~=4.4.1
 Requires-Dist: tenacity~=8.3.0
 Requires-Dist: python-keycloak~=4.7.3
-Requires-Dist: cosmotech-api~=3.2
+Requires-Dist: cosmotech-api>=3.2
 Requires-Dist: boto3~=1.35.19
 Requires-Dist: requests~=2.32.3
 Requires-Dist: singlestoredb~=1.10.0
 Requires-Dist: cosmotech-run-orchestrator~=2.0.0
 Requires-Dist: pyarrow~=20.0.0
-Requires-Dist: adbc-driver-manager~=1.1.0
-Requires-Dist: adbc-driver-sqlite~=1.1.0
-Requires-Dist: adbc-driver-postgresql~=1.1.0
+Requires-Dist: adbc-driver-manager~=1.7.0
+Requires-Dist: adbc-driver-sqlite~=1.7.0
+Requires-Dist: adbc-driver-postgresql~=1.7.0
 Requires-Dist: click~=8.1.7
 Requires-Dist: rich-click~=1.7.3
 Requires-Dist: click-log~=0.4.0
@@ -58,6 +58,8 @@ Requires-Dist: black~=24.3.0; extra == "dev"
 Requires-Dist: pre-commit~=3.3.2; extra == "dev"
 Provides-Extra: all
 Requires-Dist: CosmoTech_Acceleration_Library[dev,doc,extra,test]; extra == "all"
+Provides-Extra: past
+Requires-Dist: cosmotech-api~=3.2; extra == "past"
 Dynamic: license-file
 
 # CosmoTech-Acceleration-Library (CoAL)

{cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-1.1.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-cosmotech/coal/__init__.py,sha256=zO8tP0wjAUWBIzrSAjLECJhE7uA6CXq8k8i15ZII8SQ,436
+cosmotech/coal/__init__.py,sha256=fEmLxOs06MCBlBFoKsBUQ_ZE5aLJgmdNfzVP0Xx_Qds,436
 cosmotech/coal/aws/__init__.py,sha256=DtkCZ8Y7q5HIKui9YXCV1JXO8rRMitd0uieiW2InuXY,761
 cosmotech/coal/aws/s3.py,sha256=oq1Sl_6kk_u0Jai55p7XRfKYAK0I6y3va4IfeWNhpEE,7820
 cosmotech/coal/azure/__init__.py,sha256=1SmwP1y7quGq9cWqiWxJy1XDjWM8sAc3f63ztBnYujQ,796
@@ -13,7 +13,7 @@ cosmotech/coal/azure/adx/runner.py,sha256=FDrVaf-KDqHunPIQ7nUt57_-9pLPWXleNREgdH
 cosmotech/coal/azure/adx/store.py,sha256=IFQIi4tCmxL0NyFJhyJnUqbTt3Vwgb77900xcKprGzY,9096
 cosmotech/coal/azure/adx/tables.py,sha256=hwmiQI8OccNf3Npf8jd2wOF5hL8I2s4IkgTx8jqtSCU,4292
 cosmotech/coal/azure/adx/utils.py,sha256=nOq4WLTMDs95da-jsWJVYcv2MRcwUMvARv5Z0ivIVUo,2205
-cosmotech/coal/cosmotech_api/__init__.py,sha256=sXA-WNRBWaXZdTwOR0-sD_HOkYvDaZYoHrzBz_o5-d0,1178
+cosmotech/coal/cosmotech_api/__init__.py,sha256=ufYjA_WF0CNpU63jGlPqtY1kWiLu6Qj7TNVHbRk8n0A,1320
 cosmotech/coal/cosmotech_api/connection.py,sha256=dMI0vSv7ic1hEgtsIN92t008QI0bQrq8kmbMgqg3EF4,3953
 cosmotech/coal/cosmotech_api/parameters.py,sha256=2_ynoXfyK2pUtuZsYEamOKwfs0N6Kw2s-YE4zuPKkgM,1892
 cosmotech/coal/cosmotech_api/run.py,sha256=6Fnd3aUbWg4S-KqVG0abRfnorlNzGHY89AkheKNHprs,1001
@@ -23,16 +23,17 @@ cosmotech/coal/cosmotech_api/twin_data_layer.py,sha256=haJPx0hE84Q5pwEZlXlnrB-3u
 cosmotech/coal/cosmotech_api/workspace.py,sha256=0qPf8h23z57AUVxRoSxD9lW_dSvfsq_pawcqFYaaaok,5488
 cosmotech/coal/cosmotech_api/dataset/__init__.py,sha256=UqRrDC2AFWImJu-wTAIzj7hIhc826n_57kTtBzCDWp0,972
 cosmotech/coal/cosmotech_api/dataset/converters.py,sha256=uTqaEzXMvIH7rTBqQA5if2GML32nsiohEP3eFIVPmLc,6151
+cosmotech/coal/cosmotech_api/dataset/upload.py,sha256=Fmwx9zg5FPQcSd72YXeClB6ZCCi6IUlkAgGVs11Fvn4,1458
 cosmotech/coal/cosmotech_api/dataset/utils.py,sha256=Pb6Lh3WNZHvbM8P6RdYjVQY8Ymfcxp35YmbBX_KKdWM,4438
 cosmotech/coal/cosmotech_api/dataset/download/__init__.py,sha256=kzIe46X9cmdg83y4JBjqAasY624BTXNL1yCADQkoyR4,886
 cosmotech/coal/cosmotech_api/dataset/download/adt.py,sha256=PwfkSYTCizkE81Huh5J8BShOXQnrjUvebRsgT-nYZpw,4731
 cosmotech/coal/cosmotech_api/dataset/download/common.py,sha256=KuVEzOA_abAwtwbcdry8zmqXAuQvy-NGeg1Ot9_MMuU,5665
-cosmotech/coal/cosmotech_api/dataset/download/file.py,sha256=X4ZNVjzl9ZC8J02CjRQeptZT6FPYBuxPsw-XPssP8YE,8914
-cosmotech/coal/cosmotech_api/dataset/download/twingraph.py,sha256=iHDvsm7odQQ5gyT7a3LcvcdNA6nHTIaWJUCFb9ryjr0,7373
+cosmotech/coal/cosmotech_api/dataset/download/file.py,sha256=F_Y-O8W0VSXjDkmE9qEaM_fO7FRN_6jCND5dyCZpGz4,8410
+cosmotech/coal/cosmotech_api/dataset/download/twingraph.py,sha256=xumPNAGcB3t0ZMztvuUUnSmxoB9t-WUF3lqX8fEtWrY,7383
 cosmotech/coal/cosmotech_api/runner/__init__.py,sha256=hEfmv6nF0hxSWMOnWZF0Fy7PJtHexVf-_cXPsxoY7Ho,1081
 cosmotech/coal/cosmotech_api/runner/data.py,sha256=BtCRi_c_3aGUSBjaPt5MOajWO6WFsJdBrIrx4GMt1AE,1346
-cosmotech/coal/cosmotech_api/runner/datasets.py,sha256=ojDkHGrobsygLr2CRYROetqm9zeWv1S1Bc_4Az4z874,12271
-cosmotech/coal/cosmotech_api/runner/download.py,sha256=YZRmhsX9TVjvhtgFhY4uvsJ3If0G43OiiERkCdoOKHE,5655
+cosmotech/coal/cosmotech_api/runner/datasets.py,sha256=_jrjYGepVfjL5iCNoqg29tKNwweupfbYCUwZkNE_gxo,13555
+cosmotech/coal/cosmotech_api/runner/download.py,sha256=G02poKQrKpYQdR_JjKesINjWeqZ95uUyl-TDulWwSN8,5254
 cosmotech/coal/cosmotech_api/runner/metadata.py,sha256=eZHtlVMqsQ6WO9F-mDparjIy7tFl7kU-OqguKaI44jE,1432
 cosmotech/coal/cosmotech_api/runner/parameters.py,sha256=mKRRBPZOD_tPOTyxUAL2o9nRy_KZ5Z687MkL2uhqvIc,4651
 cosmotech/coal/csm/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
@@ -49,9 +50,10 @@ cosmotech/coal/store/pandas.py,sha256=A2eWPDsAqI2fTNmIpJa0Qzwrh3vMfvSHPLuoxQ77OP
 cosmotech/coal/store/pyarrow.py,sha256=i7f7vSd5BtOnR4oleRHTG3ff0o8NaJr4c9fRb7zgYmE,814
 cosmotech/coal/store/store.py,sha256=a9wSThk7K8tD1r_BJG1Zl2d4pfo23TqKUH82_u9H-tk,3353
 cosmotech/coal/utils/__init__.py,sha256=N2mPhli0Wj4OesuRVbt2KrzL_pgMELs4fDNIQYb_r-g,830
-cosmotech/coal/utils/api.py,sha256=OhuATEWIih4N0t6LH6dsLZHTMkV-Ko7KAk8tyKDxYfo,2777
+cosmotech/coal/utils/decorator.py,sha256=j8MGkm4tgYVmBji0Veug4uq3Ed-f4OKe9XANs47_EG0,679
 cosmotech/coal/utils/logger.py,sha256=oYVj2BtDPji4e4J-KgaDyVcbKiewkq0xey_gQPm74Xc,506
 cosmotech/coal/utils/postgresql.py,sha256=6gxk3yfkZkaJQbIG-Zxcwo7J3PI29XPLn0VdQimGSH0,8562
+cosmotech/coal/utils/semver.py,sha256=VTjixXNBeUe6A02AK6olNxeqToOCWcKih6VZun09E7A,162
 cosmotech/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
 cosmotech/csm_data/main.py,sha256=q20Gl1x7DV0gWStnWJUVnhUrvv-TPrUhv3e-5X3TDEM,2424
 cosmotech/csm_data/commands/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
@@ -71,8 +73,8 @@ cosmotech/csm_data/commands/api/run_load_data.py,sha256=oRnoIj2EXPG8Q2fcvrzt69eX
 cosmotech/csm_data/commands/api/runtemplate_load_handler.py,sha256=qWS0I1fNN6VFx7lHJoOWCv30WGI2q32B9_Zyr5KpBSM,2275
 cosmotech/csm_data/commands/api/tdl_load_files.py,sha256=O4FsRNutyEvdDUBYElLxrzEhJcXVyO9hpOdiM6FEQ5E,2394
 cosmotech/csm_data/commands/api/tdl_send_files.py,sha256=QXToEHbZvEhdh88xHamXj9mPQo1Hm2UPxaMkgNEspaw,2474
-cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=Iz3Do-pBJBTDe2P5QwINRQCyeWGHVKjh_lI610WFi1g,2307
-cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=TlDMObWGhXgX1PqIeg5OX832Lps6q5fdC_2GG5Hx3q0,2191
+cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=rDu8w4TAP7vmcbs8oh2eZOPVWEVKXsdgAgaYl980lfY,2321
+cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=M8pJxf78546onU55a0jKJ4fEnaS9Cyw4x_4NpRps0TE,2201
 cosmotech/csm_data/commands/store/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
 cosmotech/csm_data/commands/store/dump_to_azure.py,sha256=5nnK7LUATRTlfX8tmFRLBmkD15bgG37Lyn7fEuPJiNE,3313
 cosmotech/csm_data/commands/store/dump_to_postgresql.py,sha256=3ZfIwIfLXYeBb6SwBSV6-CiWdid8Udht4ILWeZNdLWM,3596
@@ -81,7 +83,7 @@ cosmotech/csm_data/commands/store/list_tables.py,sha256=epgVJAaP8QfSsEuL9n5S8sms
 cosmotech/csm_data/commands/store/load_csv_folder.py,sha256=gDKCxWIpXqhtRvlvRMM7SmyJ9_YHQ1UZHuA6NYdPTZA,1717
 cosmotech/csm_data/commands/store/load_from_singlestore.py,sha256=dWZ4jIKl8wsnOX_6qqtyQmkFQhLgjUAOZWab-sGzQ2g,3104
 cosmotech/csm_data/commands/store/reset.py,sha256=3B0E0m0nCy3vwai_D5sWWddMZWuWkWnaL1tvgR6L5Ow,1237
-cosmotech/csm_data/commands/store/store.py,sha256=_DA4QwQhfTs8bGmbLDHX_8cZZEv4FzeECuTE7d59iUc,1746
+cosmotech/csm_data/commands/store/store.py,sha256=JHfctOCOU2WJd_BpmBsIf805_hGAs2H6XysFt4hxQqI,1693
 cosmotech/csm_data/utils/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
 cosmotech/csm_data/utils/click.py,sha256=S_85cbKh3R86-FZVjTK7IXZnmp4ETjKo6K8gbK3HCgs,848
 cosmotech/csm_data/utils/decorators.py,sha256=dTcPRTYqY18mc8Ql4Qwd3gU7BxbNxfeKskQdVIsE3-g,2504
@@ -161,9 +163,9 @@ cosmotech/translation/csm_data/en-US/csm_data/commands/store/reset.yml,sha256=JM
 cosmotech/translation/csm_data/en-US/csm_data/commands/store/store.yml,sha256=N1Q8483gqJADaCe30S1M3Rj0tMJiuQiJH70-VK2x2m4,134
 cosmotech/translation/csm_data/en-US/csm_data/commons/decorators.yml,sha256=Iu59NWMfYlZZf9uUhOiLkIEGa4GY5p0nZ6vG06Xvu7k,51
 cosmotech/translation/csm_data/en-US/csm_data/commons/version.yml,sha256=7jtCV3O1S6pGjiJa63XpgPDTafjfBS0xmEVRpYNvfDg,86
-cosmotech_acceleration_library-1.0.1.dist-info/licenses/LICENSE,sha256=JXKHOQtyObmafNbQlfPYc4HkKjU9FzAP27b2qRTXNM8,1195
-cosmotech_acceleration_library-1.0.1.dist-info/METADATA,sha256=94bKFXrsdSrP6ZBfdcJeAayVOvWpZBE1OIOw072fObc,9360
-cosmotech_acceleration_library-1.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cosmotech_acceleration_library-1.0.1.dist-info/entry_points.txt,sha256=HWRqJurKuBUgqFe4jmjIAQrs768Nbb8ZTdRDLbuKM5Q,58
-cosmotech_acceleration_library-1.0.1.dist-info/top_level.txt,sha256=t2pzb8mpMUfHTa9l2SjWP0rRB8XVRjBdQK5nLx9XDDo,10
-cosmotech_acceleration_library-1.0.1.dist-info/RECORD,,
+cosmotech_acceleration_library-1.1.0.dist-info/licenses/LICENSE,sha256=JXKHOQtyObmafNbQlfPYc4HkKjU9FzAP27b2qRTXNM8,1195
+cosmotech_acceleration_library-1.1.0.dist-info/METADATA,sha256=0vbzAbv9QAqsAupcJkidNhiKPxpcST4kfalLmzOTm9g,9432
+cosmotech_acceleration_library-1.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cosmotech_acceleration_library-1.1.0.dist-info/entry_points.txt,sha256=HWRqJurKuBUgqFe4jmjIAQrs768Nbb8ZTdRDLbuKM5Q,58
+cosmotech_acceleration_library-1.1.0.dist-info/top_level.txt,sha256=t2pzb8mpMUfHTa9l2SjWP0rRB8XVRjBdQK5nLx9XDDo,10
+cosmotech_acceleration_library-1.1.0.dist-info/RECORD,,

cosmotech/coal/utils/api.py

@@ -1,68 +0,0 @@
-# Copyright (C) - 2023 - 2025 - Cosmo Tech
-# This document and all information contained herein is the exclusive property -
-# including all intellectual property rights pertaining thereto - of Cosmo Tech.
-# Any use, reproduction, translation, broadcasting, transmission, distribution,
-# etc., to any person is prohibited unless it has been previously and
-# specifically authorized by written means by Cosmo Tech.
-
-import json
-import pathlib
-from typing import Optional
-
-import cosmotech_api
-import yaml
-from cosmotech_api.api.solution_api import Solution
-from cosmotech_api.api.solution_api import SolutionApi
-from cosmotech_api.api.workspace_api import Workspace
-from cosmotech_api.api.workspace_api import WorkspaceApi
-from cosmotech_api.exceptions import ServiceException
-
-from cosmotech.coal.cosmotech_api.connection import get_api_client
-from cosmotech.coal.utils.logger import LOGGER
-from cosmotech.orchestrator.utils.translate import T
-
-
-def read_solution_file(solution_file) -> Optional[Solution]:
-    solution_path = pathlib.Path(solution_file)
-    if solution_path.suffix in [".yaml", ".yml"]:
-        open_function = yaml.safe_load
-    elif solution_path.suffix == ".json":
-        open_function = json.load
-    else:
-        LOGGER.error(T("coal.cosmotech_api.solution.invalid_file").format(file=solution_file))
-        return None
-    with solution_path.open() as _sf:
-        solution_content = open_function(_sf)
-    LOGGER.info(T("coal.cosmotech_api.solution.loaded").format(path=solution_path.absolute()))
-    _solution = Solution(
-        _configuration=cosmotech_api.Configuration(),
-        _spec_property_naming=True,
-        **solution_content,
-    )
-    LOGGER.debug(
-        T("coal.services.api.solution_debug").format(solution=json.dumps(_solution.to_dict(), indent=2, default=str))
-    )
-    return _solution
-
-
-def get_solution(organization_id, workspace_id) -> Optional[Solution]:
-    LOGGER.info(T("coal.cosmotech_api.solution.api_configured"))
-    with get_api_client()[0] as api_client:
-        api_w = WorkspaceApi(api_client)
-
-        LOGGER.info(T("coal.cosmotech_api.solution.loading_workspace"))
-        try:
-            r_data: Workspace = api_w.find_workspace_by_id(organization_id=organization_id, workspace_id=workspace_id)
-        except ServiceException as e:
-            LOGGER.error(
-                T("coal.cosmotech_api.workspace.not_found").format(
-                    workspace_id=workspace_id, organization_id=organization_id
-                )
-            )
-            LOGGER.debug(e)
-            return None
-        solution_id = r_data.solution.solution_id
-
-        api_sol = SolutionApi(api_client)
-        sol: Solution = api_sol.find_solution_by_id(organization_id=organization_id, solution_id=solution_id)
-    return sol