cosmotech-acceleration-library 1.0.0__py3-none-any.whl

cosmotech/csm_data/commands/api/tdl_send_files.py

@@ -0,0 +1,82 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@click.option(
+    "--api-url",
+    envvar="CSM_API_URL",
+    help=T("csm-data.commands.api.tdl_send_files.parameters.api_url"),
+    metavar="URI",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--organization-id",
+    envvar="CSM_ORGANIZATION_ID",
+    help=T("csm-data.commands.api.tdl_send_files.parameters.organization_id"),
+    metavar="o-XXXXXXXX",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--workspace-id",
+    envvar="CSM_WORKSPACE_ID",
+    help=T("csm-data.commands.api.tdl_send_files.parameters.workspace_id"),
+    metavar="w-XXXXXXXX",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--runner-id",
+    envvar="CSM_RUNNER_ID",
+    help=T("csm-data.commands.api.tdl_send_files.parameters.runner_id"),
+    metavar="r-XXXXXXXX",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--dir",
+    "directory_path",
+    help=T("csm-data.commands.api.tdl_send_files.parameters.dir"),
+    metavar="PATH",
+    default="./",
+    type=str,
+    envvar="CSM_DATASET_ABSOLUTE_PATH",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--clear/--keep",
+    help=T("csm-data.commands.api.tdl_send_files.parameters.clear"),
+    is_flag=True,
+    default=True,
+    show_default=True,
+    type=bool,
+)
+@web_help("csm-data/api/tdl-send-files")
+@translate_help("csm-data.commands.api.tdl_send_files.description")
+def tdl_send_files(api_url, organization_id, workspace_id, runner_id, directory_path, clear: bool):
+    # Import the function at the start of the command
+    from cosmotech.coal.cosmotech_api import send_files_to_tdl
+
+    send_files_to_tdl(
+        api_url=api_url,
+        organization_id=organization_id,
+        workspace_id=workspace_id,
+        runner_id=runner_id,
+        directory_path=directory_path,
+        clear=clear,
+    )

cosmotech/csm_data/commands/api/wsf_load_file.py

@@ -0,0 +1,66 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+import pathlib
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.coal.cosmotech_api.connection import get_api_client
+from cosmotech.coal.cosmotech_api.workspace import download_workspace_file
+from cosmotech.coal.cosmotech_api.workspace import list_workspace_files
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@click.option(
+    "--organization-id",
+    envvar="CSM_ORGANIZATION_ID",
+    help=T("csm-data.commands.api.wsf_load_file.parameters.organization_id"),
+    metavar="o-XXXXXXXX",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--workspace-id",
+    envvar="CSM_WORKSPACE_ID",
+    help=T("csm-data.commands.api.wsf_load_file.parameters.workspace_id"),
+    metavar="w-XXXXXXXX",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--workspace-path",
+    help=T("csm-data.commands.api.wsf_load_file.parameters.workspace_path"),
+    metavar="PATH",
+    default="/",
+    type=str,
+)
+@click.option(
+    "--target-folder",
+    help=T("csm-data.commands.api.wsf_load_file.parameters.target_folder"),
+    metavar="PATH",
+    default="./",
+    type=str,
+    envvar="CSM_DATASET_ABSOLUTE_PATH",
+    show_envvar=True,
+    required=True,
+)
+@web_help("csm-data/api/wsf-load-file")
+@translate_help("csm-data.commands.api.wsf_load_file.description")
+def wsf_load_file(organization_id, workspace_id, workspace_path: str, target_folder: str):
+    with get_api_client()[0] as api_client:
+        target_list = list_workspace_files(api_client, organization_id, workspace_id, workspace_path)
+
+        for target in target_list:
+            download_workspace_file(
+                api_client,
+                organization_id,
+                workspace_id,
+                target,
+                pathlib.Path(target_folder),
+            )

cosmotech/csm_data/commands/api/wsf_send_file.py

@@ -0,0 +1,68 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.coal.cosmotech_api.connection import get_api_client
+from cosmotech.coal.cosmotech_api.workspace import upload_workspace_file
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@click.option(
+    "--organization-id",
+    envvar="CSM_ORGANIZATION_ID",
+    help=T("csm-data.commands.api.wsf_send_file.parameters.organization_id"),
+    metavar="o-XXXXXXXX",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--workspace-id",
+    envvar="CSM_WORKSPACE_ID",
+    help=T("csm-data.commands.api.wsf_send_file.parameters.workspace_id"),
+    metavar="w-XXXXXXXX",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--file-path",
+    help=T("csm-data.commands.api.wsf_send_file.parameters.file_path"),
+    metavar="PATH",
+    type=str,
+    required=True,
+)
+@click.option(
+    "--workspace-path",
+    help=T("csm-data.commands.api.wsf_send_file.parameters.workspace_path"),
+    metavar="PATH",
+    default="/",
+    type=str,
+    required=True,
+)
+@click.option(
+    "--overwrite/--keep",
+    help=T("csm-data.commands.api.wsf_send_file.parameters.overwrite"),
+    is_flag=True,
+    default=True,
+    show_default=True,
+    type=bool,
+)
+@web_help("csm-data/api/wsf-send-file")
+@translate_help("csm-data.commands.api.wsf_send_file.description")
+def wsf_send_file(organization_id, workspace_id, file_path, workspace_path: str, overwrite: bool):
+    with get_api_client()[0] as api_client:
+        upload_workspace_file(
+            api_client,
+            organization_id,
+            workspace_id,
+            file_path,
+            workspace_path,
+            overwrite,
+        )

cosmotech/csm_data/commands/az_storage_upload.py

@@ -0,0 +1,76 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@click.option(
+    "--source-folder",
+    envvar="CSM_DATASET_ABSOLUTE_PATH",
+    help=T("csm-data.commands.storage.az_storage_upload.parameters.source_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--recursive/--no-recursive",
+    default=False,
+    help=T("csm-data.commands.storage.az_storage_upload.parameters.recursive"),
+    type=bool,
+    is_flag=True,
+)
+@click.option(
+    "--blob-name",
+    envvar="AZURE_STORAGE_BLOB_NAME",
+    help=T("csm-data.commands.storage.az_storage_upload.parameters.blob_name"),
+    metavar="BUCKET",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--prefix",
+    "file_prefix",
+    envvar="CSM_DATA_BLOB_PREFIX",
+    help=T("csm-data.commands.storage.az_storage_upload.parameters.prefix"),
+    metavar="PREFIX",
+    type=str,
+    show_envvar=True,
+    default="",
+)
+@click.option(
+    "--az-storage-sas-url",
+    help=T("csm-data.commands.storage.az_storage_upload.parameters.az_storage_sas_url"),
+    type=str,
+    show_envvar=True,
+    metavar="URL",
+    envvar="AZURE_STORAGE_SAS_URL",
+)
+@web_help("csm-data/az-storage-upload")
+@translate_help("csm-data.commands.storage.az_storage_upload.description")
+def az_storage_upload(
+    source_folder,
+    blob_name: str,
+    az_storage_sas_url: str,
+    file_prefix: str = "",
+    recursive: bool = False,
+):
+    # Import the function at the start of the command
+    from cosmotech.coal.azure.storage import upload_folder
+
+    # Upload files to Azure Blob Storage
+    upload_folder(
+        source_folder=source_folder,
+        blob_name=blob_name,
+        az_storage_sas_url=az_storage_sas_url,
+        file_prefix=file_prefix,
+        recursive=recursive,
+    )

cosmotech/csm_data/commands/s3_bucket_delete.py

@@ -0,0 +1,107 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from typing import Optional
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@click.option(
+    "--bucket-name",
+    envvar="CSM_DATA_BUCKET_NAME",
+    help=T("csm-data.commands.storage.s3_bucket_delete.parameters.bucket_name"),
+    metavar="BUCKET",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--prefix-filter",
+    "file_prefix",
+    envvar="CSM_DATA_BUCKET_PREFIX",
+    help=T("csm-data.commands.storage.s3_bucket_delete.parameters.prefix_filter"),
+    metavar="PREFIX",
+    type=str,
+    show_envvar=True,
+)
+@click.option(
+    "--use-ssl/--no-ssl",
+    default=True,
+    help=T("csm-data.commands.storage.s3_bucket_delete.parameters.use_ssl"),
+    type=bool,
+    is_flag=True,
+)
+@click.option(
+    "--s3-url",
+    "endpoint_url",
+    help=T("csm-data.commands.storage.s3_bucket_delete.parameters.s3_url"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="URL",
+    envvar="AWS_ENDPOINT_URL",
+)
+@click.option(
+    "--access-id",
+    "access_id",
+    help=T("csm-data.commands.storage.s3_bucket_delete.parameters.access_id"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AWS_ACCESS_KEY_ID",
+)
+@click.option(
+    "--secret-key",
+    "secret_key",
+    help=T("csm-data.commands.storage.s3_bucket_delete.parameters.secret_key"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AWS_SECRET_ACCESS_KEY",
+)
+@click.option(
+    "--ssl-cert-bundle",
+    help=T("csm-data.commands.storage.s3_bucket_delete.parameters.ssl_cert_bundle"),
+    type=str,
+    show_envvar=True,
+    metavar="PATH",
+    envvar="CSM_S3_CA_BUNDLE",
+)
+@web_help("csm-data/s3-bucket-delete")
+@translate_help("csm-data.commands.storage.s3_bucket_delete.description")
+def s3_bucket_delete(
+    bucket_name: str,
+    file_prefix: str,
+    endpoint_url: str,
+    access_id: str,
+    secret_key: str,
+    use_ssl: bool = True,
+    ssl_cert_bundle: Optional[str] = None,
+):
+    # Import the functions at the start of the command
+    from cosmotech.coal.aws.s3 import create_s3_resource, delete_objects
+
+    # Create S3 resource
+    s3_resource = create_s3_resource(
+        endpoint_url=endpoint_url,
+        access_id=access_id,
+        secret_key=secret_key,
+        use_ssl=use_ssl,
+        ssl_cert_bundle=ssl_cert_bundle,
+    )
+
+    # Delete objects
+    delete_objects(
+        bucket_name=bucket_name,
+        s3_resource=s3_resource,
+        file_prefix=file_prefix,
+    )

cosmotech/csm_data/commands/s3_bucket_download.py

@@ -0,0 +1,118 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from typing import Optional
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@click.option(
+    "--target-folder",
+    envvar="CSM_DATASET_ABSOLUTE_PATH",
+    help=T("csm-data.commands.storage.s3_bucket_download.parameters.target_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--bucket-name",
+    envvar="CSM_DATA_BUCKET_NAME",
+    help=T("csm-data.commands.storage.s3_bucket_download.parameters.bucket_name"),
+    metavar="BUCKET",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--prefix-filter",
+    "file_prefix",
+    envvar="CSM_DATA_BUCKET_PREFIX",
+    help=T("csm-data.commands.storage.s3_bucket_download.parameters.prefix_filter"),
+    metavar="PREFIX",
+    type=str,
+    show_envvar=True,
+)
+@click.option(
+    "--use-ssl/--no-ssl",
+    default=True,
+    help=T("csm-data.commands.storage.s3_bucket_download.parameters.use_ssl"),
+    type=bool,
+    is_flag=True,
+)
+@click.option(
+    "--s3-url",
+    "endpoint_url",
+    help=T("csm-data.commands.storage.s3_bucket_download.parameters.s3_url"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="URL",
+    envvar="AWS_ENDPOINT_URL",
+)
+@click.option(
+    "--access-id",
+    "access_id",
+    help=T("csm-data.commands.storage.s3_bucket_download.parameters.access_id"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AWS_ACCESS_KEY_ID",
+)
+@click.option(
+    "--secret-key",
+    "secret_key",
+    help=T("csm-data.commands.storage.s3_bucket_download.parameters.secret_key"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AWS_SECRET_ACCESS_KEY",
+)
+@click.option(
+    "--ssl-cert-bundle",
+    help=T("csm-data.commands.storage.s3_bucket_download.parameters.ssl_cert_bundle"),
+    type=str,
+    show_envvar=True,
+    metavar="PATH",
+    envvar="CSM_S3_CA_BUNDLE",
+)
+@web_help("csm-data/s3-bucket-download")
+@translate_help("csm-data.commands.storage.s3_bucket_download.description")
+def s3_bucket_download(
+    target_folder: str,
+    bucket_name: str,
+    file_prefix: str,
+    endpoint_url: str,
+    access_id: str,
+    secret_key: str,
+    use_ssl: bool = True,
+    ssl_cert_bundle: Optional[str] = None,
+):
+    # Import the functions at the start of the command
+    from cosmotech.coal.aws.s3 import create_s3_resource, download_files
+
+    # Create S3 resource
+    s3_resource = create_s3_resource(
+        endpoint_url=endpoint_url,
+        access_id=access_id,
+        secret_key=secret_key,
+        use_ssl=use_ssl,
+        ssl_cert_bundle=ssl_cert_bundle,
+    )
+
+    # Download files
+    download_files(
+        target_folder=target_folder,
+        bucket_name=bucket_name,
+        s3_resource=s3_resource,
+        file_prefix=file_prefix,
+    )

cosmotech/csm_data/commands/s3_bucket_upload.py

@@ -0,0 +1,128 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from typing import Optional
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@click.option(
+    "--source-folder",
+    envvar="CSM_DATASET_ABSOLUTE_PATH",
+    help=T("csm-data.commands.storage.s3_bucket_upload.parameters.source_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--recursive/--no-recursive",
+    default=False,
+    help=T("csm-data.commands.storage.s3_bucket_upload.parameters.recursive"),
+    type=bool,
+    is_flag=True,
+)
+@click.option(
+    "--bucket-name",
+    envvar="CSM_DATA_BUCKET_NAME",
+    help=T("csm-data.commands.storage.s3_bucket_upload.parameters.bucket_name"),
+    metavar="BUCKET",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--prefix",
+    "file_prefix",
+    envvar="CSM_DATA_BUCKET_PREFIX",
+    help=T("csm-data.commands.storage.s3_bucket_upload.parameters.prefix"),
+    metavar="PREFIX",
+    type=str,
+    show_envvar=True,
+    default="",
+)
+@click.option(
+    "--use-ssl/--no-ssl",
+    default=True,
+    help=T("csm-data.commands.storage.s3_bucket_upload.parameters.use_ssl"),
+    type=bool,
+    is_flag=True,
+)
+@click.option(
+    "--s3-url",
+    "endpoint_url",
+    help=T("csm-data.commands.storage.s3_bucket_upload.parameters.s3_url"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="URL",
+    envvar="AWS_ENDPOINT_URL",
+)
+@click.option(
+    "--access-id",
+    "access_id",
+    help=T("csm-data.commands.storage.s3_bucket_upload.parameters.access_id"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AWS_ACCESS_KEY_ID",
+)
+@click.option(
+    "--secret-key",
+    "secret_key",
+    help=T("csm-data.commands.storage.s3_bucket_upload.parameters.secret_key"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AWS_SECRET_ACCESS_KEY",
+)
+@click.option(
+    "--ssl-cert-bundle",
+    help=T("csm-data.commands.storage.s3_bucket_upload.parameters.ssl_cert_bundle"),
+    type=str,
+    show_envvar=True,
+    metavar="PATH",
+    envvar="CSM_S3_CA_BUNDLE",
+)
+@web_help("csm-data/s3-bucket-upload")
+@translate_help("csm-data.commands.storage.s3_bucket_upload.description")
+def s3_bucket_upload(
+    source_folder,
+    bucket_name: str,
+    endpoint_url: str,
+    access_id: str,
+    secret_key: str,
+    file_prefix: str = "",
+    use_ssl: bool = True,
+    ssl_cert_bundle: Optional[str] = None,
+    recursive: bool = False,
+):
+    # Import the functions at the start of the command
+    from cosmotech.coal.aws.s3 import create_s3_resource, upload_folder
+
+    # Create S3 resource
+    s3_resource = create_s3_resource(
+        endpoint_url=endpoint_url,
+        access_id=access_id,
+        secret_key=secret_key,
+        use_ssl=use_ssl,
+        ssl_cert_bundle=ssl_cert_bundle,
+    )
+
+    # Upload files
+    upload_folder(
+        source_folder=source_folder,
+        bucket_name=bucket_name,
+        s3_resource=s3_resource,
+        file_prefix=file_prefix,
+        recursive=recursive,
+    )

cosmotech/csm_data/commands/store/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.