cosmotech-acceleration-library 1.0.0__py3-none-any.whl

cosmotech/csm_data/commands/store/dump_to_azure.py

@@ -0,0 +1,120 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+VALID_TYPES = (
+    "sqlite",
+    "csv",
+    "parquet",
+)
+
+
+@click.command()
+@click.option(
+    "--store-folder",
+    envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
+    help=T("csm-data.commands.store.dump_to_azure.parameters.store_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--output-type",
+    default="sqlite",
+    help=T("csm-data.commands.store.dump_to_azure.parameters.output_type"),
+    type=click.Choice(VALID_TYPES, case_sensitive=False),
+)
+@click.option(
+    "--account-name",
+    "account_name",
+    envvar="AZURE_ACCOUNT_NAME",
+    help=T("csm-data.commands.store.dump_to_azure.parameters.account_name"),
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--container-name",
+    "container_name",
+    envvar="AZURE_CONTAINER_NAME",
+    help=T("csm-data.commands.store.dump_to_azure.parameters.container_name"),
+    type=str,
+    show_envvar=True,
+    default="",
+)
+@click.option(
+    "--prefix",
+    "file_prefix",
+    envvar="CSM_DATA_PREFIX",
+    help=T("csm-data.commands.store.dump_to_azure.parameters.prefix"),
+    metavar="PREFIX",
+    type=str,
+    show_envvar=True,
+    default="",
+)
+@click.option(
+    "--tenant-id",
+    "tenant_id",
+    help=T("csm-data.commands.store.dump_to_azure.parameters.tenant_id"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AZURE_TENANT_ID",
+)
+@click.option(
+    "--client-id",
+    "client_id",
+    help=T("csm-data.commands.store.dump_to_azure.parameters.client_id"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AZURE_CLIENT_ID",
+)
+@click.option(
+    "--client-secret",
+    "client_secret",
+    help=T("csm-data.commands.store.dump_to_azure.parameters.client_secret"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AZURE_CLIENT_SECRET",
+)
+@web_help("csm-data/store/dump-to-azure")
+@translate_help("csm-data.commands.store.dump_to_azure.description")
+def dump_to_azure(
+    store_folder,
+    account_name: str,
+    container_name: str,
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    output_type: str,
+    file_prefix: str,
+):
+    # Import the function at the start of the command
+    from cosmotech.coal.azure import dump_store_to_azure
+
+    try:
+        dump_store_to_azure(
+            store_folder=store_folder,
+            account_name=account_name,
+            container_name=container_name,
+            tenant_id=tenant_id,
+            client_id=client_id,
+            client_secret=client_secret,
+            output_type=output_type,
+            file_prefix=file_prefix,
+        )
+    except ValueError as e:
+        raise click.Abort() from e

cosmotech/csm_data/commands/store/dump_to_postgresql.py

@@ -0,0 +1,107 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@web_help("csm-data/store/dump-to-postgres")
+@translate_help("csm-data.commands.store.dump_to_postgresql.description")
+@click.option(
+    "--store-folder",
+    envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
+    help=T("csm-data.commands.store.dump_to_postgresql.parameters.store_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--table-prefix",
+    help=T("csm-data.commands.store.dump_to_postgresql.parameters.table_prefix"),
+    metavar="PREFIX",
+    type=str,
+    default="Cosmotech_",
+)
+@click.option(
+    "--postgres-host",
+    help=T("csm-data.commands.store.dump_to_postgresql.parameters.postgres_host"),
+    envvar="POSTGRES_HOST_URI",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--postgres-port",
+    help=T("csm-data.commands.store.dump_to_postgresql.parameters.postgres_port"),
+    envvar="POSTGRES_HOST_PORT",
+    show_envvar=True,
+    required=False,
+    default=5432,
+)
+@click.option(
+    "--postgres-db",
+    help=T("csm-data.commands.store.dump_to_postgresql.parameters.postgres_db"),
+    envvar="POSTGRES_DB_NAME",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--postgres-schema",
+    help=T("csm-data.commands.store.dump_to_postgresql.parameters.postgres_schema"),
+    envvar="POSTGRES_DB_SCHEMA",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--postgres-user",
+    help=T("csm-data.commands.store.dump_to_postgresql.parameters.postgres_user"),
+    envvar="POSTGRES_USER_NAME",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--postgres-password",
+    help=T("csm-data.commands.store.dump_to_postgresql.parameters.postgres_password"),
+    envvar="POSTGRES_USER_PASSWORD",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--replace/--append",
+    "replace",
+    help=T("csm-data.commands.store.dump_to_postgresql.parameters.replace"),
+    default=True,
+    is_flag=True,
+    show_default=True,
+)
+def dump_to_postgresql(
+    store_folder,
+    table_prefix: str,
+    postgres_host,
+    postgres_port,
+    postgres_db,
+    postgres_schema,
+    postgres_user,
+    postgres_password,
+    replace: bool,
+):
+    # Import the function at the start of the command
+    from cosmotech.coal.postgresql import dump_store_to_postgresql
+
+    dump_store_to_postgresql(
+        store_folder=store_folder,
+        table_prefix=table_prefix,
+        postgres_host=postgres_host,
+        postgres_port=postgres_port,
+        postgres_db=postgres_db,
+        postgres_schema=postgres_schema,
+        postgres_user=postgres_user,
+        postgres_password=postgres_password,
+        replace=replace,
+    )

cosmotech/csm_data/commands/store/dump_to_s3.py

@@ -0,0 +1,169 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from typing import Optional
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+VALID_TYPES = (
+    "sqlite",
+    "csv",
+    "parquet",
+)
+
+
+@click.command()
+@click.option(
+    "--store-folder",
+    envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
+    help=T("csm-data.commands.store.dump_to_s3.parameters.store_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--output-type",
+    default="sqlite",
+    help=T("csm-data.commands.store.dump_to_s3.parameters.output_type"),
+    type=click.Choice(VALID_TYPES, case_sensitive=False),
+)
+@click.option(
+    "--bucket-name",
+    envvar="CSM_DATA_BUCKET_NAME",
+    help=T("csm-data.commands.store.dump_to_s3.parameters.bucket_name"),
+    metavar="BUCKET",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--prefix",
+    "file_prefix",
+    envvar="CSM_DATA_BUCKET_PREFIX",
+    help=T("csm-data.commands.store.dump_to_s3.parameters.prefix"),
+    metavar="PREFIX",
+    type=str,
+    show_envvar=True,
+    default="",
+)
+@click.option(
+    "--use-ssl/--no-ssl",
+    default=True,
+    help=T("csm-data.commands.store.dump_to_s3.parameters.use_ssl"),
+    type=bool,
+    is_flag=True,
+)
+@click.option(
+    "--s3-url",
+    "endpoint_url",
+    help=T("csm-data.commands.store.dump_to_s3.parameters.s3_url"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="URL",
+    envvar="AWS_ENDPOINT_URL",
+)
+@click.option(
+    "--access-id",
+    "access_id",
+    help=T("csm-data.commands.store.dump_to_s3.parameters.access_id"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AWS_ACCESS_KEY_ID",
+)
+@click.option(
+    "--secret-key",
+    "secret_key",
+    help=T("csm-data.commands.store.dump_to_s3.parameters.secret_key"),
+    type=str,
+    required=True,
+    show_envvar=True,
+    metavar="ID",
+    envvar="AWS_SECRET_ACCESS_KEY",
+)
+@click.option(
+    "--ssl-cert-bundle",
+    help=T("csm-data.commands.store.dump_to_s3.parameters.ssl_cert_bundle"),
+    type=str,
+    show_envvar=True,
+    metavar="PATH",
+    envvar="CSM_S3_CA_BUNDLE",
+)
+@web_help("csm-data/store/dump-to-s3")
+@translate_help("csm-data.commands.store.dump_to_s3.description")
+def dump_to_s3(
+    store_folder,
+    bucket_name: str,
+    endpoint_url: str,
+    access_id: str,
+    secret_key: str,
+    output_type: str,
+    file_prefix: str = "",
+    use_ssl: bool = True,
+    ssl_cert_bundle: Optional[str] = None,
+):
+    # Import the modules and functions at the start of the command
+    from io import BytesIO
+    import pyarrow.csv as pc
+    import pyarrow.parquet as pq
+    from cosmotech.coal.aws import create_s3_client, upload_data_stream
+    from cosmotech.coal.store.store import Store
+    from cosmotech.coal.utils.logger import LOGGER
+
+    _s = Store(store_location=store_folder)
+
+    if output_type not in VALID_TYPES:
+        LOGGER.error(T("coal.common.errors.data_invalid_output_type").format(output_type=output_type))
+        raise ValueError(T("coal.common.errors.data_invalid_output_type").format(output_type=output_type))
+
+    # Create S3 client
+    s3_client = create_s3_client(
+        endpoint_url=endpoint_url,
+        access_id=access_id,
+        secret_key=secret_key,
+        use_ssl=use_ssl,
+        ssl_cert_bundle=ssl_cert_bundle,
+    )
+
+    if output_type == "sqlite":
+        _file_path = _s._database_path
+        _file_name = "db.sqlite"
+        _uploaded_file_name = file_prefix + _file_name
+        LOGGER.info(
+            T("coal.common.data_transfer.file_sent").format(file_path=_file_path, uploaded_name=_uploaded_file_name)
+        )
+        s3_client.upload_file(_file_path, bucket_name, _uploaded_file_name)
+    else:
+        tables = list(_s.list_tables())
+        for table_name in tables:
+            _data_stream = BytesIO()
+            _file_name = None
+            _data = _s.get_table(table_name)
+            if not len(_data):
+                LOGGER.info(T("coal.common.data_transfer.table_empty").format(table_name=table_name))
+                continue
+            if output_type == "csv":
+                _file_name = table_name + ".csv"
+                pc.write_csv(_data, _data_stream)
+            elif output_type == "parquet":
+                _file_name = table_name + ".parquet"
+                pq.write_table(_data, _data_stream)
+            LOGGER.info(
+                T("coal.common.data_transfer.sending_table").format(table_name=table_name, output_type=output_type)
+            )
+            upload_data_stream(
+                data_stream=_data_stream,
+                bucket_name=bucket_name,
+                s3_client=s3_client,
+                file_name=_file_name,
+                file_prefix=file_prefix,
+            )

cosmotech/csm_data/commands/store/list_tables.py

@@ -0,0 +1,48 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@web_help("csm-data/store/list-tables")
+@translate_help("csm-data.commands.store.list_tables.description")
+@click.option(
+    "--store-folder",
+    envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
+    help=T("csm-data.commands.store.list_tables.parameters.store_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--schema/--no-schema",
+    help=T("csm-data.commands.store.list_tables.parameters.schema"),
+    is_flag=True,
+    type=bool,
+    default=False,
+)
+def list_tables(store_folder, schema):
+    # Import the modules and functions at the start of the command
+    from cosmotech.coal.store.store import Store
+    from cosmotech.coal.utils.logger import LOGGER
+
+    _s = Store(store_location=store_folder)
+    tables = list(_s.list_tables())
+    if len(tables):
+        LOGGER.info(T("coal.services.database.store_tables"))
+        for table_name in tables:
+            LOGGER.info(T("coal.services.database.table_entry").format(table=table_name))
+            if schema:
+                LOGGER.info(
+                    T("coal.services.database.table_schema").format(schema=str(_s.get_table_schema(table_name)))
+                )
+    else:
+        LOGGER.info(T("coal.services.database.store_empty"))

cosmotech/csm_data/commands/store/load_csv_folder.py

@@ -0,0 +1,43 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@web_help("csm-data/store/load-csv-folder")
+@translate_help("csm-data.commands.store.load_csv_folder.description")
+@click.option(
+    "--store-folder",
+    envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
+    help=T("csm-data.commands.store.load_csv_folder.parameters.store_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--csv-folder",
+    envvar="CSM_DATASET_ABSOLUTE_PATH",
+    help=T("csm-data.commands.store.load_csv_folder.parameters.csv_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+def load_csv_folder(store_folder, csv_folder):
+    # Import the modules and functions at the start of the command
+    import pathlib
+    from cosmotech.coal.store.csv import store_csv_file
+    from cosmotech.coal.store.store import Store
+    from cosmotech.coal.utils.logger import LOGGER
+
+    for csv_path in pathlib.Path(csv_folder).glob("*.csv"):
+        LOGGER.info(T("coal.services.azure_storage.found_file").format(file=csv_path.name))
+        store_csv_file(csv_path.name[:-4], csv_path, store=Store(False, store_folder))

cosmotech/csm_data/commands/store/load_from_singlestore.py

@@ -0,0 +1,96 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@web_help("csm-data/store/load-from-singlestore")
+@translate_help("csm-data.commands.store.load_from_singlestore.description")
+@click.option(
+    "--singlestore-host",
+    "single_store_host",
+    envvar="SINGLE_STORE_HOST",
+    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_host"),
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--singlestore-port",
+    "single_store_port",
+    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_port"),
+    envvar="SINGLE_STORE_PORT",
+    show_envvar=True,
+    required=False,
+    default=3306,
+)
+@click.option(
+    "--singlestore-db",
+    "single_store_db",
+    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_db"),
+    envvar="SINGLE_STORE_DB",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--singlestore-user",
+    "single_store_user",
+    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_user"),
+    envvar="SINGLE_STORE_USERNAME",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--singlestore-password",
+    "single_store_password",
+    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_password"),
+    envvar="SINGLE_STORE_PASSWORD",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--singlestore-tables",
+    "single_store_tables",
+    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_tables"),
+    envvar="SINGLE_STORE_TABLES",
+    show_envvar=True,
+    required=True,
+)
+@click.option(
+    "--store-folder",
+    "store_folder",
+    envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
+    help=T("csm-data.commands.store.load_from_singlestore.parameters.store_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+def load_from_singlestore_command(
+    single_store_host,
+    single_store_port,
+    single_store_db,
+    single_store_user,
+    single_store_password,
+    store_folder,
+    single_store_tables: str = "",
+):
+    # Import the function at the start of the command
+    from cosmotech.coal.singlestore import load_from_singlestore
+
+    load_from_singlestore(
+        single_store_host=single_store_host,
+        single_store_port=single_store_port,
+        single_store_db=single_store_db,
+        single_store_user=single_store_user,
+        single_store_password=single_store_password,
+        store_folder=store_folder,
+        single_store_tables=single_store_tables,
+    )

cosmotech/csm_data/commands/store/reset.py

@@ -0,0 +1,31 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.command()
+@web_help("csm-data/store/reset")
+@translate_help("csm-data.commands.store.reset.description")
+@click.option(
+    "--store-folder",
+    envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
+    help=T("csm-data.commands.store.reset.parameters.store_folder"),
+    metavar="PATH",
+    type=str,
+    show_envvar=True,
+    required=True,
+)
+def reset(store_folder):
+    # Import the modules and functions at the start of the command
+    from cosmotech.coal.store.store import Store
+    from cosmotech.coal.utils.logger import LOGGER
+
+    Store(True, store_folder)
+    LOGGER.info(T("coal.services.database.store_reset").format(folder=store_folder))

cosmotech/csm_data/commands/store/store.py

@@ -0,0 +1,37 @@
+# Copyright (C) - 2023 - 2025 - Cosmo Tech
+# This document and all information contained herein is the exclusive property -
+# including all intellectual property rights pertaining thereto - of Cosmo Tech.
+# Any use, reproduction, translation, broadcasting, transmission, distribution,
+# etc., to any person is prohibited unless it has been previously and
+# specifically authorized by written means by Cosmo Tech.
+
+from cosmotech.csm_data.commands.api.rds_send_store import rds_send_store
+from cosmotech.csm_data.commands.store.dump_to_azure import dump_to_azure
+from cosmotech.csm_data.commands.store.dump_to_postgresql import dump_to_postgresql
+from cosmotech.csm_data.commands.store.dump_to_s3 import dump_to_s3
+from cosmotech.csm_data.commands.store.list_tables import list_tables
+from cosmotech.csm_data.commands.store.load_csv_folder import load_csv_folder
+from cosmotech.csm_data.commands.store.load_from_singlestore import (
+    load_from_singlestore_command,
+)
+from cosmotech.csm_data.commands.store.reset import reset
+from cosmotech.csm_data.utils.click import click
+from cosmotech.csm_data.utils.decorators import web_help, translate_help
+from cosmotech.orchestrator.utils.translate import T
+
+
+@click.group()
+@web_help("csm-data/store")
+@translate_help("csm-data.commands.store.description")
+def store():
+    pass
+
+
+store.add_command(rds_send_store, "rds-send-store")
+store.add_command(reset, "reset")
+store.add_command(list_tables, "list-tables")
+store.add_command(load_csv_folder, "load-csv-folder")
+store.add_command(load_from_singlestore_command, "load-from-singlestore")
+store.add_command(dump_to_postgresql, "dump-to-postgresql")
+store.add_command(dump_to_s3, "dump-to-s3")
+store.add_command(dump_to_azure, "dump-to-azure")