cosmotech-acceleration-library 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cosmotech/coal/__init__.py +8 -0
- cosmotech/coal/aws/__init__.py +23 -0
- cosmotech/coal/aws/s3.py +235 -0
- cosmotech/coal/azure/__init__.py +23 -0
- cosmotech/coal/azure/adx/__init__.py +26 -0
- cosmotech/coal/azure/adx/auth.py +125 -0
- cosmotech/coal/azure/adx/ingestion.py +329 -0
- cosmotech/coal/azure/adx/query.py +56 -0
- cosmotech/coal/azure/adx/runner.py +217 -0
- cosmotech/coal/azure/adx/store.py +255 -0
- cosmotech/coal/azure/adx/tables.py +118 -0
- cosmotech/coal/azure/adx/utils.py +71 -0
- cosmotech/coal/azure/blob.py +109 -0
- cosmotech/coal/azure/functions.py +72 -0
- cosmotech/coal/azure/storage.py +74 -0
- cosmotech/coal/cosmotech_api/__init__.py +36 -0
- cosmotech/coal/cosmotech_api/connection.py +96 -0
- cosmotech/coal/cosmotech_api/dataset/__init__.py +26 -0
- cosmotech/coal/cosmotech_api/dataset/converters.py +164 -0
- cosmotech/coal/cosmotech_api/dataset/download/__init__.py +19 -0
- cosmotech/coal/cosmotech_api/dataset/download/adt.py +119 -0
- cosmotech/coal/cosmotech_api/dataset/download/common.py +140 -0
- cosmotech/coal/cosmotech_api/dataset/download/file.py +216 -0
- cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +188 -0
- cosmotech/coal/cosmotech_api/dataset/utils.py +132 -0
- cosmotech/coal/cosmotech_api/parameters.py +48 -0
- cosmotech/coal/cosmotech_api/run.py +25 -0
- cosmotech/coal/cosmotech_api/run_data.py +173 -0
- cosmotech/coal/cosmotech_api/run_template.py +108 -0
- cosmotech/coal/cosmotech_api/runner/__init__.py +28 -0
- cosmotech/coal/cosmotech_api/runner/data.py +38 -0
- cosmotech/coal/cosmotech_api/runner/datasets.py +364 -0
- cosmotech/coal/cosmotech_api/runner/download.py +146 -0
- cosmotech/coal/cosmotech_api/runner/metadata.py +42 -0
- cosmotech/coal/cosmotech_api/runner/parameters.py +157 -0
- cosmotech/coal/cosmotech_api/twin_data_layer.py +512 -0
- cosmotech/coal/cosmotech_api/workspace.py +127 -0
- cosmotech/coal/csm/__init__.py +6 -0
- cosmotech/coal/csm/engine/__init__.py +47 -0
- cosmotech/coal/postgresql/__init__.py +22 -0
- cosmotech/coal/postgresql/runner.py +93 -0
- cosmotech/coal/postgresql/store.py +98 -0
- cosmotech/coal/singlestore/__init__.py +17 -0
- cosmotech/coal/singlestore/store.py +100 -0
- cosmotech/coal/store/__init__.py +42 -0
- cosmotech/coal/store/csv.py +44 -0
- cosmotech/coal/store/native_python.py +25 -0
- cosmotech/coal/store/pandas.py +26 -0
- cosmotech/coal/store/pyarrow.py +23 -0
- cosmotech/coal/store/store.py +79 -0
- cosmotech/coal/utils/__init__.py +18 -0
- cosmotech/coal/utils/api.py +68 -0
- cosmotech/coal/utils/logger.py +10 -0
- cosmotech/coal/utils/postgresql.py +236 -0
- cosmotech/csm_data/__init__.py +6 -0
- cosmotech/csm_data/commands/__init__.py +6 -0
- cosmotech/csm_data/commands/adx_send_data.py +92 -0
- cosmotech/csm_data/commands/adx_send_runnerdata.py +119 -0
- cosmotech/csm_data/commands/api/__init__.py +6 -0
- cosmotech/csm_data/commands/api/api.py +50 -0
- cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +119 -0
- cosmotech/csm_data/commands/api/rds_load_csv.py +90 -0
- cosmotech/csm_data/commands/api/rds_send_csv.py +74 -0
- cosmotech/csm_data/commands/api/rds_send_store.py +74 -0
- cosmotech/csm_data/commands/api/run_load_data.py +120 -0
- cosmotech/csm_data/commands/api/runtemplate_load_handler.py +66 -0
- cosmotech/csm_data/commands/api/tdl_load_files.py +76 -0
- cosmotech/csm_data/commands/api/tdl_send_files.py +82 -0
- cosmotech/csm_data/commands/api/wsf_load_file.py +66 -0
- cosmotech/csm_data/commands/api/wsf_send_file.py +68 -0
- cosmotech/csm_data/commands/az_storage_upload.py +76 -0
- cosmotech/csm_data/commands/s3_bucket_delete.py +107 -0
- cosmotech/csm_data/commands/s3_bucket_download.py +118 -0
- cosmotech/csm_data/commands/s3_bucket_upload.py +128 -0
- cosmotech/csm_data/commands/store/__init__.py +6 -0
- cosmotech/csm_data/commands/store/dump_to_azure.py +120 -0
- cosmotech/csm_data/commands/store/dump_to_postgresql.py +107 -0
- cosmotech/csm_data/commands/store/dump_to_s3.py +169 -0
- cosmotech/csm_data/commands/store/list_tables.py +48 -0
- cosmotech/csm_data/commands/store/load_csv_folder.py +43 -0
- cosmotech/csm_data/commands/store/load_from_singlestore.py +96 -0
- cosmotech/csm_data/commands/store/reset.py +31 -0
- cosmotech/csm_data/commands/store/store.py +37 -0
- cosmotech/csm_data/main.py +57 -0
- cosmotech/csm_data/utils/__init__.py +6 -0
- cosmotech/csm_data/utils/click.py +18 -0
- cosmotech/csm_data/utils/decorators.py +75 -0
- cosmotech/orchestrator_plugins/csm-data/__init__.py +11 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/postgres_send_runner_metadata.json +40 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +27 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +27 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +27 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/run_load_data.json +30 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +27 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +32 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +27 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/try_api_connection.json +9 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_load_file.json +36 -0
- cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_send_file.json +36 -0
- cosmotech/orchestrator_plugins/csm-data/templates/main/adx_send_runnerdata.json +29 -0
- cosmotech/orchestrator_plugins/csm-data/templates/main/az_storage_upload.json +25 -0
- cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_delete.json +31 -0
- cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_download.json +34 -0
- cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_upload.json +35 -0
- cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_azure.json +35 -0
- cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_postgresql.json +34 -0
- cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_s3.json +36 -0
- cosmotech/orchestrator_plugins/csm-data/templates/store/store_list_tables.json +15 -0
- cosmotech/orchestrator_plugins/csm-data/templates/store/store_load_csv_folder.json +18 -0
- cosmotech/orchestrator_plugins/csm-data/templates/store/store_load_from_singlestore.json +34 -0
- cosmotech/orchestrator_plugins/csm-data/templates/store/store_reset.json +15 -0
- cosmotech/translation/coal/__init__.py +6 -0
- cosmotech/translation/coal/en-US/coal/common/data_transfer.yml +6 -0
- cosmotech/translation/coal/en-US/coal/common/errors.yml +9 -0
- cosmotech/translation/coal/en-US/coal/common/file_operations.yml +6 -0
- cosmotech/translation/coal/en-US/coal/common/progress.yml +6 -0
- cosmotech/translation/coal/en-US/coal/common/timing.yml +5 -0
- cosmotech/translation/coal/en-US/coal/common/validation.yml +8 -0
- cosmotech/translation/coal/en-US/coal/cosmotech_api/connection.yml +10 -0
- cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +2 -0
- cosmotech/translation/coal/en-US/coal/cosmotech_api/run_template.yml +8 -0
- cosmotech/translation/coal/en-US/coal/cosmotech_api/runner.yml +16 -0
- cosmotech/translation/coal/en-US/coal/cosmotech_api/solution.yml +5 -0
- cosmotech/translation/coal/en-US/coal/cosmotech_api/workspace.yml +7 -0
- cosmotech/translation/coal/en-US/coal/services/adx.yml +59 -0
- cosmotech/translation/coal/en-US/coal/services/api.yml +8 -0
- cosmotech/translation/coal/en-US/coal/services/azure_storage.yml +14 -0
- cosmotech/translation/coal/en-US/coal/services/database.yml +19 -0
- cosmotech/translation/coal/en-US/coal/services/dataset.yml +68 -0
- cosmotech/translation/coal/en-US/coal/services/postgresql.yml +28 -0
- cosmotech/translation/coal/en-US/coal/services/s3.yml +9 -0
- cosmotech/translation/coal/en-US/coal/solution.yml +3 -0
- cosmotech/translation/coal/en-US/coal/web.yml +2 -0
- cosmotech/translation/csm_data/__init__.py +6 -0
- cosmotech/translation/csm_data/en-US/csm-data.yml +434 -0
- cosmotech_acceleration_library-1.0.0.dist-info/METADATA +255 -0
- cosmotech_acceleration_library-1.0.0.dist-info/RECORD +141 -0
- cosmotech_acceleration_library-1.0.0.dist-info/WHEEL +5 -0
- cosmotech_acceleration_library-1.0.0.dist-info/entry_points.txt +2 -0
- cosmotech_acceleration_library-1.0.0.dist-info/licenses/LICENSE +17 -0
- cosmotech_acceleration_library-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
+
# This document and all information contained herein is the exclusive property -
|
|
3
|
+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
+
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
+
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
from cosmotech.csm_data.commands.api.postgres_send_runner_metadata import (
|
|
8
|
+
postgres_send_runner_metadata,
|
|
9
|
+
)
|
|
10
|
+
from cosmotech.csm_data.commands.api.rds_load_csv import rds_load_csv
|
|
11
|
+
from cosmotech.csm_data.commands.api.rds_send_csv import rds_send_csv
|
|
12
|
+
from cosmotech.csm_data.commands.api.rds_send_store import rds_send_store
|
|
13
|
+
from cosmotech.csm_data.commands.api.run_load_data import run_load_data
|
|
14
|
+
from cosmotech.csm_data.commands.api.runtemplate_load_handler import (
|
|
15
|
+
runtemplate_load_handler,
|
|
16
|
+
)
|
|
17
|
+
from cosmotech.csm_data.commands.api.tdl_load_files import tdl_load_files
|
|
18
|
+
from cosmotech.csm_data.commands.api.tdl_send_files import tdl_send_files
|
|
19
|
+
from cosmotech.csm_data.commands.api.wsf_load_file import wsf_load_file
|
|
20
|
+
from cosmotech.csm_data.commands.api.wsf_send_file import wsf_send_file
|
|
21
|
+
from cosmotech.csm_data.utils.click import click
|
|
22
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
23
|
+
from cosmotech.coal.cosmotech_api.connection import get_api_client
|
|
24
|
+
from cosmotech.coal.utils.logger import LOGGER
|
|
25
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@click.group(invoke_without_command=True)
|
|
29
|
+
@web_help("csm-data/api")
|
|
30
|
+
@click.pass_context
|
|
31
|
+
@translate_help("csm-data.commands.api.description")
|
|
32
|
+
def api(ctx: click.Context):
|
|
33
|
+
if ctx.invoked_subcommand is None:
|
|
34
|
+
try:
|
|
35
|
+
api_client, description = get_api_client()
|
|
36
|
+
LOGGER.info(T("coal.cosmotech_api.connection.found_valid").format(type=description))
|
|
37
|
+
except EnvironmentError:
|
|
38
|
+
raise click.Abort()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
api.add_command(rds_send_csv, "rds-send-csv")
|
|
42
|
+
api.add_command(rds_send_store, "rds-send-store")
|
|
43
|
+
api.add_command(rds_load_csv, "rds-load-csv")
|
|
44
|
+
api.add_command(wsf_send_file, "wsf-send-file")
|
|
45
|
+
api.add_command(wsf_load_file, "wsf-load-file")
|
|
46
|
+
api.add_command(tdl_send_files, "tdl-send-files")
|
|
47
|
+
api.add_command(tdl_load_files, "tdl-load-files")
|
|
48
|
+
api.add_command(runtemplate_load_handler, "runtemplate-load-handler")
|
|
49
|
+
api.add_command(run_load_data, "run-load-data")
|
|
50
|
+
api.add_command(postgres_send_runner_metadata, "postgres-send-runner-metadata")
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
+
# This document and all information contained herein is the exclusive property -
|
|
3
|
+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
+
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
+
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
|
|
8
|
+
from cosmotech.csm_data.utils.click import click
|
|
9
|
+
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@click.command()
|
|
14
|
+
@web_help("csm-data/api/postgres-send-runner-metadata")
|
|
15
|
+
@translate_help("csm-data.commands.api.postgres_send_runner_metadata.description")
|
|
16
|
+
@click.option(
|
|
17
|
+
"--organization-id",
|
|
18
|
+
envvar="CSM_ORGANIZATION_ID",
|
|
19
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.organization_id"),
|
|
20
|
+
metavar="o-XXXXXXXX",
|
|
21
|
+
type=str,
|
|
22
|
+
show_envvar=True,
|
|
23
|
+
required=True,
|
|
24
|
+
)
|
|
25
|
+
@click.option(
|
|
26
|
+
"--workspace-id",
|
|
27
|
+
envvar="CSM_WORKSPACE_ID",
|
|
28
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.workspace_id"),
|
|
29
|
+
metavar="w-XXXXXXXX",
|
|
30
|
+
type=str,
|
|
31
|
+
show_envvar=True,
|
|
32
|
+
required=True,
|
|
33
|
+
)
|
|
34
|
+
@click.option(
|
|
35
|
+
"--runner-id",
|
|
36
|
+
envvar="CSM_RUNNER_ID",
|
|
37
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.runner_id"),
|
|
38
|
+
metavar="r-XXXXXXXX",
|
|
39
|
+
type=str,
|
|
40
|
+
show_envvar=True,
|
|
41
|
+
required=True,
|
|
42
|
+
)
|
|
43
|
+
@click.option(
|
|
44
|
+
"--table-prefix",
|
|
45
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.table_prefix"),
|
|
46
|
+
metavar="PREFIX",
|
|
47
|
+
type=str,
|
|
48
|
+
default="Cosmotech_",
|
|
49
|
+
)
|
|
50
|
+
@click.option(
|
|
51
|
+
"--postgres-host",
|
|
52
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_host"),
|
|
53
|
+
envvar="POSTGRES_HOST_URI",
|
|
54
|
+
show_envvar=True,
|
|
55
|
+
required=True,
|
|
56
|
+
)
|
|
57
|
+
@click.option(
|
|
58
|
+
"--postgres-port",
|
|
59
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_port"),
|
|
60
|
+
envvar="POSTGRES_HOST_PORT",
|
|
61
|
+
show_envvar=True,
|
|
62
|
+
required=False,
|
|
63
|
+
default=5432,
|
|
64
|
+
)
|
|
65
|
+
@click.option(
|
|
66
|
+
"--postgres-db",
|
|
67
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_db"),
|
|
68
|
+
envvar="POSTGRES_DB_NAME",
|
|
69
|
+
show_envvar=True,
|
|
70
|
+
required=True,
|
|
71
|
+
)
|
|
72
|
+
@click.option(
|
|
73
|
+
"--postgres-schema",
|
|
74
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_schema"),
|
|
75
|
+
envvar="POSTGRES_DB_SCHEMA",
|
|
76
|
+
show_envvar=True,
|
|
77
|
+
required=True,
|
|
78
|
+
)
|
|
79
|
+
@click.option(
|
|
80
|
+
"--postgres-user",
|
|
81
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_user"),
|
|
82
|
+
envvar="POSTGRES_USER_NAME",
|
|
83
|
+
show_envvar=True,
|
|
84
|
+
required=True,
|
|
85
|
+
)
|
|
86
|
+
@click.option(
|
|
87
|
+
"--postgres-password",
|
|
88
|
+
help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_password"),
|
|
89
|
+
envvar="POSTGRES_USER_PASSWORD",
|
|
90
|
+
show_envvar=True,
|
|
91
|
+
required=True,
|
|
92
|
+
)
|
|
93
|
+
def postgres_send_runner_metadata(
|
|
94
|
+
organization_id,
|
|
95
|
+
workspace_id,
|
|
96
|
+
runner_id,
|
|
97
|
+
table_prefix: str,
|
|
98
|
+
postgres_host,
|
|
99
|
+
postgres_port,
|
|
100
|
+
postgres_db,
|
|
101
|
+
postgres_schema,
|
|
102
|
+
postgres_user,
|
|
103
|
+
postgres_password,
|
|
104
|
+
):
|
|
105
|
+
# Import the function at the start of the command
|
|
106
|
+
from cosmotech.coal.postgresql import send_runner_metadata_to_postgresql
|
|
107
|
+
|
|
108
|
+
send_runner_metadata_to_postgresql(
|
|
109
|
+
organization_id=organization_id,
|
|
110
|
+
workspace_id=workspace_id,
|
|
111
|
+
runner_id=runner_id,
|
|
112
|
+
table_prefix=table_prefix,
|
|
113
|
+
postgres_host=postgres_host,
|
|
114
|
+
postgres_port=postgres_port,
|
|
115
|
+
postgres_db=postgres_db,
|
|
116
|
+
postgres_schema=postgres_schema,
|
|
117
|
+
postgres_user=postgres_user,
|
|
118
|
+
postgres_password=postgres_password,
|
|
119
|
+
)
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
+
# This document and all information contained herein is the exclusive property -
|
|
3
|
+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
+
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
+
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
|
|
8
|
+
from cosmotech.csm_data.utils.click import click
|
|
9
|
+
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@click.command()
|
|
14
|
+
@click.option(
|
|
15
|
+
"--target-folder",
|
|
16
|
+
envvar="CSM_DATASET_ABSOLUTE_PATH",
|
|
17
|
+
help=T("csm-data.commands.api.rds_load_csv.parameters.target_folder"),
|
|
18
|
+
metavar="PATH",
|
|
19
|
+
type=str,
|
|
20
|
+
show_envvar=True,
|
|
21
|
+
required=True,
|
|
22
|
+
)
|
|
23
|
+
@click.option(
|
|
24
|
+
"--organization-id",
|
|
25
|
+
envvar="CSM_ORGANIZATION_ID",
|
|
26
|
+
help=T("csm-data.commands.api.rds_load_csv.parameters.organization_id"),
|
|
27
|
+
metavar="o-XXXXXXXX",
|
|
28
|
+
type=str,
|
|
29
|
+
show_envvar=True,
|
|
30
|
+
required=True,
|
|
31
|
+
)
|
|
32
|
+
@click.option(
|
|
33
|
+
"--workspace-id",
|
|
34
|
+
envvar="CSM_WORKSPACE_ID",
|
|
35
|
+
help=T("csm-data.commands.api.rds_load_csv.parameters.workspace_id"),
|
|
36
|
+
metavar="w-XXXXXXXX",
|
|
37
|
+
type=str,
|
|
38
|
+
show_envvar=True,
|
|
39
|
+
required=True,
|
|
40
|
+
)
|
|
41
|
+
@click.option(
|
|
42
|
+
"--runner-id",
|
|
43
|
+
envvar="CSM_RUNNER_ID",
|
|
44
|
+
help=T("csm-data.commands.api.rds_load_csv.parameters.runner_id"),
|
|
45
|
+
metavar="r-XXXXXXXX",
|
|
46
|
+
type=str,
|
|
47
|
+
show_envvar=True,
|
|
48
|
+
required=True,
|
|
49
|
+
)
|
|
50
|
+
@click.option(
|
|
51
|
+
"--run-id",
|
|
52
|
+
envvar="CSM_RUN_ID",
|
|
53
|
+
help=T("csm-data.commands.api.rds_load_csv.parameters.run_id"),
|
|
54
|
+
metavar="run-XXXXXX",
|
|
55
|
+
type=str,
|
|
56
|
+
show_envvar=True,
|
|
57
|
+
required=True,
|
|
58
|
+
)
|
|
59
|
+
@click.option(
|
|
60
|
+
"--file-name",
|
|
61
|
+
help=T("csm-data.commands.api.rds_load_csv.parameters.file_name"),
|
|
62
|
+
metavar="NAME",
|
|
63
|
+
type=str,
|
|
64
|
+
default="results",
|
|
65
|
+
show_default=True,
|
|
66
|
+
required=True,
|
|
67
|
+
)
|
|
68
|
+
@click.option(
|
|
69
|
+
"--query",
|
|
70
|
+
help=T("csm-data.commands.api.rds_load_csv.parameters.query"),
|
|
71
|
+
metavar="SQL_QUERY",
|
|
72
|
+
type=str,
|
|
73
|
+
default="SELECT table_name FROM information_schema.tables WHERE table_schema='public'",
|
|
74
|
+
show_default=True,
|
|
75
|
+
)
|
|
76
|
+
@web_help("csm-data/api/rds-load-csv")
|
|
77
|
+
@translate_help("csm-data.commands.api.rds_load_csv.description")
|
|
78
|
+
def rds_load_csv(target_folder, organization_id, workspace_id, runner_id, run_id, file_name, query):
|
|
79
|
+
# Import the function at the start of the command
|
|
80
|
+
from cosmotech.coal.cosmotech_api import load_csv_from_run_data
|
|
81
|
+
|
|
82
|
+
load_csv_from_run_data(
|
|
83
|
+
target_folder=target_folder,
|
|
84
|
+
organization_id=organization_id,
|
|
85
|
+
workspace_id=workspace_id,
|
|
86
|
+
runner_id=runner_id,
|
|
87
|
+
run_id=run_id,
|
|
88
|
+
file_name=file_name,
|
|
89
|
+
query=query,
|
|
90
|
+
)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
+
# This document and all information contained herein is the exclusive property -
|
|
3
|
+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
+
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
+
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
|
|
8
|
+
from cosmotech.csm_data.utils.click import click
|
|
9
|
+
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@click.command()
|
|
14
|
+
@click.option(
|
|
15
|
+
"--source-folder",
|
|
16
|
+
envvar="CSM_DATASET_ABSOLUTE_PATH",
|
|
17
|
+
help=T("csm-data.commands.api.rds_send_csv.parameters.source_folder"),
|
|
18
|
+
metavar="PATH",
|
|
19
|
+
type=str,
|
|
20
|
+
show_envvar=True,
|
|
21
|
+
required=True,
|
|
22
|
+
)
|
|
23
|
+
@click.option(
|
|
24
|
+
"--organization-id",
|
|
25
|
+
envvar="CSM_ORGANIZATION_ID",
|
|
26
|
+
help=T("csm-data.commands.api.rds_send_csv.parameters.organization_id"),
|
|
27
|
+
metavar="o-XXXXXXXX",
|
|
28
|
+
type=str,
|
|
29
|
+
show_envvar=True,
|
|
30
|
+
required=True,
|
|
31
|
+
)
|
|
32
|
+
@click.option(
|
|
33
|
+
"--workspace-id",
|
|
34
|
+
envvar="CSM_WORKSPACE_ID",
|
|
35
|
+
help=T("csm-data.commands.api.rds_send_csv.parameters.workspace_id"),
|
|
36
|
+
metavar="w-XXXXXXXX",
|
|
37
|
+
type=str,
|
|
38
|
+
show_envvar=True,
|
|
39
|
+
required=True,
|
|
40
|
+
)
|
|
41
|
+
@click.option(
|
|
42
|
+
"--runner-id",
|
|
43
|
+
envvar="CSM_RUNNER_ID",
|
|
44
|
+
help=T("csm-data.commands.api.rds_send_csv.parameters.runner_id"),
|
|
45
|
+
metavar="r-XXXXXXXX",
|
|
46
|
+
type=str,
|
|
47
|
+
show_envvar=True,
|
|
48
|
+
required=True,
|
|
49
|
+
)
|
|
50
|
+
@click.option(
|
|
51
|
+
"--run-id",
|
|
52
|
+
envvar="CSM_RUN_ID",
|
|
53
|
+
help=T("csm-data.commands.api.rds_send_csv.parameters.run_id"),
|
|
54
|
+
metavar="run-XXXXXX",
|
|
55
|
+
type=str,
|
|
56
|
+
show_envvar=True,
|
|
57
|
+
required=True,
|
|
58
|
+
)
|
|
59
|
+
@web_help("csm-data/api/rds-send-csv")
|
|
60
|
+
@translate_help("csm-data.commands.api.rds_send_csv.description")
|
|
61
|
+
def rds_send_csv(source_folder, organization_id, workspace_id, runner_id, run_id):
|
|
62
|
+
# Import the function at the start of the command
|
|
63
|
+
from cosmotech.coal.cosmotech_api import send_csv_to_run_data
|
|
64
|
+
|
|
65
|
+
try:
|
|
66
|
+
send_csv_to_run_data(
|
|
67
|
+
source_folder=source_folder,
|
|
68
|
+
organization_id=organization_id,
|
|
69
|
+
workspace_id=workspace_id,
|
|
70
|
+
runner_id=runner_id,
|
|
71
|
+
run_id=run_id,
|
|
72
|
+
)
|
|
73
|
+
except FileNotFoundError:
|
|
74
|
+
return 1
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
+
# This document and all information contained herein is the exclusive property -
|
|
3
|
+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
+
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
+
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
|
|
8
|
+
from cosmotech.csm_data.utils.click import click
|
|
9
|
+
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@click.command()
|
|
14
|
+
@click.option(
|
|
15
|
+
"--store-folder",
|
|
16
|
+
envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
|
|
17
|
+
help=T("csm-data.commands.api.rds_send_store.parameters.store_folder"),
|
|
18
|
+
metavar="PATH",
|
|
19
|
+
type=str,
|
|
20
|
+
show_envvar=True,
|
|
21
|
+
required=True,
|
|
22
|
+
)
|
|
23
|
+
@click.option(
|
|
24
|
+
"--organization-id",
|
|
25
|
+
envvar="CSM_ORGANIZATION_ID",
|
|
26
|
+
help=T("csm-data.commands.api.rds_send_store.parameters.organization_id"),
|
|
27
|
+
metavar="o-XXXXXXXX",
|
|
28
|
+
type=str,
|
|
29
|
+
show_envvar=True,
|
|
30
|
+
required=True,
|
|
31
|
+
)
|
|
32
|
+
@click.option(
|
|
33
|
+
"--workspace-id",
|
|
34
|
+
envvar="CSM_WORKSPACE_ID",
|
|
35
|
+
help=T("csm-data.commands.api.rds_send_store.parameters.workspace_id"),
|
|
36
|
+
metavar="w-XXXXXXXX",
|
|
37
|
+
type=str,
|
|
38
|
+
show_envvar=True,
|
|
39
|
+
required=True,
|
|
40
|
+
)
|
|
41
|
+
@click.option(
|
|
42
|
+
"--runner-id",
|
|
43
|
+
envvar="CSM_RUNNER_ID",
|
|
44
|
+
help=T("csm-data.commands.api.rds_send_store.parameters.runner_id"),
|
|
45
|
+
metavar="r-XXXXXXXX",
|
|
46
|
+
type=str,
|
|
47
|
+
show_envvar=True,
|
|
48
|
+
required=True,
|
|
49
|
+
)
|
|
50
|
+
@click.option(
|
|
51
|
+
"--run-id",
|
|
52
|
+
envvar="CSM_RUN_ID",
|
|
53
|
+
help=T("csm-data.commands.api.rds_send_store.parameters.run_id"),
|
|
54
|
+
metavar="run-XXXXXX",
|
|
55
|
+
type=str,
|
|
56
|
+
show_envvar=True,
|
|
57
|
+
required=True,
|
|
58
|
+
)
|
|
59
|
+
@web_help("csm-data/api/rds-send-store")
|
|
60
|
+
@translate_help("csm-data.commands.api.rds_send_store.description")
|
|
61
|
+
def rds_send_store(store_folder, organization_id, workspace_id, runner_id, run_id):
|
|
62
|
+
# Import the function at the start of the command
|
|
63
|
+
from cosmotech.coal.cosmotech_api import send_store_to_run_data
|
|
64
|
+
|
|
65
|
+
try:
|
|
66
|
+
send_store_to_run_data(
|
|
67
|
+
store_folder=store_folder,
|
|
68
|
+
organization_id=organization_id,
|
|
69
|
+
workspace_id=workspace_id,
|
|
70
|
+
runner_id=runner_id,
|
|
71
|
+
run_id=run_id,
|
|
72
|
+
)
|
|
73
|
+
except FileNotFoundError:
|
|
74
|
+
return 1
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
+
# This document and all information contained herein is the exclusive property -
|
|
3
|
+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
+
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
+
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
|
|
8
|
+
from cosmotech.csm_data.utils.click import click
|
|
9
|
+
from cosmotech.csm_data.utils.decorators import require_env
|
|
10
|
+
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
11
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@click.command()
|
|
15
|
+
@click.option(
|
|
16
|
+
"--organization-id",
|
|
17
|
+
envvar="CSM_ORGANIZATION_ID",
|
|
18
|
+
show_envvar=True,
|
|
19
|
+
help=T("csm-data.commands.api.run_load_data.parameters.organization_id"),
|
|
20
|
+
metavar="o-##########",
|
|
21
|
+
required=True,
|
|
22
|
+
)
|
|
23
|
+
@click.option(
|
|
24
|
+
"--workspace-id",
|
|
25
|
+
envvar="CSM_WORKSPACE_ID",
|
|
26
|
+
show_envvar=True,
|
|
27
|
+
help=T("csm-data.commands.api.run_load_data.parameters.workspace_id"),
|
|
28
|
+
metavar="w-##########",
|
|
29
|
+
required=True,
|
|
30
|
+
)
|
|
31
|
+
@click.option(
|
|
32
|
+
"--runner-id",
|
|
33
|
+
envvar="CSM_RUNNER_ID",
|
|
34
|
+
show_envvar=True,
|
|
35
|
+
help=T("csm-data.commands.api.run_load_data.parameters.runner_id"),
|
|
36
|
+
metavar="s-##########",
|
|
37
|
+
required=True,
|
|
38
|
+
)
|
|
39
|
+
@click.option(
|
|
40
|
+
"--dataset-absolute-path",
|
|
41
|
+
envvar="CSM_DATASET_ABSOLUTE_PATH",
|
|
42
|
+
show_envvar=True,
|
|
43
|
+
help=T("csm-data.commands.api.run_load_data.parameters.dataset_absolute_path"),
|
|
44
|
+
metavar="PATH",
|
|
45
|
+
required=True,
|
|
46
|
+
)
|
|
47
|
+
@click.option(
|
|
48
|
+
"--parameters-absolute-path",
|
|
49
|
+
envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
|
|
50
|
+
metavar="PATH",
|
|
51
|
+
show_envvar=True,
|
|
52
|
+
help=T("csm-data.commands.api.run_load_data.parameters.parameters_absolute_path"),
|
|
53
|
+
required=True,
|
|
54
|
+
)
|
|
55
|
+
@click.option(
|
|
56
|
+
"--write-json/--no-write-json",
|
|
57
|
+
envvar="WRITE_JSON",
|
|
58
|
+
show_envvar=True,
|
|
59
|
+
default=True,
|
|
60
|
+
show_default=True,
|
|
61
|
+
help=T("csm-data.commands.api.run_load_data.parameters.write_json"),
|
|
62
|
+
)
|
|
63
|
+
@click.option(
|
|
64
|
+
"--write-csv/--no-write-csv",
|
|
65
|
+
envvar="WRITE_CSV",
|
|
66
|
+
show_envvar=True,
|
|
67
|
+
default=False,
|
|
68
|
+
show_default=True,
|
|
69
|
+
help=T("csm-data.commands.api.run_load_data.parameters.write_csv"),
|
|
70
|
+
)
|
|
71
|
+
@click.option(
|
|
72
|
+
"--fetch-dataset/--no-fetch-dataset",
|
|
73
|
+
envvar="FETCH_DATASET",
|
|
74
|
+
show_envvar=True,
|
|
75
|
+
default=True,
|
|
76
|
+
show_default=True,
|
|
77
|
+
help=T("csm-data.commands.api.run_load_data.parameters.fetch_dataset"),
|
|
78
|
+
)
|
|
79
|
+
@click.option(
|
|
80
|
+
"--parallel/--no-parallel",
|
|
81
|
+
envvar="FETCH_DATASETS_IN_PARALLEL",
|
|
82
|
+
show_envvar=True,
|
|
83
|
+
default=True,
|
|
84
|
+
show_default=True,
|
|
85
|
+
help=T("csm-data.commands.api.run_load_data.parameters.parallel"),
|
|
86
|
+
)
|
|
87
|
+
@require_env("CSM_API_SCOPE", "The identification scope of a Cosmotech API")
|
|
88
|
+
@require_env("CSM_API_URL", "The URL to a Cosmotech API")
|
|
89
|
+
@web_help("csm-data/api/run-load-data")
|
|
90
|
+
@translate_help("csm-data.commands.api.run_load_data.description")
|
|
91
|
+
def run_load_data(
|
|
92
|
+
runner_id: str,
|
|
93
|
+
workspace_id: str,
|
|
94
|
+
organization_id: str,
|
|
95
|
+
dataset_absolute_path: str,
|
|
96
|
+
parameters_absolute_path: str,
|
|
97
|
+
write_json: bool,
|
|
98
|
+
write_csv: bool,
|
|
99
|
+
fetch_dataset: bool,
|
|
100
|
+
parallel: bool,
|
|
101
|
+
):
|
|
102
|
+
# Import the function at the start of the command
|
|
103
|
+
from cosmotech.coal.cosmotech_api.runner.download import download_runner_data
|
|
104
|
+
|
|
105
|
+
return download_runner_data(
|
|
106
|
+
organization_id=organization_id,
|
|
107
|
+
workspace_id=workspace_id,
|
|
108
|
+
runner_id=runner_id,
|
|
109
|
+
parameter_folder=parameters_absolute_path,
|
|
110
|
+
dataset_folder=dataset_absolute_path,
|
|
111
|
+
read_files=False,
|
|
112
|
+
parallel=parallel,
|
|
113
|
+
write_json=write_json,
|
|
114
|
+
write_csv=write_csv,
|
|
115
|
+
fetch_dataset=fetch_dataset,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
if __name__ == "__main__":
|
|
120
|
+
run_load_data()
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
+
# This document and all information contained herein is the exclusive property -
|
|
3
|
+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
+
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
+
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
|
|
8
|
+
from cosmotech.csm_data.utils.click import click
|
|
9
|
+
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@click.command()
|
|
14
|
+
@web_help("csm-data/api/runtemplate-load-handler")
|
|
15
|
+
@translate_help("csm-data.commands.api.runtemplate_load_handler.description")
|
|
16
|
+
@click.option(
|
|
17
|
+
"--organization-id",
|
|
18
|
+
envvar="CSM_ORGANIZATION_ID",
|
|
19
|
+
show_envvar=True,
|
|
20
|
+
help=T("csm-data.commands.api.runtemplate_load_handler.parameters.organization_id"),
|
|
21
|
+
metavar="o-##########",
|
|
22
|
+
required=True,
|
|
23
|
+
)
|
|
24
|
+
@click.option(
|
|
25
|
+
"--workspace-id",
|
|
26
|
+
envvar="CSM_WORKSPACE_ID",
|
|
27
|
+
show_envvar=True,
|
|
28
|
+
help=T("csm-data.commands.api.runtemplate_load_handler.parameters.workspace_id"),
|
|
29
|
+
metavar="w-##########",
|
|
30
|
+
required=True,
|
|
31
|
+
)
|
|
32
|
+
@click.option(
|
|
33
|
+
"--run-template-id",
|
|
34
|
+
envvar="CSM_RUN_TEMPLATE_ID",
|
|
35
|
+
show_envvar=True,
|
|
36
|
+
help=T("csm-data.commands.api.runtemplate_load_handler.parameters.run_template_id"),
|
|
37
|
+
metavar="NAME",
|
|
38
|
+
required=True,
|
|
39
|
+
)
|
|
40
|
+
@click.option(
|
|
41
|
+
"--handler-list",
|
|
42
|
+
envvar="CSM_CONTAINER_MODE",
|
|
43
|
+
show_envvar=True,
|
|
44
|
+
help=T("csm-data.commands.api.runtemplate_load_handler.parameters.handler_list"),
|
|
45
|
+
metavar="HANDLER,...,HANDLER",
|
|
46
|
+
required=True,
|
|
47
|
+
)
|
|
48
|
+
def runtemplate_load_handler(workspace_id, organization_id, run_template_id, handler_list):
|
|
49
|
+
# Import the function at the start of the command
|
|
50
|
+
from cosmotech.coal.cosmotech_api import load_run_template_handlers
|
|
51
|
+
|
|
52
|
+
try:
|
|
53
|
+
success = load_run_template_handlers(
|
|
54
|
+
organization_id=organization_id,
|
|
55
|
+
workspace_id=workspace_id,
|
|
56
|
+
run_template_id=run_template_id,
|
|
57
|
+
handler_list=handler_list,
|
|
58
|
+
)
|
|
59
|
+
if not success:
|
|
60
|
+
raise click.Abort()
|
|
61
|
+
except ValueError:
|
|
62
|
+
raise click.Abort()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
if __name__ == "__main__":
|
|
66
|
+
runtemplate_load_handler()
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
+
# This document and all information contained herein is the exclusive property -
|
|
3
|
+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
+
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
+
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
|
|
8
|
+
from cosmotech.csm_data.utils.click import click
|
|
9
|
+
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@click.command()
|
|
14
|
+
@click.option(
|
|
15
|
+
"--organization-id",
|
|
16
|
+
envvar="CSM_ORGANIZATION_ID",
|
|
17
|
+
help=T("csm-data.commands.api.tdl_load_files.parameters.organization_id"),
|
|
18
|
+
metavar="o-XXXXXXXX",
|
|
19
|
+
type=str,
|
|
20
|
+
show_envvar=True,
|
|
21
|
+
required=True,
|
|
22
|
+
)
|
|
23
|
+
@click.option(
|
|
24
|
+
"--workspace-id",
|
|
25
|
+
envvar="CSM_WORKSPACE_ID",
|
|
26
|
+
help=T("csm-data.commands.api.tdl_load_files.parameters.workspace_id"),
|
|
27
|
+
metavar="w-XXXXXXXX",
|
|
28
|
+
type=str,
|
|
29
|
+
show_envvar=True,
|
|
30
|
+
required=True,
|
|
31
|
+
)
|
|
32
|
+
@click.option(
|
|
33
|
+
"--scenario-id",
|
|
34
|
+
envvar="CSM_SCENARIO_ID",
|
|
35
|
+
help=T("csm-data.commands.api.tdl_load_files.parameters.scenario_id"),
|
|
36
|
+
metavar="s-XXXXXXXX",
|
|
37
|
+
type=str,
|
|
38
|
+
show_envvar=True,
|
|
39
|
+
required=False,
|
|
40
|
+
)
|
|
41
|
+
@click.option(
|
|
42
|
+
"--runner-id",
|
|
43
|
+
envvar="CSM_RUNNER_ID",
|
|
44
|
+
help=T("csm-data.commands.api.tdl_load_files.parameters.runner_id"),
|
|
45
|
+
metavar="r-XXXXXXXX",
|
|
46
|
+
type=str,
|
|
47
|
+
show_envvar=True,
|
|
48
|
+
required=False,
|
|
49
|
+
)
|
|
50
|
+
@click.option(
|
|
51
|
+
"--dir",
|
|
52
|
+
"directory_path",
|
|
53
|
+
help=T("csm-data.commands.api.tdl_load_files.parameters.dir"),
|
|
54
|
+
metavar="PATH",
|
|
55
|
+
default="./",
|
|
56
|
+
type=str,
|
|
57
|
+
envvar="CSM_DATASET_ABSOLUTE_PATH",
|
|
58
|
+
show_envvar=True,
|
|
59
|
+
required=True,
|
|
60
|
+
)
|
|
61
|
+
@web_help("csm-data/api/tdl-load-file")
|
|
62
|
+
@translate_help("csm-data.commands.api.tdl_load_files.description")
|
|
63
|
+
def tdl_load_files(organization_id, workspace_id, scenario_id, runner_id, directory_path):
|
|
64
|
+
# Import the function at the start of the command
|
|
65
|
+
from cosmotech.coal.cosmotech_api import load_files_from_tdl
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
load_files_from_tdl(
|
|
69
|
+
organization_id=organization_id,
|
|
70
|
+
workspace_id=workspace_id,
|
|
71
|
+
directory_path=directory_path,
|
|
72
|
+
runner_id=runner_id,
|
|
73
|
+
scenario_id=scenario_id,
|
|
74
|
+
)
|
|
75
|
+
except ValueError as e:
|
|
76
|
+
raise click.Abort() from e
|