cosmotech-acceleration-library 1.1.0__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cosmotech/coal/__init__.py +1 -1
- cosmotech/coal/aws/__init__.py +1 -9
- cosmotech/coal/aws/s3.py +181 -214
- cosmotech/coal/azure/adx/auth.py +2 -2
- cosmotech/coal/azure/adx/runner.py +13 -14
- cosmotech/coal/azure/adx/store.py +5 -86
- cosmotech/coal/azure/adx/tables.py +2 -2
- cosmotech/coal/azure/blob.py +6 -6
- cosmotech/coal/azure/storage.py +3 -3
- cosmotech/coal/cosmotech_api/__init__.py +0 -28
- cosmotech/coal/cosmotech_api/apis/__init__.py +14 -0
- cosmotech/coal/cosmotech_api/apis/dataset.py +103 -0
- cosmotech/coal/cosmotech_api/apis/meta.py +25 -0
- cosmotech/coal/cosmotech_api/apis/organization.py +24 -0
- cosmotech/coal/cosmotech_api/apis/run.py +38 -0
- cosmotech/coal/cosmotech_api/apis/runner.py +71 -0
- cosmotech/coal/cosmotech_api/apis/solution.py +23 -0
- cosmotech/coal/cosmotech_api/apis/workspace.py +108 -0
- cosmotech/coal/cosmotech_api/objects/__init__.py +9 -0
- cosmotech/coal/cosmotech_api/objects/connection.py +125 -0
- cosmotech/coal/cosmotech_api/objects/parameters.py +127 -0
- cosmotech/coal/postgresql/runner.py +56 -36
- cosmotech/coal/postgresql/store.py +60 -14
- cosmotech/coal/postgresql/utils.py +254 -0
- cosmotech/coal/store/output/__init__.py +0 -0
- cosmotech/coal/store/output/aws_channel.py +73 -0
- cosmotech/coal/store/output/az_storage_channel.py +42 -0
- cosmotech/coal/store/output/channel_interface.py +23 -0
- cosmotech/coal/store/output/channel_spliter.py +55 -0
- cosmotech/coal/store/output/postgres_channel.py +40 -0
- cosmotech/coal/utils/configuration.py +169 -0
- cosmotech/coal/utils/decorator.py +4 -7
- cosmotech/csm_data/commands/api/api.py +6 -19
- cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +20 -16
- cosmotech/csm_data/commands/api/run_load_data.py +7 -46
- cosmotech/csm_data/commands/api/wsf_load_file.py +13 -16
- cosmotech/csm_data/commands/api/wsf_send_file.py +11 -14
- cosmotech/csm_data/commands/s3_bucket_delete.py +16 -15
- cosmotech/csm_data/commands/s3_bucket_download.py +16 -16
- cosmotech/csm_data/commands/s3_bucket_upload.py +16 -14
- cosmotech/csm_data/commands/store/dump_to_s3.py +18 -16
- cosmotech/csm_data/commands/store/output.py +35 -0
- cosmotech/csm_data/commands/store/store.py +3 -3
- cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml +8 -0
- cosmotech/translation/coal/en-US/coal/services/dataset.yml +4 -14
- cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml +1 -0
- cosmotech/translation/coal/en-US/coal/store/output/split.yml +6 -0
- cosmotech/translation/coal/en-US/coal/utils/configuration.yml +2 -0
- cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml +7 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/METADATA +5 -8
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/RECORD +55 -73
- cosmotech/coal/azure/functions.py +0 -72
- cosmotech/coal/cosmotech_api/connection.py +0 -96
- cosmotech/coal/cosmotech_api/dataset/__init__.py +0 -26
- cosmotech/coal/cosmotech_api/dataset/converters.py +0 -164
- cosmotech/coal/cosmotech_api/dataset/download/__init__.py +0 -19
- cosmotech/coal/cosmotech_api/dataset/download/adt.py +0 -119
- cosmotech/coal/cosmotech_api/dataset/download/common.py +0 -140
- cosmotech/coal/cosmotech_api/dataset/download/file.py +0 -229
- cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +0 -185
- cosmotech/coal/cosmotech_api/dataset/upload.py +0 -41
- cosmotech/coal/cosmotech_api/dataset/utils.py +0 -132
- cosmotech/coal/cosmotech_api/parameters.py +0 -48
- cosmotech/coal/cosmotech_api/run.py +0 -25
- cosmotech/coal/cosmotech_api/run_data.py +0 -173
- cosmotech/coal/cosmotech_api/run_template.py +0 -108
- cosmotech/coal/cosmotech_api/runner/__init__.py +0 -28
- cosmotech/coal/cosmotech_api/runner/data.py +0 -38
- cosmotech/coal/cosmotech_api/runner/datasets.py +0 -416
- cosmotech/coal/cosmotech_api/runner/download.py +0 -135
- cosmotech/coal/cosmotech_api/runner/metadata.py +0 -42
- cosmotech/coal/cosmotech_api/runner/parameters.py +0 -157
- cosmotech/coal/cosmotech_api/twin_data_layer.py +0 -512
- cosmotech/coal/cosmotech_api/workspace.py +0 -127
- cosmotech/coal/utils/postgresql.py +0 -236
- cosmotech/coal/utils/semver.py +0 -6
- cosmotech/csm_data/commands/api/rds_load_csv.py +0 -90
- cosmotech/csm_data/commands/api/rds_send_csv.py +0 -74
- cosmotech/csm_data/commands/api/rds_send_store.py +0 -74
- cosmotech/csm_data/commands/api/runtemplate_load_handler.py +0 -66
- cosmotech/csm_data/commands/api/tdl_load_files.py +0 -76
- cosmotech/csm_data/commands/api/tdl_send_files.py +0 -82
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +0 -32
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +0 -27
- cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +0 -2
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml +0 -13
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml +0 -12
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml +0 -12
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml +0 -14
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml +0 -18
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/WHEEL +0 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/entry_points.txt +0 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/licenses/LICENSE +0 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/top_level.txt +0 -0
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
import pathlib
|
|
8
|
-
|
|
9
|
-
import cosmotech_api
|
|
10
|
-
|
|
11
|
-
from cosmotech.coal.utils.logger import LOGGER
|
|
12
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def list_workspace_files(
|
|
16
|
-
api_client: cosmotech_api.api_client.ApiClient,
|
|
17
|
-
organization_id: str,
|
|
18
|
-
workspace_id: str,
|
|
19
|
-
file_prefix: str,
|
|
20
|
-
) -> list[str]:
|
|
21
|
-
"""
|
|
22
|
-
Helper function to list all workspace files using a pre-given file prefix
|
|
23
|
-
:param api_client: An api client used to connect to the Cosmo Tech API
|
|
24
|
-
:param organization_id: An ID of an Organization in the Cosmo Tech API
|
|
25
|
-
:param workspace_id: An ID of a Workspace in the Cosmo Tech API
|
|
26
|
-
:param file_prefix: The prefix of the files to find in the Workspace
|
|
27
|
-
:return: A list of existing files inside the workspace
|
|
28
|
-
"""
|
|
29
|
-
target_list = []
|
|
30
|
-
api_ws = cosmotech_api.api.workspace_api.WorkspaceApi(api_client)
|
|
31
|
-
LOGGER.info(T("coal.cosmotech_api.workspace.target_is_folder"))
|
|
32
|
-
wsf = api_ws.find_all_workspace_files(organization_id, workspace_id)
|
|
33
|
-
for workspace_file in wsf:
|
|
34
|
-
if workspace_file.file_name.startswith(file_prefix):
|
|
35
|
-
target_list.append(workspace_file.file_name)
|
|
36
|
-
|
|
37
|
-
if not target_list:
|
|
38
|
-
LOGGER.error(
|
|
39
|
-
T("coal.common.errors.data_no_workspace_files").format(file_prefix=file_prefix, workspace_id=workspace_id)
|
|
40
|
-
)
|
|
41
|
-
raise ValueError(
|
|
42
|
-
T("coal.common.errors.data_no_workspace_files").format(file_prefix=file_prefix, workspace_id=workspace_id)
|
|
43
|
-
)
|
|
44
|
-
|
|
45
|
-
return target_list
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def download_workspace_file(
|
|
49
|
-
api_client: cosmotech_api.api_client.ApiClient,
|
|
50
|
-
organization_id: str,
|
|
51
|
-
workspace_id: str,
|
|
52
|
-
file_name: str,
|
|
53
|
-
target_dir: pathlib.Path,
|
|
54
|
-
) -> pathlib.Path:
|
|
55
|
-
"""
|
|
56
|
-
Downloads a given file from a workspace to a given directory
|
|
57
|
-
If the file is inside a directory in the workspace, sub-directories will be created.
|
|
58
|
-
:param api_client: An api client used to connect to the Cosmo Tech API
|
|
59
|
-
:param organization_id: An ID of an Organization in the Cosmo Tech API
|
|
60
|
-
:param workspace_id: An ID of a Workspace in the Cosmo Tech API
|
|
61
|
-
:param file_name: The file to download to the workspace
|
|
62
|
-
:param target_dir: The directory in which to write the file
|
|
63
|
-
:return: The path to the created file
|
|
64
|
-
"""
|
|
65
|
-
if target_dir.is_file():
|
|
66
|
-
raise ValueError(T("coal.common.file_operations.not_directory").format(target_dir=target_dir))
|
|
67
|
-
api_ws = cosmotech_api.api.workspace_api.WorkspaceApi(api_client)
|
|
68
|
-
|
|
69
|
-
LOGGER.info(T("coal.cosmotech_api.workspace.loading_file").format(file_name=file_name))
|
|
70
|
-
|
|
71
|
-
_file_content = api_ws.download_workspace_file(organization_id, workspace_id, file_name)
|
|
72
|
-
|
|
73
|
-
local_target_file = target_dir / file_name
|
|
74
|
-
local_target_file.parent.mkdir(parents=True, exist_ok=True)
|
|
75
|
-
|
|
76
|
-
with open(local_target_file, "wb") as _file:
|
|
77
|
-
_file.write(_file_content)
|
|
78
|
-
|
|
79
|
-
LOGGER.info(T("coal.cosmotech_api.workspace.file_loaded").format(file=local_target_file))
|
|
80
|
-
|
|
81
|
-
return local_target_file
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
def upload_workspace_file(
|
|
85
|
-
api_client: cosmotech_api.api_client.ApiClient,
|
|
86
|
-
organization_id: str,
|
|
87
|
-
workspace_id: str,
|
|
88
|
-
file_path: str,
|
|
89
|
-
workspace_path: str,
|
|
90
|
-
overwrite: bool = True,
|
|
91
|
-
) -> str:
|
|
92
|
-
"""
|
|
93
|
-
Upload a local file to a given workspace
|
|
94
|
-
|
|
95
|
-
If workspace_path ends with a "/" it will be considered as a folder inside the workspace
|
|
96
|
-
and the file will keep its current name
|
|
97
|
-
|
|
98
|
-
:param api_client: An api client used to connect to the Cosmo Tech API
|
|
99
|
-
:param organization_id: An ID of an Organization in the Cosmo Tech API
|
|
100
|
-
:param workspace_id: An ID of a Workspace in the Cosmo Tech API
|
|
101
|
-
:param file_path: Path to the file to upload in the workspace
|
|
102
|
-
:param workspace_path: The path inside the workspace to upload the file to
|
|
103
|
-
:param overwrite: Overwrite existing file in the workspace
|
|
104
|
-
:return: The final name of the file uploaded to the workspace
|
|
105
|
-
"""
|
|
106
|
-
target_file = pathlib.Path(file_path)
|
|
107
|
-
if not target_file.exists():
|
|
108
|
-
LOGGER.error(T("coal.common.file_operations.not_exists").format(file_path=file_path))
|
|
109
|
-
raise ValueError(T("coal.common.file_operations.not_exists").format(file_path=file_path))
|
|
110
|
-
if not target_file.is_file():
|
|
111
|
-
LOGGER.error(T("coal.common.file_operations.not_single_file").format(file_path=file_path))
|
|
112
|
-
raise ValueError(T("coal.common.file_operations.not_single_file").format(file_path=file_path))
|
|
113
|
-
|
|
114
|
-
api_ws = cosmotech_api.api.workspace_api.WorkspaceApi(api_client)
|
|
115
|
-
destination = workspace_path + target_file.name if workspace_path.endswith("/") else workspace_path
|
|
116
|
-
|
|
117
|
-
LOGGER.info(T("coal.cosmotech_api.workspace.sending_to_api").format(destination=destination))
|
|
118
|
-
try:
|
|
119
|
-
_file = api_ws.upload_workspace_file(
|
|
120
|
-
organization_id, workspace_id, file_path, overwrite, destination=destination
|
|
121
|
-
)
|
|
122
|
-
except cosmotech_api.exceptions.ApiException as e:
|
|
123
|
-
LOGGER.error(T("coal.common.file_operations.already_exists").format(csv_path=destination))
|
|
124
|
-
raise e
|
|
125
|
-
|
|
126
|
-
LOGGER.info(T("coal.cosmotech_api.workspace.file_sent").format(file=_file.file_name))
|
|
127
|
-
return _file.file_name
|
|
@@ -1,236 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
from typing import Optional
|
|
9
|
-
from urllib.parse import quote
|
|
10
|
-
|
|
11
|
-
import adbc_driver_manager
|
|
12
|
-
import pyarrow as pa
|
|
13
|
-
from adbc_driver_postgresql import dbapi
|
|
14
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
15
|
-
from pyarrow import Table
|
|
16
|
-
|
|
17
|
-
from cosmotech.coal.utils.logger import LOGGER
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def generate_postgresql_full_uri(
|
|
21
|
-
postgres_host: str,
|
|
22
|
-
postgres_port: str,
|
|
23
|
-
postgres_db: str,
|
|
24
|
-
postgres_user: str,
|
|
25
|
-
postgres_password: str,
|
|
26
|
-
force_encode: bool = False,
|
|
27
|
-
) -> str:
|
|
28
|
-
# Check if password needs percent encoding (contains special characters)
|
|
29
|
-
# We don't log anything about the password for security
|
|
30
|
-
encoded_password = postgres_password
|
|
31
|
-
if force_encode:
|
|
32
|
-
encoded_password = quote(postgres_password, safe="")
|
|
33
|
-
|
|
34
|
-
return (
|
|
35
|
-
"postgresql://" + f"{postgres_user}"
|
|
36
|
-
f":{encoded_password}"
|
|
37
|
-
f"@{postgres_host}"
|
|
38
|
-
f":{postgres_port}"
|
|
39
|
-
f"/{postgres_db}"
|
|
40
|
-
)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def get_postgresql_table_schema(
|
|
44
|
-
target_table_name: str,
|
|
45
|
-
postgres_host: str,
|
|
46
|
-
postgres_port: str,
|
|
47
|
-
postgres_db: str,
|
|
48
|
-
postgres_schema: str,
|
|
49
|
-
postgres_user: str,
|
|
50
|
-
postgres_password: str,
|
|
51
|
-
force_encode: bool = False,
|
|
52
|
-
) -> Optional[pa.Schema]:
|
|
53
|
-
"""
|
|
54
|
-
Get the schema of an existing PostgreSQL table using SQL queries.
|
|
55
|
-
|
|
56
|
-
Args:
|
|
57
|
-
target_table_name: Name of the table
|
|
58
|
-
postgres_host: PostgreSQL host
|
|
59
|
-
postgres_port: PostgreSQL port
|
|
60
|
-
postgres_db: PostgreSQL database name
|
|
61
|
-
postgres_schema: PostgreSQL schema name
|
|
62
|
-
postgres_user: PostgreSQL username
|
|
63
|
-
postgres_password: PostgreSQL password
|
|
64
|
-
|
|
65
|
-
Returns:
|
|
66
|
-
PyArrow Schema if table exists, None otherwise
|
|
67
|
-
"""
|
|
68
|
-
LOGGER.debug(
|
|
69
|
-
T("coal.services.postgresql.getting_schema").format(
|
|
70
|
-
postgres_schema=postgres_schema, target_table_name=target_table_name
|
|
71
|
-
)
|
|
72
|
-
)
|
|
73
|
-
|
|
74
|
-
postgresql_full_uri = generate_postgresql_full_uri(
|
|
75
|
-
postgres_host,
|
|
76
|
-
postgres_port,
|
|
77
|
-
postgres_db,
|
|
78
|
-
postgres_user,
|
|
79
|
-
postgres_password,
|
|
80
|
-
force_encode,
|
|
81
|
-
)
|
|
82
|
-
|
|
83
|
-
with dbapi.connect(postgresql_full_uri) as conn:
|
|
84
|
-
try:
|
|
85
|
-
return conn.adbc_get_table_schema(
|
|
86
|
-
target_table_name,
|
|
87
|
-
db_schema_filter=postgres_schema,
|
|
88
|
-
)
|
|
89
|
-
except adbc_driver_manager.ProgrammingError:
|
|
90
|
-
LOGGER.warning(
|
|
91
|
-
T("coal.services.postgresql.table_not_found").format(
|
|
92
|
-
postgres_schema=postgres_schema, target_table_name=target_table_name
|
|
93
|
-
)
|
|
94
|
-
)
|
|
95
|
-
return None
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def adapt_table_to_schema(data: pa.Table, target_schema: pa.Schema) -> pa.Table:
|
|
99
|
-
"""
|
|
100
|
-
Adapt a PyArrow table to match a target schema with detailed logging.
|
|
101
|
-
"""
|
|
102
|
-
LOGGER.debug(T("coal.services.postgresql.schema_adaptation_start").format(rows=len(data)))
|
|
103
|
-
LOGGER.debug(T("coal.services.postgresql.original_schema").format(schema=data.schema))
|
|
104
|
-
LOGGER.debug(T("coal.services.postgresql.target_schema").format(schema=target_schema))
|
|
105
|
-
|
|
106
|
-
target_fields = {field.name: field.type for field in target_schema}
|
|
107
|
-
new_columns = []
|
|
108
|
-
|
|
109
|
-
# Track adaptations for summary
|
|
110
|
-
added_columns = []
|
|
111
|
-
dropped_columns = []
|
|
112
|
-
type_conversions = []
|
|
113
|
-
failed_conversions = []
|
|
114
|
-
|
|
115
|
-
# Process each field in target schema
|
|
116
|
-
for field_name, target_type in target_fields.items():
|
|
117
|
-
if field_name in data.column_names:
|
|
118
|
-
# Column exists - try to cast to target type
|
|
119
|
-
col = data[field_name]
|
|
120
|
-
original_type = col.type
|
|
121
|
-
|
|
122
|
-
if original_type != target_type:
|
|
123
|
-
LOGGER.debug(
|
|
124
|
-
T("coal.services.postgresql.casting_column").format(
|
|
125
|
-
field_name=field_name,
|
|
126
|
-
original_type=original_type,
|
|
127
|
-
target_type=target_type,
|
|
128
|
-
)
|
|
129
|
-
)
|
|
130
|
-
try:
|
|
131
|
-
new_col = pa.compute.cast(col, target_type)
|
|
132
|
-
new_columns.append(new_col)
|
|
133
|
-
type_conversions.append(f"{field_name}: {original_type} -> {target_type}")
|
|
134
|
-
except pa.ArrowInvalid as e:
|
|
135
|
-
LOGGER.warning(
|
|
136
|
-
T("coal.services.postgresql.cast_failed").format(
|
|
137
|
-
field_name=field_name,
|
|
138
|
-
original_type=original_type,
|
|
139
|
-
target_type=target_type,
|
|
140
|
-
error=str(e),
|
|
141
|
-
)
|
|
142
|
-
)
|
|
143
|
-
new_columns.append(pa.nulls(len(data), type=target_type))
|
|
144
|
-
failed_conversions.append(f"{field_name}: {original_type} -> {target_type}")
|
|
145
|
-
else:
|
|
146
|
-
new_columns.append(col)
|
|
147
|
-
else:
|
|
148
|
-
# Column doesn't exist - add nulls
|
|
149
|
-
LOGGER.debug(T("coal.services.postgresql.adding_missing_column").format(field_name=field_name))
|
|
150
|
-
new_columns.append(pa.nulls(len(data), type=target_type))
|
|
151
|
-
added_columns.append(field_name)
|
|
152
|
-
|
|
153
|
-
# Log columns that will be dropped
|
|
154
|
-
dropped_columns = [name for name in data.column_names if name not in target_fields]
|
|
155
|
-
if dropped_columns:
|
|
156
|
-
LOGGER.debug(T("coal.services.postgresql.dropping_columns").format(columns=dropped_columns))
|
|
157
|
-
|
|
158
|
-
# Create new table
|
|
159
|
-
adapted_table = pa.Table.from_arrays(new_columns, schema=target_schema)
|
|
160
|
-
|
|
161
|
-
# Log summary of adaptations
|
|
162
|
-
LOGGER.debug(T("coal.services.postgresql.adaptation_summary"))
|
|
163
|
-
if added_columns:
|
|
164
|
-
LOGGER.debug(T("coal.services.postgresql.added_columns").format(columns=added_columns))
|
|
165
|
-
if dropped_columns:
|
|
166
|
-
LOGGER.debug(T("coal.services.postgresql.dropped_columns").format(columns=dropped_columns))
|
|
167
|
-
if type_conversions:
|
|
168
|
-
LOGGER.debug(T("coal.services.postgresql.successful_conversions").format(conversions=type_conversions))
|
|
169
|
-
if failed_conversions:
|
|
170
|
-
LOGGER.debug(T("coal.services.postgresql.failed_conversions").format(conversions=failed_conversions))
|
|
171
|
-
|
|
172
|
-
LOGGER.debug(T("coal.services.postgresql.final_schema").format(schema=adapted_table.schema))
|
|
173
|
-
return adapted_table
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
def send_pyarrow_table_to_postgresql(
|
|
177
|
-
data: Table,
|
|
178
|
-
target_table_name: str,
|
|
179
|
-
postgres_host: str,
|
|
180
|
-
postgres_port: str,
|
|
181
|
-
postgres_db: str,
|
|
182
|
-
postgres_schema: str,
|
|
183
|
-
postgres_user: str,
|
|
184
|
-
postgres_password: str,
|
|
185
|
-
replace: bool,
|
|
186
|
-
force_encode: bool = False,
|
|
187
|
-
) -> int:
|
|
188
|
-
LOGGER.debug(
|
|
189
|
-
T("coal.services.postgresql.preparing_send").format(
|
|
190
|
-
postgres_schema=postgres_schema, target_table_name=target_table_name
|
|
191
|
-
)
|
|
192
|
-
)
|
|
193
|
-
LOGGER.debug(T("coal.services.postgresql.input_rows").format(rows=len(data)))
|
|
194
|
-
|
|
195
|
-
# Get existing schema if table exists
|
|
196
|
-
existing_schema = get_postgresql_table_schema(
|
|
197
|
-
target_table_name,
|
|
198
|
-
postgres_host,
|
|
199
|
-
postgres_port,
|
|
200
|
-
postgres_db,
|
|
201
|
-
postgres_schema,
|
|
202
|
-
postgres_user,
|
|
203
|
-
postgres_password,
|
|
204
|
-
force_encode,
|
|
205
|
-
)
|
|
206
|
-
|
|
207
|
-
if existing_schema is not None:
|
|
208
|
-
LOGGER.debug(T("coal.services.postgresql.found_existing_table").format(schema=existing_schema))
|
|
209
|
-
if not replace:
|
|
210
|
-
LOGGER.debug(T("coal.services.postgresql.adapting_data"))
|
|
211
|
-
data = adapt_table_to_schema(data, existing_schema)
|
|
212
|
-
else:
|
|
213
|
-
LOGGER.debug(T("coal.services.postgresql.replace_mode"))
|
|
214
|
-
else:
|
|
215
|
-
LOGGER.debug(T("coal.services.postgresql.no_existing_table"))
|
|
216
|
-
|
|
217
|
-
# Proceed with ingestion
|
|
218
|
-
total = 0
|
|
219
|
-
postgresql_full_uri = generate_postgresql_full_uri(
|
|
220
|
-
postgres_host,
|
|
221
|
-
postgres_port,
|
|
222
|
-
postgres_db,
|
|
223
|
-
postgres_user,
|
|
224
|
-
postgres_password,
|
|
225
|
-
force_encode,
|
|
226
|
-
)
|
|
227
|
-
|
|
228
|
-
LOGGER.debug(T("coal.services.postgresql.connecting"))
|
|
229
|
-
with dbapi.connect(postgresql_full_uri, autocommit=True) as conn:
|
|
230
|
-
with conn.cursor() as curs:
|
|
231
|
-
mode = "replace" if replace else "create_append"
|
|
232
|
-
LOGGER.debug(T("coal.services.postgresql.ingesting_data").format(mode=mode))
|
|
233
|
-
total += curs.adbc_ingest(target_table_name, data, mode, db_schema_name=postgres_schema)
|
|
234
|
-
|
|
235
|
-
LOGGER.debug(T("coal.services.postgresql.ingestion_success").format(rows=total))
|
|
236
|
-
return total
|
cosmotech/coal/utils/semver.py
DELETED
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
@click.command()
|
|
14
|
-
@click.option(
|
|
15
|
-
"--target-folder",
|
|
16
|
-
envvar="CSM_DATASET_ABSOLUTE_PATH",
|
|
17
|
-
help=T("csm_data.commands.api.rds_load_csv.parameters.target_folder"),
|
|
18
|
-
metavar="PATH",
|
|
19
|
-
type=str,
|
|
20
|
-
show_envvar=True,
|
|
21
|
-
required=True,
|
|
22
|
-
)
|
|
23
|
-
@click.option(
|
|
24
|
-
"--organization-id",
|
|
25
|
-
envvar="CSM_ORGANIZATION_ID",
|
|
26
|
-
help=T("csm_data.commands.api.rds_load_csv.parameters.organization_id"),
|
|
27
|
-
metavar="o-XXXXXXXX",
|
|
28
|
-
type=str,
|
|
29
|
-
show_envvar=True,
|
|
30
|
-
required=True,
|
|
31
|
-
)
|
|
32
|
-
@click.option(
|
|
33
|
-
"--workspace-id",
|
|
34
|
-
envvar="CSM_WORKSPACE_ID",
|
|
35
|
-
help=T("csm_data.commands.api.rds_load_csv.parameters.workspace_id"),
|
|
36
|
-
metavar="w-XXXXXXXX",
|
|
37
|
-
type=str,
|
|
38
|
-
show_envvar=True,
|
|
39
|
-
required=True,
|
|
40
|
-
)
|
|
41
|
-
@click.option(
|
|
42
|
-
"--runner-id",
|
|
43
|
-
envvar="CSM_RUNNER_ID",
|
|
44
|
-
help=T("csm_data.commands.api.rds_load_csv.parameters.runner_id"),
|
|
45
|
-
metavar="r-XXXXXXXX",
|
|
46
|
-
type=str,
|
|
47
|
-
show_envvar=True,
|
|
48
|
-
required=True,
|
|
49
|
-
)
|
|
50
|
-
@click.option(
|
|
51
|
-
"--run-id",
|
|
52
|
-
envvar="CSM_RUN_ID",
|
|
53
|
-
help=T("csm_data.commands.api.rds_load_csv.parameters.run_id"),
|
|
54
|
-
metavar="run-XXXXXX",
|
|
55
|
-
type=str,
|
|
56
|
-
show_envvar=True,
|
|
57
|
-
required=True,
|
|
58
|
-
)
|
|
59
|
-
@click.option(
|
|
60
|
-
"--file-name",
|
|
61
|
-
help=T("csm_data.commands.api.rds_load_csv.parameters.file_name"),
|
|
62
|
-
metavar="NAME",
|
|
63
|
-
type=str,
|
|
64
|
-
default="results",
|
|
65
|
-
show_default=True,
|
|
66
|
-
required=True,
|
|
67
|
-
)
|
|
68
|
-
@click.option(
|
|
69
|
-
"--query",
|
|
70
|
-
help=T("csm_data.commands.api.rds_load_csv.parameters.query"),
|
|
71
|
-
metavar="SQL_QUERY",
|
|
72
|
-
type=str,
|
|
73
|
-
default="SELECT table_name FROM information_schema.tables WHERE table_schema='public'",
|
|
74
|
-
show_default=True,
|
|
75
|
-
)
|
|
76
|
-
@web_help("csm-data/api/rds-load-csv")
|
|
77
|
-
@translate_help("csm_data.commands.api.rds_load_csv.description")
|
|
78
|
-
def rds_load_csv(target_folder, organization_id, workspace_id, runner_id, run_id, file_name, query):
|
|
79
|
-
# Import the function at the start of the command
|
|
80
|
-
from cosmotech.coal.cosmotech_api import load_csv_from_run_data
|
|
81
|
-
|
|
82
|
-
load_csv_from_run_data(
|
|
83
|
-
target_folder=target_folder,
|
|
84
|
-
organization_id=organization_id,
|
|
85
|
-
workspace_id=workspace_id,
|
|
86
|
-
runner_id=runner_id,
|
|
87
|
-
run_id=run_id,
|
|
88
|
-
file_name=file_name,
|
|
89
|
-
query=query,
|
|
90
|
-
)
|
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
@click.command()
|
|
14
|
-
@click.option(
|
|
15
|
-
"--source-folder",
|
|
16
|
-
envvar="CSM_DATASET_ABSOLUTE_PATH",
|
|
17
|
-
help=T("csm_data.commands.api.rds_send_csv.parameters.source_folder"),
|
|
18
|
-
metavar="PATH",
|
|
19
|
-
type=str,
|
|
20
|
-
show_envvar=True,
|
|
21
|
-
required=True,
|
|
22
|
-
)
|
|
23
|
-
@click.option(
|
|
24
|
-
"--organization-id",
|
|
25
|
-
envvar="CSM_ORGANIZATION_ID",
|
|
26
|
-
help=T("csm_data.commands.api.rds_send_csv.parameters.organization_id"),
|
|
27
|
-
metavar="o-XXXXXXXX",
|
|
28
|
-
type=str,
|
|
29
|
-
show_envvar=True,
|
|
30
|
-
required=True,
|
|
31
|
-
)
|
|
32
|
-
@click.option(
|
|
33
|
-
"--workspace-id",
|
|
34
|
-
envvar="CSM_WORKSPACE_ID",
|
|
35
|
-
help=T("csm_data.commands.api.rds_send_csv.parameters.workspace_id"),
|
|
36
|
-
metavar="w-XXXXXXXX",
|
|
37
|
-
type=str,
|
|
38
|
-
show_envvar=True,
|
|
39
|
-
required=True,
|
|
40
|
-
)
|
|
41
|
-
@click.option(
|
|
42
|
-
"--runner-id",
|
|
43
|
-
envvar="CSM_RUNNER_ID",
|
|
44
|
-
help=T("csm_data.commands.api.rds_send_csv.parameters.runner_id"),
|
|
45
|
-
metavar="r-XXXXXXXX",
|
|
46
|
-
type=str,
|
|
47
|
-
show_envvar=True,
|
|
48
|
-
required=True,
|
|
49
|
-
)
|
|
50
|
-
@click.option(
|
|
51
|
-
"--run-id",
|
|
52
|
-
envvar="CSM_RUN_ID",
|
|
53
|
-
help=T("csm_data.commands.api.rds_send_csv.parameters.run_id"),
|
|
54
|
-
metavar="run-XXXXXX",
|
|
55
|
-
type=str,
|
|
56
|
-
show_envvar=True,
|
|
57
|
-
required=True,
|
|
58
|
-
)
|
|
59
|
-
@web_help("csm-data/api/rds-send-csv")
|
|
60
|
-
@translate_help("csm_data.commands.api.rds_send_csv.description")
|
|
61
|
-
def rds_send_csv(source_folder, organization_id, workspace_id, runner_id, run_id):
|
|
62
|
-
# Import the function at the start of the command
|
|
63
|
-
from cosmotech.coal.cosmotech_api import send_csv_to_run_data
|
|
64
|
-
|
|
65
|
-
try:
|
|
66
|
-
send_csv_to_run_data(
|
|
67
|
-
source_folder=source_folder,
|
|
68
|
-
organization_id=organization_id,
|
|
69
|
-
workspace_id=workspace_id,
|
|
70
|
-
runner_id=runner_id,
|
|
71
|
-
run_id=run_id,
|
|
72
|
-
)
|
|
73
|
-
except FileNotFoundError:
|
|
74
|
-
return 1
|
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
@click.command()
|
|
14
|
-
@click.option(
|
|
15
|
-
"--store-folder",
|
|
16
|
-
envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
|
|
17
|
-
help=T("csm_data.commands.api.rds_send_store.parameters.store_folder"),
|
|
18
|
-
metavar="PATH",
|
|
19
|
-
type=str,
|
|
20
|
-
show_envvar=True,
|
|
21
|
-
required=True,
|
|
22
|
-
)
|
|
23
|
-
@click.option(
|
|
24
|
-
"--organization-id",
|
|
25
|
-
envvar="CSM_ORGANIZATION_ID",
|
|
26
|
-
help=T("csm_data.commands.api.rds_send_store.parameters.organization_id"),
|
|
27
|
-
metavar="o-XXXXXXXX",
|
|
28
|
-
type=str,
|
|
29
|
-
show_envvar=True,
|
|
30
|
-
required=True,
|
|
31
|
-
)
|
|
32
|
-
@click.option(
|
|
33
|
-
"--workspace-id",
|
|
34
|
-
envvar="CSM_WORKSPACE_ID",
|
|
35
|
-
help=T("csm_data.commands.api.rds_send_store.parameters.workspace_id"),
|
|
36
|
-
metavar="w-XXXXXXXX",
|
|
37
|
-
type=str,
|
|
38
|
-
show_envvar=True,
|
|
39
|
-
required=True,
|
|
40
|
-
)
|
|
41
|
-
@click.option(
|
|
42
|
-
"--runner-id",
|
|
43
|
-
envvar="CSM_RUNNER_ID",
|
|
44
|
-
help=T("csm_data.commands.api.rds_send_store.parameters.runner_id"),
|
|
45
|
-
metavar="r-XXXXXXXX",
|
|
46
|
-
type=str,
|
|
47
|
-
show_envvar=True,
|
|
48
|
-
required=True,
|
|
49
|
-
)
|
|
50
|
-
@click.option(
|
|
51
|
-
"--run-id",
|
|
52
|
-
envvar="CSM_RUN_ID",
|
|
53
|
-
help=T("csm_data.commands.api.rds_send_store.parameters.run_id"),
|
|
54
|
-
metavar="run-XXXXXX",
|
|
55
|
-
type=str,
|
|
56
|
-
show_envvar=True,
|
|
57
|
-
required=True,
|
|
58
|
-
)
|
|
59
|
-
@web_help("csm-data/api/rds-send-store")
|
|
60
|
-
@translate_help("csm_data.commands.api.rds_send_store.description")
|
|
61
|
-
def rds_send_store(store_folder, organization_id, workspace_id, runner_id, run_id):
|
|
62
|
-
# Import the function at the start of the command
|
|
63
|
-
from cosmotech.coal.cosmotech_api import send_store_to_run_data
|
|
64
|
-
|
|
65
|
-
try:
|
|
66
|
-
send_store_to_run_data(
|
|
67
|
-
store_folder=store_folder,
|
|
68
|
-
organization_id=organization_id,
|
|
69
|
-
workspace_id=workspace_id,
|
|
70
|
-
runner_id=runner_id,
|
|
71
|
-
run_id=run_id,
|
|
72
|
-
)
|
|
73
|
-
except FileNotFoundError:
|
|
74
|
-
return 1
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
@click.command()
|
|
14
|
-
@web_help("csm-data/api/runtemplate-load-handler")
|
|
15
|
-
@translate_help("csm_data.commands.api.runtemplate_load_handler.description")
|
|
16
|
-
@click.option(
|
|
17
|
-
"--organization-id",
|
|
18
|
-
envvar="CSM_ORGANIZATION_ID",
|
|
19
|
-
show_envvar=True,
|
|
20
|
-
help=T("csm_data.commands.api.runtemplate_load_handler.parameters.organization_id"),
|
|
21
|
-
metavar="o-##########",
|
|
22
|
-
required=True,
|
|
23
|
-
)
|
|
24
|
-
@click.option(
|
|
25
|
-
"--workspace-id",
|
|
26
|
-
envvar="CSM_WORKSPACE_ID",
|
|
27
|
-
show_envvar=True,
|
|
28
|
-
help=T("csm_data.commands.api.runtemplate_load_handler.parameters.workspace_id"),
|
|
29
|
-
metavar="w-##########",
|
|
30
|
-
required=True,
|
|
31
|
-
)
|
|
32
|
-
@click.option(
|
|
33
|
-
"--run-template-id",
|
|
34
|
-
envvar="CSM_RUN_TEMPLATE_ID",
|
|
35
|
-
show_envvar=True,
|
|
36
|
-
help=T("csm_data.commands.api.runtemplate_load_handler.parameters.run_template_id"),
|
|
37
|
-
metavar="NAME",
|
|
38
|
-
required=True,
|
|
39
|
-
)
|
|
40
|
-
@click.option(
|
|
41
|
-
"--handler-list",
|
|
42
|
-
envvar="CSM_CONTAINER_MODE",
|
|
43
|
-
show_envvar=True,
|
|
44
|
-
help=T("csm_data.commands.api.runtemplate_load_handler.parameters.handler_list"),
|
|
45
|
-
metavar="HANDLER,...,HANDLER",
|
|
46
|
-
required=True,
|
|
47
|
-
)
|
|
48
|
-
def runtemplate_load_handler(workspace_id, organization_id, run_template_id, handler_list):
|
|
49
|
-
# Import the function at the start of the command
|
|
50
|
-
from cosmotech.coal.cosmotech_api import load_run_template_handlers
|
|
51
|
-
|
|
52
|
-
try:
|
|
53
|
-
success = load_run_template_handlers(
|
|
54
|
-
organization_id=organization_id,
|
|
55
|
-
workspace_id=workspace_id,
|
|
56
|
-
run_template_id=run_template_id,
|
|
57
|
-
handler_list=handler_list,
|
|
58
|
-
)
|
|
59
|
-
if not success:
|
|
60
|
-
raise click.Abort()
|
|
61
|
-
except ValueError:
|
|
62
|
-
raise click.Abort()
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
if __name__ == "__main__":
|
|
66
|
-
runtemplate_load_handler()
|