cosmotech-acceleration-library 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. cosmotech/coal/__init__.py +1 -1
  2. cosmotech/coal/aws/__init__.py +1 -9
  3. cosmotech/coal/aws/s3.py +181 -214
  4. cosmotech/coal/azure/adx/auth.py +2 -2
  5. cosmotech/coal/azure/adx/runner.py +13 -14
  6. cosmotech/coal/azure/adx/store.py +5 -86
  7. cosmotech/coal/azure/adx/tables.py +2 -2
  8. cosmotech/coal/azure/blob.py +6 -6
  9. cosmotech/coal/azure/storage.py +3 -3
  10. cosmotech/coal/cosmotech_api/__init__.py +0 -24
  11. cosmotech/coal/cosmotech_api/apis/__init__.py +14 -0
  12. cosmotech/coal/cosmotech_api/apis/dataset.py +103 -0
  13. cosmotech/coal/cosmotech_api/apis/meta.py +25 -0
  14. cosmotech/coal/cosmotech_api/apis/organization.py +24 -0
  15. cosmotech/coal/cosmotech_api/apis/run.py +38 -0
  16. cosmotech/coal/cosmotech_api/apis/runner.py +71 -0
  17. cosmotech/coal/cosmotech_api/apis/solution.py +23 -0
  18. cosmotech/coal/cosmotech_api/apis/workspace.py +108 -0
  19. cosmotech/coal/cosmotech_api/objects/__init__.py +9 -0
  20. cosmotech/coal/cosmotech_api/objects/connection.py +125 -0
  21. cosmotech/coal/cosmotech_api/objects/parameters.py +127 -0
  22. cosmotech/coal/postgresql/runner.py +56 -36
  23. cosmotech/coal/postgresql/store.py +60 -14
  24. cosmotech/coal/postgresql/utils.py +254 -0
  25. cosmotech/coal/store/output/__init__.py +0 -0
  26. cosmotech/coal/store/output/aws_channel.py +73 -0
  27. cosmotech/coal/store/output/az_storage_channel.py +42 -0
  28. cosmotech/coal/store/output/channel_interface.py +23 -0
  29. cosmotech/coal/store/output/channel_spliter.py +55 -0
  30. cosmotech/coal/store/output/postgres_channel.py +40 -0
  31. cosmotech/coal/utils/configuration.py +169 -0
  32. cosmotech/coal/utils/decorator.py +22 -0
  33. cosmotech/csm_data/commands/api/api.py +6 -19
  34. cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +20 -16
  35. cosmotech/csm_data/commands/api/run_load_data.py +7 -46
  36. cosmotech/csm_data/commands/api/wsf_load_file.py +14 -15
  37. cosmotech/csm_data/commands/api/wsf_send_file.py +12 -13
  38. cosmotech/csm_data/commands/s3_bucket_delete.py +16 -15
  39. cosmotech/csm_data/commands/s3_bucket_download.py +16 -16
  40. cosmotech/csm_data/commands/s3_bucket_upload.py +16 -14
  41. cosmotech/csm_data/commands/store/dump_to_s3.py +18 -16
  42. cosmotech/csm_data/commands/store/output.py +35 -0
  43. cosmotech/csm_data/commands/store/store.py +3 -4
  44. cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml +8 -0
  45. cosmotech/translation/coal/en-US/coal/services/dataset.yml +4 -14
  46. cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml +1 -0
  47. cosmotech/translation/coal/en-US/coal/store/output/split.yml +6 -0
  48. cosmotech/translation/coal/en-US/coal/utils/configuration.yml +2 -0
  49. cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml +7 -0
  50. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/METADATA +8 -9
  51. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/RECORD +55 -71
  52. cosmotech/coal/azure/functions.py +0 -72
  53. cosmotech/coal/cosmotech_api/connection.py +0 -96
  54. cosmotech/coal/cosmotech_api/dataset/__init__.py +0 -26
  55. cosmotech/coal/cosmotech_api/dataset/converters.py +0 -164
  56. cosmotech/coal/cosmotech_api/dataset/download/__init__.py +0 -19
  57. cosmotech/coal/cosmotech_api/dataset/download/adt.py +0 -119
  58. cosmotech/coal/cosmotech_api/dataset/download/common.py +0 -140
  59. cosmotech/coal/cosmotech_api/dataset/download/file.py +0 -216
  60. cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +0 -188
  61. cosmotech/coal/cosmotech_api/dataset/utils.py +0 -132
  62. cosmotech/coal/cosmotech_api/parameters.py +0 -48
  63. cosmotech/coal/cosmotech_api/run.py +0 -25
  64. cosmotech/coal/cosmotech_api/run_data.py +0 -173
  65. cosmotech/coal/cosmotech_api/run_template.py +0 -108
  66. cosmotech/coal/cosmotech_api/runner/__init__.py +0 -28
  67. cosmotech/coal/cosmotech_api/runner/data.py +0 -38
  68. cosmotech/coal/cosmotech_api/runner/datasets.py +0 -364
  69. cosmotech/coal/cosmotech_api/runner/download.py +0 -146
  70. cosmotech/coal/cosmotech_api/runner/metadata.py +0 -42
  71. cosmotech/coal/cosmotech_api/runner/parameters.py +0 -157
  72. cosmotech/coal/cosmotech_api/twin_data_layer.py +0 -512
  73. cosmotech/coal/cosmotech_api/workspace.py +0 -127
  74. cosmotech/coal/utils/api.py +0 -68
  75. cosmotech/coal/utils/postgresql.py +0 -236
  76. cosmotech/csm_data/commands/api/rds_load_csv.py +0 -90
  77. cosmotech/csm_data/commands/api/rds_send_csv.py +0 -74
  78. cosmotech/csm_data/commands/api/rds_send_store.py +0 -74
  79. cosmotech/csm_data/commands/api/runtemplate_load_handler.py +0 -66
  80. cosmotech/csm_data/commands/api/tdl_load_files.py +0 -76
  81. cosmotech/csm_data/commands/api/tdl_send_files.py +0 -82
  82. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +0 -27
  83. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +0 -27
  84. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +0 -27
  85. cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +0 -27
  86. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +0 -32
  87. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +0 -27
  88. cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +0 -2
  89. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml +0 -13
  90. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml +0 -12
  91. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml +0 -12
  92. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml +0 -14
  93. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml +0 -18
  94. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/WHEEL +0 -0
  95. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/entry_points.txt +0 -0
  96. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/licenses/LICENSE +0 -0
  97. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/top_level.txt +0 -0
@@ -1,127 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
- import pathlib
8
-
9
- import cosmotech_api
10
-
11
- from cosmotech.coal.utils.logger import LOGGER
12
- from cosmotech.orchestrator.utils.translate import T
13
-
14
-
15
- def list_workspace_files(
16
- api_client: cosmotech_api.api_client.ApiClient,
17
- organization_id: str,
18
- workspace_id: str,
19
- file_prefix: str,
20
- ) -> list[str]:
21
- """
22
- Helper function to list all workspace files using a pre-given file prefix
23
- :param api_client: An api client used to connect to the Cosmo Tech API
24
- :param organization_id: An ID of an Organization in the Cosmo Tech API
25
- :param workspace_id: An ID of a Workspace in the Cosmo Tech API
26
- :param file_prefix: The prefix of the files to find in the Workspace
27
- :return: A list of existing files inside the workspace
28
- """
29
- target_list = []
30
- api_ws = cosmotech_api.api.workspace_api.WorkspaceApi(api_client)
31
- LOGGER.info(T("coal.cosmotech_api.workspace.target_is_folder"))
32
- wsf = api_ws.find_all_workspace_files(organization_id, workspace_id)
33
- for workspace_file in wsf:
34
- if workspace_file.file_name.startswith(file_prefix):
35
- target_list.append(workspace_file.file_name)
36
-
37
- if not target_list:
38
- LOGGER.error(
39
- T("coal.common.errors.data_no_workspace_files").format(file_prefix=file_prefix, workspace_id=workspace_id)
40
- )
41
- raise ValueError(
42
- T("coal.common.errors.data_no_workspace_files").format(file_prefix=file_prefix, workspace_id=workspace_id)
43
- )
44
-
45
- return target_list
46
-
47
-
48
- def download_workspace_file(
49
- api_client: cosmotech_api.api_client.ApiClient,
50
- organization_id: str,
51
- workspace_id: str,
52
- file_name: str,
53
- target_dir: pathlib.Path,
54
- ) -> pathlib.Path:
55
- """
56
- Downloads a given file from a workspace to a given directory
57
- If the file is inside a directory in the workspace, sub-directories will be created.
58
- :param api_client: An api client used to connect to the Cosmo Tech API
59
- :param organization_id: An ID of an Organization in the Cosmo Tech API
60
- :param workspace_id: An ID of a Workspace in the Cosmo Tech API
61
- :param file_name: The file to download to the workspace
62
- :param target_dir: The directory in which to write the file
63
- :return: The path to the created file
64
- """
65
- if target_dir.is_file():
66
- raise ValueError(T("coal.common.file_operations.not_directory").format(target_dir=target_dir))
67
- api_ws = cosmotech_api.api.workspace_api.WorkspaceApi(api_client)
68
-
69
- LOGGER.info(T("coal.cosmotech_api.workspace.loading_file").format(file_name=file_name))
70
-
71
- _file_content = api_ws.download_workspace_file(organization_id, workspace_id, file_name)
72
-
73
- local_target_file = target_dir / file_name
74
- local_target_file.parent.mkdir(parents=True, exist_ok=True)
75
-
76
- with open(local_target_file, "wb") as _file:
77
- _file.write(_file_content)
78
-
79
- LOGGER.info(T("coal.cosmotech_api.workspace.file_loaded").format(file=local_target_file))
80
-
81
- return local_target_file
82
-
83
-
84
- def upload_workspace_file(
85
- api_client: cosmotech_api.api_client.ApiClient,
86
- organization_id: str,
87
- workspace_id: str,
88
- file_path: str,
89
- workspace_path: str,
90
- overwrite: bool = True,
91
- ) -> str:
92
- """
93
- Upload a local file to a given workspace
94
-
95
- If workspace_path ends with a "/" it will be considered as a folder inside the workspace
96
- and the file will keep its current name
97
-
98
- :param api_client: An api client used to connect to the Cosmo Tech API
99
- :param organization_id: An ID of an Organization in the Cosmo Tech API
100
- :param workspace_id: An ID of a Workspace in the Cosmo Tech API
101
- :param file_path: Path to the file to upload in the workspace
102
- :param workspace_path: The path inside the workspace to upload the file to
103
- :param overwrite: Overwrite existing file in the workspace
104
- :return: The final name of the file uploaded to the workspace
105
- """
106
- target_file = pathlib.Path(file_path)
107
- if not target_file.exists():
108
- LOGGER.error(T("coal.common.file_operations.not_exists").format(file_path=file_path))
109
- raise ValueError(T("coal.common.file_operations.not_exists").format(file_path=file_path))
110
- if not target_file.is_file():
111
- LOGGER.error(T("coal.common.file_operations.not_single_file").format(file_path=file_path))
112
- raise ValueError(T("coal.common.file_operations.not_single_file").format(file_path=file_path))
113
-
114
- api_ws = cosmotech_api.api.workspace_api.WorkspaceApi(api_client)
115
- destination = workspace_path + target_file.name if workspace_path.endswith("/") else workspace_path
116
-
117
- LOGGER.info(T("coal.cosmotech_api.workspace.sending_to_api").format(destination=destination))
118
- try:
119
- _file = api_ws.upload_workspace_file(
120
- organization_id, workspace_id, file_path, overwrite, destination=destination
121
- )
122
- except cosmotech_api.exceptions.ApiException as e:
123
- LOGGER.error(T("coal.common.file_operations.already_exists").format(csv_path=destination))
124
- raise e
125
-
126
- LOGGER.info(T("coal.cosmotech_api.workspace.file_sent").format(file=_file.file_name))
127
- return _file.file_name
@@ -1,68 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- import json
9
- import pathlib
10
- from typing import Optional
11
-
12
- import cosmotech_api
13
- import yaml
14
- from cosmotech_api.api.solution_api import Solution
15
- from cosmotech_api.api.solution_api import SolutionApi
16
- from cosmotech_api.api.workspace_api import Workspace
17
- from cosmotech_api.api.workspace_api import WorkspaceApi
18
- from cosmotech_api.exceptions import ServiceException
19
-
20
- from cosmotech.coal.cosmotech_api.connection import get_api_client
21
- from cosmotech.coal.utils.logger import LOGGER
22
- from cosmotech.orchestrator.utils.translate import T
23
-
24
-
25
- def read_solution_file(solution_file) -> Optional[Solution]:
26
- solution_path = pathlib.Path(solution_file)
27
- if solution_path.suffix in [".yaml", ".yml"]:
28
- open_function = yaml.safe_load
29
- elif solution_path.suffix == ".json":
30
- open_function = json.load
31
- else:
32
- LOGGER.error(T("coal.cosmotech_api.solution.invalid_file").format(file=solution_file))
33
- return None
34
- with solution_path.open() as _sf:
35
- solution_content = open_function(_sf)
36
- LOGGER.info(T("coal.cosmotech_api.solution.loaded").format(path=solution_path.absolute()))
37
- _solution = Solution(
38
- _configuration=cosmotech_api.Configuration(),
39
- _spec_property_naming=True,
40
- **solution_content,
41
- )
42
- LOGGER.debug(
43
- T("coal.services.api.solution_debug").format(solution=json.dumps(_solution.to_dict(), indent=2, default=str))
44
- )
45
- return _solution
46
-
47
-
48
- def get_solution(organization_id, workspace_id) -> Optional[Solution]:
49
- LOGGER.info(T("coal.cosmotech_api.solution.api_configured"))
50
- with get_api_client()[0] as api_client:
51
- api_w = WorkspaceApi(api_client)
52
-
53
- LOGGER.info(T("coal.cosmotech_api.solution.loading_workspace"))
54
- try:
55
- r_data: Workspace = api_w.find_workspace_by_id(organization_id=organization_id, workspace_id=workspace_id)
56
- except ServiceException as e:
57
- LOGGER.error(
58
- T("coal.cosmotech_api.workspace.not_found").format(
59
- workspace_id=workspace_id, organization_id=organization_id
60
- )
61
- )
62
- LOGGER.debug(e)
63
- return None
64
- solution_id = r_data.solution.solution_id
65
-
66
- api_sol = SolutionApi(api_client)
67
- sol: Solution = api_sol.find_solution_by_id(organization_id=organization_id, solution_id=solution_id)
68
- return sol
@@ -1,236 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- from typing import Optional
9
- from urllib.parse import quote
10
-
11
- import adbc_driver_manager
12
- import pyarrow as pa
13
- from adbc_driver_postgresql import dbapi
14
- from cosmotech.orchestrator.utils.translate import T
15
- from pyarrow import Table
16
-
17
- from cosmotech.coal.utils.logger import LOGGER
18
-
19
-
20
- def generate_postgresql_full_uri(
21
- postgres_host: str,
22
- postgres_port: str,
23
- postgres_db: str,
24
- postgres_user: str,
25
- postgres_password: str,
26
- force_encode: bool = False,
27
- ) -> str:
28
- # Check if password needs percent encoding (contains special characters)
29
- # We don't log anything about the password for security
30
- encoded_password = postgres_password
31
- if force_encode:
32
- encoded_password = quote(postgres_password, safe="")
33
-
34
- return (
35
- "postgresql://" + f"{postgres_user}"
36
- f":{encoded_password}"
37
- f"@{postgres_host}"
38
- f":{postgres_port}"
39
- f"/{postgres_db}"
40
- )
41
-
42
-
43
- def get_postgresql_table_schema(
44
- target_table_name: str,
45
- postgres_host: str,
46
- postgres_port: str,
47
- postgres_db: str,
48
- postgres_schema: str,
49
- postgres_user: str,
50
- postgres_password: str,
51
- force_encode: bool = False,
52
- ) -> Optional[pa.Schema]:
53
- """
54
- Get the schema of an existing PostgreSQL table using SQL queries.
55
-
56
- Args:
57
- target_table_name: Name of the table
58
- postgres_host: PostgreSQL host
59
- postgres_port: PostgreSQL port
60
- postgres_db: PostgreSQL database name
61
- postgres_schema: PostgreSQL schema name
62
- postgres_user: PostgreSQL username
63
- postgres_password: PostgreSQL password
64
-
65
- Returns:
66
- PyArrow Schema if table exists, None otherwise
67
- """
68
- LOGGER.debug(
69
- T("coal.services.postgresql.getting_schema").format(
70
- postgres_schema=postgres_schema, target_table_name=target_table_name
71
- )
72
- )
73
-
74
- postgresql_full_uri = generate_postgresql_full_uri(
75
- postgres_host,
76
- postgres_port,
77
- postgres_db,
78
- postgres_user,
79
- postgres_password,
80
- force_encode,
81
- )
82
-
83
- with dbapi.connect(postgresql_full_uri) as conn:
84
- try:
85
- return conn.adbc_get_table_schema(
86
- target_table_name,
87
- db_schema_filter=postgres_schema,
88
- )
89
- except adbc_driver_manager.ProgrammingError:
90
- LOGGER.warning(
91
- T("coal.services.postgresql.table_not_found").format(
92
- postgres_schema=postgres_schema, target_table_name=target_table_name
93
- )
94
- )
95
- return None
96
-
97
-
98
- def adapt_table_to_schema(data: pa.Table, target_schema: pa.Schema) -> pa.Table:
99
- """
100
- Adapt a PyArrow table to match a target schema with detailed logging.
101
- """
102
- LOGGER.debug(T("coal.services.postgresql.schema_adaptation_start").format(rows=len(data)))
103
- LOGGER.debug(T("coal.services.postgresql.original_schema").format(schema=data.schema))
104
- LOGGER.debug(T("coal.services.postgresql.target_schema").format(schema=target_schema))
105
-
106
- target_fields = {field.name: field.type for field in target_schema}
107
- new_columns = []
108
-
109
- # Track adaptations for summary
110
- added_columns = []
111
- dropped_columns = []
112
- type_conversions = []
113
- failed_conversions = []
114
-
115
- # Process each field in target schema
116
- for field_name, target_type in target_fields.items():
117
- if field_name in data.column_names:
118
- # Column exists - try to cast to target type
119
- col = data[field_name]
120
- original_type = col.type
121
-
122
- if original_type != target_type:
123
- LOGGER.debug(
124
- T("coal.services.postgresql.casting_column").format(
125
- field_name=field_name,
126
- original_type=original_type,
127
- target_type=target_type,
128
- )
129
- )
130
- try:
131
- new_col = pa.compute.cast(col, target_type)
132
- new_columns.append(new_col)
133
- type_conversions.append(f"{field_name}: {original_type} -> {target_type}")
134
- except pa.ArrowInvalid as e:
135
- LOGGER.warning(
136
- T("coal.services.postgresql.cast_failed").format(
137
- field_name=field_name,
138
- original_type=original_type,
139
- target_type=target_type,
140
- error=str(e),
141
- )
142
- )
143
- new_columns.append(pa.nulls(len(data), type=target_type))
144
- failed_conversions.append(f"{field_name}: {original_type} -> {target_type}")
145
- else:
146
- new_columns.append(col)
147
- else:
148
- # Column doesn't exist - add nulls
149
- LOGGER.debug(T("coal.services.postgresql.adding_missing_column").format(field_name=field_name))
150
- new_columns.append(pa.nulls(len(data), type=target_type))
151
- added_columns.append(field_name)
152
-
153
- # Log columns that will be dropped
154
- dropped_columns = [name for name in data.column_names if name not in target_fields]
155
- if dropped_columns:
156
- LOGGER.debug(T("coal.services.postgresql.dropping_columns").format(columns=dropped_columns))
157
-
158
- # Create new table
159
- adapted_table = pa.Table.from_arrays(new_columns, schema=target_schema)
160
-
161
- # Log summary of adaptations
162
- LOGGER.debug(T("coal.services.postgresql.adaptation_summary"))
163
- if added_columns:
164
- LOGGER.debug(T("coal.services.postgresql.added_columns").format(columns=added_columns))
165
- if dropped_columns:
166
- LOGGER.debug(T("coal.services.postgresql.dropped_columns").format(columns=dropped_columns))
167
- if type_conversions:
168
- LOGGER.debug(T("coal.services.postgresql.successful_conversions").format(conversions=type_conversions))
169
- if failed_conversions:
170
- LOGGER.debug(T("coal.services.postgresql.failed_conversions").format(conversions=failed_conversions))
171
-
172
- LOGGER.debug(T("coal.services.postgresql.final_schema").format(schema=adapted_table.schema))
173
- return adapted_table
174
-
175
-
176
- def send_pyarrow_table_to_postgresql(
177
- data: Table,
178
- target_table_name: str,
179
- postgres_host: str,
180
- postgres_port: str,
181
- postgres_db: str,
182
- postgres_schema: str,
183
- postgres_user: str,
184
- postgres_password: str,
185
- replace: bool,
186
- force_encode: bool = False,
187
- ) -> int:
188
- LOGGER.debug(
189
- T("coal.services.postgresql.preparing_send").format(
190
- postgres_schema=postgres_schema, target_table_name=target_table_name
191
- )
192
- )
193
- LOGGER.debug(T("coal.services.postgresql.input_rows").format(rows=len(data)))
194
-
195
- # Get existing schema if table exists
196
- existing_schema = get_postgresql_table_schema(
197
- target_table_name,
198
- postgres_host,
199
- postgres_port,
200
- postgres_db,
201
- postgres_schema,
202
- postgres_user,
203
- postgres_password,
204
- force_encode,
205
- )
206
-
207
- if existing_schema is not None:
208
- LOGGER.debug(T("coal.services.postgresql.found_existing_table").format(schema=existing_schema))
209
- if not replace:
210
- LOGGER.debug(T("coal.services.postgresql.adapting_data"))
211
- data = adapt_table_to_schema(data, existing_schema)
212
- else:
213
- LOGGER.debug(T("coal.services.postgresql.replace_mode"))
214
- else:
215
- LOGGER.debug(T("coal.services.postgresql.no_existing_table"))
216
-
217
- # Proceed with ingestion
218
- total = 0
219
- postgresql_full_uri = generate_postgresql_full_uri(
220
- postgres_host,
221
- postgres_port,
222
- postgres_db,
223
- postgres_user,
224
- postgres_password,
225
- force_encode,
226
- )
227
-
228
- LOGGER.debug(T("coal.services.postgresql.connecting"))
229
- with dbapi.connect(postgresql_full_uri, autocommit=True) as conn:
230
- with conn.cursor() as curs:
231
- mode = "replace" if replace else "create_append"
232
- LOGGER.debug(T("coal.services.postgresql.ingesting_data").format(mode=mode))
233
- total += curs.adbc_ingest(target_table_name, data, mode, db_schema_name=postgres_schema)
234
-
235
- LOGGER.debug(T("coal.services.postgresql.ingestion_success").format(rows=total))
236
- return total
@@ -1,90 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- from cosmotech.csm_data.utils.click import click
9
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
- from cosmotech.orchestrator.utils.translate import T
11
-
12
-
13
- @click.command()
14
- @click.option(
15
- "--target-folder",
16
- envvar="CSM_DATASET_ABSOLUTE_PATH",
17
- help=T("csm_data.commands.api.rds_load_csv.parameters.target_folder"),
18
- metavar="PATH",
19
- type=str,
20
- show_envvar=True,
21
- required=True,
22
- )
23
- @click.option(
24
- "--organization-id",
25
- envvar="CSM_ORGANIZATION_ID",
26
- help=T("csm_data.commands.api.rds_load_csv.parameters.organization_id"),
27
- metavar="o-XXXXXXXX",
28
- type=str,
29
- show_envvar=True,
30
- required=True,
31
- )
32
- @click.option(
33
- "--workspace-id",
34
- envvar="CSM_WORKSPACE_ID",
35
- help=T("csm_data.commands.api.rds_load_csv.parameters.workspace_id"),
36
- metavar="w-XXXXXXXX",
37
- type=str,
38
- show_envvar=True,
39
- required=True,
40
- )
41
- @click.option(
42
- "--runner-id",
43
- envvar="CSM_RUNNER_ID",
44
- help=T("csm_data.commands.api.rds_load_csv.parameters.runner_id"),
45
- metavar="r-XXXXXXXX",
46
- type=str,
47
- show_envvar=True,
48
- required=True,
49
- )
50
- @click.option(
51
- "--run-id",
52
- envvar="CSM_RUN_ID",
53
- help=T("csm_data.commands.api.rds_load_csv.parameters.run_id"),
54
- metavar="run-XXXXXX",
55
- type=str,
56
- show_envvar=True,
57
- required=True,
58
- )
59
- @click.option(
60
- "--file-name",
61
- help=T("csm_data.commands.api.rds_load_csv.parameters.file_name"),
62
- metavar="NAME",
63
- type=str,
64
- default="results",
65
- show_default=True,
66
- required=True,
67
- )
68
- @click.option(
69
- "--query",
70
- help=T("csm_data.commands.api.rds_load_csv.parameters.query"),
71
- metavar="SQL_QUERY",
72
- type=str,
73
- default="SELECT table_name FROM information_schema.tables WHERE table_schema='public'",
74
- show_default=True,
75
- )
76
- @web_help("csm-data/api/rds-load-csv")
77
- @translate_help("csm_data.commands.api.rds_load_csv.description")
78
- def rds_load_csv(target_folder, organization_id, workspace_id, runner_id, run_id, file_name, query):
79
- # Import the function at the start of the command
80
- from cosmotech.coal.cosmotech_api import load_csv_from_run_data
81
-
82
- load_csv_from_run_data(
83
- target_folder=target_folder,
84
- organization_id=organization_id,
85
- workspace_id=workspace_id,
86
- runner_id=runner_id,
87
- run_id=run_id,
88
- file_name=file_name,
89
- query=query,
90
- )
@@ -1,74 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- from cosmotech.csm_data.utils.click import click
9
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
- from cosmotech.orchestrator.utils.translate import T
11
-
12
-
13
- @click.command()
14
- @click.option(
15
- "--source-folder",
16
- envvar="CSM_DATASET_ABSOLUTE_PATH",
17
- help=T("csm_data.commands.api.rds_send_csv.parameters.source_folder"),
18
- metavar="PATH",
19
- type=str,
20
- show_envvar=True,
21
- required=True,
22
- )
23
- @click.option(
24
- "--organization-id",
25
- envvar="CSM_ORGANIZATION_ID",
26
- help=T("csm_data.commands.api.rds_send_csv.parameters.organization_id"),
27
- metavar="o-XXXXXXXX",
28
- type=str,
29
- show_envvar=True,
30
- required=True,
31
- )
32
- @click.option(
33
- "--workspace-id",
34
- envvar="CSM_WORKSPACE_ID",
35
- help=T("csm_data.commands.api.rds_send_csv.parameters.workspace_id"),
36
- metavar="w-XXXXXXXX",
37
- type=str,
38
- show_envvar=True,
39
- required=True,
40
- )
41
- @click.option(
42
- "--runner-id",
43
- envvar="CSM_RUNNER_ID",
44
- help=T("csm_data.commands.api.rds_send_csv.parameters.runner_id"),
45
- metavar="r-XXXXXXXX",
46
- type=str,
47
- show_envvar=True,
48
- required=True,
49
- )
50
- @click.option(
51
- "--run-id",
52
- envvar="CSM_RUN_ID",
53
- help=T("csm_data.commands.api.rds_send_csv.parameters.run_id"),
54
- metavar="run-XXXXXX",
55
- type=str,
56
- show_envvar=True,
57
- required=True,
58
- )
59
- @web_help("csm-data/api/rds-send-csv")
60
- @translate_help("csm_data.commands.api.rds_send_csv.description")
61
- def rds_send_csv(source_folder, organization_id, workspace_id, runner_id, run_id):
62
- # Import the function at the start of the command
63
- from cosmotech.coal.cosmotech_api import send_csv_to_run_data
64
-
65
- try:
66
- send_csv_to_run_data(
67
- source_folder=source_folder,
68
- organization_id=organization_id,
69
- workspace_id=workspace_id,
70
- runner_id=runner_id,
71
- run_id=run_id,
72
- )
73
- except FileNotFoundError:
74
- return 1
@@ -1,74 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- from cosmotech.csm_data.utils.click import click
9
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
- from cosmotech.orchestrator.utils.translate import T
11
-
12
-
13
- @click.command()
14
- @click.option(
15
- "--store-folder",
16
- envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
17
- help=T("csm_data.commands.api.rds_send_store.parameters.store_folder"),
18
- metavar="PATH",
19
- type=str,
20
- show_envvar=True,
21
- required=True,
22
- )
23
- @click.option(
24
- "--organization-id",
25
- envvar="CSM_ORGANIZATION_ID",
26
- help=T("csm_data.commands.api.rds_send_store.parameters.organization_id"),
27
- metavar="o-XXXXXXXX",
28
- type=str,
29
- show_envvar=True,
30
- required=True,
31
- )
32
- @click.option(
33
- "--workspace-id",
34
- envvar="CSM_WORKSPACE_ID",
35
- help=T("csm_data.commands.api.rds_send_store.parameters.workspace_id"),
36
- metavar="w-XXXXXXXX",
37
- type=str,
38
- show_envvar=True,
39
- required=True,
40
- )
41
- @click.option(
42
- "--runner-id",
43
- envvar="CSM_RUNNER_ID",
44
- help=T("csm_data.commands.api.rds_send_store.parameters.runner_id"),
45
- metavar="r-XXXXXXXX",
46
- type=str,
47
- show_envvar=True,
48
- required=True,
49
- )
50
- @click.option(
51
- "--run-id",
52
- envvar="CSM_RUN_ID",
53
- help=T("csm_data.commands.api.rds_send_store.parameters.run_id"),
54
- metavar="run-XXXXXX",
55
- type=str,
56
- show_envvar=True,
57
- required=True,
58
- )
59
- @web_help("csm-data/api/rds-send-store")
60
- @translate_help("csm_data.commands.api.rds_send_store.description")
61
- def rds_send_store(store_folder, organization_id, workspace_id, runner_id, run_id):
62
- # Import the function at the start of the command
63
- from cosmotech.coal.cosmotech_api import send_store_to_run_data
64
-
65
- try:
66
- send_store_to_run_data(
67
- store_folder=store_folder,
68
- organization_id=organization_id,
69
- workspace_id=workspace_id,
70
- runner_id=runner_id,
71
- run_id=run_id,
72
- )
73
- except FileNotFoundError:
74
- return 1