cosmotech-acceleration-library 1.1.0__py3-none-any.whl → 2.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cosmotech/coal/__init__.py +1 -1
- cosmotech/coal/aws/__init__.py +1 -9
- cosmotech/coal/aws/s3.py +181 -214
- cosmotech/coal/azure/__init__.py +5 -5
- cosmotech/coal/azure/adx/__init__.py +24 -10
- cosmotech/coal/azure/adx/auth.py +2 -2
- cosmotech/coal/azure/adx/ingestion.py +10 -14
- cosmotech/coal/azure/adx/query.py +1 -1
- cosmotech/coal/azure/adx/runner.py +13 -14
- cosmotech/coal/azure/adx/store.py +5 -86
- cosmotech/coal/azure/adx/tables.py +2 -2
- cosmotech/coal/azure/adx/utils.py +2 -2
- cosmotech/coal/azure/blob.py +20 -26
- cosmotech/coal/azure/storage.py +3 -3
- cosmotech/coal/cosmotech_api/__init__.py +0 -28
- cosmotech/coal/cosmotech_api/apis/__init__.py +14 -0
- cosmotech/coal/cosmotech_api/apis/dataset.py +222 -0
- cosmotech/coal/cosmotech_api/apis/meta.py +25 -0
- cosmotech/coal/cosmotech_api/apis/organization.py +24 -0
- cosmotech/coal/cosmotech_api/apis/run.py +38 -0
- cosmotech/coal/cosmotech_api/apis/runner.py +75 -0
- cosmotech/coal/cosmotech_api/apis/solution.py +23 -0
- cosmotech/coal/cosmotech_api/apis/workspace.py +108 -0
- cosmotech/coal/cosmotech_api/objects/__init__.py +9 -0
- cosmotech/coal/cosmotech_api/objects/connection.py +125 -0
- cosmotech/coal/cosmotech_api/objects/parameters.py +127 -0
- cosmotech/coal/postgresql/runner.py +58 -41
- cosmotech/coal/postgresql/store.py +56 -15
- cosmotech/coal/postgresql/utils.py +255 -0
- cosmotech/coal/singlestore/store.py +3 -2
- cosmotech/coal/store/__init__.py +16 -13
- cosmotech/coal/store/output/__init__.py +0 -0
- cosmotech/coal/store/output/aws_channel.py +74 -0
- cosmotech/coal/store/output/az_storage_channel.py +33 -0
- cosmotech/coal/store/output/channel_interface.py +38 -0
- cosmotech/coal/store/output/channel_spliter.py +61 -0
- cosmotech/coal/store/output/postgres_channel.py +37 -0
- cosmotech/coal/store/pandas.py +1 -1
- cosmotech/coal/store/pyarrow.py +2 -2
- cosmotech/coal/store/store.py +4 -7
- cosmotech/coal/utils/configuration.py +197 -0
- cosmotech/coal/utils/decorator.py +4 -7
- cosmotech/csm_data/commands/adx_send_data.py +1 -1
- cosmotech/csm_data/commands/adx_send_runnerdata.py +3 -2
- cosmotech/csm_data/commands/api/api.py +6 -19
- cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +20 -16
- cosmotech/csm_data/commands/api/run_load_data.py +15 -52
- cosmotech/csm_data/commands/api/wsf_load_file.py +13 -16
- cosmotech/csm_data/commands/api/wsf_send_file.py +11 -14
- cosmotech/csm_data/commands/az_storage_upload.py +3 -2
- cosmotech/csm_data/commands/s3_bucket_delete.py +16 -15
- cosmotech/csm_data/commands/s3_bucket_download.py +16 -16
- cosmotech/csm_data/commands/s3_bucket_upload.py +16 -14
- cosmotech/csm_data/commands/store/dump_to_azure.py +3 -2
- cosmotech/csm_data/commands/store/dump_to_postgresql.py +3 -2
- cosmotech/csm_data/commands/store/dump_to_s3.py +18 -16
- cosmotech/csm_data/commands/store/list_tables.py +3 -2
- cosmotech/csm_data/commands/store/load_csv_folder.py +10 -4
- cosmotech/csm_data/commands/store/load_from_singlestore.py +3 -2
- cosmotech/csm_data/commands/store/output.py +35 -0
- cosmotech/csm_data/commands/store/reset.py +8 -3
- cosmotech/csm_data/commands/store/store.py +3 -3
- cosmotech/csm_data/main.py +4 -4
- cosmotech/csm_data/utils/decorators.py +4 -3
- cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml +8 -0
- cosmotech/translation/coal/en-US/coal/services/dataset.yml +10 -14
- cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml +1 -0
- cosmotech/translation/coal/en-US/coal/store/output/split.yml +6 -0
- cosmotech/translation/coal/en-US/coal/utils/configuration.yml +2 -0
- cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml +7 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/METADATA +29 -33
- cosmotech_acceleration_library-2.1.0rc1.dist-info/RECORD +153 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/WHEEL +1 -1
- cosmotech/coal/azure/functions.py +0 -72
- cosmotech/coal/cosmotech_api/connection.py +0 -96
- cosmotech/coal/cosmotech_api/dataset/__init__.py +0 -26
- cosmotech/coal/cosmotech_api/dataset/converters.py +0 -164
- cosmotech/coal/cosmotech_api/dataset/download/__init__.py +0 -19
- cosmotech/coal/cosmotech_api/dataset/download/adt.py +0 -119
- cosmotech/coal/cosmotech_api/dataset/download/common.py +0 -140
- cosmotech/coal/cosmotech_api/dataset/download/file.py +0 -229
- cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +0 -185
- cosmotech/coal/cosmotech_api/dataset/upload.py +0 -41
- cosmotech/coal/cosmotech_api/dataset/utils.py +0 -132
- cosmotech/coal/cosmotech_api/parameters.py +0 -48
- cosmotech/coal/cosmotech_api/run.py +0 -25
- cosmotech/coal/cosmotech_api/run_data.py +0 -173
- cosmotech/coal/cosmotech_api/run_template.py +0 -108
- cosmotech/coal/cosmotech_api/runner/__init__.py +0 -28
- cosmotech/coal/cosmotech_api/runner/data.py +0 -38
- cosmotech/coal/cosmotech_api/runner/datasets.py +0 -416
- cosmotech/coal/cosmotech_api/runner/download.py +0 -135
- cosmotech/coal/cosmotech_api/runner/metadata.py +0 -42
- cosmotech/coal/cosmotech_api/runner/parameters.py +0 -157
- cosmotech/coal/cosmotech_api/twin_data_layer.py +0 -512
- cosmotech/coal/cosmotech_api/workspace.py +0 -127
- cosmotech/coal/utils/postgresql.py +0 -236
- cosmotech/coal/utils/semver.py +0 -6
- cosmotech/csm_data/commands/api/rds_load_csv.py +0 -90
- cosmotech/csm_data/commands/api/rds_send_csv.py +0 -74
- cosmotech/csm_data/commands/api/rds_send_store.py +0 -74
- cosmotech/csm_data/commands/api/runtemplate_load_handler.py +0 -66
- cosmotech/csm_data/commands/api/tdl_load_files.py +0 -76
- cosmotech/csm_data/commands/api/tdl_send_files.py +0 -82
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +0 -32
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +0 -27
- cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +0 -2
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml +0 -13
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml +0 -12
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml +0 -12
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml +0 -14
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml +0 -18
- cosmotech_acceleration_library-1.1.0.dist-info/RECORD +0 -171
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/entry_points.txt +0 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/licenses/LICENSE +0 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -7,10 +7,11 @@
|
|
|
7
7
|
|
|
8
8
|
from typing import Optional
|
|
9
9
|
|
|
10
|
-
from cosmotech.csm_data.utils.click import click
|
|
11
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
12
10
|
from cosmotech.orchestrator.utils.translate import T
|
|
13
11
|
|
|
12
|
+
from cosmotech.csm_data.utils.click import click
|
|
13
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
14
|
+
|
|
14
15
|
|
|
15
16
|
@click.command()
|
|
16
17
|
@click.option(
|
|
@@ -98,21 +99,20 @@ def s3_bucket_download(
|
|
|
98
99
|
ssl_cert_bundle: Optional[str] = None,
|
|
99
100
|
):
|
|
100
101
|
# Import the functions at the start of the command
|
|
101
|
-
from cosmotech.coal.aws
|
|
102
|
+
from cosmotech.coal.aws import S3
|
|
103
|
+
from cosmotech.coal.utils.configuration import Configuration
|
|
102
104
|
|
|
103
105
|
# Create S3 resource
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
106
|
+
_c = Configuration()
|
|
107
|
+
_c.s3.bucket_name = bucket_name
|
|
108
|
+
_c.s3.endpoint_url = endpoint_url
|
|
109
|
+
_c.s3.access_key_id = access_id
|
|
110
|
+
_c.s3.secret_access_key = secret_key
|
|
111
|
+
_c.s3.bucket_prefix = file_prefix
|
|
112
|
+
_c.s3.use_ssl = use_ssl
|
|
113
|
+
_c.s3.ssl_cert_bundle = ssl_cert_bundle
|
|
114
|
+
|
|
115
|
+
_s3 = S3(_c)
|
|
111
116
|
|
|
112
117
|
# Download files
|
|
113
|
-
download_files(
|
|
114
|
-
target_folder=target_folder,
|
|
115
|
-
bucket_name=bucket_name,
|
|
116
|
-
s3_resource=s3_resource,
|
|
117
|
-
file_prefix=file_prefix,
|
|
118
|
-
)
|
|
118
|
+
_s3.download_files(target_folder=target_folder)
|
|
@@ -7,10 +7,11 @@
|
|
|
7
7
|
|
|
8
8
|
from typing import Optional
|
|
9
9
|
|
|
10
|
-
from cosmotech.csm_data.utils.click import click
|
|
11
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
12
10
|
from cosmotech.orchestrator.utils.translate import T
|
|
13
11
|
|
|
12
|
+
from cosmotech.csm_data.utils.click import click
|
|
13
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
14
|
+
|
|
14
15
|
|
|
15
16
|
@click.command()
|
|
16
17
|
@click.option(
|
|
@@ -107,22 +108,23 @@ def s3_bucket_upload(
|
|
|
107
108
|
recursive: bool = False,
|
|
108
109
|
):
|
|
109
110
|
# Import the functions at the start of the command
|
|
110
|
-
from cosmotech.coal.aws
|
|
111
|
+
from cosmotech.coal.aws import S3
|
|
112
|
+
from cosmotech.coal.utils.configuration import Configuration
|
|
111
113
|
|
|
112
114
|
# Create S3 resource
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
115
|
+
_c = Configuration()
|
|
116
|
+
_c.s3.bucket_name = bucket_name
|
|
117
|
+
_c.s3.endpoint_url = endpoint_url
|
|
118
|
+
_c.s3.access_key_id = access_id
|
|
119
|
+
_c.s3.secret_access_key = secret_key
|
|
120
|
+
_c.s3.bucket_prefix = file_prefix
|
|
121
|
+
_c.s3.use_ssl = use_ssl
|
|
122
|
+
_c.s3.ssl_cert_bundle = ssl_cert_bundle
|
|
123
|
+
|
|
124
|
+
_s3 = S3(_c)
|
|
120
125
|
|
|
121
126
|
# Upload files
|
|
122
|
-
upload_folder(
|
|
127
|
+
_s3.upload_folder(
|
|
123
128
|
source_folder=source_folder,
|
|
124
|
-
bucket_name=bucket_name,
|
|
125
|
-
s3_resource=s3_resource,
|
|
126
|
-
file_prefix=file_prefix,
|
|
127
129
|
recursive=recursive,
|
|
128
130
|
)
|
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
# etc., to any person is prohibited unless it has been previously and
|
|
6
6
|
# specifically authorized by written means by Cosmo Tech.
|
|
7
7
|
|
|
8
|
-
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
8
|
from cosmotech.orchestrator.utils.translate import T
|
|
11
9
|
|
|
10
|
+
from cosmotech.csm_data.utils.click import click
|
|
11
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
12
|
+
|
|
12
13
|
VALID_TYPES = (
|
|
13
14
|
"sqlite",
|
|
14
15
|
"csv",
|
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
# etc., to any person is prohibited unless it has been previously and
|
|
6
6
|
# specifically authorized by written means by Cosmo Tech.
|
|
7
7
|
|
|
8
|
-
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
8
|
from cosmotech.orchestrator.utils.translate import T
|
|
11
9
|
|
|
10
|
+
from cosmotech.csm_data.utils.click import click
|
|
11
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
12
|
+
|
|
12
13
|
|
|
13
14
|
@click.command()
|
|
14
15
|
@web_help("csm-data/store/dump-to-postgres")
|
|
@@ -7,10 +7,11 @@
|
|
|
7
7
|
|
|
8
8
|
from typing import Optional
|
|
9
9
|
|
|
10
|
-
from cosmotech.csm_data.utils.click import click
|
|
11
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
12
10
|
from cosmotech.orchestrator.utils.translate import T
|
|
13
11
|
|
|
12
|
+
from cosmotech.csm_data.utils.click import click
|
|
13
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
14
|
+
|
|
14
15
|
VALID_TYPES = (
|
|
15
16
|
"sqlite",
|
|
16
17
|
"csv",
|
|
@@ -113,10 +114,13 @@ def dump_to_s3(
|
|
|
113
114
|
):
|
|
114
115
|
# Import the modules and functions at the start of the command
|
|
115
116
|
from io import BytesIO
|
|
117
|
+
|
|
116
118
|
import pyarrow.csv as pc
|
|
117
119
|
import pyarrow.parquet as pq
|
|
118
|
-
|
|
120
|
+
|
|
121
|
+
from cosmotech.coal.aws import S3
|
|
119
122
|
from cosmotech.coal.store.store import Store
|
|
123
|
+
from cosmotech.coal.utils.configuration import Configuration
|
|
120
124
|
from cosmotech.coal.utils.logger import LOGGER
|
|
121
125
|
|
|
122
126
|
_s = Store(store_location=store_folder)
|
|
@@ -125,14 +129,15 @@ def dump_to_s3(
|
|
|
125
129
|
LOGGER.error(T("coal.common.errors.data_invalid_output_type").format(output_type=output_type))
|
|
126
130
|
raise ValueError(T("coal.common.errors.data_invalid_output_type").format(output_type=output_type))
|
|
127
131
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
132
|
+
_configuration = Configuration()
|
|
133
|
+
_configuration.s3.bucket_name = bucket_name
|
|
134
|
+
_configuration.s3.endpoint_url = endpoint_url
|
|
135
|
+
_configuration.s3.access_key_id = access_id
|
|
136
|
+
_configuration.s3.secret_access_key = secret_key
|
|
137
|
+
_configuration.s3.bucket_prefix_prefix = file_prefix
|
|
138
|
+
_configuration.s3.use_ssl = use_ssl
|
|
139
|
+
_configuration.s3.ssl_cert_bundle = ssl_cert_bundle
|
|
140
|
+
_s3 = S3(_configuration)
|
|
136
141
|
|
|
137
142
|
if output_type == "sqlite":
|
|
138
143
|
_file_path = _s._database_path
|
|
@@ -141,7 +146,7 @@ def dump_to_s3(
|
|
|
141
146
|
LOGGER.info(
|
|
142
147
|
T("coal.common.data_transfer.file_sent").format(file_path=_file_path, uploaded_name=_uploaded_file_name)
|
|
143
148
|
)
|
|
144
|
-
|
|
149
|
+
_s3.upload_file(_file_path, _uploaded_file_name)
|
|
145
150
|
else:
|
|
146
151
|
tables = list(_s.list_tables())
|
|
147
152
|
for table_name in tables:
|
|
@@ -160,10 +165,7 @@ def dump_to_s3(
|
|
|
160
165
|
LOGGER.info(
|
|
161
166
|
T("coal.common.data_transfer.sending_table").format(table_name=table_name, output_type=output_type)
|
|
162
167
|
)
|
|
163
|
-
upload_data_stream(
|
|
168
|
+
_s3.upload_data_stream(
|
|
164
169
|
data_stream=_data_stream,
|
|
165
|
-
bucket_name=bucket_name,
|
|
166
|
-
s3_client=s3_client,
|
|
167
170
|
file_name=_file_name,
|
|
168
|
-
file_prefix=file_prefix,
|
|
169
171
|
)
|
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
# etc., to any person is prohibited unless it has been previously and
|
|
6
6
|
# specifically authorized by written means by Cosmo Tech.
|
|
7
7
|
|
|
8
|
-
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
8
|
from cosmotech.orchestrator.utils.translate import T
|
|
11
9
|
|
|
10
|
+
from cosmotech.csm_data.utils.click import click
|
|
11
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
12
|
+
|
|
12
13
|
|
|
13
14
|
@click.command()
|
|
14
15
|
@web_help("csm-data/store/list-tables")
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
5
|
# etc., to any person is prohibited unless it has been previously and
|
|
6
6
|
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
7
8
|
|
|
8
9
|
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import
|
|
10
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
10
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
@click.command()
|
|
@@ -24,7 +24,7 @@ from cosmotech.orchestrator.utils.translate import T
|
|
|
24
24
|
)
|
|
25
25
|
@click.option(
|
|
26
26
|
"--csv-folder",
|
|
27
|
-
envvar="
|
|
27
|
+
envvar="CSM_OUTPUT_ABSOLUTE_PATH",
|
|
28
28
|
help=T("csm_data.commands.store.load_csv_folder.parameters.csv_folder"),
|
|
29
29
|
metavar="PATH",
|
|
30
30
|
type=str,
|
|
@@ -34,10 +34,16 @@ from cosmotech.orchestrator.utils.translate import T
|
|
|
34
34
|
def load_csv_folder(store_folder, csv_folder):
|
|
35
35
|
# Import the modules and functions at the start of the command
|
|
36
36
|
import pathlib
|
|
37
|
+
|
|
37
38
|
from cosmotech.coal.store.csv import store_csv_file
|
|
38
39
|
from cosmotech.coal.store.store import Store
|
|
40
|
+
from cosmotech.coal.utils.configuration import Configuration
|
|
39
41
|
from cosmotech.coal.utils.logger import LOGGER
|
|
40
42
|
|
|
43
|
+
_conf = Configuration()
|
|
44
|
+
|
|
45
|
+
_conf.coal.store = store_folder
|
|
46
|
+
|
|
41
47
|
for csv_path in pathlib.Path(csv_folder).glob("*.csv"):
|
|
42
48
|
LOGGER.info(T("coal.services.azure_storage.found_file").format(file=csv_path.name))
|
|
43
|
-
store_csv_file(csv_path.name[:-4], csv_path, store=Store(False,
|
|
49
|
+
store_csv_file(csv_path.name[:-4], csv_path, store=Store(False, _conf))
|
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
# etc., to any person is prohibited unless it has been previously and
|
|
6
6
|
# specifically authorized by written means by Cosmo Tech.
|
|
7
7
|
|
|
8
|
-
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import web_help, translate_help
|
|
10
8
|
from cosmotech.orchestrator.utils.translate import T
|
|
11
9
|
|
|
10
|
+
from cosmotech.csm_data.utils.click import click
|
|
11
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
12
|
+
|
|
12
13
|
|
|
13
14
|
@click.command()
|
|
14
15
|
@web_help("csm-data/store/load-from-singlestore")
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
+
# This document and all information contained herein is the exclusive property -
|
|
3
|
+
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
+
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
+
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
+
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
|
|
8
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
9
|
+
|
|
10
|
+
from cosmotech.csm_data.utils.click import click
|
|
11
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@click.command()
|
|
15
|
+
@click.option(
|
|
16
|
+
"--filter",
|
|
17
|
+
"-f",
|
|
18
|
+
multiple=True,
|
|
19
|
+
help=T("csm_data.commands.store.output.parameters.filter"),
|
|
20
|
+
type=str,
|
|
21
|
+
)
|
|
22
|
+
@web_help("csm-data/store/output")
|
|
23
|
+
@translate_help("csm_data.commands.store.output.description")
|
|
24
|
+
def output(
|
|
25
|
+
filter,
|
|
26
|
+
):
|
|
27
|
+
# Import the function at the start of the command
|
|
28
|
+
from cosmotech.coal.store.output import channel_spliter
|
|
29
|
+
from cosmotech.coal.utils.configuration import Configuration
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
_cs = channel_spliter.ChannelSpliter(Configuration())
|
|
33
|
+
_cs.send(list(filter))
|
|
34
|
+
except ValueError as e:
|
|
35
|
+
raise click.Abort() from e
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
5
|
# etc., to any person is prohibited unless it has been previously and
|
|
6
6
|
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
7
8
|
|
|
8
9
|
from cosmotech.csm_data.utils.click import click
|
|
9
|
-
from cosmotech.csm_data.utils.decorators import
|
|
10
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
10
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
@click.command()
|
|
@@ -25,7 +25,12 @@ from cosmotech.orchestrator.utils.translate import T
|
|
|
25
25
|
def reset(store_folder):
|
|
26
26
|
# Import the modules and functions at the start of the command
|
|
27
27
|
from cosmotech.coal.store.store import Store
|
|
28
|
+
from cosmotech.coal.utils.configuration import Configuration
|
|
28
29
|
from cosmotech.coal.utils.logger import LOGGER
|
|
29
30
|
|
|
30
|
-
|
|
31
|
+
_conf = Configuration()
|
|
32
|
+
|
|
33
|
+
_conf.coal.store = store_folder
|
|
34
|
+
|
|
35
|
+
Store(True, _conf)
|
|
31
36
|
LOGGER.info(T("coal.services.database.store_reset").format(folder=store_folder))
|
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
# etc., to any person is prohibited unless it has been previously and
|
|
6
6
|
# specifically authorized by written means by Cosmo Tech.
|
|
7
7
|
|
|
8
|
-
from cosmotech.csm_data.commands.api.rds_send_store import rds_send_store
|
|
9
8
|
from cosmotech.csm_data.commands.store.dump_to_azure import dump_to_azure
|
|
10
9
|
from cosmotech.csm_data.commands.store.dump_to_postgresql import dump_to_postgresql
|
|
11
10
|
from cosmotech.csm_data.commands.store.dump_to_s3 import dump_to_s3
|
|
@@ -14,9 +13,10 @@ from cosmotech.csm_data.commands.store.load_csv_folder import load_csv_folder
|
|
|
14
13
|
from cosmotech.csm_data.commands.store.load_from_singlestore import (
|
|
15
14
|
load_from_singlestore_command,
|
|
16
15
|
)
|
|
16
|
+
from cosmotech.csm_data.commands.store.output import output
|
|
17
17
|
from cosmotech.csm_data.commands.store.reset import reset
|
|
18
18
|
from cosmotech.csm_data.utils.click import click
|
|
19
|
-
from cosmotech.csm_data.utils.decorators import
|
|
19
|
+
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
@click.group()
|
|
@@ -26,7 +26,6 @@ def store():
|
|
|
26
26
|
pass
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
store.add_command(rds_send_store, "rds-send-store")
|
|
30
29
|
store.add_command(reset, "reset")
|
|
31
30
|
store.add_command(list_tables, "list-tables")
|
|
32
31
|
store.add_command(load_csv_folder, "load-csv-folder")
|
|
@@ -34,3 +33,4 @@ store.add_command(load_from_singlestore_command, "load-from-singlestore")
|
|
|
34
33
|
store.add_command(dump_to_postgresql, "dump-to-postgresql")
|
|
35
34
|
store.add_command(dump_to_s3, "dump-to-s3")
|
|
36
35
|
store.add_command(dump_to_azure, "dump-to-azure")
|
|
36
|
+
store.add_command(output, "output")
|
cosmotech/csm_data/main.py
CHANGED
|
@@ -5,20 +5,20 @@
|
|
|
5
5
|
# etc., to any person is prohibited unless it has been previously and
|
|
6
6
|
# specifically authorized by written means by Cosmo Tech.
|
|
7
7
|
import click_log
|
|
8
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
8
9
|
|
|
9
10
|
from cosmotech.coal import __version__
|
|
11
|
+
from cosmotech.coal.utils.logger import LOGGER
|
|
12
|
+
from cosmotech.csm_data.commands.adx_send_data import adx_send_data
|
|
10
13
|
from cosmotech.csm_data.commands.adx_send_runnerdata import adx_send_runnerdata
|
|
11
14
|
from cosmotech.csm_data.commands.api.api import api
|
|
12
15
|
from cosmotech.csm_data.commands.az_storage_upload import az_storage_upload
|
|
16
|
+
from cosmotech.csm_data.commands.s3_bucket_delete import s3_bucket_delete
|
|
13
17
|
from cosmotech.csm_data.commands.s3_bucket_download import s3_bucket_download
|
|
14
18
|
from cosmotech.csm_data.commands.s3_bucket_upload import s3_bucket_upload
|
|
15
|
-
from cosmotech.csm_data.commands.s3_bucket_delete import s3_bucket_delete
|
|
16
|
-
from cosmotech.csm_data.commands.adx_send_data import adx_send_data
|
|
17
19
|
from cosmotech.csm_data.commands.store.store import store
|
|
18
20
|
from cosmotech.csm_data.utils.click import click
|
|
19
21
|
from cosmotech.csm_data.utils.decorators import translate_help, web_help
|
|
20
|
-
from cosmotech.coal.utils.logger import LOGGER
|
|
21
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def print_version(ctx, param, value):
|
|
@@ -9,10 +9,11 @@ import os
|
|
|
9
9
|
import webbrowser
|
|
10
10
|
from functools import wraps
|
|
11
11
|
|
|
12
|
+
from cosmotech.orchestrator.utils.translate import T
|
|
13
|
+
|
|
12
14
|
from cosmotech.coal.utils import WEB_DOCUMENTATION_ROOT
|
|
13
|
-
from cosmotech.csm_data.utils.click import click
|
|
14
15
|
from cosmotech.coal.utils.logger import LOGGER
|
|
15
|
-
from cosmotech.
|
|
16
|
+
from cosmotech.csm_data.utils.click import click
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
def translate_help(translation_key):
|
|
@@ -34,7 +35,7 @@ def require_env(envvar, envvar_desc):
|
|
|
34
35
|
@wraps(func)
|
|
35
36
|
def f(*args, **kwargs):
|
|
36
37
|
if envvar not in os.environ:
|
|
37
|
-
raise EnvironmentError(T("coal.errors.
|
|
38
|
+
raise EnvironmentError(T("coal.common.errors.missing_var").format(envvar=envvar))
|
|
38
39
|
return func(*args, **kwargs)
|
|
39
40
|
|
|
40
41
|
f.__doc__ = "\n".join([f.__doc__ or "", f"Requires env var `{envvar:<15}` *{envvar_desc}* "])
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# API initialization messages
|
|
2
|
+
dataset_api_initialized: "Initialized DatasetApi"
|
|
3
|
+
organization_api_initialized: "Initialized OrganizationApi"
|
|
4
|
+
run_api_initialized: "Initialized RunApi"
|
|
5
|
+
runner_api_initialized: "Initialized RunnerApi"
|
|
6
|
+
solution_api_initialized: "Initialized SolutionApi"
|
|
7
|
+
meta_api_initialized: "Initialized MetaApi"
|
|
8
|
+
workspace_api_initialized: "Initialized WorkspaceApi"
|
|
@@ -35,20 +35,6 @@ adt_twins_found: "Found {count} digital twins"
|
|
|
35
35
|
adt_querying_relations: "Querying relationships"
|
|
36
36
|
adt_relations_found: "Found {count} relationships"
|
|
37
37
|
|
|
38
|
-
# TwinGraph specific
|
|
39
|
-
twingraph_downloading: "Downloading TwinGraph dataset (organization: {organization_id}, dataset: {dataset_id})"
|
|
40
|
-
twingraph_querying_nodes: "Querying TwinGraph nodes for dataset {dataset_id}"
|
|
41
|
-
twingraph_nodes_found: "Found {count} nodes in TwinGraph"
|
|
42
|
-
twingraph_querying_edges: "Querying TwinGraph edges for dataset {dataset_id}"
|
|
43
|
-
twingraph_edges_found: "Found {count} edges in TwinGraph"
|
|
44
|
-
|
|
45
|
-
# Legacy TwinGraph specific
|
|
46
|
-
legacy_twingraph_downloading: "Downloading legacy TwinGraph dataset (organization: {organization_id}, cache: {cache_name})"
|
|
47
|
-
legacy_twingraph_querying_nodes: "Querying legacy TwinGraph nodes for cache {cache_name}"
|
|
48
|
-
legacy_twingraph_nodes_found: "Found {count} nodes in legacy TwinGraph"
|
|
49
|
-
legacy_twingraph_querying_relations: "Querying legacy TwinGraph relationships for cache {cache_name}"
|
|
50
|
-
legacy_twingraph_relations_found: "Found {count} relationships in legacy TwinGraph"
|
|
51
|
-
|
|
52
38
|
# File specific
|
|
53
39
|
file_downloading: "Downloading file dataset (organization: {organization_id}, workspace: {workspace_id}, file: {file_name})"
|
|
54
40
|
listing_workspace_files: "Listing workspace files"
|
|
@@ -66,3 +52,13 @@ processing_json: "Processing JSON file: {file_name}"
|
|
|
66
52
|
json_processed: "Processed JSON file {file_name} with {items} items"
|
|
67
53
|
processing_text: "Processing text file: {file_name}"
|
|
68
54
|
text_processed: "Processed text file {file_name} with {lines} lines"
|
|
55
|
+
|
|
56
|
+
# Dataset API operations
|
|
57
|
+
part_downloaded: "Downloaded part {part_name} to {file_path}"
|
|
58
|
+
dataset_created: "Created dataset {dataset_id}"
|
|
59
|
+
|
|
60
|
+
# Dataset parts operations
|
|
61
|
+
part_uploaded: "Uploaded part {part_name}"
|
|
62
|
+
part_replaced: "Replaced existing part {part_name}"
|
|
63
|
+
part_skipped: "Skipped existing part {part_name} (use replace_existing=True to overwrite)"
|
|
64
|
+
parts_uploaded: "Successfully uploaded parts to dataset {dataset_id}"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
requirements: "Not a valid interface, need to be implemented for a specific service"
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
send:
|
|
2
|
+
error: "Class {interface_name} was not able to send data"
|
|
3
|
+
delete:
|
|
4
|
+
error: "Class {interface_name} was not able to delete data"
|
|
5
|
+
requirements: "Class {interface_name} has to following requirements : {requirements}"
|
|
6
|
+
no_targets: "No valid interface was found"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cosmotech_acceleration_library
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.1.0rc1
|
|
4
4
|
Summary: Acceleration libraries for CosmoTech cloud based solution development
|
|
5
5
|
Author-email: Cosmo Tech <platform@cosmotech.com>
|
|
6
6
|
Project-URL: Homepage, https://www.cosmotech.com
|
|
@@ -8,41 +8,40 @@ Project-URL: Source, https://github.com/Cosmo-Tech/CosmoTech-Acceleration-Librar
|
|
|
8
8
|
Project-URL: Documentation, https://cosmo-tech.github.io/CosmoTech-Acceleration-Library
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
10
|
License-File: LICENSE
|
|
11
|
-
Requires-Dist: azure-
|
|
12
|
-
Requires-Dist: azure-
|
|
13
|
-
Requires-Dist: azure-
|
|
14
|
-
Requires-Dist: azure-
|
|
15
|
-
Requires-Dist: azure-kusto-
|
|
16
|
-
Requires-Dist:
|
|
17
|
-
Requires-Dist:
|
|
18
|
-
Requires-Dist:
|
|
19
|
-
Requires-Dist:
|
|
20
|
-
Requires-Dist:
|
|
21
|
-
Requires-Dist:
|
|
11
|
+
Requires-Dist: azure-core~=1.38
|
|
12
|
+
Requires-Dist: azure-functions~=1.20
|
|
13
|
+
Requires-Dist: azure-digitaltwins-core~=1.2
|
|
14
|
+
Requires-Dist: azure-identity~=1.16
|
|
15
|
+
Requires-Dist: azure-kusto-data~=4.4
|
|
16
|
+
Requires-Dist: azure-kusto-ingest~=4.4
|
|
17
|
+
Requires-Dist: tenacity~=9.1
|
|
18
|
+
Requires-Dist: python-keycloak~=5.8
|
|
19
|
+
Requires-Dist: cosmotech-api~=5.0.0rc1
|
|
20
|
+
Requires-Dist: boto3~=1.41
|
|
21
|
+
Requires-Dist: requests~=2.32
|
|
22
|
+
Requires-Dist: singlestoredb~=1.16
|
|
22
23
|
Requires-Dist: cosmotech-run-orchestrator~=2.0.0
|
|
23
|
-
Requires-Dist: pyarrow~=
|
|
24
|
-
Requires-Dist: adbc-driver-manager~=1.7
|
|
25
|
-
Requires-Dist: adbc-driver-sqlite~=1.7
|
|
26
|
-
Requires-Dist: adbc-driver-postgresql~=1.7
|
|
27
|
-
Requires-Dist: click~=8.1
|
|
28
|
-
Requires-Dist: rich-click~=1.7
|
|
29
|
-
Requires-Dist: click-log~=0.4
|
|
30
|
-
Requires-Dist: tqdm~=4.67
|
|
24
|
+
Requires-Dist: pyarrow~=22.0
|
|
25
|
+
Requires-Dist: adbc-driver-manager~=1.7
|
|
26
|
+
Requires-Dist: adbc-driver-sqlite~=1.7
|
|
27
|
+
Requires-Dist: adbc-driver-postgresql~=1.7
|
|
28
|
+
Requires-Dist: click~=8.1
|
|
29
|
+
Requires-Dist: rich-click~=1.7
|
|
30
|
+
Requires-Dist: click-log~=0.4
|
|
31
|
+
Requires-Dist: tqdm~=4.67
|
|
31
32
|
Requires-Dist: openpyxl~=3.1
|
|
32
|
-
Requires-Dist: pandas~=2.
|
|
33
|
+
Requires-Dist: pandas~=2.3
|
|
33
34
|
Requires-Dist: python-dateutil~=2.8
|
|
34
35
|
Requires-Dist: rich~=13.7
|
|
35
36
|
Requires-Dist: setuptools
|
|
36
37
|
Provides-Extra: doc
|
|
37
38
|
Requires-Dist: mkdocs~=1.6.1; extra == "doc"
|
|
38
|
-
Requires-Dist: mkdocs-click~=0.8
|
|
39
|
-
Requires-Dist: mkdocs-gen-files~=0.5
|
|
40
|
-
Requires-Dist: mkdocstrings[python]~=0.28.2; extra == "doc"
|
|
39
|
+
Requires-Dist: mkdocs-click~=0.8; extra == "doc"
|
|
40
|
+
Requires-Dist: mkdocs-gen-files~=0.5; extra == "doc"
|
|
41
41
|
Requires-Dist: mkdocs-awesome-nav~=3.1.2; extra == "doc"
|
|
42
42
|
Requires-Dist: pymdown-extensions~=10.7; extra == "doc"
|
|
43
|
-
Requires-Dist: requirements-parser~=0.11
|
|
44
|
-
Requires-Dist: mike~=2.
|
|
45
|
-
Requires-Dist: griffe~=1.5.7; extra == "doc"
|
|
43
|
+
Requires-Dist: requirements-parser~=0.11; extra == "doc"
|
|
44
|
+
Requires-Dist: mike~=2.1.0; extra == "doc"
|
|
46
45
|
Requires-Dist: mkdocs-include-dir-to-nav~=1.2.0; extra == "doc"
|
|
47
46
|
Requires-Dist: mkdocs-material[imaging]~=9.6.14; extra == "doc"
|
|
48
47
|
Requires-Dist: mkdocs-table-reader-plugin~=2.0.3; extra == "doc"
|
|
@@ -51,15 +50,12 @@ Provides-Extra: test
|
|
|
51
50
|
Requires-Dist: pytest; extra == "test"
|
|
52
51
|
Requires-Dist: pytest-docker; extra == "test"
|
|
53
52
|
Requires-Dist: pytest-cov; extra == "test"
|
|
54
|
-
Provides-Extra: extra
|
|
55
|
-
Requires-Dist: pandas~=2.2.2; extra == "extra"
|
|
56
53
|
Provides-Extra: dev
|
|
57
|
-
Requires-Dist: black~=
|
|
54
|
+
Requires-Dist: black~=25.0; extra == "dev"
|
|
58
55
|
Requires-Dist: pre-commit~=3.3.2; extra == "dev"
|
|
56
|
+
Requires-Dist: isort~=7.0; extra == "dev"
|
|
59
57
|
Provides-Extra: all
|
|
60
|
-
Requires-Dist: CosmoTech_Acceleration_Library[dev,doc,
|
|
61
|
-
Provides-Extra: past
|
|
62
|
-
Requires-Dist: cosmotech-api~=3.2; extra == "past"
|
|
58
|
+
Requires-Dist: CosmoTech_Acceleration_Library[dev,doc,test]; extra == "all"
|
|
63
59
|
Dynamic: license-file
|
|
64
60
|
|
|
65
61
|
# CosmoTech-Acceleration-Library (CoAL)
|