cosmotech-acceleration-library 1.1.0__py3-none-any.whl → 2.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. cosmotech/coal/__init__.py +1 -1
  2. cosmotech/coal/aws/__init__.py +1 -9
  3. cosmotech/coal/aws/s3.py +181 -214
  4. cosmotech/coal/azure/__init__.py +5 -5
  5. cosmotech/coal/azure/adx/__init__.py +24 -10
  6. cosmotech/coal/azure/adx/auth.py +2 -2
  7. cosmotech/coal/azure/adx/ingestion.py +10 -14
  8. cosmotech/coal/azure/adx/query.py +1 -1
  9. cosmotech/coal/azure/adx/runner.py +13 -14
  10. cosmotech/coal/azure/adx/store.py +5 -86
  11. cosmotech/coal/azure/adx/tables.py +2 -2
  12. cosmotech/coal/azure/adx/utils.py +2 -2
  13. cosmotech/coal/azure/blob.py +20 -26
  14. cosmotech/coal/azure/storage.py +3 -3
  15. cosmotech/coal/cosmotech_api/__init__.py +0 -28
  16. cosmotech/coal/cosmotech_api/apis/__init__.py +14 -0
  17. cosmotech/coal/cosmotech_api/apis/dataset.py +222 -0
  18. cosmotech/coal/cosmotech_api/apis/meta.py +25 -0
  19. cosmotech/coal/cosmotech_api/apis/organization.py +24 -0
  20. cosmotech/coal/cosmotech_api/apis/run.py +38 -0
  21. cosmotech/coal/cosmotech_api/apis/runner.py +75 -0
  22. cosmotech/coal/cosmotech_api/apis/solution.py +23 -0
  23. cosmotech/coal/cosmotech_api/apis/workspace.py +108 -0
  24. cosmotech/coal/cosmotech_api/objects/__init__.py +9 -0
  25. cosmotech/coal/cosmotech_api/objects/connection.py +125 -0
  26. cosmotech/coal/cosmotech_api/objects/parameters.py +127 -0
  27. cosmotech/coal/postgresql/runner.py +58 -41
  28. cosmotech/coal/postgresql/store.py +56 -15
  29. cosmotech/coal/postgresql/utils.py +255 -0
  30. cosmotech/coal/singlestore/store.py +3 -2
  31. cosmotech/coal/store/__init__.py +16 -13
  32. cosmotech/coal/store/output/__init__.py +0 -0
  33. cosmotech/coal/store/output/aws_channel.py +74 -0
  34. cosmotech/coal/store/output/az_storage_channel.py +33 -0
  35. cosmotech/coal/store/output/channel_interface.py +38 -0
  36. cosmotech/coal/store/output/channel_spliter.py +61 -0
  37. cosmotech/coal/store/output/postgres_channel.py +37 -0
  38. cosmotech/coal/store/pandas.py +1 -1
  39. cosmotech/coal/store/pyarrow.py +2 -2
  40. cosmotech/coal/store/store.py +4 -7
  41. cosmotech/coal/utils/configuration.py +197 -0
  42. cosmotech/coal/utils/decorator.py +4 -7
  43. cosmotech/csm_data/commands/adx_send_data.py +1 -1
  44. cosmotech/csm_data/commands/adx_send_runnerdata.py +3 -2
  45. cosmotech/csm_data/commands/api/api.py +6 -19
  46. cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +20 -16
  47. cosmotech/csm_data/commands/api/run_load_data.py +15 -52
  48. cosmotech/csm_data/commands/api/wsf_load_file.py +13 -16
  49. cosmotech/csm_data/commands/api/wsf_send_file.py +11 -14
  50. cosmotech/csm_data/commands/az_storage_upload.py +3 -2
  51. cosmotech/csm_data/commands/s3_bucket_delete.py +16 -15
  52. cosmotech/csm_data/commands/s3_bucket_download.py +16 -16
  53. cosmotech/csm_data/commands/s3_bucket_upload.py +16 -14
  54. cosmotech/csm_data/commands/store/dump_to_azure.py +3 -2
  55. cosmotech/csm_data/commands/store/dump_to_postgresql.py +3 -2
  56. cosmotech/csm_data/commands/store/dump_to_s3.py +18 -16
  57. cosmotech/csm_data/commands/store/list_tables.py +3 -2
  58. cosmotech/csm_data/commands/store/load_csv_folder.py +10 -4
  59. cosmotech/csm_data/commands/store/load_from_singlestore.py +3 -2
  60. cosmotech/csm_data/commands/store/output.py +35 -0
  61. cosmotech/csm_data/commands/store/reset.py +8 -3
  62. cosmotech/csm_data/commands/store/store.py +3 -3
  63. cosmotech/csm_data/main.py +4 -4
  64. cosmotech/csm_data/utils/decorators.py +4 -3
  65. cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml +8 -0
  66. cosmotech/translation/coal/en-US/coal/services/dataset.yml +10 -14
  67. cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml +1 -0
  68. cosmotech/translation/coal/en-US/coal/store/output/split.yml +6 -0
  69. cosmotech/translation/coal/en-US/coal/utils/configuration.yml +2 -0
  70. cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml +7 -0
  71. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/METADATA +29 -33
  72. cosmotech_acceleration_library-2.1.0rc1.dist-info/RECORD +153 -0
  73. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/WHEEL +1 -1
  74. cosmotech/coal/azure/functions.py +0 -72
  75. cosmotech/coal/cosmotech_api/connection.py +0 -96
  76. cosmotech/coal/cosmotech_api/dataset/__init__.py +0 -26
  77. cosmotech/coal/cosmotech_api/dataset/converters.py +0 -164
  78. cosmotech/coal/cosmotech_api/dataset/download/__init__.py +0 -19
  79. cosmotech/coal/cosmotech_api/dataset/download/adt.py +0 -119
  80. cosmotech/coal/cosmotech_api/dataset/download/common.py +0 -140
  81. cosmotech/coal/cosmotech_api/dataset/download/file.py +0 -229
  82. cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +0 -185
  83. cosmotech/coal/cosmotech_api/dataset/upload.py +0 -41
  84. cosmotech/coal/cosmotech_api/dataset/utils.py +0 -132
  85. cosmotech/coal/cosmotech_api/parameters.py +0 -48
  86. cosmotech/coal/cosmotech_api/run.py +0 -25
  87. cosmotech/coal/cosmotech_api/run_data.py +0 -173
  88. cosmotech/coal/cosmotech_api/run_template.py +0 -108
  89. cosmotech/coal/cosmotech_api/runner/__init__.py +0 -28
  90. cosmotech/coal/cosmotech_api/runner/data.py +0 -38
  91. cosmotech/coal/cosmotech_api/runner/datasets.py +0 -416
  92. cosmotech/coal/cosmotech_api/runner/download.py +0 -135
  93. cosmotech/coal/cosmotech_api/runner/metadata.py +0 -42
  94. cosmotech/coal/cosmotech_api/runner/parameters.py +0 -157
  95. cosmotech/coal/cosmotech_api/twin_data_layer.py +0 -512
  96. cosmotech/coal/cosmotech_api/workspace.py +0 -127
  97. cosmotech/coal/utils/postgresql.py +0 -236
  98. cosmotech/coal/utils/semver.py +0 -6
  99. cosmotech/csm_data/commands/api/rds_load_csv.py +0 -90
  100. cosmotech/csm_data/commands/api/rds_send_csv.py +0 -74
  101. cosmotech/csm_data/commands/api/rds_send_store.py +0 -74
  102. cosmotech/csm_data/commands/api/runtemplate_load_handler.py +0 -66
  103. cosmotech/csm_data/commands/api/tdl_load_files.py +0 -76
  104. cosmotech/csm_data/commands/api/tdl_send_files.py +0 -82
  105. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +0 -27
  106. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +0 -27
  107. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +0 -27
  108. cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +0 -27
  109. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +0 -32
  110. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +0 -27
  111. cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +0 -2
  112. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml +0 -13
  113. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml +0 -12
  114. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml +0 -12
  115. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml +0 -14
  116. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml +0 -18
  117. cosmotech_acceleration_library-1.1.0.dist-info/RECORD +0 -171
  118. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/entry_points.txt +0 -0
  119. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/licenses/LICENSE +0 -0
  120. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/top_level.txt +0 -0
@@ -7,10 +7,11 @@
7
7
 
8
8
  from typing import Optional
9
9
 
10
- from cosmotech.csm_data.utils.click import click
11
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
12
10
  from cosmotech.orchestrator.utils.translate import T
13
11
 
12
+ from cosmotech.csm_data.utils.click import click
13
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
14
+
14
15
 
15
16
  @click.command()
16
17
  @click.option(
@@ -98,21 +99,20 @@ def s3_bucket_download(
98
99
  ssl_cert_bundle: Optional[str] = None,
99
100
  ):
100
101
  # Import the functions at the start of the command
101
- from cosmotech.coal.aws.s3 import create_s3_resource, download_files
102
+ from cosmotech.coal.aws import S3
103
+ from cosmotech.coal.utils.configuration import Configuration
102
104
 
103
105
  # Create S3 resource
104
- s3_resource = create_s3_resource(
105
- endpoint_url=endpoint_url,
106
- access_id=access_id,
107
- secret_key=secret_key,
108
- use_ssl=use_ssl,
109
- ssl_cert_bundle=ssl_cert_bundle,
110
- )
106
+ _c = Configuration()
107
+ _c.s3.bucket_name = bucket_name
108
+ _c.s3.endpoint_url = endpoint_url
109
+ _c.s3.access_key_id = access_id
110
+ _c.s3.secret_access_key = secret_key
111
+ _c.s3.bucket_prefix = file_prefix
112
+ _c.s3.use_ssl = use_ssl
113
+ _c.s3.ssl_cert_bundle = ssl_cert_bundle
114
+
115
+ _s3 = S3(_c)
111
116
 
112
117
  # Download files
113
- download_files(
114
- target_folder=target_folder,
115
- bucket_name=bucket_name,
116
- s3_resource=s3_resource,
117
- file_prefix=file_prefix,
118
- )
118
+ _s3.download_files(target_folder=target_folder)
@@ -7,10 +7,11 @@
7
7
 
8
8
  from typing import Optional
9
9
 
10
- from cosmotech.csm_data.utils.click import click
11
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
12
10
  from cosmotech.orchestrator.utils.translate import T
13
11
 
12
+ from cosmotech.csm_data.utils.click import click
13
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
14
+
14
15
 
15
16
  @click.command()
16
17
  @click.option(
@@ -107,22 +108,23 @@ def s3_bucket_upload(
107
108
  recursive: bool = False,
108
109
  ):
109
110
  # Import the functions at the start of the command
110
- from cosmotech.coal.aws.s3 import create_s3_resource, upload_folder
111
+ from cosmotech.coal.aws import S3
112
+ from cosmotech.coal.utils.configuration import Configuration
111
113
 
112
114
  # Create S3 resource
113
- s3_resource = create_s3_resource(
114
- endpoint_url=endpoint_url,
115
- access_id=access_id,
116
- secret_key=secret_key,
117
- use_ssl=use_ssl,
118
- ssl_cert_bundle=ssl_cert_bundle,
119
- )
115
+ _c = Configuration()
116
+ _c.s3.bucket_name = bucket_name
117
+ _c.s3.endpoint_url = endpoint_url
118
+ _c.s3.access_key_id = access_id
119
+ _c.s3.secret_access_key = secret_key
120
+ _c.s3.bucket_prefix = file_prefix
121
+ _c.s3.use_ssl = use_ssl
122
+ _c.s3.ssl_cert_bundle = ssl_cert_bundle
123
+
124
+ _s3 = S3(_c)
120
125
 
121
126
  # Upload files
122
- upload_folder(
127
+ _s3.upload_folder(
123
128
  source_folder=source_folder,
124
- bucket_name=bucket_name,
125
- s3_resource=s3_resource,
126
- file_prefix=file_prefix,
127
129
  recursive=recursive,
128
130
  )
@@ -5,10 +5,11 @@
5
5
  # etc., to any person is prohibited unless it has been previously and
6
6
  # specifically authorized by written means by Cosmo Tech.
7
7
 
8
- from cosmotech.csm_data.utils.click import click
9
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
8
  from cosmotech.orchestrator.utils.translate import T
11
9
 
10
+ from cosmotech.csm_data.utils.click import click
11
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
12
+
12
13
  VALID_TYPES = (
13
14
  "sqlite",
14
15
  "csv",
@@ -5,10 +5,11 @@
5
5
  # etc., to any person is prohibited unless it has been previously and
6
6
  # specifically authorized by written means by Cosmo Tech.
7
7
 
8
- from cosmotech.csm_data.utils.click import click
9
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
8
  from cosmotech.orchestrator.utils.translate import T
11
9
 
10
+ from cosmotech.csm_data.utils.click import click
11
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
12
+
12
13
 
13
14
  @click.command()
14
15
  @web_help("csm-data/store/dump-to-postgres")
@@ -7,10 +7,11 @@
7
7
 
8
8
  from typing import Optional
9
9
 
10
- from cosmotech.csm_data.utils.click import click
11
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
12
10
  from cosmotech.orchestrator.utils.translate import T
13
11
 
12
+ from cosmotech.csm_data.utils.click import click
13
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
14
+
14
15
  VALID_TYPES = (
15
16
  "sqlite",
16
17
  "csv",
@@ -113,10 +114,13 @@ def dump_to_s3(
113
114
  ):
114
115
  # Import the modules and functions at the start of the command
115
116
  from io import BytesIO
117
+
116
118
  import pyarrow.csv as pc
117
119
  import pyarrow.parquet as pq
118
- from cosmotech.coal.aws import create_s3_client, upload_data_stream
120
+
121
+ from cosmotech.coal.aws import S3
119
122
  from cosmotech.coal.store.store import Store
123
+ from cosmotech.coal.utils.configuration import Configuration
120
124
  from cosmotech.coal.utils.logger import LOGGER
121
125
 
122
126
  _s = Store(store_location=store_folder)
@@ -125,14 +129,15 @@ def dump_to_s3(
125
129
  LOGGER.error(T("coal.common.errors.data_invalid_output_type").format(output_type=output_type))
126
130
  raise ValueError(T("coal.common.errors.data_invalid_output_type").format(output_type=output_type))
127
131
 
128
- # Create S3 client
129
- s3_client = create_s3_client(
130
- endpoint_url=endpoint_url,
131
- access_id=access_id,
132
- secret_key=secret_key,
133
- use_ssl=use_ssl,
134
- ssl_cert_bundle=ssl_cert_bundle,
135
- )
132
+ _configuration = Configuration()
133
+ _configuration.s3.bucket_name = bucket_name
134
+ _configuration.s3.endpoint_url = endpoint_url
135
+ _configuration.s3.access_key_id = access_id
136
+ _configuration.s3.secret_access_key = secret_key
137
+ _configuration.s3.bucket_prefix_prefix = file_prefix
138
+ _configuration.s3.use_ssl = use_ssl
139
+ _configuration.s3.ssl_cert_bundle = ssl_cert_bundle
140
+ _s3 = S3(_configuration)
136
141
 
137
142
  if output_type == "sqlite":
138
143
  _file_path = _s._database_path
@@ -141,7 +146,7 @@ def dump_to_s3(
141
146
  LOGGER.info(
142
147
  T("coal.common.data_transfer.file_sent").format(file_path=_file_path, uploaded_name=_uploaded_file_name)
143
148
  )
144
- s3_client.upload_file(_file_path, bucket_name, _uploaded_file_name)
149
+ _s3.upload_file(_file_path, _uploaded_file_name)
145
150
  else:
146
151
  tables = list(_s.list_tables())
147
152
  for table_name in tables:
@@ -160,10 +165,7 @@ def dump_to_s3(
160
165
  LOGGER.info(
161
166
  T("coal.common.data_transfer.sending_table").format(table_name=table_name, output_type=output_type)
162
167
  )
163
- upload_data_stream(
168
+ _s3.upload_data_stream(
164
169
  data_stream=_data_stream,
165
- bucket_name=bucket_name,
166
- s3_client=s3_client,
167
170
  file_name=_file_name,
168
- file_prefix=file_prefix,
169
171
  )
@@ -5,10 +5,11 @@
5
5
  # etc., to any person is prohibited unless it has been previously and
6
6
  # specifically authorized by written means by Cosmo Tech.
7
7
 
8
- from cosmotech.csm_data.utils.click import click
9
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
8
  from cosmotech.orchestrator.utils.translate import T
11
9
 
10
+ from cosmotech.csm_data.utils.click import click
11
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
12
+
12
13
 
13
14
  @click.command()
14
15
  @web_help("csm-data/store/list-tables")
@@ -4,10 +4,10 @@
4
4
  # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
5
  # etc., to any person is prohibited unless it has been previously and
6
6
  # specifically authorized by written means by Cosmo Tech.
7
+ from cosmotech.orchestrator.utils.translate import T
7
8
 
8
9
  from cosmotech.csm_data.utils.click import click
9
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
- from cosmotech.orchestrator.utils.translate import T
10
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
11
11
 
12
12
 
13
13
  @click.command()
@@ -24,7 +24,7 @@ from cosmotech.orchestrator.utils.translate import T
24
24
  )
25
25
  @click.option(
26
26
  "--csv-folder",
27
- envvar="CSM_DATASET_ABSOLUTE_PATH",
27
+ envvar="CSM_OUTPUT_ABSOLUTE_PATH",
28
28
  help=T("csm_data.commands.store.load_csv_folder.parameters.csv_folder"),
29
29
  metavar="PATH",
30
30
  type=str,
@@ -34,10 +34,16 @@ from cosmotech.orchestrator.utils.translate import T
34
34
  def load_csv_folder(store_folder, csv_folder):
35
35
  # Import the modules and functions at the start of the command
36
36
  import pathlib
37
+
37
38
  from cosmotech.coal.store.csv import store_csv_file
38
39
  from cosmotech.coal.store.store import Store
40
+ from cosmotech.coal.utils.configuration import Configuration
39
41
  from cosmotech.coal.utils.logger import LOGGER
40
42
 
43
+ _conf = Configuration()
44
+
45
+ _conf.coal.store = store_folder
46
+
41
47
  for csv_path in pathlib.Path(csv_folder).glob("*.csv"):
42
48
  LOGGER.info(T("coal.services.azure_storage.found_file").format(file=csv_path.name))
43
- store_csv_file(csv_path.name[:-4], csv_path, store=Store(False, store_folder))
49
+ store_csv_file(csv_path.name[:-4], csv_path, store=Store(False, _conf))
@@ -5,10 +5,11 @@
5
5
  # etc., to any person is prohibited unless it has been previously and
6
6
  # specifically authorized by written means by Cosmo Tech.
7
7
 
8
- from cosmotech.csm_data.utils.click import click
9
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
8
  from cosmotech.orchestrator.utils.translate import T
11
9
 
10
+ from cosmotech.csm_data.utils.click import click
11
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
12
+
12
13
 
13
14
  @click.command()
14
15
  @web_help("csm-data/store/load-from-singlestore")
@@ -0,0 +1,35 @@
1
+ # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
+ # This document and all information contained herein is the exclusive property -
3
+ # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
+ # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
+ # etc., to any person is prohibited unless it has been previously and
6
+ # specifically authorized by written means by Cosmo Tech.
7
+
8
+ from cosmotech.orchestrator.utils.translate import T
9
+
10
+ from cosmotech.csm_data.utils.click import click
11
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
12
+
13
+
14
+ @click.command()
15
+ @click.option(
16
+ "--filter",
17
+ "-f",
18
+ multiple=True,
19
+ help=T("csm_data.commands.store.output.parameters.filter"),
20
+ type=str,
21
+ )
22
+ @web_help("csm-data/store/output")
23
+ @translate_help("csm_data.commands.store.output.description")
24
+ def output(
25
+ filter,
26
+ ):
27
+ # Import the function at the start of the command
28
+ from cosmotech.coal.store.output import channel_spliter
29
+ from cosmotech.coal.utils.configuration import Configuration
30
+
31
+ try:
32
+ _cs = channel_spliter.ChannelSpliter(Configuration())
33
+ _cs.send(list(filter))
34
+ except ValueError as e:
35
+ raise click.Abort() from e
@@ -4,10 +4,10 @@
4
4
  # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
5
  # etc., to any person is prohibited unless it has been previously and
6
6
  # specifically authorized by written means by Cosmo Tech.
7
+ from cosmotech.orchestrator.utils.translate import T
7
8
 
8
9
  from cosmotech.csm_data.utils.click import click
9
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
- from cosmotech.orchestrator.utils.translate import T
10
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
11
11
 
12
12
 
13
13
  @click.command()
@@ -25,7 +25,12 @@ from cosmotech.orchestrator.utils.translate import T
25
25
  def reset(store_folder):
26
26
  # Import the modules and functions at the start of the command
27
27
  from cosmotech.coal.store.store import Store
28
+ from cosmotech.coal.utils.configuration import Configuration
28
29
  from cosmotech.coal.utils.logger import LOGGER
29
30
 
30
- Store(True, store_folder)
31
+ _conf = Configuration()
32
+
33
+ _conf.coal.store = store_folder
34
+
35
+ Store(True, _conf)
31
36
  LOGGER.info(T("coal.services.database.store_reset").format(folder=store_folder))
@@ -5,7 +5,6 @@
5
5
  # etc., to any person is prohibited unless it has been previously and
6
6
  # specifically authorized by written means by Cosmo Tech.
7
7
 
8
- from cosmotech.csm_data.commands.api.rds_send_store import rds_send_store
9
8
  from cosmotech.csm_data.commands.store.dump_to_azure import dump_to_azure
10
9
  from cosmotech.csm_data.commands.store.dump_to_postgresql import dump_to_postgresql
11
10
  from cosmotech.csm_data.commands.store.dump_to_s3 import dump_to_s3
@@ -14,9 +13,10 @@ from cosmotech.csm_data.commands.store.load_csv_folder import load_csv_folder
14
13
  from cosmotech.csm_data.commands.store.load_from_singlestore import (
15
14
  load_from_singlestore_command,
16
15
  )
16
+ from cosmotech.csm_data.commands.store.output import output
17
17
  from cosmotech.csm_data.commands.store.reset import reset
18
18
  from cosmotech.csm_data.utils.click import click
19
- from cosmotech.csm_data.utils.decorators import web_help, translate_help
19
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
20
20
 
21
21
 
22
22
  @click.group()
@@ -26,7 +26,6 @@ def store():
26
26
  pass
27
27
 
28
28
 
29
- store.add_command(rds_send_store, "rds-send-store")
30
29
  store.add_command(reset, "reset")
31
30
  store.add_command(list_tables, "list-tables")
32
31
  store.add_command(load_csv_folder, "load-csv-folder")
@@ -34,3 +33,4 @@ store.add_command(load_from_singlestore_command, "load-from-singlestore")
34
33
  store.add_command(dump_to_postgresql, "dump-to-postgresql")
35
34
  store.add_command(dump_to_s3, "dump-to-s3")
36
35
  store.add_command(dump_to_azure, "dump-to-azure")
36
+ store.add_command(output, "output")
@@ -5,20 +5,20 @@
5
5
  # etc., to any person is prohibited unless it has been previously and
6
6
  # specifically authorized by written means by Cosmo Tech.
7
7
  import click_log
8
+ from cosmotech.orchestrator.utils.translate import T
8
9
 
9
10
  from cosmotech.coal import __version__
11
+ from cosmotech.coal.utils.logger import LOGGER
12
+ from cosmotech.csm_data.commands.adx_send_data import adx_send_data
10
13
  from cosmotech.csm_data.commands.adx_send_runnerdata import adx_send_runnerdata
11
14
  from cosmotech.csm_data.commands.api.api import api
12
15
  from cosmotech.csm_data.commands.az_storage_upload import az_storage_upload
16
+ from cosmotech.csm_data.commands.s3_bucket_delete import s3_bucket_delete
13
17
  from cosmotech.csm_data.commands.s3_bucket_download import s3_bucket_download
14
18
  from cosmotech.csm_data.commands.s3_bucket_upload import s3_bucket_upload
15
- from cosmotech.csm_data.commands.s3_bucket_delete import s3_bucket_delete
16
- from cosmotech.csm_data.commands.adx_send_data import adx_send_data
17
19
  from cosmotech.csm_data.commands.store.store import store
18
20
  from cosmotech.csm_data.utils.click import click
19
21
  from cosmotech.csm_data.utils.decorators import translate_help, web_help
20
- from cosmotech.coal.utils.logger import LOGGER
21
- from cosmotech.orchestrator.utils.translate import T
22
22
 
23
23
 
24
24
  def print_version(ctx, param, value):
@@ -9,10 +9,11 @@ import os
9
9
  import webbrowser
10
10
  from functools import wraps
11
11
 
12
+ from cosmotech.orchestrator.utils.translate import T
13
+
12
14
  from cosmotech.coal.utils import WEB_DOCUMENTATION_ROOT
13
- from cosmotech.csm_data.utils.click import click
14
15
  from cosmotech.coal.utils.logger import LOGGER
15
- from cosmotech.orchestrator.utils.translate import T
16
+ from cosmotech.csm_data.utils.click import click
16
17
 
17
18
 
18
19
  def translate_help(translation_key):
@@ -34,7 +35,7 @@ def require_env(envvar, envvar_desc):
34
35
  @wraps(func)
35
36
  def f(*args, **kwargs):
36
37
  if envvar not in os.environ:
37
- raise EnvironmentError(T("coal.errors.environment.missing_var").format(envvar=envvar))
38
+ raise EnvironmentError(T("coal.common.errors.missing_var").format(envvar=envvar))
38
39
  return func(*args, **kwargs)
39
40
 
40
41
  f.__doc__ = "\n".join([f.__doc__ or "", f"Requires env var `{envvar:<15}` *{envvar_desc}* "])
@@ -0,0 +1,8 @@
1
+ # API initialization messages
2
+ dataset_api_initialized: "Initialized DatasetApi"
3
+ organization_api_initialized: "Initialized OrganizationApi"
4
+ run_api_initialized: "Initialized RunApi"
5
+ runner_api_initialized: "Initialized RunnerApi"
6
+ solution_api_initialized: "Initialized SolutionApi"
7
+ meta_api_initialized: "Initialized MetaApi"
8
+ workspace_api_initialized: "Initialized WorkspaceApi"
@@ -35,20 +35,6 @@ adt_twins_found: "Found {count} digital twins"
35
35
  adt_querying_relations: "Querying relationships"
36
36
  adt_relations_found: "Found {count} relationships"
37
37
 
38
- # TwinGraph specific
39
- twingraph_downloading: "Downloading TwinGraph dataset (organization: {organization_id}, dataset: {dataset_id})"
40
- twingraph_querying_nodes: "Querying TwinGraph nodes for dataset {dataset_id}"
41
- twingraph_nodes_found: "Found {count} nodes in TwinGraph"
42
- twingraph_querying_edges: "Querying TwinGraph edges for dataset {dataset_id}"
43
- twingraph_edges_found: "Found {count} edges in TwinGraph"
44
-
45
- # Legacy TwinGraph specific
46
- legacy_twingraph_downloading: "Downloading legacy TwinGraph dataset (organization: {organization_id}, cache: {cache_name})"
47
- legacy_twingraph_querying_nodes: "Querying legacy TwinGraph nodes for cache {cache_name}"
48
- legacy_twingraph_nodes_found: "Found {count} nodes in legacy TwinGraph"
49
- legacy_twingraph_querying_relations: "Querying legacy TwinGraph relationships for cache {cache_name}"
50
- legacy_twingraph_relations_found: "Found {count} relationships in legacy TwinGraph"
51
-
52
38
  # File specific
53
39
  file_downloading: "Downloading file dataset (organization: {organization_id}, workspace: {workspace_id}, file: {file_name})"
54
40
  listing_workspace_files: "Listing workspace files"
@@ -66,3 +52,13 @@ processing_json: "Processing JSON file: {file_name}"
66
52
  json_processed: "Processed JSON file {file_name} with {items} items"
67
53
  processing_text: "Processing text file: {file_name}"
68
54
  text_processed: "Processed text file {file_name} with {lines} lines"
55
+
56
+ # Dataset API operations
57
+ part_downloaded: "Downloaded part {part_name} to {file_path}"
58
+ dataset_created: "Created dataset {dataset_id}"
59
+
60
+ # Dataset parts operations
61
+ part_uploaded: "Uploaded part {part_name}"
62
+ part_replaced: "Replaced existing part {part_name}"
63
+ part_skipped: "Skipped existing part {part_name} (use replace_existing=True to overwrite)"
64
+ parts_uploaded: "Successfully uploaded parts to dataset {dataset_id}"
@@ -0,0 +1 @@
1
+ requirements: "Not a valid interface, need to be implemented for a specific service"
@@ -0,0 +1,6 @@
1
+ send:
2
+ error: "Class {interface_name} was not able to send data"
3
+ delete:
4
+ error: "Class {interface_name} was not able to delete data"
5
+ requirements: "Class {interface_name} has to following requirements : {requirements}"
6
+ no_targets: "No valid interface was found"
@@ -0,0 +1,2 @@
1
+ # Configuration messages
2
+ no_config_file: "no configuration file set. setting up default values"
@@ -0,0 +1,7 @@
1
+ description: |
2
+ Dump simulation from the coal store to the configured output.
3
+
4
+ This output is base on the configuration. Default dconfiguration will try to output in postgres.
5
+
6
+ parameters:
7
+ filter: names of the tables to output
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cosmotech_acceleration_library
3
- Version: 1.1.0
3
+ Version: 2.1.0rc1
4
4
  Summary: Acceleration libraries for CosmoTech cloud based solution development
5
5
  Author-email: Cosmo Tech <platform@cosmotech.com>
6
6
  Project-URL: Homepage, https://www.cosmotech.com
@@ -8,41 +8,40 @@ Project-URL: Source, https://github.com/Cosmo-Tech/CosmoTech-Acceleration-Librar
8
8
  Project-URL: Documentation, https://cosmo-tech.github.io/CosmoTech-Acceleration-Library
9
9
  Description-Content-Type: text/markdown
10
10
  License-File: LICENSE
11
- Requires-Dist: azure-functions~=1.20.0
12
- Requires-Dist: azure-digitaltwins-core~=1.2.0
13
- Requires-Dist: azure-identity~=1.16.1
14
- Requires-Dist: azure-kusto-data~=4.4.1
15
- Requires-Dist: azure-kusto-ingest~=4.4.1
16
- Requires-Dist: tenacity~=8.3.0
17
- Requires-Dist: python-keycloak~=4.7.3
18
- Requires-Dist: cosmotech-api>=3.2
19
- Requires-Dist: boto3~=1.35.19
20
- Requires-Dist: requests~=2.32.3
21
- Requires-Dist: singlestoredb~=1.10.0
11
+ Requires-Dist: azure-core~=1.38
12
+ Requires-Dist: azure-functions~=1.20
13
+ Requires-Dist: azure-digitaltwins-core~=1.2
14
+ Requires-Dist: azure-identity~=1.16
15
+ Requires-Dist: azure-kusto-data~=4.4
16
+ Requires-Dist: azure-kusto-ingest~=4.4
17
+ Requires-Dist: tenacity~=9.1
18
+ Requires-Dist: python-keycloak~=5.8
19
+ Requires-Dist: cosmotech-api~=5.0.0rc1
20
+ Requires-Dist: boto3~=1.41
21
+ Requires-Dist: requests~=2.32
22
+ Requires-Dist: singlestoredb~=1.16
22
23
  Requires-Dist: cosmotech-run-orchestrator~=2.0.0
23
- Requires-Dist: pyarrow~=20.0.0
24
- Requires-Dist: adbc-driver-manager~=1.7.0
25
- Requires-Dist: adbc-driver-sqlite~=1.7.0
26
- Requires-Dist: adbc-driver-postgresql~=1.7.0
27
- Requires-Dist: click~=8.1.7
28
- Requires-Dist: rich-click~=1.7.3
29
- Requires-Dist: click-log~=0.4.0
30
- Requires-Dist: tqdm~=4.67.1
24
+ Requires-Dist: pyarrow~=22.0
25
+ Requires-Dist: adbc-driver-manager~=1.7
26
+ Requires-Dist: adbc-driver-sqlite~=1.7
27
+ Requires-Dist: adbc-driver-postgresql~=1.7
28
+ Requires-Dist: click~=8.1
29
+ Requires-Dist: rich-click~=1.7
30
+ Requires-Dist: click-log~=0.4
31
+ Requires-Dist: tqdm~=4.67
31
32
  Requires-Dist: openpyxl~=3.1
32
- Requires-Dist: pandas~=2.1
33
+ Requires-Dist: pandas~=2.3
33
34
  Requires-Dist: python-dateutil~=2.8
34
35
  Requires-Dist: rich~=13.7
35
36
  Requires-Dist: setuptools
36
37
  Provides-Extra: doc
37
38
  Requires-Dist: mkdocs~=1.6.1; extra == "doc"
38
- Requires-Dist: mkdocs-click~=0.8.1; extra == "doc"
39
- Requires-Dist: mkdocs-gen-files~=0.5.0; extra == "doc"
40
- Requires-Dist: mkdocstrings[python]~=0.28.2; extra == "doc"
39
+ Requires-Dist: mkdocs-click~=0.8; extra == "doc"
40
+ Requires-Dist: mkdocs-gen-files~=0.5; extra == "doc"
41
41
  Requires-Dist: mkdocs-awesome-nav~=3.1.2; extra == "doc"
42
42
  Requires-Dist: pymdown-extensions~=10.7; extra == "doc"
43
- Requires-Dist: requirements-parser~=0.11.0; extra == "doc"
44
- Requires-Dist: mike~=2.0.0; extra == "doc"
45
- Requires-Dist: griffe~=1.5.7; extra == "doc"
43
+ Requires-Dist: requirements-parser~=0.11; extra == "doc"
44
+ Requires-Dist: mike~=2.1.0; extra == "doc"
46
45
  Requires-Dist: mkdocs-include-dir-to-nav~=1.2.0; extra == "doc"
47
46
  Requires-Dist: mkdocs-material[imaging]~=9.6.14; extra == "doc"
48
47
  Requires-Dist: mkdocs-table-reader-plugin~=2.0.3; extra == "doc"
@@ -51,15 +50,12 @@ Provides-Extra: test
51
50
  Requires-Dist: pytest; extra == "test"
52
51
  Requires-Dist: pytest-docker; extra == "test"
53
52
  Requires-Dist: pytest-cov; extra == "test"
54
- Provides-Extra: extra
55
- Requires-Dist: pandas~=2.2.2; extra == "extra"
56
53
  Provides-Extra: dev
57
- Requires-Dist: black~=24.3.0; extra == "dev"
54
+ Requires-Dist: black~=25.0; extra == "dev"
58
55
  Requires-Dist: pre-commit~=3.3.2; extra == "dev"
56
+ Requires-Dist: isort~=7.0; extra == "dev"
59
57
  Provides-Extra: all
60
- Requires-Dist: CosmoTech_Acceleration_Library[dev,doc,extra,test]; extra == "all"
61
- Provides-Extra: past
62
- Requires-Dist: cosmotech-api~=3.2; extra == "past"
58
+ Requires-Dist: CosmoTech_Acceleration_Library[dev,doc,test]; extra == "all"
63
59
  Dynamic: license-file
64
60
 
65
61
  # CosmoTech-Acceleration-Library (CoAL)