cosmotech-acceleration-library 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. cosmotech/coal/__init__.py +8 -0
  2. cosmotech/coal/aws/__init__.py +23 -0
  3. cosmotech/coal/aws/s3.py +235 -0
  4. cosmotech/coal/azure/__init__.py +23 -0
  5. cosmotech/coal/azure/adx/__init__.py +26 -0
  6. cosmotech/coal/azure/adx/auth.py +125 -0
  7. cosmotech/coal/azure/adx/ingestion.py +329 -0
  8. cosmotech/coal/azure/adx/query.py +56 -0
  9. cosmotech/coal/azure/adx/runner.py +217 -0
  10. cosmotech/coal/azure/adx/store.py +255 -0
  11. cosmotech/coal/azure/adx/tables.py +118 -0
  12. cosmotech/coal/azure/adx/utils.py +71 -0
  13. cosmotech/coal/azure/blob.py +109 -0
  14. cosmotech/coal/azure/functions.py +72 -0
  15. cosmotech/coal/azure/storage.py +74 -0
  16. cosmotech/coal/cosmotech_api/__init__.py +36 -0
  17. cosmotech/coal/cosmotech_api/connection.py +96 -0
  18. cosmotech/coal/cosmotech_api/dataset/__init__.py +26 -0
  19. cosmotech/coal/cosmotech_api/dataset/converters.py +164 -0
  20. cosmotech/coal/cosmotech_api/dataset/download/__init__.py +19 -0
  21. cosmotech/coal/cosmotech_api/dataset/download/adt.py +119 -0
  22. cosmotech/coal/cosmotech_api/dataset/download/common.py +140 -0
  23. cosmotech/coal/cosmotech_api/dataset/download/file.py +216 -0
  24. cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +188 -0
  25. cosmotech/coal/cosmotech_api/dataset/utils.py +132 -0
  26. cosmotech/coal/cosmotech_api/parameters.py +48 -0
  27. cosmotech/coal/cosmotech_api/run.py +25 -0
  28. cosmotech/coal/cosmotech_api/run_data.py +173 -0
  29. cosmotech/coal/cosmotech_api/run_template.py +108 -0
  30. cosmotech/coal/cosmotech_api/runner/__init__.py +28 -0
  31. cosmotech/coal/cosmotech_api/runner/data.py +38 -0
  32. cosmotech/coal/cosmotech_api/runner/datasets.py +364 -0
  33. cosmotech/coal/cosmotech_api/runner/download.py +146 -0
  34. cosmotech/coal/cosmotech_api/runner/metadata.py +42 -0
  35. cosmotech/coal/cosmotech_api/runner/parameters.py +157 -0
  36. cosmotech/coal/cosmotech_api/twin_data_layer.py +512 -0
  37. cosmotech/coal/cosmotech_api/workspace.py +127 -0
  38. cosmotech/coal/csm/__init__.py +6 -0
  39. cosmotech/coal/csm/engine/__init__.py +47 -0
  40. cosmotech/coal/postgresql/__init__.py +22 -0
  41. cosmotech/coal/postgresql/runner.py +93 -0
  42. cosmotech/coal/postgresql/store.py +98 -0
  43. cosmotech/coal/singlestore/__init__.py +17 -0
  44. cosmotech/coal/singlestore/store.py +100 -0
  45. cosmotech/coal/store/__init__.py +42 -0
  46. cosmotech/coal/store/csv.py +44 -0
  47. cosmotech/coal/store/native_python.py +25 -0
  48. cosmotech/coal/store/pandas.py +26 -0
  49. cosmotech/coal/store/pyarrow.py +23 -0
  50. cosmotech/coal/store/store.py +79 -0
  51. cosmotech/coal/utils/__init__.py +18 -0
  52. cosmotech/coal/utils/api.py +68 -0
  53. cosmotech/coal/utils/logger.py +10 -0
  54. cosmotech/coal/utils/postgresql.py +236 -0
  55. cosmotech/csm_data/__init__.py +6 -0
  56. cosmotech/csm_data/commands/__init__.py +6 -0
  57. cosmotech/csm_data/commands/adx_send_data.py +92 -0
  58. cosmotech/csm_data/commands/adx_send_runnerdata.py +119 -0
  59. cosmotech/csm_data/commands/api/__init__.py +6 -0
  60. cosmotech/csm_data/commands/api/api.py +50 -0
  61. cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +119 -0
  62. cosmotech/csm_data/commands/api/rds_load_csv.py +90 -0
  63. cosmotech/csm_data/commands/api/rds_send_csv.py +74 -0
  64. cosmotech/csm_data/commands/api/rds_send_store.py +74 -0
  65. cosmotech/csm_data/commands/api/run_load_data.py +120 -0
  66. cosmotech/csm_data/commands/api/runtemplate_load_handler.py +66 -0
  67. cosmotech/csm_data/commands/api/tdl_load_files.py +76 -0
  68. cosmotech/csm_data/commands/api/tdl_send_files.py +82 -0
  69. cosmotech/csm_data/commands/api/wsf_load_file.py +66 -0
  70. cosmotech/csm_data/commands/api/wsf_send_file.py +68 -0
  71. cosmotech/csm_data/commands/az_storage_upload.py +76 -0
  72. cosmotech/csm_data/commands/s3_bucket_delete.py +107 -0
  73. cosmotech/csm_data/commands/s3_bucket_download.py +118 -0
  74. cosmotech/csm_data/commands/s3_bucket_upload.py +128 -0
  75. cosmotech/csm_data/commands/store/__init__.py +6 -0
  76. cosmotech/csm_data/commands/store/dump_to_azure.py +120 -0
  77. cosmotech/csm_data/commands/store/dump_to_postgresql.py +107 -0
  78. cosmotech/csm_data/commands/store/dump_to_s3.py +169 -0
  79. cosmotech/csm_data/commands/store/list_tables.py +48 -0
  80. cosmotech/csm_data/commands/store/load_csv_folder.py +43 -0
  81. cosmotech/csm_data/commands/store/load_from_singlestore.py +96 -0
  82. cosmotech/csm_data/commands/store/reset.py +31 -0
  83. cosmotech/csm_data/commands/store/store.py +37 -0
  84. cosmotech/csm_data/main.py +57 -0
  85. cosmotech/csm_data/utils/__init__.py +6 -0
  86. cosmotech/csm_data/utils/click.py +18 -0
  87. cosmotech/csm_data/utils/decorators.py +75 -0
  88. cosmotech/orchestrator_plugins/csm-data/__init__.py +11 -0
  89. cosmotech/orchestrator_plugins/csm-data/templates/api/postgres_send_runner_metadata.json +40 -0
  90. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +27 -0
  91. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +27 -0
  92. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +27 -0
  93. cosmotech/orchestrator_plugins/csm-data/templates/api/run_load_data.json +30 -0
  94. cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +27 -0
  95. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +32 -0
  96. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +27 -0
  97. cosmotech/orchestrator_plugins/csm-data/templates/api/try_api_connection.json +9 -0
  98. cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_load_file.json +36 -0
  99. cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_send_file.json +36 -0
  100. cosmotech/orchestrator_plugins/csm-data/templates/main/adx_send_runnerdata.json +29 -0
  101. cosmotech/orchestrator_plugins/csm-data/templates/main/az_storage_upload.json +25 -0
  102. cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_delete.json +31 -0
  103. cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_download.json +34 -0
  104. cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_upload.json +35 -0
  105. cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_azure.json +35 -0
  106. cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_postgresql.json +34 -0
  107. cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_s3.json +36 -0
  108. cosmotech/orchestrator_plugins/csm-data/templates/store/store_list_tables.json +15 -0
  109. cosmotech/orchestrator_plugins/csm-data/templates/store/store_load_csv_folder.json +18 -0
  110. cosmotech/orchestrator_plugins/csm-data/templates/store/store_load_from_singlestore.json +34 -0
  111. cosmotech/orchestrator_plugins/csm-data/templates/store/store_reset.json +15 -0
  112. cosmotech/translation/coal/__init__.py +6 -0
  113. cosmotech/translation/coal/en-US/coal/common/data_transfer.yml +6 -0
  114. cosmotech/translation/coal/en-US/coal/common/errors.yml +9 -0
  115. cosmotech/translation/coal/en-US/coal/common/file_operations.yml +6 -0
  116. cosmotech/translation/coal/en-US/coal/common/progress.yml +6 -0
  117. cosmotech/translation/coal/en-US/coal/common/timing.yml +5 -0
  118. cosmotech/translation/coal/en-US/coal/common/validation.yml +8 -0
  119. cosmotech/translation/coal/en-US/coal/cosmotech_api/connection.yml +10 -0
  120. cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +2 -0
  121. cosmotech/translation/coal/en-US/coal/cosmotech_api/run_template.yml +8 -0
  122. cosmotech/translation/coal/en-US/coal/cosmotech_api/runner.yml +16 -0
  123. cosmotech/translation/coal/en-US/coal/cosmotech_api/solution.yml +5 -0
  124. cosmotech/translation/coal/en-US/coal/cosmotech_api/workspace.yml +7 -0
  125. cosmotech/translation/coal/en-US/coal/services/adx.yml +59 -0
  126. cosmotech/translation/coal/en-US/coal/services/api.yml +8 -0
  127. cosmotech/translation/coal/en-US/coal/services/azure_storage.yml +14 -0
  128. cosmotech/translation/coal/en-US/coal/services/database.yml +19 -0
  129. cosmotech/translation/coal/en-US/coal/services/dataset.yml +68 -0
  130. cosmotech/translation/coal/en-US/coal/services/postgresql.yml +28 -0
  131. cosmotech/translation/coal/en-US/coal/services/s3.yml +9 -0
  132. cosmotech/translation/coal/en-US/coal/solution.yml +3 -0
  133. cosmotech/translation/coal/en-US/coal/web.yml +2 -0
  134. cosmotech/translation/csm_data/__init__.py +6 -0
  135. cosmotech/translation/csm_data/en-US/csm-data.yml +434 -0
  136. cosmotech_acceleration_library-1.0.0.dist-info/METADATA +255 -0
  137. cosmotech_acceleration_library-1.0.0.dist-info/RECORD +141 -0
  138. cosmotech_acceleration_library-1.0.0.dist-info/WHEEL +5 -0
  139. cosmotech_acceleration_library-1.0.0.dist-info/entry_points.txt +2 -0
  140. cosmotech_acceleration_library-1.0.0.dist-info/licenses/LICENSE +17 -0
  141. cosmotech_acceleration_library-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,50 @@
1
+ # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
+ # This document and all information contained herein is the exclusive property -
3
+ # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
+ # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
+ # etc., to any person is prohibited unless it has been previously and
6
+ # specifically authorized by written means by Cosmo Tech.
7
+ from cosmotech.csm_data.commands.api.postgres_send_runner_metadata import (
8
+ postgres_send_runner_metadata,
9
+ )
10
+ from cosmotech.csm_data.commands.api.rds_load_csv import rds_load_csv
11
+ from cosmotech.csm_data.commands.api.rds_send_csv import rds_send_csv
12
+ from cosmotech.csm_data.commands.api.rds_send_store import rds_send_store
13
+ from cosmotech.csm_data.commands.api.run_load_data import run_load_data
14
+ from cosmotech.csm_data.commands.api.runtemplate_load_handler import (
15
+ runtemplate_load_handler,
16
+ )
17
+ from cosmotech.csm_data.commands.api.tdl_load_files import tdl_load_files
18
+ from cosmotech.csm_data.commands.api.tdl_send_files import tdl_send_files
19
+ from cosmotech.csm_data.commands.api.wsf_load_file import wsf_load_file
20
+ from cosmotech.csm_data.commands.api.wsf_send_file import wsf_send_file
21
+ from cosmotech.csm_data.utils.click import click
22
+ from cosmotech.csm_data.utils.decorators import translate_help, web_help
23
+ from cosmotech.coal.cosmotech_api.connection import get_api_client
24
+ from cosmotech.coal.utils.logger import LOGGER
25
+ from cosmotech.orchestrator.utils.translate import T
26
+
27
+
28
+ @click.group(invoke_without_command=True)
29
+ @web_help("csm-data/api")
30
+ @click.pass_context
31
+ @translate_help("csm-data.commands.api.description")
32
+ def api(ctx: click.Context):
33
+ if ctx.invoked_subcommand is None:
34
+ try:
35
+ api_client, description = get_api_client()
36
+ LOGGER.info(T("coal.cosmotech_api.connection.found_valid").format(type=description))
37
+ except EnvironmentError:
38
+ raise click.Abort()
39
+
40
+
41
+ api.add_command(rds_send_csv, "rds-send-csv")
42
+ api.add_command(rds_send_store, "rds-send-store")
43
+ api.add_command(rds_load_csv, "rds-load-csv")
44
+ api.add_command(wsf_send_file, "wsf-send-file")
45
+ api.add_command(wsf_load_file, "wsf-load-file")
46
+ api.add_command(tdl_send_files, "tdl-send-files")
47
+ api.add_command(tdl_load_files, "tdl-load-files")
48
+ api.add_command(runtemplate_load_handler, "runtemplate-load-handler")
49
+ api.add_command(run_load_data, "run-load-data")
50
+ api.add_command(postgres_send_runner_metadata, "postgres-send-runner-metadata")
@@ -0,0 +1,119 @@
1
+ # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
+ # This document and all information contained herein is the exclusive property -
3
+ # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
+ # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
+ # etc., to any person is prohibited unless it has been previously and
6
+ # specifically authorized by written means by Cosmo Tech.
7
+
8
+ from cosmotech.csm_data.utils.click import click
9
+ from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
+ from cosmotech.orchestrator.utils.translate import T
11
+
12
+
13
+ @click.command()
14
+ @web_help("csm-data/api/postgres-send-runner-metadata")
15
+ @translate_help("csm-data.commands.api.postgres_send_runner_metadata.description")
16
+ @click.option(
17
+ "--organization-id",
18
+ envvar="CSM_ORGANIZATION_ID",
19
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.organization_id"),
20
+ metavar="o-XXXXXXXX",
21
+ type=str,
22
+ show_envvar=True,
23
+ required=True,
24
+ )
25
+ @click.option(
26
+ "--workspace-id",
27
+ envvar="CSM_WORKSPACE_ID",
28
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.workspace_id"),
29
+ metavar="w-XXXXXXXX",
30
+ type=str,
31
+ show_envvar=True,
32
+ required=True,
33
+ )
34
+ @click.option(
35
+ "--runner-id",
36
+ envvar="CSM_RUNNER_ID",
37
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.runner_id"),
38
+ metavar="r-XXXXXXXX",
39
+ type=str,
40
+ show_envvar=True,
41
+ required=True,
42
+ )
43
+ @click.option(
44
+ "--table-prefix",
45
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.table_prefix"),
46
+ metavar="PREFIX",
47
+ type=str,
48
+ default="Cosmotech_",
49
+ )
50
+ @click.option(
51
+ "--postgres-host",
52
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_host"),
53
+ envvar="POSTGRES_HOST_URI",
54
+ show_envvar=True,
55
+ required=True,
56
+ )
57
+ @click.option(
58
+ "--postgres-port",
59
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_port"),
60
+ envvar="POSTGRES_HOST_PORT",
61
+ show_envvar=True,
62
+ required=False,
63
+ default=5432,
64
+ )
65
+ @click.option(
66
+ "--postgres-db",
67
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_db"),
68
+ envvar="POSTGRES_DB_NAME",
69
+ show_envvar=True,
70
+ required=True,
71
+ )
72
+ @click.option(
73
+ "--postgres-schema",
74
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_schema"),
75
+ envvar="POSTGRES_DB_SCHEMA",
76
+ show_envvar=True,
77
+ required=True,
78
+ )
79
+ @click.option(
80
+ "--postgres-user",
81
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_user"),
82
+ envvar="POSTGRES_USER_NAME",
83
+ show_envvar=True,
84
+ required=True,
85
+ )
86
+ @click.option(
87
+ "--postgres-password",
88
+ help=T("csm-data.commands.api.postgres_send_runner_metadata.parameters.postgres_password"),
89
+ envvar="POSTGRES_USER_PASSWORD",
90
+ show_envvar=True,
91
+ required=True,
92
+ )
93
+ def postgres_send_runner_metadata(
94
+ organization_id,
95
+ workspace_id,
96
+ runner_id,
97
+ table_prefix: str,
98
+ postgres_host,
99
+ postgres_port,
100
+ postgres_db,
101
+ postgres_schema,
102
+ postgres_user,
103
+ postgres_password,
104
+ ):
105
+ # Import the function at the start of the command
106
+ from cosmotech.coal.postgresql import send_runner_metadata_to_postgresql
107
+
108
+ send_runner_metadata_to_postgresql(
109
+ organization_id=organization_id,
110
+ workspace_id=workspace_id,
111
+ runner_id=runner_id,
112
+ table_prefix=table_prefix,
113
+ postgres_host=postgres_host,
114
+ postgres_port=postgres_port,
115
+ postgres_db=postgres_db,
116
+ postgres_schema=postgres_schema,
117
+ postgres_user=postgres_user,
118
+ postgres_password=postgres_password,
119
+ )
@@ -0,0 +1,90 @@
1
+ # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
+ # This document and all information contained herein is the exclusive property -
3
+ # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
+ # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
+ # etc., to any person is prohibited unless it has been previously and
6
+ # specifically authorized by written means by Cosmo Tech.
7
+
8
+ from cosmotech.csm_data.utils.click import click
9
+ from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
+ from cosmotech.orchestrator.utils.translate import T
11
+
12
+
13
+ @click.command()
14
+ @click.option(
15
+ "--target-folder",
16
+ envvar="CSM_DATASET_ABSOLUTE_PATH",
17
+ help=T("csm-data.commands.api.rds_load_csv.parameters.target_folder"),
18
+ metavar="PATH",
19
+ type=str,
20
+ show_envvar=True,
21
+ required=True,
22
+ )
23
+ @click.option(
24
+ "--organization-id",
25
+ envvar="CSM_ORGANIZATION_ID",
26
+ help=T("csm-data.commands.api.rds_load_csv.parameters.organization_id"),
27
+ metavar="o-XXXXXXXX",
28
+ type=str,
29
+ show_envvar=True,
30
+ required=True,
31
+ )
32
+ @click.option(
33
+ "--workspace-id",
34
+ envvar="CSM_WORKSPACE_ID",
35
+ help=T("csm-data.commands.api.rds_load_csv.parameters.workspace_id"),
36
+ metavar="w-XXXXXXXX",
37
+ type=str,
38
+ show_envvar=True,
39
+ required=True,
40
+ )
41
+ @click.option(
42
+ "--runner-id",
43
+ envvar="CSM_RUNNER_ID",
44
+ help=T("csm-data.commands.api.rds_load_csv.parameters.runner_id"),
45
+ metavar="r-XXXXXXXX",
46
+ type=str,
47
+ show_envvar=True,
48
+ required=True,
49
+ )
50
+ @click.option(
51
+ "--run-id",
52
+ envvar="CSM_RUN_ID",
53
+ help=T("csm-data.commands.api.rds_load_csv.parameters.run_id"),
54
+ metavar="run-XXXXXX",
55
+ type=str,
56
+ show_envvar=True,
57
+ required=True,
58
+ )
59
+ @click.option(
60
+ "--file-name",
61
+ help=T("csm-data.commands.api.rds_load_csv.parameters.file_name"),
62
+ metavar="NAME",
63
+ type=str,
64
+ default="results",
65
+ show_default=True,
66
+ required=True,
67
+ )
68
+ @click.option(
69
+ "--query",
70
+ help=T("csm-data.commands.api.rds_load_csv.parameters.query"),
71
+ metavar="SQL_QUERY",
72
+ type=str,
73
+ default="SELECT table_name FROM information_schema.tables WHERE table_schema='public'",
74
+ show_default=True,
75
+ )
76
+ @web_help("csm-data/api/rds-load-csv")
77
+ @translate_help("csm-data.commands.api.rds_load_csv.description")
78
+ def rds_load_csv(target_folder, organization_id, workspace_id, runner_id, run_id, file_name, query):
79
+ # Import the function at the start of the command
80
+ from cosmotech.coal.cosmotech_api import load_csv_from_run_data
81
+
82
+ load_csv_from_run_data(
83
+ target_folder=target_folder,
84
+ organization_id=organization_id,
85
+ workspace_id=workspace_id,
86
+ runner_id=runner_id,
87
+ run_id=run_id,
88
+ file_name=file_name,
89
+ query=query,
90
+ )
@@ -0,0 +1,74 @@
1
+ # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
+ # This document and all information contained herein is the exclusive property -
3
+ # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
+ # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
+ # etc., to any person is prohibited unless it has been previously and
6
+ # specifically authorized by written means by Cosmo Tech.
7
+
8
+ from cosmotech.csm_data.utils.click import click
9
+ from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
+ from cosmotech.orchestrator.utils.translate import T
11
+
12
+
13
+ @click.command()
14
+ @click.option(
15
+ "--source-folder",
16
+ envvar="CSM_DATASET_ABSOLUTE_PATH",
17
+ help=T("csm-data.commands.api.rds_send_csv.parameters.source_folder"),
18
+ metavar="PATH",
19
+ type=str,
20
+ show_envvar=True,
21
+ required=True,
22
+ )
23
+ @click.option(
24
+ "--organization-id",
25
+ envvar="CSM_ORGANIZATION_ID",
26
+ help=T("csm-data.commands.api.rds_send_csv.parameters.organization_id"),
27
+ metavar="o-XXXXXXXX",
28
+ type=str,
29
+ show_envvar=True,
30
+ required=True,
31
+ )
32
+ @click.option(
33
+ "--workspace-id",
34
+ envvar="CSM_WORKSPACE_ID",
35
+ help=T("csm-data.commands.api.rds_send_csv.parameters.workspace_id"),
36
+ metavar="w-XXXXXXXX",
37
+ type=str,
38
+ show_envvar=True,
39
+ required=True,
40
+ )
41
+ @click.option(
42
+ "--runner-id",
43
+ envvar="CSM_RUNNER_ID",
44
+ help=T("csm-data.commands.api.rds_send_csv.parameters.runner_id"),
45
+ metavar="r-XXXXXXXX",
46
+ type=str,
47
+ show_envvar=True,
48
+ required=True,
49
+ )
50
+ @click.option(
51
+ "--run-id",
52
+ envvar="CSM_RUN_ID",
53
+ help=T("csm-data.commands.api.rds_send_csv.parameters.run_id"),
54
+ metavar="run-XXXXXX",
55
+ type=str,
56
+ show_envvar=True,
57
+ required=True,
58
+ )
59
+ @web_help("csm-data/api/rds-send-csv")
60
+ @translate_help("csm-data.commands.api.rds_send_csv.description")
61
+ def rds_send_csv(source_folder, organization_id, workspace_id, runner_id, run_id):
62
+ # Import the function at the start of the command
63
+ from cosmotech.coal.cosmotech_api import send_csv_to_run_data
64
+
65
+ try:
66
+ send_csv_to_run_data(
67
+ source_folder=source_folder,
68
+ organization_id=organization_id,
69
+ workspace_id=workspace_id,
70
+ runner_id=runner_id,
71
+ run_id=run_id,
72
+ )
73
+ except FileNotFoundError:
74
+ return 1
@@ -0,0 +1,74 @@
1
+ # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
+ # This document and all information contained herein is the exclusive property -
3
+ # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
+ # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
+ # etc., to any person is prohibited unless it has been previously and
6
+ # specifically authorized by written means by Cosmo Tech.
7
+
8
+ from cosmotech.csm_data.utils.click import click
9
+ from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
+ from cosmotech.orchestrator.utils.translate import T
11
+
12
+
13
+ @click.command()
14
+ @click.option(
15
+ "--store-folder",
16
+ envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
17
+ help=T("csm-data.commands.api.rds_send_store.parameters.store_folder"),
18
+ metavar="PATH",
19
+ type=str,
20
+ show_envvar=True,
21
+ required=True,
22
+ )
23
+ @click.option(
24
+ "--organization-id",
25
+ envvar="CSM_ORGANIZATION_ID",
26
+ help=T("csm-data.commands.api.rds_send_store.parameters.organization_id"),
27
+ metavar="o-XXXXXXXX",
28
+ type=str,
29
+ show_envvar=True,
30
+ required=True,
31
+ )
32
+ @click.option(
33
+ "--workspace-id",
34
+ envvar="CSM_WORKSPACE_ID",
35
+ help=T("csm-data.commands.api.rds_send_store.parameters.workspace_id"),
36
+ metavar="w-XXXXXXXX",
37
+ type=str,
38
+ show_envvar=True,
39
+ required=True,
40
+ )
41
+ @click.option(
42
+ "--runner-id",
43
+ envvar="CSM_RUNNER_ID",
44
+ help=T("csm-data.commands.api.rds_send_store.parameters.runner_id"),
45
+ metavar="r-XXXXXXXX",
46
+ type=str,
47
+ show_envvar=True,
48
+ required=True,
49
+ )
50
+ @click.option(
51
+ "--run-id",
52
+ envvar="CSM_RUN_ID",
53
+ help=T("csm-data.commands.api.rds_send_store.parameters.run_id"),
54
+ metavar="run-XXXXXX",
55
+ type=str,
56
+ show_envvar=True,
57
+ required=True,
58
+ )
59
+ @web_help("csm-data/api/rds-send-store")
60
+ @translate_help("csm-data.commands.api.rds_send_store.description")
61
+ def rds_send_store(store_folder, organization_id, workspace_id, runner_id, run_id):
62
+ # Import the function at the start of the command
63
+ from cosmotech.coal.cosmotech_api import send_store_to_run_data
64
+
65
+ try:
66
+ send_store_to_run_data(
67
+ store_folder=store_folder,
68
+ organization_id=organization_id,
69
+ workspace_id=workspace_id,
70
+ runner_id=runner_id,
71
+ run_id=run_id,
72
+ )
73
+ except FileNotFoundError:
74
+ return 1
@@ -0,0 +1,120 @@
1
+ # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
+ # This document and all information contained herein is the exclusive property -
3
+ # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
+ # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
+ # etc., to any person is prohibited unless it has been previously and
6
+ # specifically authorized by written means by Cosmo Tech.
7
+
8
+ from cosmotech.csm_data.utils.click import click
9
+ from cosmotech.csm_data.utils.decorators import require_env
10
+ from cosmotech.csm_data.utils.decorators import web_help, translate_help
11
+ from cosmotech.orchestrator.utils.translate import T
12
+
13
+
14
+ @click.command()
15
+ @click.option(
16
+ "--organization-id",
17
+ envvar="CSM_ORGANIZATION_ID",
18
+ show_envvar=True,
19
+ help=T("csm-data.commands.api.run_load_data.parameters.organization_id"),
20
+ metavar="o-##########",
21
+ required=True,
22
+ )
23
+ @click.option(
24
+ "--workspace-id",
25
+ envvar="CSM_WORKSPACE_ID",
26
+ show_envvar=True,
27
+ help=T("csm-data.commands.api.run_load_data.parameters.workspace_id"),
28
+ metavar="w-##########",
29
+ required=True,
30
+ )
31
+ @click.option(
32
+ "--runner-id",
33
+ envvar="CSM_RUNNER_ID",
34
+ show_envvar=True,
35
+ help=T("csm-data.commands.api.run_load_data.parameters.runner_id"),
36
+ metavar="s-##########",
37
+ required=True,
38
+ )
39
+ @click.option(
40
+ "--dataset-absolute-path",
41
+ envvar="CSM_DATASET_ABSOLUTE_PATH",
42
+ show_envvar=True,
43
+ help=T("csm-data.commands.api.run_load_data.parameters.dataset_absolute_path"),
44
+ metavar="PATH",
45
+ required=True,
46
+ )
47
+ @click.option(
48
+ "--parameters-absolute-path",
49
+ envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
50
+ metavar="PATH",
51
+ show_envvar=True,
52
+ help=T("csm-data.commands.api.run_load_data.parameters.parameters_absolute_path"),
53
+ required=True,
54
+ )
55
+ @click.option(
56
+ "--write-json/--no-write-json",
57
+ envvar="WRITE_JSON",
58
+ show_envvar=True,
59
+ default=True,
60
+ show_default=True,
61
+ help=T("csm-data.commands.api.run_load_data.parameters.write_json"),
62
+ )
63
+ @click.option(
64
+ "--write-csv/--no-write-csv",
65
+ envvar="WRITE_CSV",
66
+ show_envvar=True,
67
+ default=False,
68
+ show_default=True,
69
+ help=T("csm-data.commands.api.run_load_data.parameters.write_csv"),
70
+ )
71
+ @click.option(
72
+ "--fetch-dataset/--no-fetch-dataset",
73
+ envvar="FETCH_DATASET",
74
+ show_envvar=True,
75
+ default=True,
76
+ show_default=True,
77
+ help=T("csm-data.commands.api.run_load_data.parameters.fetch_dataset"),
78
+ )
79
+ @click.option(
80
+ "--parallel/--no-parallel",
81
+ envvar="FETCH_DATASETS_IN_PARALLEL",
82
+ show_envvar=True,
83
+ default=True,
84
+ show_default=True,
85
+ help=T("csm-data.commands.api.run_load_data.parameters.parallel"),
86
+ )
87
+ @require_env("CSM_API_SCOPE", "The identification scope of a Cosmotech API")
88
+ @require_env("CSM_API_URL", "The URL to a Cosmotech API")
89
+ @web_help("csm-data/api/run-load-data")
90
+ @translate_help("csm-data.commands.api.run_load_data.description")
91
+ def run_load_data(
92
+ runner_id: str,
93
+ workspace_id: str,
94
+ organization_id: str,
95
+ dataset_absolute_path: str,
96
+ parameters_absolute_path: str,
97
+ write_json: bool,
98
+ write_csv: bool,
99
+ fetch_dataset: bool,
100
+ parallel: bool,
101
+ ):
102
+ # Import the function at the start of the command
103
+ from cosmotech.coal.cosmotech_api.runner.download import download_runner_data
104
+
105
+ return download_runner_data(
106
+ organization_id=organization_id,
107
+ workspace_id=workspace_id,
108
+ runner_id=runner_id,
109
+ parameter_folder=parameters_absolute_path,
110
+ dataset_folder=dataset_absolute_path,
111
+ read_files=False,
112
+ parallel=parallel,
113
+ write_json=write_json,
114
+ write_csv=write_csv,
115
+ fetch_dataset=fetch_dataset,
116
+ )
117
+
118
+
119
+ if __name__ == "__main__":
120
+ run_load_data()
@@ -0,0 +1,66 @@
1
+ # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
+ # This document and all information contained herein is the exclusive property -
3
+ # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
+ # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
+ # etc., to any person is prohibited unless it has been previously and
6
+ # specifically authorized by written means by Cosmo Tech.
7
+
8
+ from cosmotech.csm_data.utils.click import click
9
+ from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
+ from cosmotech.orchestrator.utils.translate import T
11
+
12
+
13
+ @click.command()
14
+ @web_help("csm-data/api/runtemplate-load-handler")
15
+ @translate_help("csm-data.commands.api.runtemplate_load_handler.description")
16
+ @click.option(
17
+ "--organization-id",
18
+ envvar="CSM_ORGANIZATION_ID",
19
+ show_envvar=True,
20
+ help=T("csm-data.commands.api.runtemplate_load_handler.parameters.organization_id"),
21
+ metavar="o-##########",
22
+ required=True,
23
+ )
24
+ @click.option(
25
+ "--workspace-id",
26
+ envvar="CSM_WORKSPACE_ID",
27
+ show_envvar=True,
28
+ help=T("csm-data.commands.api.runtemplate_load_handler.parameters.workspace_id"),
29
+ metavar="w-##########",
30
+ required=True,
31
+ )
32
+ @click.option(
33
+ "--run-template-id",
34
+ envvar="CSM_RUN_TEMPLATE_ID",
35
+ show_envvar=True,
36
+ help=T("csm-data.commands.api.runtemplate_load_handler.parameters.run_template_id"),
37
+ metavar="NAME",
38
+ required=True,
39
+ )
40
+ @click.option(
41
+ "--handler-list",
42
+ envvar="CSM_CONTAINER_MODE",
43
+ show_envvar=True,
44
+ help=T("csm-data.commands.api.runtemplate_load_handler.parameters.handler_list"),
45
+ metavar="HANDLER,...,HANDLER",
46
+ required=True,
47
+ )
48
+ def runtemplate_load_handler(workspace_id, organization_id, run_template_id, handler_list):
49
+ # Import the function at the start of the command
50
+ from cosmotech.coal.cosmotech_api import load_run_template_handlers
51
+
52
+ try:
53
+ success = load_run_template_handlers(
54
+ organization_id=organization_id,
55
+ workspace_id=workspace_id,
56
+ run_template_id=run_template_id,
57
+ handler_list=handler_list,
58
+ )
59
+ if not success:
60
+ raise click.Abort()
61
+ except ValueError:
62
+ raise click.Abort()
63
+
64
+
65
+ if __name__ == "__main__":
66
+ runtemplate_load_handler()
@@ -0,0 +1,76 @@
1
+ # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
+ # This document and all information contained herein is the exclusive property -
3
+ # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
+ # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
+ # etc., to any person is prohibited unless it has been previously and
6
+ # specifically authorized by written means by Cosmo Tech.
7
+
8
+ from cosmotech.csm_data.utils.click import click
9
+ from cosmotech.csm_data.utils.decorators import web_help, translate_help
10
+ from cosmotech.orchestrator.utils.translate import T
11
+
12
+
13
+ @click.command()
14
+ @click.option(
15
+ "--organization-id",
16
+ envvar="CSM_ORGANIZATION_ID",
17
+ help=T("csm-data.commands.api.tdl_load_files.parameters.organization_id"),
18
+ metavar="o-XXXXXXXX",
19
+ type=str,
20
+ show_envvar=True,
21
+ required=True,
22
+ )
23
+ @click.option(
24
+ "--workspace-id",
25
+ envvar="CSM_WORKSPACE_ID",
26
+ help=T("csm-data.commands.api.tdl_load_files.parameters.workspace_id"),
27
+ metavar="w-XXXXXXXX",
28
+ type=str,
29
+ show_envvar=True,
30
+ required=True,
31
+ )
32
+ @click.option(
33
+ "--scenario-id",
34
+ envvar="CSM_SCENARIO_ID",
35
+ help=T("csm-data.commands.api.tdl_load_files.parameters.scenario_id"),
36
+ metavar="s-XXXXXXXX",
37
+ type=str,
38
+ show_envvar=True,
39
+ required=False,
40
+ )
41
+ @click.option(
42
+ "--runner-id",
43
+ envvar="CSM_RUNNER_ID",
44
+ help=T("csm-data.commands.api.tdl_load_files.parameters.runner_id"),
45
+ metavar="r-XXXXXXXX",
46
+ type=str,
47
+ show_envvar=True,
48
+ required=False,
49
+ )
50
+ @click.option(
51
+ "--dir",
52
+ "directory_path",
53
+ help=T("csm-data.commands.api.tdl_load_files.parameters.dir"),
54
+ metavar="PATH",
55
+ default="./",
56
+ type=str,
57
+ envvar="CSM_DATASET_ABSOLUTE_PATH",
58
+ show_envvar=True,
59
+ required=True,
60
+ )
61
+ @web_help("csm-data/api/tdl-load-file")
62
+ @translate_help("csm-data.commands.api.tdl_load_files.description")
63
+ def tdl_load_files(organization_id, workspace_id, scenario_id, runner_id, directory_path):
64
+ # Import the function at the start of the command
65
+ from cosmotech.coal.cosmotech_api import load_files_from_tdl
66
+
67
+ try:
68
+ load_files_from_tdl(
69
+ organization_id=organization_id,
70
+ workspace_id=workspace_id,
71
+ directory_path=directory_path,
72
+ runner_id=runner_id,
73
+ scenario_id=scenario_id,
74
+ )
75
+ except ValueError as e:
76
+ raise click.Abort() from e