cosmotech-acceleration-library 1.1.0__py3-none-any.whl → 2.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cosmotech/coal/__init__.py +1 -1
- cosmotech/coal/aws/__init__.py +1 -9
- cosmotech/coal/aws/s3.py +181 -214
- cosmotech/coal/azure/__init__.py +5 -5
- cosmotech/coal/azure/adx/__init__.py +24 -10
- cosmotech/coal/azure/adx/auth.py +2 -2
- cosmotech/coal/azure/adx/ingestion.py +10 -14
- cosmotech/coal/azure/adx/query.py +1 -1
- cosmotech/coal/azure/adx/runner.py +13 -14
- cosmotech/coal/azure/adx/store.py +5 -86
- cosmotech/coal/azure/adx/tables.py +2 -2
- cosmotech/coal/azure/adx/utils.py +2 -2
- cosmotech/coal/azure/blob.py +20 -26
- cosmotech/coal/azure/storage.py +3 -3
- cosmotech/coal/cosmotech_api/__init__.py +0 -28
- cosmotech/coal/cosmotech_api/apis/__init__.py +14 -0
- cosmotech/coal/cosmotech_api/apis/dataset.py +222 -0
- cosmotech/coal/cosmotech_api/apis/meta.py +25 -0
- cosmotech/coal/cosmotech_api/apis/organization.py +24 -0
- cosmotech/coal/cosmotech_api/apis/run.py +38 -0
- cosmotech/coal/cosmotech_api/apis/runner.py +75 -0
- cosmotech/coal/cosmotech_api/apis/solution.py +23 -0
- cosmotech/coal/cosmotech_api/apis/workspace.py +108 -0
- cosmotech/coal/cosmotech_api/objects/__init__.py +9 -0
- cosmotech/coal/cosmotech_api/objects/connection.py +125 -0
- cosmotech/coal/cosmotech_api/objects/parameters.py +127 -0
- cosmotech/coal/postgresql/runner.py +58 -41
- cosmotech/coal/postgresql/store.py +56 -15
- cosmotech/coal/postgresql/utils.py +255 -0
- cosmotech/coal/singlestore/store.py +3 -2
- cosmotech/coal/store/__init__.py +16 -13
- cosmotech/coal/store/output/__init__.py +0 -0
- cosmotech/coal/store/output/aws_channel.py +74 -0
- cosmotech/coal/store/output/az_storage_channel.py +33 -0
- cosmotech/coal/store/output/channel_interface.py +38 -0
- cosmotech/coal/store/output/channel_spliter.py +61 -0
- cosmotech/coal/store/output/postgres_channel.py +37 -0
- cosmotech/coal/store/pandas.py +1 -1
- cosmotech/coal/store/pyarrow.py +2 -2
- cosmotech/coal/store/store.py +4 -7
- cosmotech/coal/utils/configuration.py +197 -0
- cosmotech/coal/utils/decorator.py +4 -7
- cosmotech/csm_data/commands/adx_send_data.py +1 -1
- cosmotech/csm_data/commands/adx_send_runnerdata.py +3 -2
- cosmotech/csm_data/commands/api/api.py +6 -19
- cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +20 -16
- cosmotech/csm_data/commands/api/run_load_data.py +15 -52
- cosmotech/csm_data/commands/api/wsf_load_file.py +13 -16
- cosmotech/csm_data/commands/api/wsf_send_file.py +11 -14
- cosmotech/csm_data/commands/az_storage_upload.py +3 -2
- cosmotech/csm_data/commands/s3_bucket_delete.py +16 -15
- cosmotech/csm_data/commands/s3_bucket_download.py +16 -16
- cosmotech/csm_data/commands/s3_bucket_upload.py +16 -14
- cosmotech/csm_data/commands/store/dump_to_azure.py +3 -2
- cosmotech/csm_data/commands/store/dump_to_postgresql.py +3 -2
- cosmotech/csm_data/commands/store/dump_to_s3.py +18 -16
- cosmotech/csm_data/commands/store/list_tables.py +3 -2
- cosmotech/csm_data/commands/store/load_csv_folder.py +10 -4
- cosmotech/csm_data/commands/store/load_from_singlestore.py +3 -2
- cosmotech/csm_data/commands/store/output.py +35 -0
- cosmotech/csm_data/commands/store/reset.py +8 -3
- cosmotech/csm_data/commands/store/store.py +3 -3
- cosmotech/csm_data/main.py +4 -4
- cosmotech/csm_data/utils/decorators.py +4 -3
- cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml +8 -0
- cosmotech/translation/coal/en-US/coal/services/dataset.yml +10 -14
- cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml +1 -0
- cosmotech/translation/coal/en-US/coal/store/output/split.yml +6 -0
- cosmotech/translation/coal/en-US/coal/utils/configuration.yml +2 -0
- cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml +7 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/METADATA +29 -33
- cosmotech_acceleration_library-2.1.0rc1.dist-info/RECORD +153 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/WHEEL +1 -1
- cosmotech/coal/azure/functions.py +0 -72
- cosmotech/coal/cosmotech_api/connection.py +0 -96
- cosmotech/coal/cosmotech_api/dataset/__init__.py +0 -26
- cosmotech/coal/cosmotech_api/dataset/converters.py +0 -164
- cosmotech/coal/cosmotech_api/dataset/download/__init__.py +0 -19
- cosmotech/coal/cosmotech_api/dataset/download/adt.py +0 -119
- cosmotech/coal/cosmotech_api/dataset/download/common.py +0 -140
- cosmotech/coal/cosmotech_api/dataset/download/file.py +0 -229
- cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +0 -185
- cosmotech/coal/cosmotech_api/dataset/upload.py +0 -41
- cosmotech/coal/cosmotech_api/dataset/utils.py +0 -132
- cosmotech/coal/cosmotech_api/parameters.py +0 -48
- cosmotech/coal/cosmotech_api/run.py +0 -25
- cosmotech/coal/cosmotech_api/run_data.py +0 -173
- cosmotech/coal/cosmotech_api/run_template.py +0 -108
- cosmotech/coal/cosmotech_api/runner/__init__.py +0 -28
- cosmotech/coal/cosmotech_api/runner/data.py +0 -38
- cosmotech/coal/cosmotech_api/runner/datasets.py +0 -416
- cosmotech/coal/cosmotech_api/runner/download.py +0 -135
- cosmotech/coal/cosmotech_api/runner/metadata.py +0 -42
- cosmotech/coal/cosmotech_api/runner/parameters.py +0 -157
- cosmotech/coal/cosmotech_api/twin_data_layer.py +0 -512
- cosmotech/coal/cosmotech_api/workspace.py +0 -127
- cosmotech/coal/utils/postgresql.py +0 -236
- cosmotech/coal/utils/semver.py +0 -6
- cosmotech/csm_data/commands/api/rds_load_csv.py +0 -90
- cosmotech/csm_data/commands/api/rds_send_csv.py +0 -74
- cosmotech/csm_data/commands/api/rds_send_store.py +0 -74
- cosmotech/csm_data/commands/api/runtemplate_load_handler.py +0 -66
- cosmotech/csm_data/commands/api/tdl_load_files.py +0 -76
- cosmotech/csm_data/commands/api/tdl_send_files.py +0 -82
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +0 -32
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +0 -27
- cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +0 -2
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml +0 -13
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml +0 -12
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml +0 -12
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml +0 -14
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml +0 -18
- cosmotech_acceleration_library-1.1.0.dist-info/RECORD +0 -171
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/entry_points.txt +0 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/licenses/LICENSE +0 -0
- {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
cosmotech/coal/__init__.py,sha256=-QWX8vGspoMFG2cOvq21Sg4KBpIZUzhd76Kv7-6-6yg,440
|
|
2
|
+
cosmotech/coal/aws/__init__.py,sha256=qZZyxNCwGa1yPpIEc7CS6sVcv00PNcUnUOsKhsHBSdc,614
|
|
3
|
+
cosmotech/coal/aws/s3.py,sha256=5az9BtyUUBlfqFV3BG32V5-9yDpm-6q_O4YV79HcmMA,7522
|
|
4
|
+
cosmotech/coal/azure/__init__.py,sha256=Zp7v4g2Tp6h8j6_-_drMNHdEaq8FqfKhlZOwvfxhiMc,796
|
|
5
|
+
cosmotech/coal/azure/blob.py,sha256=p1iG2_lWA2UDZQrYIkz1TCK1--5IVeaSmyDC0RenFt0,4019
|
|
6
|
+
cosmotech/coal/azure/storage.py,sha256=TItajcil0Ht44kru44T9HIP2cizOvAS_H9uRdSgaBG0,2799
|
|
7
|
+
cosmotech/coal/azure/adx/__init__.py,sha256=-nDfPlb1lC1zy_1llxi82FT0hWVYGfYPMgAaQNa6Dak,1286
|
|
8
|
+
cosmotech/coal/azure/adx/auth.py,sha256=IFNaB8eoc99_j3Ml6rueYfuFK36rze4_8LG7B0gYdIQ,4748
|
|
9
|
+
cosmotech/coal/azure/adx/ingestion.py,sha256=pDksvQzzONSSbFRTcf4EzgNhwe7MkhY3lCkNvlgG8U4,11266
|
|
10
|
+
cosmotech/coal/azure/adx/query.py,sha256=JrTqZaQ6Ody21rbjkEIx4J3lYJ1-gOJxFKJRrbEpdi0,1870
|
|
11
|
+
cosmotech/coal/azure/adx/runner.py,sha256=IXgciGxHjSqkiqm23CuyUk3RgCUgd1G-e0eMyohjygM,8066
|
|
12
|
+
cosmotech/coal/azure/adx/store.py,sha256=7fxG3yVxQ9_6sLaSU1Z4cxmsMcSO58Ni371FD2m_4LA,6214
|
|
13
|
+
cosmotech/coal/azure/adx/tables.py,sha256=y6GBrjXAKAOn7q4Ekf-hV6K5P3ejdV2L57FbFDvp22c,4287
|
|
14
|
+
cosmotech/coal/azure/adx/utils.py,sha256=0MCnIJ-UijovA04byFbSwO4QVUMVn7v3P7dxGpdN_zM,2205
|
|
15
|
+
cosmotech/coal/cosmotech_api/__init__.py,sha256=YlIyLwoqd4CveVnfrA9UDunlhO9HSvrBAfUwzcBay5M,528
|
|
16
|
+
cosmotech/coal/cosmotech_api/apis/__init__.py,sha256=RDe2FXBCrgUoSAONfpf_Udyc3M3w2y9K9mjFzUX_SMY,869
|
|
17
|
+
cosmotech/coal/cosmotech_api/apis/dataset.py,sha256=CtIrRvykUhJ9cFZqz8WEj-gn1trx9NXp9NAfKQklteU,8874
|
|
18
|
+
cosmotech/coal/cosmotech_api/apis/meta.py,sha256=lCi5Jo7c_gXewob9B_NgvRyKAmPSJ_bVK7Z_pIVUyP8,1055
|
|
19
|
+
cosmotech/coal/cosmotech_api/apis/organization.py,sha256=JA55Bu4mJL5aE7UNtnPDWyjB6YZktSsnCzOvs9bW34g,1102
|
|
20
|
+
cosmotech/coal/cosmotech_api/apis/run.py,sha256=TOVD0E4QIR26rBZr3MjsWuSbYcKcuTVSJsq_zD4nzD0,1528
|
|
21
|
+
cosmotech/coal/cosmotech_api/apis/runner.py,sha256=Q9y3DDKFYRMcibMPkU7K0pv6M5z2On5oFG2JrMIHkPY,3115
|
|
22
|
+
cosmotech/coal/cosmotech_api/apis/solution.py,sha256=rGsbtkVkhrt0vAano4HcIYQJFaXGSpWCHLku2TEpEHI,1077
|
|
23
|
+
cosmotech/coal/cosmotech_api/apis/workspace.py,sha256=YeXuODKJnwpdO2g2yaJzg9095VyzAuGlMV1JL1hNMsU,4330
|
|
24
|
+
cosmotech/coal/cosmotech_api/objects/__init__.py,sha256=6cWMAAyhCK478_Je9Opsu0NBv617aTQo3r9mhbV2wwo,556
|
|
25
|
+
cosmotech/coal/cosmotech_api/objects/connection.py,sha256=z0hd2uJNZWZdTyL7NlGuXt_HeZMenSODNJatNdmhk0o,5292
|
|
26
|
+
cosmotech/coal/cosmotech_api/objects/parameters.py,sha256=6T4yJMm_yvularJwMIeg8qoqBYdpWEdbQx0Q2dCieDw,4227
|
|
27
|
+
cosmotech/coal/csm/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
28
|
+
cosmotech/coal/csm/engine/__init__.py,sha256=EG3iArAy3OFGPmClsYXQmb77YYrEkeVC5uwPg_TbAYI,2120
|
|
29
|
+
cosmotech/coal/postgresql/__init__.py,sha256=766LBZyWCQUI6rlzXDY_C9Yrxg551VX2L4lOuhDiS8w,786
|
|
30
|
+
cosmotech/coal/postgresql/runner.py,sha256=DqXM_GBiVQQ_QcNitEC4t9jthoNxUKsPvuaAWQ3BCQ8,4091
|
|
31
|
+
cosmotech/coal/postgresql/store.py,sha256=dyKwqngpSm40fZ5Fo4vHyU3gnRdC_xdk8USc_Qs06xM,5176
|
|
32
|
+
cosmotech/coal/postgresql/utils.py,sha256=zM69jq2XLiS8KgX6Zgo3W3GOZ8kb8goq-HMIRyx0EBU,9663
|
|
33
|
+
cosmotech/coal/singlestore/__init__.py,sha256=gXQaa8OAHnDcI7iHy-5zzGG7ZqNq-uOtw60qASjwu9I,651
|
|
34
|
+
cosmotech/coal/singlestore/store.py,sha256=C6EbXRAxsZmW1It3FTb9NEOK4H_oMBfNOTvhwPjrE8o,3578
|
|
35
|
+
cosmotech/coal/store/__init__.py,sha256=3UicSottHGQ3yjWv_ib8GBRUYcZ9-cwSq_wvXq7TpxQ,1395
|
|
36
|
+
cosmotech/coal/store/csv.py,sha256=2toydK0aSs1WLhO2_mSKpCJsaMGs7aBlL7RLASNalto,1494
|
|
37
|
+
cosmotech/coal/store/native_python.py,sha256=Mp2U7tkUjH1wT76U4WzekaW4H5nN2VHTrAKnJ4NzB8E,847
|
|
38
|
+
cosmotech/coal/store/pandas.py,sha256=ZvyK1APmcGscdt48401eBfm3zgvX3FKFfARsbV7kWcI,906
|
|
39
|
+
cosmotech/coal/store/pyarrow.py,sha256=tchLfthMRYiEXFrkjLDNXdDOOJYO7l3hSBH35MaNVg0,814
|
|
40
|
+
cosmotech/coal/store/store.py,sha256=1XIIRQL4Af-R1UdJlo-AzmDxtKoIYOtm7KvPdhTWJ-o,3390
|
|
41
|
+
cosmotech/coal/store/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
42
|
+
cosmotech/coal/store/output/aws_channel.py,sha256=0YZ0OjAEnDGOu48ZAFZP-y2IU_quqY0zSxhFwp509MM,2881
|
|
43
|
+
cosmotech/coal/store/output/az_storage_channel.py,sha256=WYMJD1dovg1SLXJFdaJnBIG86cHaFc9Y4avAX1143hk,842
|
|
44
|
+
cosmotech/coal/store/output/channel_interface.py,sha256=C_rbZnGOf-5YO8rnxTRe_Pd0EZ_2FNwLr3qSiKyUpWw,1241
|
|
45
|
+
cosmotech/coal/store/output/channel_spliter.py,sha256=f7ljfkqcuURLSeok7jr2nW5sqMSnMeuLonQEJTQlrX8,2441
|
|
46
|
+
cosmotech/coal/store/output/postgres_channel.py,sha256=HOqGSlAbRFN6yIneKjyxqAcTri83HmJkZVu74L3l99Y,1184
|
|
47
|
+
cosmotech/coal/utils/__init__.py,sha256=N2mPhli0Wj4OesuRVbt2KrzL_pgMELs4fDNIQYb_r-g,830
|
|
48
|
+
cosmotech/coal/utils/configuration.py,sha256=dFfjJHW1WPLqhg6kBsYOAwvk3h04i-YL2oAuzTAT-Xk,7460
|
|
49
|
+
cosmotech/coal/utils/decorator.py,sha256=p4yqkbln-YTppzeKvi3K4ZZ6AibVYGD6pnJQk0ynWbE,646
|
|
50
|
+
cosmotech/coal/utils/logger.py,sha256=oYVj2BtDPji4e4J-KgaDyVcbKiewkq0xey_gQPm74Xc,506
|
|
51
|
+
cosmotech/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
52
|
+
cosmotech/csm_data/main.py,sha256=Io-qKkUt7BT9ovA-_rQCATnZHWhgpGLP7n0en7L1SiU,2424
|
|
53
|
+
cosmotech/csm_data/commands/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
54
|
+
cosmotech/csm_data/commands/adx_send_data.py,sha256=FH8-_DRtM1qccHRCIRFqaZq2YiHz5pWLJUOTfPDSYOI,2638
|
|
55
|
+
cosmotech/csm_data/commands/adx_send_runnerdata.py,sha256=Pg40TQw38W40qcqvqf9B7y9w2uFj8KtH0_9_n0onvG8,3714
|
|
56
|
+
cosmotech/csm_data/commands/az_storage_upload.py,sha256=EvPOLvVL4vteaPbBHbC0gR2drnYbSppP6RlaklmZHXw,2345
|
|
57
|
+
cosmotech/csm_data/commands/s3_bucket_delete.py,sha256=2Sx3sLkiQxTd9MNL85zUZCvNTi6bZKxusQmu8xG0EYY,3056
|
|
58
|
+
cosmotech/csm_data/commands/s3_bucket_download.py,sha256=X0IahwMTzPUelTLZvNKIKkV2NMK6W8zw_Qp2hjFSsaA,3367
|
|
59
|
+
cosmotech/csm_data/commands/s3_bucket_upload.py,sha256=MKRFoYS9-MMCdk_jVRDF-Kx3YitjlmqUxX6-kd2CyxQ,3599
|
|
60
|
+
cosmotech/csm_data/commands/api/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
61
|
+
cosmotech/csm_data/commands/api/api.py,sha256=v8qBRjI1he8o6hDXmjmptTUdO8F92sgwr9glszsy49o,1698
|
|
62
|
+
cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py,sha256=h_24m7sBrA4Dej7UyIsAzlpJi9mGIM5MBiUHxN8PT08,4235
|
|
63
|
+
cosmotech/csm_data/commands/api/run_load_data.py,sha256=IXCCXb3wKPZTeQybteMqXWreQHggQ3GALfNIw2bphys,2831
|
|
64
|
+
cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=wnNvRY1D7GBE5lQz5KOQ9x5T_wxs-4wIfGPO4jiMcwQ,2103
|
|
65
|
+
cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=AC-Li-2tN5KAtIE5TnElRbSgUTNCa7uWkn90MlOozjg,2045
|
|
66
|
+
cosmotech/csm_data/commands/store/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
67
|
+
cosmotech/csm_data/commands/store/dump_to_azure.py,sha256=9vavbHHueqdGbSjbt2GLnM2adzm7bDpmt_zUeX9ArZU,3314
|
|
68
|
+
cosmotech/csm_data/commands/store/dump_to_postgresql.py,sha256=7SB9e9TT8Kn3xeK0P4COTww-dsTJdG0vwjHAKkPob18,3597
|
|
69
|
+
cosmotech/csm_data/commands/store/dump_to_s3.py,sha256=6EUwtkP8_k9Hlq65R9h-MYsH58e-UAYmSR95JKoZuvY,5342
|
|
70
|
+
cosmotech/csm_data/commands/store/list_tables.py,sha256=dkrVANYhSD2FD1iny60ZXkaKZP5dPuK_eJ3R0gvlJtE,1861
|
|
71
|
+
cosmotech/csm_data/commands/store/load_csv_folder.py,sha256=aiMWmrC87kqTRJK06g16KlA90eXjPaFGswr2yZXJ2nM,1841
|
|
72
|
+
cosmotech/csm_data/commands/store/load_from_singlestore.py,sha256=cMPHnIfcjl4L7whwhs1sOkHf9eJbsj5s9ltzXO34KcM,3105
|
|
73
|
+
cosmotech/csm_data/commands/store/output.py,sha256=RkBQxZ6c-p6F3A0Ci7wW-ODHsrJJO8ivzQCumvjoqXI,1218
|
|
74
|
+
cosmotech/csm_data/commands/store/reset.py,sha256=At33wH3r_bITlUphp84NkBQNfz_8kW5FRlQDELopsUA,1361
|
|
75
|
+
cosmotech/csm_data/commands/store/store.py,sha256=oQhTIF1WKKGKdFp1rNm1vuTgblmh27r_jQpPhNFmesE,1663
|
|
76
|
+
cosmotech/csm_data/utils/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
77
|
+
cosmotech/csm_data/utils/click.py,sha256=S_85cbKh3R86-FZVjTK7IXZnmp4ETjKo6K8gbK3HCgs,848
|
|
78
|
+
cosmotech/csm_data/utils/decorators.py,sha256=HlteYuqP8WWIIErKRhvT1wYsUT5ZCZAQNTAXs70Tc8g,2500
|
|
79
|
+
cosmotech/orchestrator_plugins/csm-data/__init__.py,sha256=UPzNgaNBMoBaxSlnofxl988dAswMp_tv8DzaR_IxkiE,519
|
|
80
|
+
cosmotech/orchestrator_plugins/csm-data/templates/api/postgres_send_runner_metadata.json,sha256=alXzvOa985PvN5nf3uoTsFXgY29EJWeK94CcBjoLFUk,1085
|
|
81
|
+
cosmotech/orchestrator_plugins/csm-data/templates/api/run_load_data.json,sha256=_ztHPGu5UbbdtQHUCaOuKqwtjB0DrZSZwcUePoOgS40,894
|
|
82
|
+
cosmotech/orchestrator_plugins/csm-data/templates/api/try_api_connection.json,sha256=ApkzR0HQc0J6sxxTh91_DHPRI2cCtx_-qjpYFKGkGVw,215
|
|
83
|
+
cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_load_file.json,sha256=yX7yUrq7epQTp0PaQElFC-elof8W6jaMtek6uQ0V6Og,1142
|
|
84
|
+
cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_send_file.json,sha256=7pyYOUPg_lvD4kw3sqd_zBonru7g-DVatQmu4EJw0Ps,1120
|
|
85
|
+
cosmotech/orchestrator_plugins/csm-data/templates/main/adx_send_runnerdata.json,sha256=thIyXyOqtnJFh-JWHijPytp_X5CG4194PS3Q_WgMbd4,1009
|
|
86
|
+
cosmotech/orchestrator_plugins/csm-data/templates/main/az_storage_upload.json,sha256=WS0MuXq5AbxkpOPUShmnBS1rHA3clfjXpTsdhooWMfk,804
|
|
87
|
+
cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_delete.json,sha256=iVgLJVDLiI44fLL6ejId-Du3T38bEPnS_j0ZWDSp1OY,1017
|
|
88
|
+
cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_download.json,sha256=aaFO8tWcN_x4ZMugC0peUxSEbarGOHnODh4vbbCXgCA,1131
|
|
89
|
+
cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_upload.json,sha256=pndJLHV-_QWAu0WEvsAs5TJf7G9f_oC-iOLNQsSjYEY,1129
|
|
90
|
+
cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_azure.json,sha256=iO3rSR80YiOb7xH1yl36jGzF65-qOp4NB9hdcYxKsX4,907
|
|
91
|
+
cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_postgresql.json,sha256=g0uG8FfxtLfjadolncbpUw-ZdQVY7uavpEj_DkPzqm8,898
|
|
92
|
+
cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_s3.json,sha256=4r9i4HEGgcAO0r8rlKJq7x6hTtt7dSO2cFeW74-BQbk,1050
|
|
93
|
+
cosmotech/orchestrator_plugins/csm-data/templates/store/store_list_tables.json,sha256=1xbjPqR6B-5o6lEQ6MXSPxuKy-px-85qhnRRYOcUXcQ,349
|
|
94
|
+
cosmotech/orchestrator_plugins/csm-data/templates/store/store_load_csv_folder.json,sha256=Vwgho85XordZV_RhAAhclJDgD727f7t9yQs0gn-MDPs,495
|
|
95
|
+
cosmotech/orchestrator_plugins/csm-data/templates/store/store_load_from_singlestore.json,sha256=AOWnKQ1C6p8hf6HNQ712N59h5xZCVaaqEL2QdhV3vj4,937
|
|
96
|
+
cosmotech/orchestrator_plugins/csm-data/templates/store/store_reset.json,sha256=jWKRDVW9HrUXHPG-MgVU_PljLyJkeYf6Tzi9wmL8yGc,337
|
|
97
|
+
cosmotech/translation/coal/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
98
|
+
cosmotech/translation/coal/en-US/coal/solution.yml,sha256=gHf26F5XX3BEiHvvV0bUFjje9dwlTZsl9KWm1M0brkU,141
|
|
99
|
+
cosmotech/translation/coal/en-US/coal/web.yml,sha256=I4Hu_zBm55cUsJVSSGQ0sk5KfX6GZPMmaXsLd2rG9u0,78
|
|
100
|
+
cosmotech/translation/coal/en-US/coal/common/data_transfer.yml,sha256=dSoheA2bEpaO5nRXhlg8XSQOJsfa9A7AGHCg6nQ82cM,299
|
|
101
|
+
cosmotech/translation/coal/en-US/coal/common/errors.yml,sha256=l1fiCZdTjff4R55PFLhj3hHx61u52DxHaKzJzc5ddA4,535
|
|
102
|
+
cosmotech/translation/coal/en-US/coal/common/file_operations.yml,sha256=DxggJM431fnFveUemHRM0Y18GPn_Vgv-LRSZXVRNCaI,278
|
|
103
|
+
cosmotech/translation/coal/en-US/coal/common/progress.yml,sha256=G_BTYgAYVD_JqU3rJ5fVTySQa0HtUcLNFoHRjSnBCwk,272
|
|
104
|
+
cosmotech/translation/coal/en-US/coal/common/timing.yml,sha256=YtxIT8EKZDWD4hJ0tJOwkReUsrOY67Wq0MFkEZjzANE,274
|
|
105
|
+
cosmotech/translation/coal/en-US/coal/common/validation.yml,sha256=OAHeXrvMo2_S-D0z-ESIS--U7bU0z1i0wta4ibaqk3c,528
|
|
106
|
+
cosmotech/translation/coal/en-US/coal/cosmotech_api/connection.yml,sha256=2Om0DZmOI8u0hzlWWhWnGx-o0mzMk_vgE3hGRBNaWWU,518
|
|
107
|
+
cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml,sha256=NGD5bbbS037QosTmHxXOiUJ_Zl3gyIr1iR6hCaSlolY,380
|
|
108
|
+
cosmotech/translation/coal/en-US/coal/cosmotech_api/run_template.yml,sha256=JIQ0S-9_YhchAPVFkcMpJTVUBlhPo4yS-nBoMysus5o,441
|
|
109
|
+
cosmotech/translation/coal/en-US/coal/cosmotech_api/runner.yml,sha256=fSoFiJFJcowXwBVgfAkZKL8j3iKzyVDLWm48cFpiYMA,804
|
|
110
|
+
cosmotech/translation/coal/en-US/coal/cosmotech_api/solution.yml,sha256=dKOTcv8bBYl3ZxyvKsQ8msO--PyLTfEOodb4rHl2bq4,216
|
|
111
|
+
cosmotech/translation/coal/en-US/coal/cosmotech_api/workspace.yml,sha256=4RHIHZi_ziSYB_U-YoLxNTm3jD67yN1Cxr14W9XoTok,307
|
|
112
|
+
cosmotech/translation/coal/en-US/coal/services/adx.yml,sha256=9yxR0qYKyA5mqgW-pX2J65JkkMcGFC3cPJChGmikJso,3502
|
|
113
|
+
cosmotech/translation/coal/en-US/coal/services/api.yml,sha256=HPyKJaZrS41Qo9upRvPsLdn29QgS1xw3S-bgipHeO4s,342
|
|
114
|
+
cosmotech/translation/coal/en-US/coal/services/azure_storage.yml,sha256=p0alxlJA_-NH9MxKXRKGgiLbBvHuQdCROhJKiqvuX0c,595
|
|
115
|
+
cosmotech/translation/coal/en-US/coal/services/database.yml,sha256=S1pAu4OrVd9nAVb3oYSAurzA4cgD43UDlwK4qWhy2g4,860
|
|
116
|
+
cosmotech/translation/coal/en-US/coal/services/dataset.yml,sha256=W-MHnp9E3udMGXdPriVZF7xgPSh1ksA3tFc7M7xMCkE,3265
|
|
117
|
+
cosmotech/translation/coal/en-US/coal/services/postgresql.yml,sha256=tRGNTTayyoxEHVyM-nzudEFdQ5HUeRIHPWIuvFmJGfM,1774
|
|
118
|
+
cosmotech/translation/coal/en-US/coal/services/s3.yml,sha256=362wt7SdOu-MyMj9MgjQNTNWzIq-RJ9rUqc2VKljGFE,482
|
|
119
|
+
cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml,sha256=4pVJPAXYeIq9Gpmg_0VpSrB17AsGWA9hvk8Qi8-kZvw,85
|
|
120
|
+
cosmotech/translation/coal/en-US/coal/store/output/split.yml,sha256=h89v89sQsK5pGDJP5EE_KNbQbaS8003VPZNIgU2dqWw,265
|
|
121
|
+
cosmotech/translation/coal/en-US/coal/utils/configuration.yml,sha256=q3kEzPTP81SfsdrhAZa2JusHAjbdYqjs0DcvjhewQ-4,96
|
|
122
|
+
cosmotech/translation/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
123
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/main.yml,sha256=3XbXR5_Uo7PxJOWzm7XRMvPVxK-fzOL4FxWWY7rX3eg,396
|
|
124
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/api/api.yml,sha256=eMT4s1H9LuNUh_sAlnEtg7Co4ie3bFfB1vkwnLW28dQ,340
|
|
125
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml,sha256=avckcrp2aYz7rzGCKEdL81hwHDVbcUn2csQ_krKLcaM,844
|
|
126
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/api/run_load_data.yml,sha256=k5J3HTA8vCmI_7CFHvMP51EAbmefubuoEhXZf2V3XDo,765
|
|
127
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/api/runtemplate_load_handler.yml,sha256=9jvMa6fb99CPzTMx7nnuoNNzJw7_7rapee7P80Plxjs,353
|
|
128
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_load_file.yml,sha256=cnR_0PMkbrj8GROgvIUyeLXiB90IBbkcTvM9WJDEM0k,511
|
|
129
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_send_file.yml,sha256=yvsD364GND1jLe_SJxuNXjDgGZFGlJlC5xUTRqV0fr0,632
|
|
130
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/storage/adx_send_data.yml,sha256=oNQMHhHMRTl7EfFLuEy5agctzgwGHsyKzAu6hlSUyU4,448
|
|
131
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/storage/adx_send_runnerdata.yml,sha256=_IAZO1VVc2eVznA2VLb0_o3sEc8cSO-xw3H90pUTVGg,884
|
|
132
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/storage/az_storage_upload.yml,sha256=KmVGI1RwFfFQnMUPAlRTJ6K3OXAYTy7GAitZfcWKdrw,473
|
|
133
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/storage/s3_bucket_delete.yml,sha256=GUCNXpiRui-BRwP8A7M4dXGr3LMBJw99z-b1ToFuESY,792
|
|
134
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/storage/s3_bucket_download.yml,sha256=GQPNFcdB0hIldX0kDBFNzHs11RckPBAA697BAJugZx8,872
|
|
135
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/storage/s3_bucket_upload.yml,sha256=jIrdisG9FcvTrCM5jvTaCYccKKXtah7huSvXDK8bdEA,1100
|
|
136
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/storage/storage.yml,sha256=NZrRzfnOpXrX5wBpEqr40CCmNGP5MXJ-aO7ks1YM58s,144
|
|
137
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_azure.yml,sha256=JqT9Cu1m4CLmZWvXTHAIS3xS8rK_BeKL041OEliBbn8,1299
|
|
138
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_postgresql.yml,sha256=QEQVHuXrSMsT2uOkiF-oMYlir1mINXXL-Irro_Ra3RM,900
|
|
139
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_s3.yml,sha256=7pbLObdjZjcKeSfYI_3oAzp9AZOrwT5WiMcLOTj5rCc,1268
|
|
140
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/list_tables.yml,sha256=-x8zwCjAHILFaJv7VLPskipSoQTwmI7jUsxlx6_OKlc,195
|
|
141
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_csv_folder.yml,sha256=9HjW9tOq5o31jmADp_QIync7G74c6XGiPqofXh8pvv8,228
|
|
142
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_from_singlestore.yml,sha256=_OUXlxC_wi-GPgHU7EVdU9g0WIdblVpEA4WIzw4aQcQ,824
|
|
143
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml,sha256=odkR9pzeel5l0wW_F4kx4HhNpw_TzEvok-OKyh25ZkQ,232
|
|
144
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/reset.yml,sha256=JMspZ25Q2HDpH8MYqUaQcveUnCgdH21RmfTCrPf2gW8,139
|
|
145
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/store.yml,sha256=N1Q8483gqJADaCe30S1M3Rj0tMJiuQiJH70-VK2x2m4,134
|
|
146
|
+
cosmotech/translation/csm_data/en-US/csm_data/commons/decorators.yml,sha256=Iu59NWMfYlZZf9uUhOiLkIEGa4GY5p0nZ6vG06Xvu7k,51
|
|
147
|
+
cosmotech/translation/csm_data/en-US/csm_data/commons/version.yml,sha256=7jtCV3O1S6pGjiJa63XpgPDTafjfBS0xmEVRpYNvfDg,86
|
|
148
|
+
cosmotech_acceleration_library-2.1.0rc1.dist-info/licenses/LICENSE,sha256=JXKHOQtyObmafNbQlfPYc4HkKjU9FzAP27b2qRTXNM8,1195
|
|
149
|
+
cosmotech_acceleration_library-2.1.0rc1.dist-info/METADATA,sha256=OLnJ8oFZytFr3FZL0R-gPy2lUwoVA4KosLZt5U3Y77c,9217
|
|
150
|
+
cosmotech_acceleration_library-2.1.0rc1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
151
|
+
cosmotech_acceleration_library-2.1.0rc1.dist-info/entry_points.txt,sha256=HWRqJurKuBUgqFe4jmjIAQrs768Nbb8ZTdRDLbuKM5Q,58
|
|
152
|
+
cosmotech_acceleration_library-2.1.0rc1.dist-info/top_level.txt,sha256=t2pzb8mpMUfHTa9l2SjWP0rRB8XVRjBdQK5nLx9XDDo,10
|
|
153
|
+
cosmotech_acceleration_library-2.1.0rc1.dist-info/RECORD,,
|
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
import azure.functions as func
|
|
8
|
-
from cosmotech.coal.cosmotech_api.runner.download import download_runner_data
|
|
9
|
-
from cosmotech_api.api.runner_api import RunnerApi
|
|
10
|
-
|
|
11
|
-
import json
|
|
12
|
-
import http
|
|
13
|
-
import traceback
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def generate_main(apply_update, parallel=True):
|
|
17
|
-
def main(req: func.HttpRequest) -> func.HttpResponse:
|
|
18
|
-
try:
|
|
19
|
-
runner_id = req.params.get("scenario-id") # Keep parameter name for backward compatibility
|
|
20
|
-
organization_id = req.params.get("organization-id")
|
|
21
|
-
workspace_id = req.params.get("workspace-id")
|
|
22
|
-
access_token: str = req.headers.get("authorization", None)
|
|
23
|
-
if access_token:
|
|
24
|
-
access_token = access_token.split(" ")[1]
|
|
25
|
-
|
|
26
|
-
if runner_id is None or organization_id is None or workspace_id is None:
|
|
27
|
-
return func.HttpResponse(
|
|
28
|
-
body=f"Invalid request: organization-id={organization_id}, workspace-id={workspace_id}, scenario-id={runner_id}",
|
|
29
|
-
status_code=http.HTTPStatus.BAD_REQUEST,
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
# Get runner data
|
|
33
|
-
result = download_runner_data(
|
|
34
|
-
organization_id=organization_id,
|
|
35
|
-
workspace_id=workspace_id,
|
|
36
|
-
runner_id=runner_id,
|
|
37
|
-
parameter_folder=None, # We don't need to save to files
|
|
38
|
-
read_files=True,
|
|
39
|
-
parallel=parallel,
|
|
40
|
-
write_json=False,
|
|
41
|
-
write_csv=False,
|
|
42
|
-
fetch_dataset=True,
|
|
43
|
-
)
|
|
44
|
-
|
|
45
|
-
content = {
|
|
46
|
-
"datasets": result["datasets"],
|
|
47
|
-
"parameters": result["parameters"],
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
runner_data = result["runner_data"]
|
|
51
|
-
|
|
52
|
-
updated_content = apply_update(
|
|
53
|
-
content=content, scenario_data=runner_data
|
|
54
|
-
) # Keep parameter name for backward compatibility
|
|
55
|
-
|
|
56
|
-
return func.HttpResponse(
|
|
57
|
-
body=json.dumps(updated_content),
|
|
58
|
-
headers={"Content-Type": "application/json"},
|
|
59
|
-
)
|
|
60
|
-
except Exception as e:
|
|
61
|
-
response = {
|
|
62
|
-
"error": getattr(e, "message", str(e)),
|
|
63
|
-
"type": type(e).__name__,
|
|
64
|
-
"trace": traceback.format_exc(),
|
|
65
|
-
}
|
|
66
|
-
return func.HttpResponse(
|
|
67
|
-
status_code=http.HTTPStatus.INTERNAL_SERVER_ERROR,
|
|
68
|
-
body=json.dumps(response),
|
|
69
|
-
headers={"Content-Type": "application/json"},
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
return main
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
import os
|
|
8
|
-
import pathlib
|
|
9
|
-
import ssl
|
|
10
|
-
|
|
11
|
-
import cosmotech_api
|
|
12
|
-
|
|
13
|
-
from cosmotech.coal.utils.logger import LOGGER
|
|
14
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
15
|
-
|
|
16
|
-
api_env_keys = {"CSM_API_KEY", "CSM_API_URL"}
|
|
17
|
-
azure_env_keys = {
|
|
18
|
-
"AZURE_CLIENT_ID",
|
|
19
|
-
"AZURE_CLIENT_SECRET",
|
|
20
|
-
"AZURE_TENANT_ID",
|
|
21
|
-
"CSM_API_URL",
|
|
22
|
-
"CSM_API_SCOPE",
|
|
23
|
-
}
|
|
24
|
-
keycloak_env_keys = {
|
|
25
|
-
"IDP_TENANT_ID",
|
|
26
|
-
"IDP_CLIENT_ID",
|
|
27
|
-
"IDP_CLIENT_SECRET",
|
|
28
|
-
"IDP_BASE_URL",
|
|
29
|
-
"CSM_API_URL",
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def get_api_client() -> (cosmotech_api.ApiClient, str):
|
|
34
|
-
existing_keys = set(os.environ.keys())
|
|
35
|
-
missing_azure_keys = azure_env_keys - existing_keys
|
|
36
|
-
missing_api_keys = api_env_keys - existing_keys
|
|
37
|
-
missing_keycloak_keys = keycloak_env_keys - existing_keys
|
|
38
|
-
if all((missing_api_keys, missing_azure_keys, missing_keycloak_keys)):
|
|
39
|
-
LOGGER.error(T("coal.common.errors.no_env_vars"))
|
|
40
|
-
LOGGER.error(T("coal.cosmotech_api.connection.existing_sets"))
|
|
41
|
-
LOGGER.error(T("coal.cosmotech_api.connection.azure_connection").format(keys=", ".join(azure_env_keys)))
|
|
42
|
-
LOGGER.error(T("coal.cosmotech_api.connection.api_key_connection").format(keys=", ".join(api_env_keys)))
|
|
43
|
-
LOGGER.error(T("coal.cosmotech_api.connection.keycloak_connection").format(keys=", ".join(keycloak_env_keys)))
|
|
44
|
-
raise EnvironmentError(T("coal.common.errors.no_env_vars"))
|
|
45
|
-
|
|
46
|
-
if not missing_keycloak_keys:
|
|
47
|
-
LOGGER.info(T("coal.cosmotech_api.connection.found_keycloak"))
|
|
48
|
-
from keycloak import KeycloakOpenID
|
|
49
|
-
|
|
50
|
-
server_url = os.environ.get("IDP_BASE_URL")
|
|
51
|
-
if server_url[-1] != "/":
|
|
52
|
-
server_url = server_url + "/"
|
|
53
|
-
keycloack_parameters = dict(
|
|
54
|
-
server_url=server_url,
|
|
55
|
-
client_id=os.environ.get("IDP_CLIENT_ID"),
|
|
56
|
-
realm_name=os.environ.get("IDP_TENANT_ID"),
|
|
57
|
-
client_secret_key=os.environ.get("IDP_CLIENT_SECRET"),
|
|
58
|
-
)
|
|
59
|
-
if (ca_cert_path := os.environ.get("IDP_CA_CERT")) and pathlib.Path(ca_cert_path).exists():
|
|
60
|
-
LOGGER.info(T("coal.cosmotech_api.connection.found_cert_authority"))
|
|
61
|
-
keycloack_parameters["verify"] = ca_cert_path
|
|
62
|
-
keycloak_openid = KeycloakOpenID(**keycloack_parameters)
|
|
63
|
-
|
|
64
|
-
access_token = keycloak_openid.token(grant_type="client_credentials")
|
|
65
|
-
|
|
66
|
-
configuration = cosmotech_api.Configuration(
|
|
67
|
-
host=os.environ.get("CSM_API_URL"),
|
|
68
|
-
access_token=access_token["access_token"],
|
|
69
|
-
)
|
|
70
|
-
return cosmotech_api.ApiClient(configuration), "Keycloak Connection"
|
|
71
|
-
|
|
72
|
-
if not missing_api_keys:
|
|
73
|
-
LOGGER.info(T("coal.cosmotech_api.connection.found_api_key"))
|
|
74
|
-
configuration = cosmotech_api.Configuration(
|
|
75
|
-
host=os.environ.get("CSM_API_URL"),
|
|
76
|
-
)
|
|
77
|
-
return (
|
|
78
|
-
cosmotech_api.ApiClient(
|
|
79
|
-
configuration,
|
|
80
|
-
os.environ.get("CSM_API_KEY_HEADER", "X-CSM-API-KEY"),
|
|
81
|
-
os.environ.get("CSM_API_KEY"),
|
|
82
|
-
),
|
|
83
|
-
"Cosmo Tech API Key",
|
|
84
|
-
)
|
|
85
|
-
|
|
86
|
-
if not missing_azure_keys:
|
|
87
|
-
LOGGER.info(T("coal.cosmotech_api.connection.found_azure"))
|
|
88
|
-
from azure.identity import EnvironmentCredential
|
|
89
|
-
|
|
90
|
-
credentials = EnvironmentCredential()
|
|
91
|
-
token = credentials.get_token(os.environ.get("CSM_API_SCOPE"))
|
|
92
|
-
|
|
93
|
-
configuration = cosmotech_api.Configuration(host=os.environ.get("CSM_API_URL"), access_token=token.token)
|
|
94
|
-
return cosmotech_api.ApiClient(configuration), "Azure Entra Connection"
|
|
95
|
-
|
|
96
|
-
raise EnvironmentError(T("coal.common.errors.no_valid_connection"))
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
# Re-export all download functions from download submodule
|
|
9
|
-
from cosmotech.coal.cosmotech_api.dataset.download import (
|
|
10
|
-
download_adt_dataset,
|
|
11
|
-
download_twingraph_dataset,
|
|
12
|
-
download_legacy_twingraph_dataset,
|
|
13
|
-
download_file_dataset,
|
|
14
|
-
download_dataset_by_id,
|
|
15
|
-
)
|
|
16
|
-
|
|
17
|
-
from cosmotech.coal.cosmotech_api.dataset.converters import (
|
|
18
|
-
convert_dataset_to_files,
|
|
19
|
-
convert_graph_dataset_to_files,
|
|
20
|
-
convert_file_dataset_to_files,
|
|
21
|
-
)
|
|
22
|
-
|
|
23
|
-
from cosmotech.coal.cosmotech_api.dataset.utils import (
|
|
24
|
-
get_content_from_twin_graph_data,
|
|
25
|
-
sheet_to_header,
|
|
26
|
-
)
|
|
@@ -1,164 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
import csv
|
|
9
|
-
import json
|
|
10
|
-
import os
|
|
11
|
-
import tempfile
|
|
12
|
-
from pathlib import Path
|
|
13
|
-
from typing import Dict, List, Any, Optional, Union
|
|
14
|
-
|
|
15
|
-
from cosmotech.coal.utils.logger import LOGGER
|
|
16
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
17
|
-
from cosmotech.coal.cosmotech_api.dataset.utils import sheet_to_header
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def convert_dataset_to_files(dataset_info: Dict[str, Any], target_folder: Optional[Union[str, Path]] = None) -> Path:
|
|
21
|
-
"""
|
|
22
|
-
Convert dataset info to files.
|
|
23
|
-
|
|
24
|
-
Args:
|
|
25
|
-
dataset_info: Dataset info dict with type, content, name
|
|
26
|
-
target_folder: Optional folder to save files (if None, uses temp dir)
|
|
27
|
-
|
|
28
|
-
Returns:
|
|
29
|
-
Path to folder containing files
|
|
30
|
-
"""
|
|
31
|
-
dataset_type = dataset_info["type"]
|
|
32
|
-
content = dataset_info["content"]
|
|
33
|
-
name = dataset_info["name"]
|
|
34
|
-
|
|
35
|
-
LOGGER.info(T("coal.services.dataset.converting_to_files").format(dataset_type=dataset_type, dataset_name=name))
|
|
36
|
-
|
|
37
|
-
if target_folder is None:
|
|
38
|
-
target_folder = Path(tempfile.mkdtemp())
|
|
39
|
-
LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
|
|
40
|
-
else:
|
|
41
|
-
target_folder = Path(target_folder)
|
|
42
|
-
target_folder.mkdir(parents=True, exist_ok=True)
|
|
43
|
-
LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
|
|
44
|
-
|
|
45
|
-
if dataset_type in ["adt", "twincache"]:
|
|
46
|
-
return convert_graph_dataset_to_files(content, target_folder)
|
|
47
|
-
else:
|
|
48
|
-
return convert_file_dataset_to_files(content, target_folder, dataset_type)
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def convert_graph_dataset_to_files(
|
|
52
|
-
content: Dict[str, List[Dict]], target_folder: Optional[Union[str, Path]] = None
|
|
53
|
-
) -> Path:
|
|
54
|
-
"""
|
|
55
|
-
Convert graph dataset content to CSV files.
|
|
56
|
-
|
|
57
|
-
Args:
|
|
58
|
-
content: Dictionary mapping entity types to lists of entities
|
|
59
|
-
target_folder: Folder to save files (if None, uses temp dir)
|
|
60
|
-
|
|
61
|
-
Returns:
|
|
62
|
-
Path to folder containing files
|
|
63
|
-
"""
|
|
64
|
-
if target_folder is None:
|
|
65
|
-
target_folder = Path(tempfile.mkdtemp())
|
|
66
|
-
LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
|
|
67
|
-
else:
|
|
68
|
-
target_folder = Path(target_folder)
|
|
69
|
-
target_folder.mkdir(parents=True, exist_ok=True)
|
|
70
|
-
LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
|
|
71
|
-
file_count = 0
|
|
72
|
-
|
|
73
|
-
LOGGER.info(
|
|
74
|
-
T("coal.services.dataset.converting_graph_data").format(entity_types=len(content), folder=target_folder)
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
-
for entity_type, entities in content.items():
|
|
78
|
-
if not entities:
|
|
79
|
-
LOGGER.debug(T("coal.services.dataset.skipping_empty_entity").format(entity_type=entity_type))
|
|
80
|
-
continue
|
|
81
|
-
|
|
82
|
-
file_path = target_folder / f"{entity_type}.csv"
|
|
83
|
-
LOGGER.debug(T("coal.services.dataset.writing_csv").format(file_name=file_path.name, count=len(entities)))
|
|
84
|
-
|
|
85
|
-
fieldnames = sheet_to_header(entities)
|
|
86
|
-
|
|
87
|
-
with open(file_path, "w", newline="") as file:
|
|
88
|
-
writer = csv.DictWriter(file, fieldnames=fieldnames, dialect="unix", quoting=csv.QUOTE_MINIMAL)
|
|
89
|
-
writer.writeheader()
|
|
90
|
-
|
|
91
|
-
for entity in entities:
|
|
92
|
-
# Convert values to strings and handle boolean values
|
|
93
|
-
row = {
|
|
94
|
-
k: str(v).replace("'", '"').replace("True", "true").replace("False", "false")
|
|
95
|
-
for k, v in entity.items()
|
|
96
|
-
}
|
|
97
|
-
writer.writerow(row)
|
|
98
|
-
|
|
99
|
-
file_count += 1
|
|
100
|
-
LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
|
|
101
|
-
|
|
102
|
-
LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
|
|
103
|
-
|
|
104
|
-
return target_folder
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def convert_file_dataset_to_files(
|
|
108
|
-
content: Dict[str, Any],
|
|
109
|
-
target_folder: Optional[Union[str, Path]] = None,
|
|
110
|
-
file_type: str = "",
|
|
111
|
-
) -> Path:
|
|
112
|
-
"""
|
|
113
|
-
Convert file dataset content to files.
|
|
114
|
-
|
|
115
|
-
Args:
|
|
116
|
-
content: Dictionary mapping file names to content
|
|
117
|
-
target_folder: Folder to save files (if None, uses temp dir)
|
|
118
|
-
file_type: Type of file (csv, json, etc.)
|
|
119
|
-
|
|
120
|
-
Returns:
|
|
121
|
-
Path to folder containing files
|
|
122
|
-
"""
|
|
123
|
-
if target_folder is None:
|
|
124
|
-
target_folder = Path(tempfile.mkdtemp())
|
|
125
|
-
LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
|
|
126
|
-
else:
|
|
127
|
-
target_folder = Path(target_folder)
|
|
128
|
-
target_folder.mkdir(parents=True, exist_ok=True)
|
|
129
|
-
LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
|
|
130
|
-
file_count = 0
|
|
131
|
-
|
|
132
|
-
LOGGER.info(
|
|
133
|
-
T("coal.services.dataset.converting_file_data").format(
|
|
134
|
-
file_count=len(content), file_type=file_type, folder=target_folder
|
|
135
|
-
)
|
|
136
|
-
)
|
|
137
|
-
|
|
138
|
-
for file_name, file_content in content.items():
|
|
139
|
-
file_path = target_folder / file_name
|
|
140
|
-
|
|
141
|
-
# Ensure parent directories exist
|
|
142
|
-
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
143
|
-
|
|
144
|
-
LOGGER.debug(T("coal.services.dataset.writing_file").format(file_name=file_path.name, file_type=file_type))
|
|
145
|
-
|
|
146
|
-
if isinstance(file_content, str):
|
|
147
|
-
# Text content
|
|
148
|
-
with open(file_path, "w") as file:
|
|
149
|
-
file.write(file_content)
|
|
150
|
-
elif isinstance(file_content, dict) or isinstance(file_content, list):
|
|
151
|
-
# JSON content
|
|
152
|
-
with open(file_path, "w") as file:
|
|
153
|
-
json.dump(file_content, file, indent=2)
|
|
154
|
-
else:
|
|
155
|
-
# Other content types
|
|
156
|
-
with open(file_path, "w") as file:
|
|
157
|
-
file.write(str(file_content))
|
|
158
|
-
|
|
159
|
-
file_count += 1
|
|
160
|
-
LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
|
|
161
|
-
|
|
162
|
-
LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
|
|
163
|
-
|
|
164
|
-
return target_folder
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
"""
|
|
9
|
-
Dataset download submodules.
|
|
10
|
-
"""
|
|
11
|
-
|
|
12
|
-
# Re-export all download functions
|
|
13
|
-
from cosmotech.coal.cosmotech_api.dataset.download.adt import download_adt_dataset
|
|
14
|
-
from cosmotech.coal.cosmotech_api.dataset.download.twingraph import (
|
|
15
|
-
download_twingraph_dataset,
|
|
16
|
-
download_legacy_twingraph_dataset,
|
|
17
|
-
)
|
|
18
|
-
from cosmotech.coal.cosmotech_api.dataset.download.file import download_file_dataset
|
|
19
|
-
from cosmotech.coal.cosmotech_api.dataset.download.common import download_dataset_by_id
|