cosmotech-acceleration-library 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cosmotech/coal/__init__.py +1 -1
- cosmotech/coal/aws/__init__.py +1 -9
- cosmotech/coal/aws/s3.py +181 -214
- cosmotech/coal/azure/adx/auth.py +2 -2
- cosmotech/coal/azure/adx/runner.py +13 -14
- cosmotech/coal/azure/adx/store.py +5 -86
- cosmotech/coal/azure/adx/tables.py +2 -2
- cosmotech/coal/azure/blob.py +6 -6
- cosmotech/coal/azure/storage.py +3 -3
- cosmotech/coal/cosmotech_api/__init__.py +0 -24
- cosmotech/coal/cosmotech_api/apis/__init__.py +14 -0
- cosmotech/coal/cosmotech_api/apis/dataset.py +103 -0
- cosmotech/coal/cosmotech_api/apis/meta.py +25 -0
- cosmotech/coal/cosmotech_api/apis/organization.py +24 -0
- cosmotech/coal/cosmotech_api/apis/run.py +38 -0
- cosmotech/coal/cosmotech_api/apis/runner.py +71 -0
- cosmotech/coal/cosmotech_api/apis/solution.py +23 -0
- cosmotech/coal/cosmotech_api/apis/workspace.py +108 -0
- cosmotech/coal/cosmotech_api/objects/__init__.py +9 -0
- cosmotech/coal/cosmotech_api/objects/connection.py +125 -0
- cosmotech/coal/cosmotech_api/objects/parameters.py +127 -0
- cosmotech/coal/postgresql/runner.py +56 -36
- cosmotech/coal/postgresql/store.py +60 -14
- cosmotech/coal/postgresql/utils.py +254 -0
- cosmotech/coal/store/output/__init__.py +0 -0
- cosmotech/coal/store/output/aws_channel.py +73 -0
- cosmotech/coal/store/output/az_storage_channel.py +42 -0
- cosmotech/coal/store/output/channel_interface.py +23 -0
- cosmotech/coal/store/output/channel_spliter.py +55 -0
- cosmotech/coal/store/output/postgres_channel.py +40 -0
- cosmotech/coal/utils/configuration.py +169 -0
- cosmotech/coal/utils/decorator.py +22 -0
- cosmotech/csm_data/commands/api/api.py +6 -19
- cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +20 -16
- cosmotech/csm_data/commands/api/run_load_data.py +7 -46
- cosmotech/csm_data/commands/api/wsf_load_file.py +14 -15
- cosmotech/csm_data/commands/api/wsf_send_file.py +12 -13
- cosmotech/csm_data/commands/s3_bucket_delete.py +16 -15
- cosmotech/csm_data/commands/s3_bucket_download.py +16 -16
- cosmotech/csm_data/commands/s3_bucket_upload.py +16 -14
- cosmotech/csm_data/commands/store/dump_to_s3.py +18 -16
- cosmotech/csm_data/commands/store/output.py +35 -0
- cosmotech/csm_data/commands/store/store.py +3 -4
- cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml +8 -0
- cosmotech/translation/coal/en-US/coal/services/dataset.yml +4 -14
- cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml +1 -0
- cosmotech/translation/coal/en-US/coal/store/output/split.yml +6 -0
- cosmotech/translation/coal/en-US/coal/utils/configuration.yml +2 -0
- cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml +7 -0
- {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/METADATA +8 -9
- {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/RECORD +55 -71
- cosmotech/coal/azure/functions.py +0 -72
- cosmotech/coal/cosmotech_api/connection.py +0 -96
- cosmotech/coal/cosmotech_api/dataset/__init__.py +0 -26
- cosmotech/coal/cosmotech_api/dataset/converters.py +0 -164
- cosmotech/coal/cosmotech_api/dataset/download/__init__.py +0 -19
- cosmotech/coal/cosmotech_api/dataset/download/adt.py +0 -119
- cosmotech/coal/cosmotech_api/dataset/download/common.py +0 -140
- cosmotech/coal/cosmotech_api/dataset/download/file.py +0 -216
- cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +0 -188
- cosmotech/coal/cosmotech_api/dataset/utils.py +0 -132
- cosmotech/coal/cosmotech_api/parameters.py +0 -48
- cosmotech/coal/cosmotech_api/run.py +0 -25
- cosmotech/coal/cosmotech_api/run_data.py +0 -173
- cosmotech/coal/cosmotech_api/run_template.py +0 -108
- cosmotech/coal/cosmotech_api/runner/__init__.py +0 -28
- cosmotech/coal/cosmotech_api/runner/data.py +0 -38
- cosmotech/coal/cosmotech_api/runner/datasets.py +0 -364
- cosmotech/coal/cosmotech_api/runner/download.py +0 -146
- cosmotech/coal/cosmotech_api/runner/metadata.py +0 -42
- cosmotech/coal/cosmotech_api/runner/parameters.py +0 -157
- cosmotech/coal/cosmotech_api/twin_data_layer.py +0 -512
- cosmotech/coal/cosmotech_api/workspace.py +0 -127
- cosmotech/coal/utils/api.py +0 -68
- cosmotech/coal/utils/postgresql.py +0 -236
- cosmotech/csm_data/commands/api/rds_load_csv.py +0 -90
- cosmotech/csm_data/commands/api/rds_send_csv.py +0 -74
- cosmotech/csm_data/commands/api/rds_send_store.py +0 -74
- cosmotech/csm_data/commands/api/runtemplate_load_handler.py +0 -66
- cosmotech/csm_data/commands/api/tdl_load_files.py +0 -76
- cosmotech/csm_data/commands/api/tdl_send_files.py +0 -82
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +0 -27
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +0 -32
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +0 -27
- cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +0 -2
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml +0 -13
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml +0 -12
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml +0 -12
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml +0 -14
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml +0 -18
- {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/WHEEL +0 -0
- {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/entry_points.txt +0 -0
- {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/licenses/LICENSE +0 -0
- {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/top_level.txt +0 -0
|
@@ -1,45 +1,35 @@
|
|
|
1
|
-
cosmotech/coal/__init__.py,sha256=
|
|
2
|
-
cosmotech/coal/aws/__init__.py,sha256=
|
|
3
|
-
cosmotech/coal/aws/s3.py,sha256=
|
|
1
|
+
cosmotech/coal/__init__.py,sha256=5GlLyV1soguj1xNP_KdhUDoCa5hTAunpZAKbapYdsKk,436
|
|
2
|
+
cosmotech/coal/aws/__init__.py,sha256=qZZyxNCwGa1yPpIEc7CS6sVcv00PNcUnUOsKhsHBSdc,614
|
|
3
|
+
cosmotech/coal/aws/s3.py,sha256=5az9BtyUUBlfqFV3BG32V5-9yDpm-6q_O4YV79HcmMA,7522
|
|
4
4
|
cosmotech/coal/azure/__init__.py,sha256=1SmwP1y7quGq9cWqiWxJy1XDjWM8sAc3f63ztBnYujQ,796
|
|
5
|
-
cosmotech/coal/azure/blob.py,sha256=
|
|
6
|
-
cosmotech/coal/azure/
|
|
7
|
-
cosmotech/coal/azure/storage.py,sha256=JocO3YJ4WnXouHja6Ipra0d06tcKu_ecoQfqNV8WTU8,2795
|
|
5
|
+
cosmotech/coal/azure/blob.py,sha256=C2i1JIC7Z4Iv_RLzbyFqbHao9JsJI9jMiBAo6Bix6Ns,4082
|
|
6
|
+
cosmotech/coal/azure/storage.py,sha256=TItajcil0Ht44kru44T9HIP2cizOvAS_H9uRdSgaBG0,2799
|
|
8
7
|
cosmotech/coal/azure/adx/__init__.py,sha256=zRYssRscq30jJ1q1CPUK6S3Sq4EBqZ_uL0M1ju3vktw,1227
|
|
9
|
-
cosmotech/coal/azure/adx/auth.py,sha256=
|
|
8
|
+
cosmotech/coal/azure/adx/auth.py,sha256=IFNaB8eoc99_j3Ml6rueYfuFK36rze4_8LG7B0gYdIQ,4748
|
|
10
9
|
cosmotech/coal/azure/adx/ingestion.py,sha256=--hOB5iUWe-RwXpFC5EaBDmWbJrIAtYula59wrUzCM8,11455
|
|
11
10
|
cosmotech/coal/azure/adx/query.py,sha256=SHMeGX-TNBfQE_DV-J5lrHxqDXgftYiLCasDhgO79rQ,1870
|
|
12
|
-
cosmotech/coal/azure/adx/runner.py,sha256=
|
|
13
|
-
cosmotech/coal/azure/adx/store.py,sha256=
|
|
14
|
-
cosmotech/coal/azure/adx/tables.py,sha256=
|
|
11
|
+
cosmotech/coal/azure/adx/runner.py,sha256=IXgciGxHjSqkiqm23CuyUk3RgCUgd1G-e0eMyohjygM,8066
|
|
12
|
+
cosmotech/coal/azure/adx/store.py,sha256=7fxG3yVxQ9_6sLaSU1Z4cxmsMcSO58Ni371FD2m_4LA,6214
|
|
13
|
+
cosmotech/coal/azure/adx/tables.py,sha256=y6GBrjXAKAOn7q4Ekf-hV6K5P3ejdV2L57FbFDvp22c,4287
|
|
15
14
|
cosmotech/coal/azure/adx/utils.py,sha256=nOq4WLTMDs95da-jsWJVYcv2MRcwUMvARv5Z0ivIVUo,2205
|
|
16
|
-
cosmotech/coal/cosmotech_api/__init__.py,sha256=
|
|
17
|
-
cosmotech/coal/cosmotech_api/
|
|
18
|
-
cosmotech/coal/cosmotech_api/
|
|
19
|
-
cosmotech/coal/cosmotech_api/
|
|
20
|
-
cosmotech/coal/cosmotech_api/
|
|
21
|
-
cosmotech/coal/cosmotech_api/
|
|
22
|
-
cosmotech/coal/cosmotech_api/
|
|
23
|
-
cosmotech/coal/cosmotech_api/
|
|
24
|
-
cosmotech/coal/cosmotech_api/
|
|
25
|
-
cosmotech/coal/cosmotech_api/
|
|
26
|
-
cosmotech/coal/cosmotech_api/
|
|
27
|
-
cosmotech/coal/cosmotech_api/
|
|
28
|
-
cosmotech/coal/cosmotech_api/dataset/download/adt.py,sha256=PwfkSYTCizkE81Huh5J8BShOXQnrjUvebRsgT-nYZpw,4731
|
|
29
|
-
cosmotech/coal/cosmotech_api/dataset/download/common.py,sha256=KuVEzOA_abAwtwbcdry8zmqXAuQvy-NGeg1Ot9_MMuU,5665
|
|
30
|
-
cosmotech/coal/cosmotech_api/dataset/download/file.py,sha256=X4ZNVjzl9ZC8J02CjRQeptZT6FPYBuxPsw-XPssP8YE,8914
|
|
31
|
-
cosmotech/coal/cosmotech_api/dataset/download/twingraph.py,sha256=iHDvsm7odQQ5gyT7a3LcvcdNA6nHTIaWJUCFb9ryjr0,7373
|
|
32
|
-
cosmotech/coal/cosmotech_api/runner/__init__.py,sha256=hEfmv6nF0hxSWMOnWZF0Fy7PJtHexVf-_cXPsxoY7Ho,1081
|
|
33
|
-
cosmotech/coal/cosmotech_api/runner/data.py,sha256=BtCRi_c_3aGUSBjaPt5MOajWO6WFsJdBrIrx4GMt1AE,1346
|
|
34
|
-
cosmotech/coal/cosmotech_api/runner/datasets.py,sha256=ojDkHGrobsygLr2CRYROetqm9zeWv1S1Bc_4Az4z874,12271
|
|
35
|
-
cosmotech/coal/cosmotech_api/runner/download.py,sha256=YZRmhsX9TVjvhtgFhY4uvsJ3If0G43OiiERkCdoOKHE,5655
|
|
36
|
-
cosmotech/coal/cosmotech_api/runner/metadata.py,sha256=eZHtlVMqsQ6WO9F-mDparjIy7tFl7kU-OqguKaI44jE,1432
|
|
37
|
-
cosmotech/coal/cosmotech_api/runner/parameters.py,sha256=mKRRBPZOD_tPOTyxUAL2o9nRy_KZ5Z687MkL2uhqvIc,4651
|
|
15
|
+
cosmotech/coal/cosmotech_api/__init__.py,sha256=YlIyLwoqd4CveVnfrA9UDunlhO9HSvrBAfUwzcBay5M,528
|
|
16
|
+
cosmotech/coal/cosmotech_api/apis/__init__.py,sha256=RDe2FXBCrgUoSAONfpf_Udyc3M3w2y9K9mjFzUX_SMY,869
|
|
17
|
+
cosmotech/coal/cosmotech_api/apis/dataset.py,sha256=Ymo5NtIARiZtuzHOciRh4WzDaGMSo92KanxqptfVdzc,3946
|
|
18
|
+
cosmotech/coal/cosmotech_api/apis/meta.py,sha256=lCi5Jo7c_gXewob9B_NgvRyKAmPSJ_bVK7Z_pIVUyP8,1055
|
|
19
|
+
cosmotech/coal/cosmotech_api/apis/organization.py,sha256=JA55Bu4mJL5aE7UNtnPDWyjB6YZktSsnCzOvs9bW34g,1102
|
|
20
|
+
cosmotech/coal/cosmotech_api/apis/run.py,sha256=TOVD0E4QIR26rBZr3MjsWuSbYcKcuTVSJsq_zD4nzD0,1528
|
|
21
|
+
cosmotech/coal/cosmotech_api/apis/runner.py,sha256=ls-xuAfiCFa-2m7DgL_W_iB_y_y1q4JBt1eOOHSMdqE,2834
|
|
22
|
+
cosmotech/coal/cosmotech_api/apis/solution.py,sha256=rGsbtkVkhrt0vAano4HcIYQJFaXGSpWCHLku2TEpEHI,1077
|
|
23
|
+
cosmotech/coal/cosmotech_api/apis/workspace.py,sha256=YeXuODKJnwpdO2g2yaJzg9095VyzAuGlMV1JL1hNMsU,4330
|
|
24
|
+
cosmotech/coal/cosmotech_api/objects/__init__.py,sha256=6cWMAAyhCK478_Je9Opsu0NBv617aTQo3r9mhbV2wwo,556
|
|
25
|
+
cosmotech/coal/cosmotech_api/objects/connection.py,sha256=z0hd2uJNZWZdTyL7NlGuXt_HeZMenSODNJatNdmhk0o,5292
|
|
26
|
+
cosmotech/coal/cosmotech_api/objects/parameters.py,sha256=6T4yJMm_yvularJwMIeg8qoqBYdpWEdbQx0Q2dCieDw,4227
|
|
38
27
|
cosmotech/coal/csm/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
39
28
|
cosmotech/coal/csm/engine/__init__.py,sha256=EG3iArAy3OFGPmClsYXQmb77YYrEkeVC5uwPg_TbAYI,2120
|
|
40
29
|
cosmotech/coal/postgresql/__init__.py,sha256=766LBZyWCQUI6rlzXDY_C9Yrxg551VX2L4lOuhDiS8w,786
|
|
41
|
-
cosmotech/coal/postgresql/runner.py,sha256=
|
|
42
|
-
cosmotech/coal/postgresql/store.py,sha256=
|
|
30
|
+
cosmotech/coal/postgresql/runner.py,sha256=9UVVvX0FX6X-ZwaUGZiGLso410YdR28BF_bb45Urfww,4189
|
|
31
|
+
cosmotech/coal/postgresql/store.py,sha256=bJ-tUjJt8-MFi5GcmkE7sYZFzXDvKkwoo1rbXayvOzo,5371
|
|
32
|
+
cosmotech/coal/postgresql/utils.py,sha256=F-qHUxCS4BXOiOTuA5YVQdkOO10T-BMWTdE70L9UIfM,9642
|
|
43
33
|
cosmotech/coal/singlestore/__init__.py,sha256=gXQaa8OAHnDcI7iHy-5zzGG7ZqNq-uOtw60qASjwu9I,651
|
|
44
34
|
cosmotech/coal/singlestore/store.py,sha256=jF3gkg2qW_SrtbVKkBJ-SCtV6ukd2eLnFySOvMZFvMY,3577
|
|
45
35
|
cosmotech/coal/store/__init__.py,sha256=kWGM9B-Y-M257OYbouJ-1JLES5qecQbGeUBbcgyrsEo,1307
|
|
@@ -48,52 +38,47 @@ cosmotech/coal/store/native_python.py,sha256=Mp2U7tkUjH1wT76U4WzekaW4H5nN2VHTrAK
|
|
|
48
38
|
cosmotech/coal/store/pandas.py,sha256=A2eWPDsAqI2fTNmIpJa0Qzwrh3vMfvSHPLuoxQ77OPM,906
|
|
49
39
|
cosmotech/coal/store/pyarrow.py,sha256=i7f7vSd5BtOnR4oleRHTG3ff0o8NaJr4c9fRb7zgYmE,814
|
|
50
40
|
cosmotech/coal/store/store.py,sha256=a9wSThk7K8tD1r_BJG1Zl2d4pfo23TqKUH82_u9H-tk,3353
|
|
41
|
+
cosmotech/coal/store/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
42
|
+
cosmotech/coal/store/output/aws_channel.py,sha256=31m--qcnIyyCXpd7G4072tmDaHHx7RzLdVtl1XmaPMQ,2948
|
|
43
|
+
cosmotech/coal/store/output/az_storage_channel.py,sha256=zD8BsjuWsAr9BMq7sLYrlbMOoKDIl792_tqKJwegdsM,1425
|
|
44
|
+
cosmotech/coal/store/output/channel_interface.py,sha256=ojLDvpsBz2xOVhVxgeyQkNx0w-Rw66hcPesp24QQ5TI,670
|
|
45
|
+
cosmotech/coal/store/output/channel_spliter.py,sha256=HU0KssksIPI_cBruVy-jQmS2t8w74mHiy-kYrbCZdEs,2203
|
|
46
|
+
cosmotech/coal/store/output/postgres_channel.py,sha256=DXYVATnvIaCUtjTw3VrKc7EdbvjLOD69WotJESfUC30,1384
|
|
51
47
|
cosmotech/coal/utils/__init__.py,sha256=N2mPhli0Wj4OesuRVbt2KrzL_pgMELs4fDNIQYb_r-g,830
|
|
52
|
-
cosmotech/coal/utils/
|
|
48
|
+
cosmotech/coal/utils/configuration.py,sha256=0awBiD0d2m0zgMBV4IAm47rTAgHaV9xska2UUa8kVF0,6533
|
|
49
|
+
cosmotech/coal/utils/decorator.py,sha256=p4yqkbln-YTppzeKvi3K4ZZ6AibVYGD6pnJQk0ynWbE,646
|
|
53
50
|
cosmotech/coal/utils/logger.py,sha256=oYVj2BtDPji4e4J-KgaDyVcbKiewkq0xey_gQPm74Xc,506
|
|
54
|
-
cosmotech/coal/utils/postgresql.py,sha256=6gxk3yfkZkaJQbIG-Zxcwo7J3PI29XPLn0VdQimGSH0,8562
|
|
55
51
|
cosmotech/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
56
52
|
cosmotech/csm_data/main.py,sha256=q20Gl1x7DV0gWStnWJUVnhUrvv-TPrUhv3e-5X3TDEM,2424
|
|
57
53
|
cosmotech/csm_data/commands/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
58
54
|
cosmotech/csm_data/commands/adx_send_data.py,sha256=wKt30RmuS1FdnhgW0mJZVXifnI_YLRLaj4w5DYlPl7s,2638
|
|
59
55
|
cosmotech/csm_data/commands/adx_send_runnerdata.py,sha256=SHZ0RijCnYRrAdCgdbW_YTaN3joihtNtbLzZAICCzJ4,3713
|
|
60
56
|
cosmotech/csm_data/commands/az_storage_upload.py,sha256=7qXhFxftfil72tG23HTvHiW708t4QkM4rymcFJSPCj4,2344
|
|
61
|
-
cosmotech/csm_data/commands/s3_bucket_delete.py,sha256=
|
|
62
|
-
cosmotech/csm_data/commands/s3_bucket_download.py,sha256=
|
|
63
|
-
cosmotech/csm_data/commands/s3_bucket_upload.py,sha256=
|
|
57
|
+
cosmotech/csm_data/commands/s3_bucket_delete.py,sha256=2Sx3sLkiQxTd9MNL85zUZCvNTi6bZKxusQmu8xG0EYY,3056
|
|
58
|
+
cosmotech/csm_data/commands/s3_bucket_download.py,sha256=X0IahwMTzPUelTLZvNKIKkV2NMK6W8zw_Qp2hjFSsaA,3367
|
|
59
|
+
cosmotech/csm_data/commands/s3_bucket_upload.py,sha256=MKRFoYS9-MMCdk_jVRDF-Kx3YitjlmqUxX6-kd2CyxQ,3599
|
|
64
60
|
cosmotech/csm_data/commands/api/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
65
|
-
cosmotech/csm_data/commands/api/api.py,sha256=
|
|
66
|
-
cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py,sha256=
|
|
67
|
-
cosmotech/csm_data/commands/api/
|
|
68
|
-
cosmotech/csm_data/commands/api/
|
|
69
|
-
cosmotech/csm_data/commands/api/
|
|
70
|
-
cosmotech/csm_data/commands/api/run_load_data.py,sha256=oRnoIj2EXPG8Q2fcvrzt69eX4LNCVCO612qejsOivBM,3725
|
|
71
|
-
cosmotech/csm_data/commands/api/runtemplate_load_handler.py,sha256=qWS0I1fNN6VFx7lHJoOWCv30WGI2q32B9_Zyr5KpBSM,2275
|
|
72
|
-
cosmotech/csm_data/commands/api/tdl_load_files.py,sha256=O4FsRNutyEvdDUBYElLxrzEhJcXVyO9hpOdiM6FEQ5E,2394
|
|
73
|
-
cosmotech/csm_data/commands/api/tdl_send_files.py,sha256=QXToEHbZvEhdh88xHamXj9mPQo1Hm2UPxaMkgNEspaw,2474
|
|
74
|
-
cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=Iz3Do-pBJBTDe2P5QwINRQCyeWGHVKjh_lI610WFi1g,2307
|
|
75
|
-
cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=TlDMObWGhXgX1PqIeg5OX832Lps6q5fdC_2GG5Hx3q0,2191
|
|
61
|
+
cosmotech/csm_data/commands/api/api.py,sha256=v8qBRjI1he8o6hDXmjmptTUdO8F92sgwr9glszsy49o,1698
|
|
62
|
+
cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py,sha256=h_24m7sBrA4Dej7UyIsAzlpJi9mGIM5MBiUHxN8PT08,4235
|
|
63
|
+
cosmotech/csm_data/commands/api/run_load_data.py,sha256=XL_txDSMdNLePa2ScidF8CzUVTasn9SAtKZXE4Gmo8A,2576
|
|
64
|
+
cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=wnNvRY1D7GBE5lQz5KOQ9x5T_wxs-4wIfGPO4jiMcwQ,2103
|
|
65
|
+
cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=AC-Li-2tN5KAtIE5TnElRbSgUTNCa7uWkn90MlOozjg,2045
|
|
76
66
|
cosmotech/csm_data/commands/store/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
77
67
|
cosmotech/csm_data/commands/store/dump_to_azure.py,sha256=5nnK7LUATRTlfX8tmFRLBmkD15bgG37Lyn7fEuPJiNE,3313
|
|
78
68
|
cosmotech/csm_data/commands/store/dump_to_postgresql.py,sha256=3ZfIwIfLXYeBb6SwBSV6-CiWdid8Udht4ILWeZNdLWM,3596
|
|
79
|
-
cosmotech/csm_data/commands/store/dump_to_s3.py,sha256=
|
|
69
|
+
cosmotech/csm_data/commands/store/dump_to_s3.py,sha256=6EUwtkP8_k9Hlq65R9h-MYsH58e-UAYmSR95JKoZuvY,5342
|
|
80
70
|
cosmotech/csm_data/commands/store/list_tables.py,sha256=epgVJAaP8QfSsEuL9n5S8smsfIxK-Cr0bLv88FAnkWw,1860
|
|
81
71
|
cosmotech/csm_data/commands/store/load_csv_folder.py,sha256=gDKCxWIpXqhtRvlvRMM7SmyJ9_YHQ1UZHuA6NYdPTZA,1717
|
|
82
72
|
cosmotech/csm_data/commands/store/load_from_singlestore.py,sha256=dWZ4jIKl8wsnOX_6qqtyQmkFQhLgjUAOZWab-sGzQ2g,3104
|
|
73
|
+
cosmotech/csm_data/commands/store/output.py,sha256=RkBQxZ6c-p6F3A0Ci7wW-ODHsrJJO8ivzQCumvjoqXI,1218
|
|
83
74
|
cosmotech/csm_data/commands/store/reset.py,sha256=3B0E0m0nCy3vwai_D5sWWddMZWuWkWnaL1tvgR6L5Ow,1237
|
|
84
|
-
cosmotech/csm_data/commands/store/store.py,sha256=
|
|
75
|
+
cosmotech/csm_data/commands/store/store.py,sha256=oQhTIF1WKKGKdFp1rNm1vuTgblmh27r_jQpPhNFmesE,1663
|
|
85
76
|
cosmotech/csm_data/utils/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
86
77
|
cosmotech/csm_data/utils/click.py,sha256=S_85cbKh3R86-FZVjTK7IXZnmp4ETjKo6K8gbK3HCgs,848
|
|
87
78
|
cosmotech/csm_data/utils/decorators.py,sha256=dTcPRTYqY18mc8Ql4Qwd3gU7BxbNxfeKskQdVIsE3-g,2504
|
|
88
79
|
cosmotech/orchestrator_plugins/csm-data/__init__.py,sha256=UPzNgaNBMoBaxSlnofxl988dAswMp_tv8DzaR_IxkiE,519
|
|
89
80
|
cosmotech/orchestrator_plugins/csm-data/templates/api/postgres_send_runner_metadata.json,sha256=alXzvOa985PvN5nf3uoTsFXgY29EJWeK94CcBjoLFUk,1085
|
|
90
|
-
cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json,sha256=nM2hVFqE6Mja6u6becVStJTD8POcu76Ys0oW-mU6-TU,819
|
|
91
|
-
cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json,sha256=Z8uh4-C1TNVjGxUkuGlHCoSqRPNL4DVn-wVMZXSAOto,790
|
|
92
|
-
cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json,sha256=AutPCsUfK0gHGjojDZQdQ0pgHsfC6uLtUtQ80eSHD6o,772
|
|
93
81
|
cosmotech/orchestrator_plugins/csm-data/templates/api/run_load_data.json,sha256=_ztHPGu5UbbdtQHUCaOuKqwtjB0DrZSZwcUePoOgS40,894
|
|
94
|
-
cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json,sha256=aL-4GIUi2dV9jh3vuRI0BkAQJdHzfbf53Sxd91Lfj_U,761
|
|
95
|
-
cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json,sha256=LxrwHU-UxtP0GB7FpNUw2o_HIM6VK2PBH7P928SFk_8,889
|
|
96
|
-
cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json,sha256=A9Rt3wiiBly01ugEXGo1Qejp7UV3t8P13tLNLseqgZU,751
|
|
97
82
|
cosmotech/orchestrator_plugins/csm-data/templates/api/try_api_connection.json,sha256=ApkzR0HQc0J6sxxTh91_DHPRI2cCtx_-qjpYFKGkGVw,215
|
|
98
83
|
cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_load_file.json,sha256=yX7yUrq7epQTp0PaQElFC-elof8W6jaMtek6uQ0V6Og,1142
|
|
99
84
|
cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_send_file.json,sha256=7pyYOUPg_lvD4kw3sqd_zBonru7g-DVatQmu4EJw0Ps,1120
|
|
@@ -119,7 +104,7 @@ cosmotech/translation/coal/en-US/coal/common/progress.yml,sha256=G_BTYgAYVD_JqU3
|
|
|
119
104
|
cosmotech/translation/coal/en-US/coal/common/timing.yml,sha256=YtxIT8EKZDWD4hJ0tJOwkReUsrOY67Wq0MFkEZjzANE,274
|
|
120
105
|
cosmotech/translation/coal/en-US/coal/common/validation.yml,sha256=OAHeXrvMo2_S-D0z-ESIS--U7bU0z1i0wta4ibaqk3c,528
|
|
121
106
|
cosmotech/translation/coal/en-US/coal/cosmotech_api/connection.yml,sha256=2Om0DZmOI8u0hzlWWhWnGx-o0mzMk_vgE3hGRBNaWWU,518
|
|
122
|
-
cosmotech/translation/coal/en-US/coal/cosmotech_api/
|
|
107
|
+
cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml,sha256=NGD5bbbS037QosTmHxXOiUJ_Zl3gyIr1iR6hCaSlolY,380
|
|
123
108
|
cosmotech/translation/coal/en-US/coal/cosmotech_api/run_template.yml,sha256=JIQ0S-9_YhchAPVFkcMpJTVUBlhPo4yS-nBoMysus5o,441
|
|
124
109
|
cosmotech/translation/coal/en-US/coal/cosmotech_api/runner.yml,sha256=fSoFiJFJcowXwBVgfAkZKL8j3iKzyVDLWm48cFpiYMA,804
|
|
125
110
|
cosmotech/translation/coal/en-US/coal/cosmotech_api/solution.yml,sha256=dKOTcv8bBYl3ZxyvKsQ8msO--PyLTfEOodb4rHl2bq4,216
|
|
@@ -128,20 +113,18 @@ cosmotech/translation/coal/en-US/coal/services/adx.yml,sha256=9yxR0qYKyA5mqgW-pX
|
|
|
128
113
|
cosmotech/translation/coal/en-US/coal/services/api.yml,sha256=HPyKJaZrS41Qo9upRvPsLdn29QgS1xw3S-bgipHeO4s,342
|
|
129
114
|
cosmotech/translation/coal/en-US/coal/services/azure_storage.yml,sha256=p0alxlJA_-NH9MxKXRKGgiLbBvHuQdCROhJKiqvuX0c,595
|
|
130
115
|
cosmotech/translation/coal/en-US/coal/services/database.yml,sha256=S1pAu4OrVd9nAVb3oYSAurzA4cgD43UDlwK4qWhy2g4,860
|
|
131
|
-
cosmotech/translation/coal/en-US/coal/services/dataset.yml,sha256=
|
|
116
|
+
cosmotech/translation/coal/en-US/coal/services/dataset.yml,sha256=aaFBSwkoakMbyjMVeljp7RfC-h4wEvclZqMOBD8ynSo,2981
|
|
132
117
|
cosmotech/translation/coal/en-US/coal/services/postgresql.yml,sha256=tRGNTTayyoxEHVyM-nzudEFdQ5HUeRIHPWIuvFmJGfM,1774
|
|
133
118
|
cosmotech/translation/coal/en-US/coal/services/s3.yml,sha256=362wt7SdOu-MyMj9MgjQNTNWzIq-RJ9rUqc2VKljGFE,482
|
|
119
|
+
cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml,sha256=4pVJPAXYeIq9Gpmg_0VpSrB17AsGWA9hvk8Qi8-kZvw,85
|
|
120
|
+
cosmotech/translation/coal/en-US/coal/store/output/split.yml,sha256=h89v89sQsK5pGDJP5EE_KNbQbaS8003VPZNIgU2dqWw,265
|
|
121
|
+
cosmotech/translation/coal/en-US/coal/utils/configuration.yml,sha256=q3kEzPTP81SfsdrhAZa2JusHAjbdYqjs0DcvjhewQ-4,96
|
|
134
122
|
cosmotech/translation/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
|
|
135
123
|
cosmotech/translation/csm_data/en-US/csm_data/commands/main.yml,sha256=3XbXR5_Uo7PxJOWzm7XRMvPVxK-fzOL4FxWWY7rX3eg,396
|
|
136
124
|
cosmotech/translation/csm_data/en-US/csm_data/commands/api/api.yml,sha256=eMT4s1H9LuNUh_sAlnEtg7Co4ie3bFfB1vkwnLW28dQ,340
|
|
137
125
|
cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml,sha256=avckcrp2aYz7rzGCKEdL81hwHDVbcUn2csQ_krKLcaM,844
|
|
138
|
-
cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml,sha256=jaCmdHY0fJp9Sh2PcOizMyPIdvZU2iTIoSLNdfuLCXE,668
|
|
139
|
-
cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml,sha256=0I9MAGgdCmNHGN3sdRVSWzEobAABFsb9qVG_l0Vk7Sg,580
|
|
140
|
-
cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml,sha256=9vRnrqYYzSYMPNzR-Q7Gw5G7LOWlEd5GWepSyzg49i4,574
|
|
141
126
|
cosmotech/translation/csm_data/en-US/csm_data/commands/api/run_load_data.yml,sha256=k5J3HTA8vCmI_7CFHvMP51EAbmefubuoEhXZf2V3XDo,765
|
|
142
127
|
cosmotech/translation/csm_data/en-US/csm_data/commands/api/runtemplate_load_handler.yml,sha256=9jvMa6fb99CPzTMx7nnuoNNzJw7_7rapee7P80Plxjs,353
|
|
143
|
-
cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml,sha256=ko4Nw96w6vBhE-68swd9h4edWgXCWIfMGqYc8Fvh2LM,569
|
|
144
|
-
cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml,sha256=13eacNdU6i92XemHyx-bzIdXbQUIs6eg2YiT3SxivyM,836
|
|
145
128
|
cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_load_file.yml,sha256=cnR_0PMkbrj8GROgvIUyeLXiB90IBbkcTvM9WJDEM0k,511
|
|
146
129
|
cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_send_file.yml,sha256=yvsD364GND1jLe_SJxuNXjDgGZFGlJlC5xUTRqV0fr0,632
|
|
147
130
|
cosmotech/translation/csm_data/en-US/csm_data/commands/storage/adx_send_data.yml,sha256=oNQMHhHMRTl7EfFLuEy5agctzgwGHsyKzAu6hlSUyU4,448
|
|
@@ -157,13 +140,14 @@ cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_s3.yml,sha2
|
|
|
157
140
|
cosmotech/translation/csm_data/en-US/csm_data/commands/store/list_tables.yml,sha256=-x8zwCjAHILFaJv7VLPskipSoQTwmI7jUsxlx6_OKlc,195
|
|
158
141
|
cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_csv_folder.yml,sha256=9HjW9tOq5o31jmADp_QIync7G74c6XGiPqofXh8pvv8,228
|
|
159
142
|
cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_from_singlestore.yml,sha256=_OUXlxC_wi-GPgHU7EVdU9g0WIdblVpEA4WIzw4aQcQ,824
|
|
143
|
+
cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml,sha256=odkR9pzeel5l0wW_F4kx4HhNpw_TzEvok-OKyh25ZkQ,232
|
|
160
144
|
cosmotech/translation/csm_data/en-US/csm_data/commands/store/reset.yml,sha256=JMspZ25Q2HDpH8MYqUaQcveUnCgdH21RmfTCrPf2gW8,139
|
|
161
145
|
cosmotech/translation/csm_data/en-US/csm_data/commands/store/store.yml,sha256=N1Q8483gqJADaCe30S1M3Rj0tMJiuQiJH70-VK2x2m4,134
|
|
162
146
|
cosmotech/translation/csm_data/en-US/csm_data/commons/decorators.yml,sha256=Iu59NWMfYlZZf9uUhOiLkIEGa4GY5p0nZ6vG06Xvu7k,51
|
|
163
147
|
cosmotech/translation/csm_data/en-US/csm_data/commons/version.yml,sha256=7jtCV3O1S6pGjiJa63XpgPDTafjfBS0xmEVRpYNvfDg,86
|
|
164
|
-
cosmotech_acceleration_library-
|
|
165
|
-
cosmotech_acceleration_library-
|
|
166
|
-
cosmotech_acceleration_library-
|
|
167
|
-
cosmotech_acceleration_library-
|
|
168
|
-
cosmotech_acceleration_library-
|
|
169
|
-
cosmotech_acceleration_library-
|
|
148
|
+
cosmotech_acceleration_library-2.0.0.dist-info/licenses/LICENSE,sha256=JXKHOQtyObmafNbQlfPYc4HkKjU9FzAP27b2qRTXNM8,1195
|
|
149
|
+
cosmotech_acceleration_library-2.0.0.dist-info/METADATA,sha256=LVDVBJn0B1rE1z4rKbm3wUG5UKKt3Qj6Jkm2lGcLDxk,9334
|
|
150
|
+
cosmotech_acceleration_library-2.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
151
|
+
cosmotech_acceleration_library-2.0.0.dist-info/entry_points.txt,sha256=HWRqJurKuBUgqFe4jmjIAQrs768Nbb8ZTdRDLbuKM5Q,58
|
|
152
|
+
cosmotech_acceleration_library-2.0.0.dist-info/top_level.txt,sha256=t2pzb8mpMUfHTa9l2SjWP0rRB8XVRjBdQK5nLx9XDDo,10
|
|
153
|
+
cosmotech_acceleration_library-2.0.0.dist-info/RECORD,,
|
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
import azure.functions as func
|
|
8
|
-
from cosmotech.coal.cosmotech_api.runner.download import download_runner_data
|
|
9
|
-
from cosmotech_api.api.runner_api import RunnerApi
|
|
10
|
-
|
|
11
|
-
import json
|
|
12
|
-
import http
|
|
13
|
-
import traceback
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def generate_main(apply_update, parallel=True):
|
|
17
|
-
def main(req: func.HttpRequest) -> func.HttpResponse:
|
|
18
|
-
try:
|
|
19
|
-
runner_id = req.params.get("scenario-id") # Keep parameter name for backward compatibility
|
|
20
|
-
organization_id = req.params.get("organization-id")
|
|
21
|
-
workspace_id = req.params.get("workspace-id")
|
|
22
|
-
access_token: str = req.headers.get("authorization", None)
|
|
23
|
-
if access_token:
|
|
24
|
-
access_token = access_token.split(" ")[1]
|
|
25
|
-
|
|
26
|
-
if runner_id is None or organization_id is None or workspace_id is None:
|
|
27
|
-
return func.HttpResponse(
|
|
28
|
-
body=f"Invalid request: organization-id={organization_id}, workspace-id={workspace_id}, scenario-id={runner_id}",
|
|
29
|
-
status_code=http.HTTPStatus.BAD_REQUEST,
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
# Get runner data
|
|
33
|
-
result = download_runner_data(
|
|
34
|
-
organization_id=organization_id,
|
|
35
|
-
workspace_id=workspace_id,
|
|
36
|
-
runner_id=runner_id,
|
|
37
|
-
parameter_folder=None, # We don't need to save to files
|
|
38
|
-
read_files=True,
|
|
39
|
-
parallel=parallel,
|
|
40
|
-
write_json=False,
|
|
41
|
-
write_csv=False,
|
|
42
|
-
fetch_dataset=True,
|
|
43
|
-
)
|
|
44
|
-
|
|
45
|
-
content = {
|
|
46
|
-
"datasets": result["datasets"],
|
|
47
|
-
"parameters": result["parameters"],
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
runner_data = result["runner_data"]
|
|
51
|
-
|
|
52
|
-
updated_content = apply_update(
|
|
53
|
-
content=content, scenario_data=runner_data
|
|
54
|
-
) # Keep parameter name for backward compatibility
|
|
55
|
-
|
|
56
|
-
return func.HttpResponse(
|
|
57
|
-
body=json.dumps(updated_content),
|
|
58
|
-
headers={"Content-Type": "application/json"},
|
|
59
|
-
)
|
|
60
|
-
except Exception as e:
|
|
61
|
-
response = {
|
|
62
|
-
"error": getattr(e, "message", str(e)),
|
|
63
|
-
"type": type(e).__name__,
|
|
64
|
-
"trace": traceback.format_exc(),
|
|
65
|
-
}
|
|
66
|
-
return func.HttpResponse(
|
|
67
|
-
status_code=http.HTTPStatus.INTERNAL_SERVER_ERROR,
|
|
68
|
-
body=json.dumps(response),
|
|
69
|
-
headers={"Content-Type": "application/json"},
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
return main
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
import os
|
|
8
|
-
import pathlib
|
|
9
|
-
import ssl
|
|
10
|
-
|
|
11
|
-
import cosmotech_api
|
|
12
|
-
|
|
13
|
-
from cosmotech.coal.utils.logger import LOGGER
|
|
14
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
15
|
-
|
|
16
|
-
api_env_keys = {"CSM_API_KEY", "CSM_API_URL"}
|
|
17
|
-
azure_env_keys = {
|
|
18
|
-
"AZURE_CLIENT_ID",
|
|
19
|
-
"AZURE_CLIENT_SECRET",
|
|
20
|
-
"AZURE_TENANT_ID",
|
|
21
|
-
"CSM_API_URL",
|
|
22
|
-
"CSM_API_SCOPE",
|
|
23
|
-
}
|
|
24
|
-
keycloak_env_keys = {
|
|
25
|
-
"IDP_TENANT_ID",
|
|
26
|
-
"IDP_CLIENT_ID",
|
|
27
|
-
"IDP_CLIENT_SECRET",
|
|
28
|
-
"IDP_BASE_URL",
|
|
29
|
-
"CSM_API_URL",
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def get_api_client() -> (cosmotech_api.ApiClient, str):
|
|
34
|
-
existing_keys = set(os.environ.keys())
|
|
35
|
-
missing_azure_keys = azure_env_keys - existing_keys
|
|
36
|
-
missing_api_keys = api_env_keys - existing_keys
|
|
37
|
-
missing_keycloak_keys = keycloak_env_keys - existing_keys
|
|
38
|
-
if all((missing_api_keys, missing_azure_keys, missing_keycloak_keys)):
|
|
39
|
-
LOGGER.error(T("coal.common.errors.no_env_vars"))
|
|
40
|
-
LOGGER.error(T("coal.cosmotech_api.connection.existing_sets"))
|
|
41
|
-
LOGGER.error(T("coal.cosmotech_api.connection.azure_connection").format(keys=", ".join(azure_env_keys)))
|
|
42
|
-
LOGGER.error(T("coal.cosmotech_api.connection.api_key_connection").format(keys=", ".join(api_env_keys)))
|
|
43
|
-
LOGGER.error(T("coal.cosmotech_api.connection.keycloak_connection").format(keys=", ".join(keycloak_env_keys)))
|
|
44
|
-
raise EnvironmentError(T("coal.common.errors.no_env_vars"))
|
|
45
|
-
|
|
46
|
-
if not missing_keycloak_keys:
|
|
47
|
-
LOGGER.info(T("coal.cosmotech_api.connection.found_keycloak"))
|
|
48
|
-
from keycloak import KeycloakOpenID
|
|
49
|
-
|
|
50
|
-
server_url = os.environ.get("IDP_BASE_URL")
|
|
51
|
-
if server_url[-1] != "/":
|
|
52
|
-
server_url = server_url + "/"
|
|
53
|
-
keycloack_parameters = dict(
|
|
54
|
-
server_url=server_url,
|
|
55
|
-
client_id=os.environ.get("IDP_CLIENT_ID"),
|
|
56
|
-
realm_name=os.environ.get("IDP_TENANT_ID"),
|
|
57
|
-
client_secret_key=os.environ.get("IDP_CLIENT_SECRET"),
|
|
58
|
-
)
|
|
59
|
-
if (ca_cert_path := os.environ.get("IDP_CA_CERT")) and pathlib.Path(ca_cert_path).exists():
|
|
60
|
-
LOGGER.info(T("coal.cosmotech_api.connection.found_cert_authority"))
|
|
61
|
-
keycloack_parameters["verify"] = ca_cert_path
|
|
62
|
-
keycloak_openid = KeycloakOpenID(**keycloack_parameters)
|
|
63
|
-
|
|
64
|
-
access_token = keycloak_openid.token(grant_type="client_credentials")
|
|
65
|
-
|
|
66
|
-
configuration = cosmotech_api.Configuration(
|
|
67
|
-
host=os.environ.get("CSM_API_URL"),
|
|
68
|
-
access_token=access_token["access_token"],
|
|
69
|
-
)
|
|
70
|
-
return cosmotech_api.ApiClient(configuration), "Keycloak Connection"
|
|
71
|
-
|
|
72
|
-
if not missing_api_keys:
|
|
73
|
-
LOGGER.info(T("coal.cosmotech_api.connection.found_api_key"))
|
|
74
|
-
configuration = cosmotech_api.Configuration(
|
|
75
|
-
host=os.environ.get("CSM_API_URL"),
|
|
76
|
-
)
|
|
77
|
-
return (
|
|
78
|
-
cosmotech_api.ApiClient(
|
|
79
|
-
configuration,
|
|
80
|
-
os.environ.get("CSM_API_KEY_HEADER", "X-CSM-API-KEY"),
|
|
81
|
-
os.environ.get("CSM_API_KEY"),
|
|
82
|
-
),
|
|
83
|
-
"Cosmo Tech API Key",
|
|
84
|
-
)
|
|
85
|
-
|
|
86
|
-
if not missing_azure_keys:
|
|
87
|
-
LOGGER.info(T("coal.cosmotech_api.connection.found_azure"))
|
|
88
|
-
from azure.identity import EnvironmentCredential
|
|
89
|
-
|
|
90
|
-
credentials = EnvironmentCredential()
|
|
91
|
-
token = credentials.get_token(os.environ.get("CSM_API_SCOPE"))
|
|
92
|
-
|
|
93
|
-
configuration = cosmotech_api.Configuration(host=os.environ.get("CSM_API_URL"), access_token=token.token)
|
|
94
|
-
return cosmotech_api.ApiClient(configuration), "Azure Entra Connection"
|
|
95
|
-
|
|
96
|
-
raise EnvironmentError(T("coal.common.errors.no_valid_connection"))
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
# Re-export all download functions from download submodule
|
|
9
|
-
from cosmotech.coal.cosmotech_api.dataset.download import (
|
|
10
|
-
download_adt_dataset,
|
|
11
|
-
download_twingraph_dataset,
|
|
12
|
-
download_legacy_twingraph_dataset,
|
|
13
|
-
download_file_dataset,
|
|
14
|
-
download_dataset_by_id,
|
|
15
|
-
)
|
|
16
|
-
|
|
17
|
-
from cosmotech.coal.cosmotech_api.dataset.converters import (
|
|
18
|
-
convert_dataset_to_files,
|
|
19
|
-
convert_graph_dataset_to_files,
|
|
20
|
-
convert_file_dataset_to_files,
|
|
21
|
-
)
|
|
22
|
-
|
|
23
|
-
from cosmotech.coal.cosmotech_api.dataset.utils import (
|
|
24
|
-
get_content_from_twin_graph_data,
|
|
25
|
-
sheet_to_header,
|
|
26
|
-
)
|
|
@@ -1,164 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
import csv
|
|
9
|
-
import json
|
|
10
|
-
import os
|
|
11
|
-
import tempfile
|
|
12
|
-
from pathlib import Path
|
|
13
|
-
from typing import Dict, List, Any, Optional, Union
|
|
14
|
-
|
|
15
|
-
from cosmotech.coal.utils.logger import LOGGER
|
|
16
|
-
from cosmotech.orchestrator.utils.translate import T
|
|
17
|
-
from cosmotech.coal.cosmotech_api.dataset.utils import sheet_to_header
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def convert_dataset_to_files(dataset_info: Dict[str, Any], target_folder: Optional[Union[str, Path]] = None) -> Path:
|
|
21
|
-
"""
|
|
22
|
-
Convert dataset info to files.
|
|
23
|
-
|
|
24
|
-
Args:
|
|
25
|
-
dataset_info: Dataset info dict with type, content, name
|
|
26
|
-
target_folder: Optional folder to save files (if None, uses temp dir)
|
|
27
|
-
|
|
28
|
-
Returns:
|
|
29
|
-
Path to folder containing files
|
|
30
|
-
"""
|
|
31
|
-
dataset_type = dataset_info["type"]
|
|
32
|
-
content = dataset_info["content"]
|
|
33
|
-
name = dataset_info["name"]
|
|
34
|
-
|
|
35
|
-
LOGGER.info(T("coal.services.dataset.converting_to_files").format(dataset_type=dataset_type, dataset_name=name))
|
|
36
|
-
|
|
37
|
-
if target_folder is None:
|
|
38
|
-
target_folder = Path(tempfile.mkdtemp())
|
|
39
|
-
LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
|
|
40
|
-
else:
|
|
41
|
-
target_folder = Path(target_folder)
|
|
42
|
-
target_folder.mkdir(parents=True, exist_ok=True)
|
|
43
|
-
LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
|
|
44
|
-
|
|
45
|
-
if dataset_type in ["adt", "twincache"]:
|
|
46
|
-
return convert_graph_dataset_to_files(content, target_folder)
|
|
47
|
-
else:
|
|
48
|
-
return convert_file_dataset_to_files(content, target_folder, dataset_type)
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def convert_graph_dataset_to_files(
|
|
52
|
-
content: Dict[str, List[Dict]], target_folder: Optional[Union[str, Path]] = None
|
|
53
|
-
) -> Path:
|
|
54
|
-
"""
|
|
55
|
-
Convert graph dataset content to CSV files.
|
|
56
|
-
|
|
57
|
-
Args:
|
|
58
|
-
content: Dictionary mapping entity types to lists of entities
|
|
59
|
-
target_folder: Folder to save files (if None, uses temp dir)
|
|
60
|
-
|
|
61
|
-
Returns:
|
|
62
|
-
Path to folder containing files
|
|
63
|
-
"""
|
|
64
|
-
if target_folder is None:
|
|
65
|
-
target_folder = Path(tempfile.mkdtemp())
|
|
66
|
-
LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
|
|
67
|
-
else:
|
|
68
|
-
target_folder = Path(target_folder)
|
|
69
|
-
target_folder.mkdir(parents=True, exist_ok=True)
|
|
70
|
-
LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
|
|
71
|
-
file_count = 0
|
|
72
|
-
|
|
73
|
-
LOGGER.info(
|
|
74
|
-
T("coal.services.dataset.converting_graph_data").format(entity_types=len(content), folder=target_folder)
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
-
for entity_type, entities in content.items():
|
|
78
|
-
if not entities:
|
|
79
|
-
LOGGER.debug(T("coal.services.dataset.skipping_empty_entity").format(entity_type=entity_type))
|
|
80
|
-
continue
|
|
81
|
-
|
|
82
|
-
file_path = target_folder / f"{entity_type}.csv"
|
|
83
|
-
LOGGER.debug(T("coal.services.dataset.writing_csv").format(file_name=file_path.name, count=len(entities)))
|
|
84
|
-
|
|
85
|
-
fieldnames = sheet_to_header(entities)
|
|
86
|
-
|
|
87
|
-
with open(file_path, "w", newline="") as file:
|
|
88
|
-
writer = csv.DictWriter(file, fieldnames=fieldnames, dialect="unix", quoting=csv.QUOTE_MINIMAL)
|
|
89
|
-
writer.writeheader()
|
|
90
|
-
|
|
91
|
-
for entity in entities:
|
|
92
|
-
# Convert values to strings and handle boolean values
|
|
93
|
-
row = {
|
|
94
|
-
k: str(v).replace("'", '"').replace("True", "true").replace("False", "false")
|
|
95
|
-
for k, v in entity.items()
|
|
96
|
-
}
|
|
97
|
-
writer.writerow(row)
|
|
98
|
-
|
|
99
|
-
file_count += 1
|
|
100
|
-
LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
|
|
101
|
-
|
|
102
|
-
LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
|
|
103
|
-
|
|
104
|
-
return target_folder
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def convert_file_dataset_to_files(
|
|
108
|
-
content: Dict[str, Any],
|
|
109
|
-
target_folder: Optional[Union[str, Path]] = None,
|
|
110
|
-
file_type: str = "",
|
|
111
|
-
) -> Path:
|
|
112
|
-
"""
|
|
113
|
-
Convert file dataset content to files.
|
|
114
|
-
|
|
115
|
-
Args:
|
|
116
|
-
content: Dictionary mapping file names to content
|
|
117
|
-
target_folder: Folder to save files (if None, uses temp dir)
|
|
118
|
-
file_type: Type of file (csv, json, etc.)
|
|
119
|
-
|
|
120
|
-
Returns:
|
|
121
|
-
Path to folder containing files
|
|
122
|
-
"""
|
|
123
|
-
if target_folder is None:
|
|
124
|
-
target_folder = Path(tempfile.mkdtemp())
|
|
125
|
-
LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
|
|
126
|
-
else:
|
|
127
|
-
target_folder = Path(target_folder)
|
|
128
|
-
target_folder.mkdir(parents=True, exist_ok=True)
|
|
129
|
-
LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
|
|
130
|
-
file_count = 0
|
|
131
|
-
|
|
132
|
-
LOGGER.info(
|
|
133
|
-
T("coal.services.dataset.converting_file_data").format(
|
|
134
|
-
file_count=len(content), file_type=file_type, folder=target_folder
|
|
135
|
-
)
|
|
136
|
-
)
|
|
137
|
-
|
|
138
|
-
for file_name, file_content in content.items():
|
|
139
|
-
file_path = target_folder / file_name
|
|
140
|
-
|
|
141
|
-
# Ensure parent directories exist
|
|
142
|
-
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
143
|
-
|
|
144
|
-
LOGGER.debug(T("coal.services.dataset.writing_file").format(file_name=file_path.name, file_type=file_type))
|
|
145
|
-
|
|
146
|
-
if isinstance(file_content, str):
|
|
147
|
-
# Text content
|
|
148
|
-
with open(file_path, "w") as file:
|
|
149
|
-
file.write(file_content)
|
|
150
|
-
elif isinstance(file_content, dict) or isinstance(file_content, list):
|
|
151
|
-
# JSON content
|
|
152
|
-
with open(file_path, "w") as file:
|
|
153
|
-
json.dump(file_content, file, indent=2)
|
|
154
|
-
else:
|
|
155
|
-
# Other content types
|
|
156
|
-
with open(file_path, "w") as file:
|
|
157
|
-
file.write(str(file_content))
|
|
158
|
-
|
|
159
|
-
file_count += 1
|
|
160
|
-
LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
|
|
161
|
-
|
|
162
|
-
LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
|
|
163
|
-
|
|
164
|
-
return target_folder
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
# Copyright (C) - 2023 - 2025 - Cosmo Tech
|
|
2
|
-
# This document and all information contained herein is the exclusive property -
|
|
3
|
-
# including all intellectual property rights pertaining thereto - of Cosmo Tech.
|
|
4
|
-
# Any use, reproduction, translation, broadcasting, transmission, distribution,
|
|
5
|
-
# etc., to any person is prohibited unless it has been previously and
|
|
6
|
-
# specifically authorized by written means by Cosmo Tech.
|
|
7
|
-
|
|
8
|
-
"""
|
|
9
|
-
Dataset download submodules.
|
|
10
|
-
"""
|
|
11
|
-
|
|
12
|
-
# Re-export all download functions
|
|
13
|
-
from cosmotech.coal.cosmotech_api.dataset.download.adt import download_adt_dataset
|
|
14
|
-
from cosmotech.coal.cosmotech_api.dataset.download.twingraph import (
|
|
15
|
-
download_twingraph_dataset,
|
|
16
|
-
download_legacy_twingraph_dataset,
|
|
17
|
-
)
|
|
18
|
-
from cosmotech.coal.cosmotech_api.dataset.download.file import download_file_dataset
|
|
19
|
-
from cosmotech.coal.cosmotech_api.dataset.download.common import download_dataset_by_id
|