cosmotech-acceleration-library 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. cosmotech/coal/__init__.py +1 -1
  2. cosmotech/coal/aws/__init__.py +1 -9
  3. cosmotech/coal/aws/s3.py +181 -214
  4. cosmotech/coal/azure/adx/auth.py +2 -2
  5. cosmotech/coal/azure/adx/runner.py +13 -14
  6. cosmotech/coal/azure/adx/store.py +5 -86
  7. cosmotech/coal/azure/adx/tables.py +2 -2
  8. cosmotech/coal/azure/blob.py +6 -6
  9. cosmotech/coal/azure/storage.py +3 -3
  10. cosmotech/coal/cosmotech_api/__init__.py +0 -24
  11. cosmotech/coal/cosmotech_api/apis/__init__.py +14 -0
  12. cosmotech/coal/cosmotech_api/apis/dataset.py +103 -0
  13. cosmotech/coal/cosmotech_api/apis/meta.py +25 -0
  14. cosmotech/coal/cosmotech_api/apis/organization.py +24 -0
  15. cosmotech/coal/cosmotech_api/apis/run.py +38 -0
  16. cosmotech/coal/cosmotech_api/apis/runner.py +71 -0
  17. cosmotech/coal/cosmotech_api/apis/solution.py +23 -0
  18. cosmotech/coal/cosmotech_api/apis/workspace.py +108 -0
  19. cosmotech/coal/cosmotech_api/objects/__init__.py +9 -0
  20. cosmotech/coal/cosmotech_api/objects/connection.py +125 -0
  21. cosmotech/coal/cosmotech_api/objects/parameters.py +127 -0
  22. cosmotech/coal/postgresql/runner.py +56 -36
  23. cosmotech/coal/postgresql/store.py +60 -14
  24. cosmotech/coal/postgresql/utils.py +254 -0
  25. cosmotech/coal/store/output/__init__.py +0 -0
  26. cosmotech/coal/store/output/aws_channel.py +73 -0
  27. cosmotech/coal/store/output/az_storage_channel.py +42 -0
  28. cosmotech/coal/store/output/channel_interface.py +23 -0
  29. cosmotech/coal/store/output/channel_spliter.py +55 -0
  30. cosmotech/coal/store/output/postgres_channel.py +40 -0
  31. cosmotech/coal/utils/configuration.py +169 -0
  32. cosmotech/coal/utils/decorator.py +22 -0
  33. cosmotech/csm_data/commands/api/api.py +6 -19
  34. cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +20 -16
  35. cosmotech/csm_data/commands/api/run_load_data.py +7 -46
  36. cosmotech/csm_data/commands/api/wsf_load_file.py +14 -15
  37. cosmotech/csm_data/commands/api/wsf_send_file.py +12 -13
  38. cosmotech/csm_data/commands/s3_bucket_delete.py +16 -15
  39. cosmotech/csm_data/commands/s3_bucket_download.py +16 -16
  40. cosmotech/csm_data/commands/s3_bucket_upload.py +16 -14
  41. cosmotech/csm_data/commands/store/dump_to_s3.py +18 -16
  42. cosmotech/csm_data/commands/store/output.py +35 -0
  43. cosmotech/csm_data/commands/store/store.py +3 -4
  44. cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml +8 -0
  45. cosmotech/translation/coal/en-US/coal/services/dataset.yml +4 -14
  46. cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml +1 -0
  47. cosmotech/translation/coal/en-US/coal/store/output/split.yml +6 -0
  48. cosmotech/translation/coal/en-US/coal/utils/configuration.yml +2 -0
  49. cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml +7 -0
  50. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/METADATA +8 -9
  51. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/RECORD +55 -71
  52. cosmotech/coal/azure/functions.py +0 -72
  53. cosmotech/coal/cosmotech_api/connection.py +0 -96
  54. cosmotech/coal/cosmotech_api/dataset/__init__.py +0 -26
  55. cosmotech/coal/cosmotech_api/dataset/converters.py +0 -164
  56. cosmotech/coal/cosmotech_api/dataset/download/__init__.py +0 -19
  57. cosmotech/coal/cosmotech_api/dataset/download/adt.py +0 -119
  58. cosmotech/coal/cosmotech_api/dataset/download/common.py +0 -140
  59. cosmotech/coal/cosmotech_api/dataset/download/file.py +0 -216
  60. cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +0 -188
  61. cosmotech/coal/cosmotech_api/dataset/utils.py +0 -132
  62. cosmotech/coal/cosmotech_api/parameters.py +0 -48
  63. cosmotech/coal/cosmotech_api/run.py +0 -25
  64. cosmotech/coal/cosmotech_api/run_data.py +0 -173
  65. cosmotech/coal/cosmotech_api/run_template.py +0 -108
  66. cosmotech/coal/cosmotech_api/runner/__init__.py +0 -28
  67. cosmotech/coal/cosmotech_api/runner/data.py +0 -38
  68. cosmotech/coal/cosmotech_api/runner/datasets.py +0 -364
  69. cosmotech/coal/cosmotech_api/runner/download.py +0 -146
  70. cosmotech/coal/cosmotech_api/runner/metadata.py +0 -42
  71. cosmotech/coal/cosmotech_api/runner/parameters.py +0 -157
  72. cosmotech/coal/cosmotech_api/twin_data_layer.py +0 -512
  73. cosmotech/coal/cosmotech_api/workspace.py +0 -127
  74. cosmotech/coal/utils/api.py +0 -68
  75. cosmotech/coal/utils/postgresql.py +0 -236
  76. cosmotech/csm_data/commands/api/rds_load_csv.py +0 -90
  77. cosmotech/csm_data/commands/api/rds_send_csv.py +0 -74
  78. cosmotech/csm_data/commands/api/rds_send_store.py +0 -74
  79. cosmotech/csm_data/commands/api/runtemplate_load_handler.py +0 -66
  80. cosmotech/csm_data/commands/api/tdl_load_files.py +0 -76
  81. cosmotech/csm_data/commands/api/tdl_send_files.py +0 -82
  82. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +0 -27
  83. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +0 -27
  84. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +0 -27
  85. cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +0 -27
  86. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +0 -32
  87. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +0 -27
  88. cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +0 -2
  89. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml +0 -13
  90. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml +0 -12
  91. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml +0 -12
  92. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml +0 -14
  93. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml +0 -18
  94. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/WHEEL +0 -0
  95. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/entry_points.txt +0 -0
  96. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/licenses/LICENSE +0 -0
  97. {cosmotech_acceleration_library-1.0.1.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/top_level.txt +0 -0
@@ -1,45 +1,35 @@
1
- cosmotech/coal/__init__.py,sha256=zO8tP0wjAUWBIzrSAjLECJhE7uA6CXq8k8i15ZII8SQ,436
2
- cosmotech/coal/aws/__init__.py,sha256=DtkCZ8Y7q5HIKui9YXCV1JXO8rRMitd0uieiW2InuXY,761
3
- cosmotech/coal/aws/s3.py,sha256=oq1Sl_6kk_u0Jai55p7XRfKYAK0I6y3va4IfeWNhpEE,7820
1
+ cosmotech/coal/__init__.py,sha256=5GlLyV1soguj1xNP_KdhUDoCa5hTAunpZAKbapYdsKk,436
2
+ cosmotech/coal/aws/__init__.py,sha256=qZZyxNCwGa1yPpIEc7CS6sVcv00PNcUnUOsKhsHBSdc,614
3
+ cosmotech/coal/aws/s3.py,sha256=5az9BtyUUBlfqFV3BG32V5-9yDpm-6q_O4YV79HcmMA,7522
4
4
  cosmotech/coal/azure/__init__.py,sha256=1SmwP1y7quGq9cWqiWxJy1XDjWM8sAc3f63ztBnYujQ,796
5
- cosmotech/coal/azure/blob.py,sha256=O6UwQ9AKns53yM5AqkxgZQX7k-I4dM6yRPL-3RzOm9s,4002
6
- cosmotech/coal/azure/functions.py,sha256=HPxKslag2ANeDyd5y5AVk-pQLZkUffGMGoIh-5C4dvU,2901
7
- cosmotech/coal/azure/storage.py,sha256=JocO3YJ4WnXouHja6Ipra0d06tcKu_ecoQfqNV8WTU8,2795
5
+ cosmotech/coal/azure/blob.py,sha256=C2i1JIC7Z4Iv_RLzbyFqbHao9JsJI9jMiBAo6Bix6Ns,4082
6
+ cosmotech/coal/azure/storage.py,sha256=TItajcil0Ht44kru44T9HIP2cizOvAS_H9uRdSgaBG0,2799
8
7
  cosmotech/coal/azure/adx/__init__.py,sha256=zRYssRscq30jJ1q1CPUK6S3Sq4EBqZ_uL0M1ju3vktw,1227
9
- cosmotech/coal/azure/adx/auth.py,sha256=BafFyLaHuiC3M0Mzp4ZcVAzzG1GUDsnOGn1Chf_vVMk,4755
8
+ cosmotech/coal/azure/adx/auth.py,sha256=IFNaB8eoc99_j3Ml6rueYfuFK36rze4_8LG7B0gYdIQ,4748
10
9
  cosmotech/coal/azure/adx/ingestion.py,sha256=--hOB5iUWe-RwXpFC5EaBDmWbJrIAtYula59wrUzCM8,11455
11
10
  cosmotech/coal/azure/adx/query.py,sha256=SHMeGX-TNBfQE_DV-J5lrHxqDXgftYiLCasDhgO79rQ,1870
12
- cosmotech/coal/azure/adx/runner.py,sha256=FDrVaf-KDqHunPIQ7nUt57_-9pLPWXleNREgdHn9qeI,8333
13
- cosmotech/coal/azure/adx/store.py,sha256=IFQIi4tCmxL0NyFJhyJnUqbTt3Vwgb77900xcKprGzY,9096
14
- cosmotech/coal/azure/adx/tables.py,sha256=hwmiQI8OccNf3Npf8jd2wOF5hL8I2s4IkgTx8jqtSCU,4292
11
+ cosmotech/coal/azure/adx/runner.py,sha256=IXgciGxHjSqkiqm23CuyUk3RgCUgd1G-e0eMyohjygM,8066
12
+ cosmotech/coal/azure/adx/store.py,sha256=7fxG3yVxQ9_6sLaSU1Z4cxmsMcSO58Ni371FD2m_4LA,6214
13
+ cosmotech/coal/azure/adx/tables.py,sha256=y6GBrjXAKAOn7q4Ekf-hV6K5P3ejdV2L57FbFDvp22c,4287
15
14
  cosmotech/coal/azure/adx/utils.py,sha256=nOq4WLTMDs95da-jsWJVYcv2MRcwUMvARv5Z0ivIVUo,2205
16
- cosmotech/coal/cosmotech_api/__init__.py,sha256=sXA-WNRBWaXZdTwOR0-sD_HOkYvDaZYoHrzBz_o5-d0,1178
17
- cosmotech/coal/cosmotech_api/connection.py,sha256=dMI0vSv7ic1hEgtsIN92t008QI0bQrq8kmbMgqg3EF4,3953
18
- cosmotech/coal/cosmotech_api/parameters.py,sha256=2_ynoXfyK2pUtuZsYEamOKwfs0N6Kw2s-YE4zuPKkgM,1892
19
- cosmotech/coal/cosmotech_api/run.py,sha256=6Fnd3aUbWg4S-KqVG0abRfnorlNzGHY89AkheKNHprs,1001
20
- cosmotech/coal/cosmotech_api/run_data.py,sha256=W_egpjnic8aZ4R25JzQIEzMDOALm1tpTFnnvOfa17-8,6375
21
- cosmotech/coal/cosmotech_api/run_template.py,sha256=UJAa3WpxQqiBb4OokOHsGeHIEQzAcphVMXv0eMgGfSA,4359
22
- cosmotech/coal/cosmotech_api/twin_data_layer.py,sha256=haJPx0hE84Q5pwEZlXlnrB-3ugMJ9S66Iix3D19WxUs,18104
23
- cosmotech/coal/cosmotech_api/workspace.py,sha256=0qPf8h23z57AUVxRoSxD9lW_dSvfsq_pawcqFYaaaok,5488
24
- cosmotech/coal/cosmotech_api/dataset/__init__.py,sha256=UqRrDC2AFWImJu-wTAIzj7hIhc826n_57kTtBzCDWp0,972
25
- cosmotech/coal/cosmotech_api/dataset/converters.py,sha256=uTqaEzXMvIH7rTBqQA5if2GML32nsiohEP3eFIVPmLc,6151
26
- cosmotech/coal/cosmotech_api/dataset/utils.py,sha256=Pb6Lh3WNZHvbM8P6RdYjVQY8Ymfcxp35YmbBX_KKdWM,4438
27
- cosmotech/coal/cosmotech_api/dataset/download/__init__.py,sha256=kzIe46X9cmdg83y4JBjqAasY624BTXNL1yCADQkoyR4,886
28
- cosmotech/coal/cosmotech_api/dataset/download/adt.py,sha256=PwfkSYTCizkE81Huh5J8BShOXQnrjUvebRsgT-nYZpw,4731
29
- cosmotech/coal/cosmotech_api/dataset/download/common.py,sha256=KuVEzOA_abAwtwbcdry8zmqXAuQvy-NGeg1Ot9_MMuU,5665
30
- cosmotech/coal/cosmotech_api/dataset/download/file.py,sha256=X4ZNVjzl9ZC8J02CjRQeptZT6FPYBuxPsw-XPssP8YE,8914
31
- cosmotech/coal/cosmotech_api/dataset/download/twingraph.py,sha256=iHDvsm7odQQ5gyT7a3LcvcdNA6nHTIaWJUCFb9ryjr0,7373
32
- cosmotech/coal/cosmotech_api/runner/__init__.py,sha256=hEfmv6nF0hxSWMOnWZF0Fy7PJtHexVf-_cXPsxoY7Ho,1081
33
- cosmotech/coal/cosmotech_api/runner/data.py,sha256=BtCRi_c_3aGUSBjaPt5MOajWO6WFsJdBrIrx4GMt1AE,1346
34
- cosmotech/coal/cosmotech_api/runner/datasets.py,sha256=ojDkHGrobsygLr2CRYROetqm9zeWv1S1Bc_4Az4z874,12271
35
- cosmotech/coal/cosmotech_api/runner/download.py,sha256=YZRmhsX9TVjvhtgFhY4uvsJ3If0G43OiiERkCdoOKHE,5655
36
- cosmotech/coal/cosmotech_api/runner/metadata.py,sha256=eZHtlVMqsQ6WO9F-mDparjIy7tFl7kU-OqguKaI44jE,1432
37
- cosmotech/coal/cosmotech_api/runner/parameters.py,sha256=mKRRBPZOD_tPOTyxUAL2o9nRy_KZ5Z687MkL2uhqvIc,4651
15
+ cosmotech/coal/cosmotech_api/__init__.py,sha256=YlIyLwoqd4CveVnfrA9UDunlhO9HSvrBAfUwzcBay5M,528
16
+ cosmotech/coal/cosmotech_api/apis/__init__.py,sha256=RDe2FXBCrgUoSAONfpf_Udyc3M3w2y9K9mjFzUX_SMY,869
17
+ cosmotech/coal/cosmotech_api/apis/dataset.py,sha256=Ymo5NtIARiZtuzHOciRh4WzDaGMSo92KanxqptfVdzc,3946
18
+ cosmotech/coal/cosmotech_api/apis/meta.py,sha256=lCi5Jo7c_gXewob9B_NgvRyKAmPSJ_bVK7Z_pIVUyP8,1055
19
+ cosmotech/coal/cosmotech_api/apis/organization.py,sha256=JA55Bu4mJL5aE7UNtnPDWyjB6YZktSsnCzOvs9bW34g,1102
20
+ cosmotech/coal/cosmotech_api/apis/run.py,sha256=TOVD0E4QIR26rBZr3MjsWuSbYcKcuTVSJsq_zD4nzD0,1528
21
+ cosmotech/coal/cosmotech_api/apis/runner.py,sha256=ls-xuAfiCFa-2m7DgL_W_iB_y_y1q4JBt1eOOHSMdqE,2834
22
+ cosmotech/coal/cosmotech_api/apis/solution.py,sha256=rGsbtkVkhrt0vAano4HcIYQJFaXGSpWCHLku2TEpEHI,1077
23
+ cosmotech/coal/cosmotech_api/apis/workspace.py,sha256=YeXuODKJnwpdO2g2yaJzg9095VyzAuGlMV1JL1hNMsU,4330
24
+ cosmotech/coal/cosmotech_api/objects/__init__.py,sha256=6cWMAAyhCK478_Je9Opsu0NBv617aTQo3r9mhbV2wwo,556
25
+ cosmotech/coal/cosmotech_api/objects/connection.py,sha256=z0hd2uJNZWZdTyL7NlGuXt_HeZMenSODNJatNdmhk0o,5292
26
+ cosmotech/coal/cosmotech_api/objects/parameters.py,sha256=6T4yJMm_yvularJwMIeg8qoqBYdpWEdbQx0Q2dCieDw,4227
38
27
  cosmotech/coal/csm/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
39
28
  cosmotech/coal/csm/engine/__init__.py,sha256=EG3iArAy3OFGPmClsYXQmb77YYrEkeVC5uwPg_TbAYI,2120
40
29
  cosmotech/coal/postgresql/__init__.py,sha256=766LBZyWCQUI6rlzXDY_C9Yrxg551VX2L4lOuhDiS8w,786
41
- cosmotech/coal/postgresql/runner.py,sha256=lxUDrbwW-YhWgPiEBRMt_I5zR8DfDNJyvbRwzk5ixDU,3647
42
- cosmotech/coal/postgresql/store.py,sha256=uH1MooLfEDmNVO6KNQDrzUpC7YaLGzfdLt3KqBoD_Go,3643
30
+ cosmotech/coal/postgresql/runner.py,sha256=9UVVvX0FX6X-ZwaUGZiGLso410YdR28BF_bb45Urfww,4189
31
+ cosmotech/coal/postgresql/store.py,sha256=bJ-tUjJt8-MFi5GcmkE7sYZFzXDvKkwoo1rbXayvOzo,5371
32
+ cosmotech/coal/postgresql/utils.py,sha256=F-qHUxCS4BXOiOTuA5YVQdkOO10T-BMWTdE70L9UIfM,9642
43
33
  cosmotech/coal/singlestore/__init__.py,sha256=gXQaa8OAHnDcI7iHy-5zzGG7ZqNq-uOtw60qASjwu9I,651
44
34
  cosmotech/coal/singlestore/store.py,sha256=jF3gkg2qW_SrtbVKkBJ-SCtV6ukd2eLnFySOvMZFvMY,3577
45
35
  cosmotech/coal/store/__init__.py,sha256=kWGM9B-Y-M257OYbouJ-1JLES5qecQbGeUBbcgyrsEo,1307
@@ -48,52 +38,47 @@ cosmotech/coal/store/native_python.py,sha256=Mp2U7tkUjH1wT76U4WzekaW4H5nN2VHTrAK
48
38
  cosmotech/coal/store/pandas.py,sha256=A2eWPDsAqI2fTNmIpJa0Qzwrh3vMfvSHPLuoxQ77OPM,906
49
39
  cosmotech/coal/store/pyarrow.py,sha256=i7f7vSd5BtOnR4oleRHTG3ff0o8NaJr4c9fRb7zgYmE,814
50
40
  cosmotech/coal/store/store.py,sha256=a9wSThk7K8tD1r_BJG1Zl2d4pfo23TqKUH82_u9H-tk,3353
41
+ cosmotech/coal/store/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
42
+ cosmotech/coal/store/output/aws_channel.py,sha256=31m--qcnIyyCXpd7G4072tmDaHHx7RzLdVtl1XmaPMQ,2948
43
+ cosmotech/coal/store/output/az_storage_channel.py,sha256=zD8BsjuWsAr9BMq7sLYrlbMOoKDIl792_tqKJwegdsM,1425
44
+ cosmotech/coal/store/output/channel_interface.py,sha256=ojLDvpsBz2xOVhVxgeyQkNx0w-Rw66hcPesp24QQ5TI,670
45
+ cosmotech/coal/store/output/channel_spliter.py,sha256=HU0KssksIPI_cBruVy-jQmS2t8w74mHiy-kYrbCZdEs,2203
46
+ cosmotech/coal/store/output/postgres_channel.py,sha256=DXYVATnvIaCUtjTw3VrKc7EdbvjLOD69WotJESfUC30,1384
51
47
  cosmotech/coal/utils/__init__.py,sha256=N2mPhli0Wj4OesuRVbt2KrzL_pgMELs4fDNIQYb_r-g,830
52
- cosmotech/coal/utils/api.py,sha256=OhuATEWIih4N0t6LH6dsLZHTMkV-Ko7KAk8tyKDxYfo,2777
48
+ cosmotech/coal/utils/configuration.py,sha256=0awBiD0d2m0zgMBV4IAm47rTAgHaV9xska2UUa8kVF0,6533
49
+ cosmotech/coal/utils/decorator.py,sha256=p4yqkbln-YTppzeKvi3K4ZZ6AibVYGD6pnJQk0ynWbE,646
53
50
  cosmotech/coal/utils/logger.py,sha256=oYVj2BtDPji4e4J-KgaDyVcbKiewkq0xey_gQPm74Xc,506
54
- cosmotech/coal/utils/postgresql.py,sha256=6gxk3yfkZkaJQbIG-Zxcwo7J3PI29XPLn0VdQimGSH0,8562
55
51
  cosmotech/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
56
52
  cosmotech/csm_data/main.py,sha256=q20Gl1x7DV0gWStnWJUVnhUrvv-TPrUhv3e-5X3TDEM,2424
57
53
  cosmotech/csm_data/commands/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
58
54
  cosmotech/csm_data/commands/adx_send_data.py,sha256=wKt30RmuS1FdnhgW0mJZVXifnI_YLRLaj4w5DYlPl7s,2638
59
55
  cosmotech/csm_data/commands/adx_send_runnerdata.py,sha256=SHZ0RijCnYRrAdCgdbW_YTaN3joihtNtbLzZAICCzJ4,3713
60
56
  cosmotech/csm_data/commands/az_storage_upload.py,sha256=7qXhFxftfil72tG23HTvHiW708t4QkM4rymcFJSPCj4,2344
61
- cosmotech/csm_data/commands/s3_bucket_delete.py,sha256=0k56xKOPp7CRP-OGL5qqETgdCmcSnx6PjiGGE5ha1Bs,3026
62
- cosmotech/csm_data/commands/s3_bucket_download.py,sha256=vyEmKA2EUbb0glBXI1IXx-GKG6Fgoqys1hZONjb_qpI,3347
63
- cosmotech/csm_data/commands/s3_bucket_upload.py,sha256=eKIQd-RDwYKjGADYsoMYuT9-I6jYT2-16BH2rVEMbuM,3563
57
+ cosmotech/csm_data/commands/s3_bucket_delete.py,sha256=2Sx3sLkiQxTd9MNL85zUZCvNTi6bZKxusQmu8xG0EYY,3056
58
+ cosmotech/csm_data/commands/s3_bucket_download.py,sha256=X0IahwMTzPUelTLZvNKIKkV2NMK6W8zw_Qp2hjFSsaA,3367
59
+ cosmotech/csm_data/commands/s3_bucket_upload.py,sha256=MKRFoYS9-MMCdk_jVRDF-Kx3YitjlmqUxX6-kd2CyxQ,3599
64
60
  cosmotech/csm_data/commands/api/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
65
- cosmotech/csm_data/commands/api/api.py,sha256=rzxb7zX_CJCzrOUxIN75mwARmQWhCmha4ogh1r3JAmg,2479
66
- cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py,sha256=ZRcTICk_3DKyl3Gic-L6mqfWeT8hEcU5AvY6c8UQ8_E,4076
67
- cosmotech/csm_data/commands/api/rds_load_csv.py,sha256=9GMK1ViazuZIuwfuRj6SyhqMa39Oc0b0yRtX60_G0mw,2757
68
- cosmotech/csm_data/commands/api/rds_send_csv.py,sha256=_YeKCnKHEtxFluGRjRqZKdUMVn2bO3pAGBJQFYHUj7Y,2310
69
- cosmotech/csm_data/commands/api/rds_send_store.py,sha256=7_pgWtHfmFF-evuO7EOzFX6N7V5d2EsO-KG46B-F13Q,2328
70
- cosmotech/csm_data/commands/api/run_load_data.py,sha256=oRnoIj2EXPG8Q2fcvrzt69eX4LNCVCO612qejsOivBM,3725
71
- cosmotech/csm_data/commands/api/runtemplate_load_handler.py,sha256=qWS0I1fNN6VFx7lHJoOWCv30WGI2q32B9_Zyr5KpBSM,2275
72
- cosmotech/csm_data/commands/api/tdl_load_files.py,sha256=O4FsRNutyEvdDUBYElLxrzEhJcXVyO9hpOdiM6FEQ5E,2394
73
- cosmotech/csm_data/commands/api/tdl_send_files.py,sha256=QXToEHbZvEhdh88xHamXj9mPQo1Hm2UPxaMkgNEspaw,2474
74
- cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=Iz3Do-pBJBTDe2P5QwINRQCyeWGHVKjh_lI610WFi1g,2307
75
- cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=TlDMObWGhXgX1PqIeg5OX832Lps6q5fdC_2GG5Hx3q0,2191
61
+ cosmotech/csm_data/commands/api/api.py,sha256=v8qBRjI1he8o6hDXmjmptTUdO8F92sgwr9glszsy49o,1698
62
+ cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py,sha256=h_24m7sBrA4Dej7UyIsAzlpJi9mGIM5MBiUHxN8PT08,4235
63
+ cosmotech/csm_data/commands/api/run_load_data.py,sha256=XL_txDSMdNLePa2ScidF8CzUVTasn9SAtKZXE4Gmo8A,2576
64
+ cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=wnNvRY1D7GBE5lQz5KOQ9x5T_wxs-4wIfGPO4jiMcwQ,2103
65
+ cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=AC-Li-2tN5KAtIE5TnElRbSgUTNCa7uWkn90MlOozjg,2045
76
66
  cosmotech/csm_data/commands/store/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
77
67
  cosmotech/csm_data/commands/store/dump_to_azure.py,sha256=5nnK7LUATRTlfX8tmFRLBmkD15bgG37Lyn7fEuPJiNE,3313
78
68
  cosmotech/csm_data/commands/store/dump_to_postgresql.py,sha256=3ZfIwIfLXYeBb6SwBSV6-CiWdid8Udht4ILWeZNdLWM,3596
79
- cosmotech/csm_data/commands/store/dump_to_s3.py,sha256=TUOTtIsx-n8c2hHglQ8yr6diqpS-LYKF41a8RzW_qkc,5248
69
+ cosmotech/csm_data/commands/store/dump_to_s3.py,sha256=6EUwtkP8_k9Hlq65R9h-MYsH58e-UAYmSR95JKoZuvY,5342
80
70
  cosmotech/csm_data/commands/store/list_tables.py,sha256=epgVJAaP8QfSsEuL9n5S8smsfIxK-Cr0bLv88FAnkWw,1860
81
71
  cosmotech/csm_data/commands/store/load_csv_folder.py,sha256=gDKCxWIpXqhtRvlvRMM7SmyJ9_YHQ1UZHuA6NYdPTZA,1717
82
72
  cosmotech/csm_data/commands/store/load_from_singlestore.py,sha256=dWZ4jIKl8wsnOX_6qqtyQmkFQhLgjUAOZWab-sGzQ2g,3104
73
+ cosmotech/csm_data/commands/store/output.py,sha256=RkBQxZ6c-p6F3A0Ci7wW-ODHsrJJO8ivzQCumvjoqXI,1218
83
74
  cosmotech/csm_data/commands/store/reset.py,sha256=3B0E0m0nCy3vwai_D5sWWddMZWuWkWnaL1tvgR6L5Ow,1237
84
- cosmotech/csm_data/commands/store/store.py,sha256=_DA4QwQhfTs8bGmbLDHX_8cZZEv4FzeECuTE7d59iUc,1746
75
+ cosmotech/csm_data/commands/store/store.py,sha256=oQhTIF1WKKGKdFp1rNm1vuTgblmh27r_jQpPhNFmesE,1663
85
76
  cosmotech/csm_data/utils/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
86
77
  cosmotech/csm_data/utils/click.py,sha256=S_85cbKh3R86-FZVjTK7IXZnmp4ETjKo6K8gbK3HCgs,848
87
78
  cosmotech/csm_data/utils/decorators.py,sha256=dTcPRTYqY18mc8Ql4Qwd3gU7BxbNxfeKskQdVIsE3-g,2504
88
79
  cosmotech/orchestrator_plugins/csm-data/__init__.py,sha256=UPzNgaNBMoBaxSlnofxl988dAswMp_tv8DzaR_IxkiE,519
89
80
  cosmotech/orchestrator_plugins/csm-data/templates/api/postgres_send_runner_metadata.json,sha256=alXzvOa985PvN5nf3uoTsFXgY29EJWeK94CcBjoLFUk,1085
90
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json,sha256=nM2hVFqE6Mja6u6becVStJTD8POcu76Ys0oW-mU6-TU,819
91
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json,sha256=Z8uh4-C1TNVjGxUkuGlHCoSqRPNL4DVn-wVMZXSAOto,790
92
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json,sha256=AutPCsUfK0gHGjojDZQdQ0pgHsfC6uLtUtQ80eSHD6o,772
93
81
  cosmotech/orchestrator_plugins/csm-data/templates/api/run_load_data.json,sha256=_ztHPGu5UbbdtQHUCaOuKqwtjB0DrZSZwcUePoOgS40,894
94
- cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json,sha256=aL-4GIUi2dV9jh3vuRI0BkAQJdHzfbf53Sxd91Lfj_U,761
95
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json,sha256=LxrwHU-UxtP0GB7FpNUw2o_HIM6VK2PBH7P928SFk_8,889
96
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json,sha256=A9Rt3wiiBly01ugEXGo1Qejp7UV3t8P13tLNLseqgZU,751
97
82
  cosmotech/orchestrator_plugins/csm-data/templates/api/try_api_connection.json,sha256=ApkzR0HQc0J6sxxTh91_DHPRI2cCtx_-qjpYFKGkGVw,215
98
83
  cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_load_file.json,sha256=yX7yUrq7epQTp0PaQElFC-elof8W6jaMtek6uQ0V6Og,1142
99
84
  cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_send_file.json,sha256=7pyYOUPg_lvD4kw3sqd_zBonru7g-DVatQmu4EJw0Ps,1120
@@ -119,7 +104,7 @@ cosmotech/translation/coal/en-US/coal/common/progress.yml,sha256=G_BTYgAYVD_JqU3
119
104
  cosmotech/translation/coal/en-US/coal/common/timing.yml,sha256=YtxIT8EKZDWD4hJ0tJOwkReUsrOY67Wq0MFkEZjzANE,274
120
105
  cosmotech/translation/coal/en-US/coal/common/validation.yml,sha256=OAHeXrvMo2_S-D0z-ESIS--U7bU0z1i0wta4ibaqk3c,528
121
106
  cosmotech/translation/coal/en-US/coal/cosmotech_api/connection.yml,sha256=2Om0DZmOI8u0hzlWWhWnGx-o0mzMk_vgE3hGRBNaWWU,518
122
- cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml,sha256=HogZqb_Mhu4yyJmklBzb44Uk0cDnM5ssb8NZMWQMw6c,84
107
+ cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml,sha256=NGD5bbbS037QosTmHxXOiUJ_Zl3gyIr1iR6hCaSlolY,380
123
108
  cosmotech/translation/coal/en-US/coal/cosmotech_api/run_template.yml,sha256=JIQ0S-9_YhchAPVFkcMpJTVUBlhPo4yS-nBoMysus5o,441
124
109
  cosmotech/translation/coal/en-US/coal/cosmotech_api/runner.yml,sha256=fSoFiJFJcowXwBVgfAkZKL8j3iKzyVDLWm48cFpiYMA,804
125
110
  cosmotech/translation/coal/en-US/coal/cosmotech_api/solution.yml,sha256=dKOTcv8bBYl3ZxyvKsQ8msO--PyLTfEOodb4rHl2bq4,216
@@ -128,20 +113,18 @@ cosmotech/translation/coal/en-US/coal/services/adx.yml,sha256=9yxR0qYKyA5mqgW-pX
128
113
  cosmotech/translation/coal/en-US/coal/services/api.yml,sha256=HPyKJaZrS41Qo9upRvPsLdn29QgS1xw3S-bgipHeO4s,342
129
114
  cosmotech/translation/coal/en-US/coal/services/azure_storage.yml,sha256=p0alxlJA_-NH9MxKXRKGgiLbBvHuQdCROhJKiqvuX0c,595
130
115
  cosmotech/translation/coal/en-US/coal/services/database.yml,sha256=S1pAu4OrVd9nAVb3oYSAurzA4cgD43UDlwK4qWhy2g4,860
131
- cosmotech/translation/coal/en-US/coal/services/dataset.yml,sha256=J-lP802jTb7J6CYBQ5iUXSFF44yd7P230NosQz9tnzk,3752
116
+ cosmotech/translation/coal/en-US/coal/services/dataset.yml,sha256=aaFBSwkoakMbyjMVeljp7RfC-h4wEvclZqMOBD8ynSo,2981
132
117
  cosmotech/translation/coal/en-US/coal/services/postgresql.yml,sha256=tRGNTTayyoxEHVyM-nzudEFdQ5HUeRIHPWIuvFmJGfM,1774
133
118
  cosmotech/translation/coal/en-US/coal/services/s3.yml,sha256=362wt7SdOu-MyMj9MgjQNTNWzIq-RJ9rUqc2VKljGFE,482
119
+ cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml,sha256=4pVJPAXYeIq9Gpmg_0VpSrB17AsGWA9hvk8Qi8-kZvw,85
120
+ cosmotech/translation/coal/en-US/coal/store/output/split.yml,sha256=h89v89sQsK5pGDJP5EE_KNbQbaS8003VPZNIgU2dqWw,265
121
+ cosmotech/translation/coal/en-US/coal/utils/configuration.yml,sha256=q3kEzPTP81SfsdrhAZa2JusHAjbdYqjs0DcvjhewQ-4,96
134
122
  cosmotech/translation/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
135
123
  cosmotech/translation/csm_data/en-US/csm_data/commands/main.yml,sha256=3XbXR5_Uo7PxJOWzm7XRMvPVxK-fzOL4FxWWY7rX3eg,396
136
124
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/api.yml,sha256=eMT4s1H9LuNUh_sAlnEtg7Co4ie3bFfB1vkwnLW28dQ,340
137
125
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml,sha256=avckcrp2aYz7rzGCKEdL81hwHDVbcUn2csQ_krKLcaM,844
138
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml,sha256=jaCmdHY0fJp9Sh2PcOizMyPIdvZU2iTIoSLNdfuLCXE,668
139
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml,sha256=0I9MAGgdCmNHGN3sdRVSWzEobAABFsb9qVG_l0Vk7Sg,580
140
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml,sha256=9vRnrqYYzSYMPNzR-Q7Gw5G7LOWlEd5GWepSyzg49i4,574
141
126
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/run_load_data.yml,sha256=k5J3HTA8vCmI_7CFHvMP51EAbmefubuoEhXZf2V3XDo,765
142
127
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/runtemplate_load_handler.yml,sha256=9jvMa6fb99CPzTMx7nnuoNNzJw7_7rapee7P80Plxjs,353
143
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml,sha256=ko4Nw96w6vBhE-68swd9h4edWgXCWIfMGqYc8Fvh2LM,569
144
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml,sha256=13eacNdU6i92XemHyx-bzIdXbQUIs6eg2YiT3SxivyM,836
145
128
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_load_file.yml,sha256=cnR_0PMkbrj8GROgvIUyeLXiB90IBbkcTvM9WJDEM0k,511
146
129
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_send_file.yml,sha256=yvsD364GND1jLe_SJxuNXjDgGZFGlJlC5xUTRqV0fr0,632
147
130
  cosmotech/translation/csm_data/en-US/csm_data/commands/storage/adx_send_data.yml,sha256=oNQMHhHMRTl7EfFLuEy5agctzgwGHsyKzAu6hlSUyU4,448
@@ -157,13 +140,14 @@ cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_s3.yml,sha2
157
140
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/list_tables.yml,sha256=-x8zwCjAHILFaJv7VLPskipSoQTwmI7jUsxlx6_OKlc,195
158
141
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_csv_folder.yml,sha256=9HjW9tOq5o31jmADp_QIync7G74c6XGiPqofXh8pvv8,228
159
142
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_from_singlestore.yml,sha256=_OUXlxC_wi-GPgHU7EVdU9g0WIdblVpEA4WIzw4aQcQ,824
143
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml,sha256=odkR9pzeel5l0wW_F4kx4HhNpw_TzEvok-OKyh25ZkQ,232
160
144
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/reset.yml,sha256=JMspZ25Q2HDpH8MYqUaQcveUnCgdH21RmfTCrPf2gW8,139
161
145
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/store.yml,sha256=N1Q8483gqJADaCe30S1M3Rj0tMJiuQiJH70-VK2x2m4,134
162
146
  cosmotech/translation/csm_data/en-US/csm_data/commons/decorators.yml,sha256=Iu59NWMfYlZZf9uUhOiLkIEGa4GY5p0nZ6vG06Xvu7k,51
163
147
  cosmotech/translation/csm_data/en-US/csm_data/commons/version.yml,sha256=7jtCV3O1S6pGjiJa63XpgPDTafjfBS0xmEVRpYNvfDg,86
164
- cosmotech_acceleration_library-1.0.1.dist-info/licenses/LICENSE,sha256=JXKHOQtyObmafNbQlfPYc4HkKjU9FzAP27b2qRTXNM8,1195
165
- cosmotech_acceleration_library-1.0.1.dist-info/METADATA,sha256=94bKFXrsdSrP6ZBfdcJeAayVOvWpZBE1OIOw072fObc,9360
166
- cosmotech_acceleration_library-1.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
167
- cosmotech_acceleration_library-1.0.1.dist-info/entry_points.txt,sha256=HWRqJurKuBUgqFe4jmjIAQrs768Nbb8ZTdRDLbuKM5Q,58
168
- cosmotech_acceleration_library-1.0.1.dist-info/top_level.txt,sha256=t2pzb8mpMUfHTa9l2SjWP0rRB8XVRjBdQK5nLx9XDDo,10
169
- cosmotech_acceleration_library-1.0.1.dist-info/RECORD,,
148
+ cosmotech_acceleration_library-2.0.0.dist-info/licenses/LICENSE,sha256=JXKHOQtyObmafNbQlfPYc4HkKjU9FzAP27b2qRTXNM8,1195
149
+ cosmotech_acceleration_library-2.0.0.dist-info/METADATA,sha256=LVDVBJn0B1rE1z4rKbm3wUG5UKKt3Qj6Jkm2lGcLDxk,9334
150
+ cosmotech_acceleration_library-2.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
151
+ cosmotech_acceleration_library-2.0.0.dist-info/entry_points.txt,sha256=HWRqJurKuBUgqFe4jmjIAQrs768Nbb8ZTdRDLbuKM5Q,58
152
+ cosmotech_acceleration_library-2.0.0.dist-info/top_level.txt,sha256=t2pzb8mpMUfHTa9l2SjWP0rRB8XVRjBdQK5nLx9XDDo,10
153
+ cosmotech_acceleration_library-2.0.0.dist-info/RECORD,,
@@ -1,72 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
- import azure.functions as func
8
- from cosmotech.coal.cosmotech_api.runner.download import download_runner_data
9
- from cosmotech_api.api.runner_api import RunnerApi
10
-
11
- import json
12
- import http
13
- import traceback
14
-
15
-
16
- def generate_main(apply_update, parallel=True):
17
- def main(req: func.HttpRequest) -> func.HttpResponse:
18
- try:
19
- runner_id = req.params.get("scenario-id") # Keep parameter name for backward compatibility
20
- organization_id = req.params.get("organization-id")
21
- workspace_id = req.params.get("workspace-id")
22
- access_token: str = req.headers.get("authorization", None)
23
- if access_token:
24
- access_token = access_token.split(" ")[1]
25
-
26
- if runner_id is None or organization_id is None or workspace_id is None:
27
- return func.HttpResponse(
28
- body=f"Invalid request: organization-id={organization_id}, workspace-id={workspace_id}, scenario-id={runner_id}",
29
- status_code=http.HTTPStatus.BAD_REQUEST,
30
- )
31
-
32
- # Get runner data
33
- result = download_runner_data(
34
- organization_id=organization_id,
35
- workspace_id=workspace_id,
36
- runner_id=runner_id,
37
- parameter_folder=None, # We don't need to save to files
38
- read_files=True,
39
- parallel=parallel,
40
- write_json=False,
41
- write_csv=False,
42
- fetch_dataset=True,
43
- )
44
-
45
- content = {
46
- "datasets": result["datasets"],
47
- "parameters": result["parameters"],
48
- }
49
-
50
- runner_data = result["runner_data"]
51
-
52
- updated_content = apply_update(
53
- content=content, scenario_data=runner_data
54
- ) # Keep parameter name for backward compatibility
55
-
56
- return func.HttpResponse(
57
- body=json.dumps(updated_content),
58
- headers={"Content-Type": "application/json"},
59
- )
60
- except Exception as e:
61
- response = {
62
- "error": getattr(e, "message", str(e)),
63
- "type": type(e).__name__,
64
- "trace": traceback.format_exc(),
65
- }
66
- return func.HttpResponse(
67
- status_code=http.HTTPStatus.INTERNAL_SERVER_ERROR,
68
- body=json.dumps(response),
69
- headers={"Content-Type": "application/json"},
70
- )
71
-
72
- return main
@@ -1,96 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
- import os
8
- import pathlib
9
- import ssl
10
-
11
- import cosmotech_api
12
-
13
- from cosmotech.coal.utils.logger import LOGGER
14
- from cosmotech.orchestrator.utils.translate import T
15
-
16
- api_env_keys = {"CSM_API_KEY", "CSM_API_URL"}
17
- azure_env_keys = {
18
- "AZURE_CLIENT_ID",
19
- "AZURE_CLIENT_SECRET",
20
- "AZURE_TENANT_ID",
21
- "CSM_API_URL",
22
- "CSM_API_SCOPE",
23
- }
24
- keycloak_env_keys = {
25
- "IDP_TENANT_ID",
26
- "IDP_CLIENT_ID",
27
- "IDP_CLIENT_SECRET",
28
- "IDP_BASE_URL",
29
- "CSM_API_URL",
30
- }
31
-
32
-
33
- def get_api_client() -> (cosmotech_api.ApiClient, str):
34
- existing_keys = set(os.environ.keys())
35
- missing_azure_keys = azure_env_keys - existing_keys
36
- missing_api_keys = api_env_keys - existing_keys
37
- missing_keycloak_keys = keycloak_env_keys - existing_keys
38
- if all((missing_api_keys, missing_azure_keys, missing_keycloak_keys)):
39
- LOGGER.error(T("coal.common.errors.no_env_vars"))
40
- LOGGER.error(T("coal.cosmotech_api.connection.existing_sets"))
41
- LOGGER.error(T("coal.cosmotech_api.connection.azure_connection").format(keys=", ".join(azure_env_keys)))
42
- LOGGER.error(T("coal.cosmotech_api.connection.api_key_connection").format(keys=", ".join(api_env_keys)))
43
- LOGGER.error(T("coal.cosmotech_api.connection.keycloak_connection").format(keys=", ".join(keycloak_env_keys)))
44
- raise EnvironmentError(T("coal.common.errors.no_env_vars"))
45
-
46
- if not missing_keycloak_keys:
47
- LOGGER.info(T("coal.cosmotech_api.connection.found_keycloak"))
48
- from keycloak import KeycloakOpenID
49
-
50
- server_url = os.environ.get("IDP_BASE_URL")
51
- if server_url[-1] != "/":
52
- server_url = server_url + "/"
53
- keycloack_parameters = dict(
54
- server_url=server_url,
55
- client_id=os.environ.get("IDP_CLIENT_ID"),
56
- realm_name=os.environ.get("IDP_TENANT_ID"),
57
- client_secret_key=os.environ.get("IDP_CLIENT_SECRET"),
58
- )
59
- if (ca_cert_path := os.environ.get("IDP_CA_CERT")) and pathlib.Path(ca_cert_path).exists():
60
- LOGGER.info(T("coal.cosmotech_api.connection.found_cert_authority"))
61
- keycloack_parameters["verify"] = ca_cert_path
62
- keycloak_openid = KeycloakOpenID(**keycloack_parameters)
63
-
64
- access_token = keycloak_openid.token(grant_type="client_credentials")
65
-
66
- configuration = cosmotech_api.Configuration(
67
- host=os.environ.get("CSM_API_URL"),
68
- access_token=access_token["access_token"],
69
- )
70
- return cosmotech_api.ApiClient(configuration), "Keycloak Connection"
71
-
72
- if not missing_api_keys:
73
- LOGGER.info(T("coal.cosmotech_api.connection.found_api_key"))
74
- configuration = cosmotech_api.Configuration(
75
- host=os.environ.get("CSM_API_URL"),
76
- )
77
- return (
78
- cosmotech_api.ApiClient(
79
- configuration,
80
- os.environ.get("CSM_API_KEY_HEADER", "X-CSM-API-KEY"),
81
- os.environ.get("CSM_API_KEY"),
82
- ),
83
- "Cosmo Tech API Key",
84
- )
85
-
86
- if not missing_azure_keys:
87
- LOGGER.info(T("coal.cosmotech_api.connection.found_azure"))
88
- from azure.identity import EnvironmentCredential
89
-
90
- credentials = EnvironmentCredential()
91
- token = credentials.get_token(os.environ.get("CSM_API_SCOPE"))
92
-
93
- configuration = cosmotech_api.Configuration(host=os.environ.get("CSM_API_URL"), access_token=token.token)
94
- return cosmotech_api.ApiClient(configuration), "Azure Entra Connection"
95
-
96
- raise EnvironmentError(T("coal.common.errors.no_valid_connection"))
@@ -1,26 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- # Re-export all download functions from download submodule
9
- from cosmotech.coal.cosmotech_api.dataset.download import (
10
- download_adt_dataset,
11
- download_twingraph_dataset,
12
- download_legacy_twingraph_dataset,
13
- download_file_dataset,
14
- download_dataset_by_id,
15
- )
16
-
17
- from cosmotech.coal.cosmotech_api.dataset.converters import (
18
- convert_dataset_to_files,
19
- convert_graph_dataset_to_files,
20
- convert_file_dataset_to_files,
21
- )
22
-
23
- from cosmotech.coal.cosmotech_api.dataset.utils import (
24
- get_content_from_twin_graph_data,
25
- sheet_to_header,
26
- )
@@ -1,164 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- import csv
9
- import json
10
- import os
11
- import tempfile
12
- from pathlib import Path
13
- from typing import Dict, List, Any, Optional, Union
14
-
15
- from cosmotech.coal.utils.logger import LOGGER
16
- from cosmotech.orchestrator.utils.translate import T
17
- from cosmotech.coal.cosmotech_api.dataset.utils import sheet_to_header
18
-
19
-
20
- def convert_dataset_to_files(dataset_info: Dict[str, Any], target_folder: Optional[Union[str, Path]] = None) -> Path:
21
- """
22
- Convert dataset info to files.
23
-
24
- Args:
25
- dataset_info: Dataset info dict with type, content, name
26
- target_folder: Optional folder to save files (if None, uses temp dir)
27
-
28
- Returns:
29
- Path to folder containing files
30
- """
31
- dataset_type = dataset_info["type"]
32
- content = dataset_info["content"]
33
- name = dataset_info["name"]
34
-
35
- LOGGER.info(T("coal.services.dataset.converting_to_files").format(dataset_type=dataset_type, dataset_name=name))
36
-
37
- if target_folder is None:
38
- target_folder = Path(tempfile.mkdtemp())
39
- LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
40
- else:
41
- target_folder = Path(target_folder)
42
- target_folder.mkdir(parents=True, exist_ok=True)
43
- LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
44
-
45
- if dataset_type in ["adt", "twincache"]:
46
- return convert_graph_dataset_to_files(content, target_folder)
47
- else:
48
- return convert_file_dataset_to_files(content, target_folder, dataset_type)
49
-
50
-
51
- def convert_graph_dataset_to_files(
52
- content: Dict[str, List[Dict]], target_folder: Optional[Union[str, Path]] = None
53
- ) -> Path:
54
- """
55
- Convert graph dataset content to CSV files.
56
-
57
- Args:
58
- content: Dictionary mapping entity types to lists of entities
59
- target_folder: Folder to save files (if None, uses temp dir)
60
-
61
- Returns:
62
- Path to folder containing files
63
- """
64
- if target_folder is None:
65
- target_folder = Path(tempfile.mkdtemp())
66
- LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
67
- else:
68
- target_folder = Path(target_folder)
69
- target_folder.mkdir(parents=True, exist_ok=True)
70
- LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
71
- file_count = 0
72
-
73
- LOGGER.info(
74
- T("coal.services.dataset.converting_graph_data").format(entity_types=len(content), folder=target_folder)
75
- )
76
-
77
- for entity_type, entities in content.items():
78
- if not entities:
79
- LOGGER.debug(T("coal.services.dataset.skipping_empty_entity").format(entity_type=entity_type))
80
- continue
81
-
82
- file_path = target_folder / f"{entity_type}.csv"
83
- LOGGER.debug(T("coal.services.dataset.writing_csv").format(file_name=file_path.name, count=len(entities)))
84
-
85
- fieldnames = sheet_to_header(entities)
86
-
87
- with open(file_path, "w", newline="") as file:
88
- writer = csv.DictWriter(file, fieldnames=fieldnames, dialect="unix", quoting=csv.QUOTE_MINIMAL)
89
- writer.writeheader()
90
-
91
- for entity in entities:
92
- # Convert values to strings and handle boolean values
93
- row = {
94
- k: str(v).replace("'", '"').replace("True", "true").replace("False", "false")
95
- for k, v in entity.items()
96
- }
97
- writer.writerow(row)
98
-
99
- file_count += 1
100
- LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
101
-
102
- LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
103
-
104
- return target_folder
105
-
106
-
107
- def convert_file_dataset_to_files(
108
- content: Dict[str, Any],
109
- target_folder: Optional[Union[str, Path]] = None,
110
- file_type: str = "",
111
- ) -> Path:
112
- """
113
- Convert file dataset content to files.
114
-
115
- Args:
116
- content: Dictionary mapping file names to content
117
- target_folder: Folder to save files (if None, uses temp dir)
118
- file_type: Type of file (csv, json, etc.)
119
-
120
- Returns:
121
- Path to folder containing files
122
- """
123
- if target_folder is None:
124
- target_folder = Path(tempfile.mkdtemp())
125
- LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
126
- else:
127
- target_folder = Path(target_folder)
128
- target_folder.mkdir(parents=True, exist_ok=True)
129
- LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
130
- file_count = 0
131
-
132
- LOGGER.info(
133
- T("coal.services.dataset.converting_file_data").format(
134
- file_count=len(content), file_type=file_type, folder=target_folder
135
- )
136
- )
137
-
138
- for file_name, file_content in content.items():
139
- file_path = target_folder / file_name
140
-
141
- # Ensure parent directories exist
142
- file_path.parent.mkdir(parents=True, exist_ok=True)
143
-
144
- LOGGER.debug(T("coal.services.dataset.writing_file").format(file_name=file_path.name, file_type=file_type))
145
-
146
- if isinstance(file_content, str):
147
- # Text content
148
- with open(file_path, "w") as file:
149
- file.write(file_content)
150
- elif isinstance(file_content, dict) or isinstance(file_content, list):
151
- # JSON content
152
- with open(file_path, "w") as file:
153
- json.dump(file_content, file, indent=2)
154
- else:
155
- # Other content types
156
- with open(file_path, "w") as file:
157
- file.write(str(file_content))
158
-
159
- file_count += 1
160
- LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
161
-
162
- LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
163
-
164
- return target_folder
@@ -1,19 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- """
9
- Dataset download submodules.
10
- """
11
-
12
- # Re-export all download functions
13
- from cosmotech.coal.cosmotech_api.dataset.download.adt import download_adt_dataset
14
- from cosmotech.coal.cosmotech_api.dataset.download.twingraph import (
15
- download_twingraph_dataset,
16
- download_legacy_twingraph_dataset,
17
- )
18
- from cosmotech.coal.cosmotech_api.dataset.download.file import download_file_dataset
19
- from cosmotech.coal.cosmotech_api.dataset.download.common import download_dataset_by_id