cosmotech-acceleration-library 1.1.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. cosmotech/coal/__init__.py +1 -1
  2. cosmotech/coal/aws/__init__.py +1 -9
  3. cosmotech/coal/aws/s3.py +181 -214
  4. cosmotech/coal/azure/adx/auth.py +2 -2
  5. cosmotech/coal/azure/adx/runner.py +13 -14
  6. cosmotech/coal/azure/adx/store.py +5 -86
  7. cosmotech/coal/azure/adx/tables.py +2 -2
  8. cosmotech/coal/azure/blob.py +6 -6
  9. cosmotech/coal/azure/storage.py +3 -3
  10. cosmotech/coal/cosmotech_api/__init__.py +0 -28
  11. cosmotech/coal/cosmotech_api/apis/__init__.py +14 -0
  12. cosmotech/coal/cosmotech_api/apis/dataset.py +103 -0
  13. cosmotech/coal/cosmotech_api/apis/meta.py +25 -0
  14. cosmotech/coal/cosmotech_api/apis/organization.py +24 -0
  15. cosmotech/coal/cosmotech_api/apis/run.py +38 -0
  16. cosmotech/coal/cosmotech_api/apis/runner.py +71 -0
  17. cosmotech/coal/cosmotech_api/apis/solution.py +23 -0
  18. cosmotech/coal/cosmotech_api/apis/workspace.py +108 -0
  19. cosmotech/coal/cosmotech_api/objects/__init__.py +9 -0
  20. cosmotech/coal/cosmotech_api/objects/connection.py +125 -0
  21. cosmotech/coal/cosmotech_api/objects/parameters.py +127 -0
  22. cosmotech/coal/postgresql/runner.py +56 -36
  23. cosmotech/coal/postgresql/store.py +60 -14
  24. cosmotech/coal/postgresql/utils.py +254 -0
  25. cosmotech/coal/store/output/__init__.py +0 -0
  26. cosmotech/coal/store/output/aws_channel.py +73 -0
  27. cosmotech/coal/store/output/az_storage_channel.py +42 -0
  28. cosmotech/coal/store/output/channel_interface.py +23 -0
  29. cosmotech/coal/store/output/channel_spliter.py +55 -0
  30. cosmotech/coal/store/output/postgres_channel.py +40 -0
  31. cosmotech/coal/utils/configuration.py +169 -0
  32. cosmotech/coal/utils/decorator.py +4 -7
  33. cosmotech/csm_data/commands/api/api.py +6 -19
  34. cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +20 -16
  35. cosmotech/csm_data/commands/api/run_load_data.py +7 -46
  36. cosmotech/csm_data/commands/api/wsf_load_file.py +13 -16
  37. cosmotech/csm_data/commands/api/wsf_send_file.py +11 -14
  38. cosmotech/csm_data/commands/s3_bucket_delete.py +16 -15
  39. cosmotech/csm_data/commands/s3_bucket_download.py +16 -16
  40. cosmotech/csm_data/commands/s3_bucket_upload.py +16 -14
  41. cosmotech/csm_data/commands/store/dump_to_s3.py +18 -16
  42. cosmotech/csm_data/commands/store/output.py +35 -0
  43. cosmotech/csm_data/commands/store/store.py +3 -3
  44. cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml +8 -0
  45. cosmotech/translation/coal/en-US/coal/services/dataset.yml +4 -14
  46. cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml +1 -0
  47. cosmotech/translation/coal/en-US/coal/store/output/split.yml +6 -0
  48. cosmotech/translation/coal/en-US/coal/utils/configuration.yml +2 -0
  49. cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml +7 -0
  50. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/METADATA +5 -8
  51. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/RECORD +55 -73
  52. cosmotech/coal/azure/functions.py +0 -72
  53. cosmotech/coal/cosmotech_api/connection.py +0 -96
  54. cosmotech/coal/cosmotech_api/dataset/__init__.py +0 -26
  55. cosmotech/coal/cosmotech_api/dataset/converters.py +0 -164
  56. cosmotech/coal/cosmotech_api/dataset/download/__init__.py +0 -19
  57. cosmotech/coal/cosmotech_api/dataset/download/adt.py +0 -119
  58. cosmotech/coal/cosmotech_api/dataset/download/common.py +0 -140
  59. cosmotech/coal/cosmotech_api/dataset/download/file.py +0 -229
  60. cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +0 -185
  61. cosmotech/coal/cosmotech_api/dataset/upload.py +0 -41
  62. cosmotech/coal/cosmotech_api/dataset/utils.py +0 -132
  63. cosmotech/coal/cosmotech_api/parameters.py +0 -48
  64. cosmotech/coal/cosmotech_api/run.py +0 -25
  65. cosmotech/coal/cosmotech_api/run_data.py +0 -173
  66. cosmotech/coal/cosmotech_api/run_template.py +0 -108
  67. cosmotech/coal/cosmotech_api/runner/__init__.py +0 -28
  68. cosmotech/coal/cosmotech_api/runner/data.py +0 -38
  69. cosmotech/coal/cosmotech_api/runner/datasets.py +0 -416
  70. cosmotech/coal/cosmotech_api/runner/download.py +0 -135
  71. cosmotech/coal/cosmotech_api/runner/metadata.py +0 -42
  72. cosmotech/coal/cosmotech_api/runner/parameters.py +0 -157
  73. cosmotech/coal/cosmotech_api/twin_data_layer.py +0 -512
  74. cosmotech/coal/cosmotech_api/workspace.py +0 -127
  75. cosmotech/coal/utils/postgresql.py +0 -236
  76. cosmotech/coal/utils/semver.py +0 -6
  77. cosmotech/csm_data/commands/api/rds_load_csv.py +0 -90
  78. cosmotech/csm_data/commands/api/rds_send_csv.py +0 -74
  79. cosmotech/csm_data/commands/api/rds_send_store.py +0 -74
  80. cosmotech/csm_data/commands/api/runtemplate_load_handler.py +0 -66
  81. cosmotech/csm_data/commands/api/tdl_load_files.py +0 -76
  82. cosmotech/csm_data/commands/api/tdl_send_files.py +0 -82
  83. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +0 -27
  84. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +0 -27
  85. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +0 -27
  86. cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +0 -27
  87. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +0 -32
  88. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +0 -27
  89. cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +0 -2
  90. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml +0 -13
  91. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml +0 -12
  92. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml +0 -12
  93. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml +0 -14
  94. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml +0 -18
  95. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/WHEEL +0 -0
  96. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/entry_points.txt +0 -0
  97. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/licenses/LICENSE +0 -0
  98. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.0.0.dist-info}/top_level.txt +0 -0
@@ -1,46 +1,35 @@
1
- cosmotech/coal/__init__.py,sha256=fEmLxOs06MCBlBFoKsBUQ_ZE5aLJgmdNfzVP0Xx_Qds,436
2
- cosmotech/coal/aws/__init__.py,sha256=DtkCZ8Y7q5HIKui9YXCV1JXO8rRMitd0uieiW2InuXY,761
3
- cosmotech/coal/aws/s3.py,sha256=oq1Sl_6kk_u0Jai55p7XRfKYAK0I6y3va4IfeWNhpEE,7820
1
+ cosmotech/coal/__init__.py,sha256=5GlLyV1soguj1xNP_KdhUDoCa5hTAunpZAKbapYdsKk,436
2
+ cosmotech/coal/aws/__init__.py,sha256=qZZyxNCwGa1yPpIEc7CS6sVcv00PNcUnUOsKhsHBSdc,614
3
+ cosmotech/coal/aws/s3.py,sha256=5az9BtyUUBlfqFV3BG32V5-9yDpm-6q_O4YV79HcmMA,7522
4
4
  cosmotech/coal/azure/__init__.py,sha256=1SmwP1y7quGq9cWqiWxJy1XDjWM8sAc3f63ztBnYujQ,796
5
- cosmotech/coal/azure/blob.py,sha256=O6UwQ9AKns53yM5AqkxgZQX7k-I4dM6yRPL-3RzOm9s,4002
6
- cosmotech/coal/azure/functions.py,sha256=HPxKslag2ANeDyd5y5AVk-pQLZkUffGMGoIh-5C4dvU,2901
7
- cosmotech/coal/azure/storage.py,sha256=JocO3YJ4WnXouHja6Ipra0d06tcKu_ecoQfqNV8WTU8,2795
5
+ cosmotech/coal/azure/blob.py,sha256=C2i1JIC7Z4Iv_RLzbyFqbHao9JsJI9jMiBAo6Bix6Ns,4082
6
+ cosmotech/coal/azure/storage.py,sha256=TItajcil0Ht44kru44T9HIP2cizOvAS_H9uRdSgaBG0,2799
8
7
  cosmotech/coal/azure/adx/__init__.py,sha256=zRYssRscq30jJ1q1CPUK6S3Sq4EBqZ_uL0M1ju3vktw,1227
9
- cosmotech/coal/azure/adx/auth.py,sha256=BafFyLaHuiC3M0Mzp4ZcVAzzG1GUDsnOGn1Chf_vVMk,4755
8
+ cosmotech/coal/azure/adx/auth.py,sha256=IFNaB8eoc99_j3Ml6rueYfuFK36rze4_8LG7B0gYdIQ,4748
10
9
  cosmotech/coal/azure/adx/ingestion.py,sha256=--hOB5iUWe-RwXpFC5EaBDmWbJrIAtYula59wrUzCM8,11455
11
10
  cosmotech/coal/azure/adx/query.py,sha256=SHMeGX-TNBfQE_DV-J5lrHxqDXgftYiLCasDhgO79rQ,1870
12
- cosmotech/coal/azure/adx/runner.py,sha256=FDrVaf-KDqHunPIQ7nUt57_-9pLPWXleNREgdHn9qeI,8333
13
- cosmotech/coal/azure/adx/store.py,sha256=IFQIi4tCmxL0NyFJhyJnUqbTt3Vwgb77900xcKprGzY,9096
14
- cosmotech/coal/azure/adx/tables.py,sha256=hwmiQI8OccNf3Npf8jd2wOF5hL8I2s4IkgTx8jqtSCU,4292
11
+ cosmotech/coal/azure/adx/runner.py,sha256=IXgciGxHjSqkiqm23CuyUk3RgCUgd1G-e0eMyohjygM,8066
12
+ cosmotech/coal/azure/adx/store.py,sha256=7fxG3yVxQ9_6sLaSU1Z4cxmsMcSO58Ni371FD2m_4LA,6214
13
+ cosmotech/coal/azure/adx/tables.py,sha256=y6GBrjXAKAOn7q4Ekf-hV6K5P3ejdV2L57FbFDvp22c,4287
15
14
  cosmotech/coal/azure/adx/utils.py,sha256=nOq4WLTMDs95da-jsWJVYcv2MRcwUMvARv5Z0ivIVUo,2205
16
- cosmotech/coal/cosmotech_api/__init__.py,sha256=ufYjA_WF0CNpU63jGlPqtY1kWiLu6Qj7TNVHbRk8n0A,1320
17
- cosmotech/coal/cosmotech_api/connection.py,sha256=dMI0vSv7ic1hEgtsIN92t008QI0bQrq8kmbMgqg3EF4,3953
18
- cosmotech/coal/cosmotech_api/parameters.py,sha256=2_ynoXfyK2pUtuZsYEamOKwfs0N6Kw2s-YE4zuPKkgM,1892
19
- cosmotech/coal/cosmotech_api/run.py,sha256=6Fnd3aUbWg4S-KqVG0abRfnorlNzGHY89AkheKNHprs,1001
20
- cosmotech/coal/cosmotech_api/run_data.py,sha256=W_egpjnic8aZ4R25JzQIEzMDOALm1tpTFnnvOfa17-8,6375
21
- cosmotech/coal/cosmotech_api/run_template.py,sha256=UJAa3WpxQqiBb4OokOHsGeHIEQzAcphVMXv0eMgGfSA,4359
22
- cosmotech/coal/cosmotech_api/twin_data_layer.py,sha256=haJPx0hE84Q5pwEZlXlnrB-3ugMJ9S66Iix3D19WxUs,18104
23
- cosmotech/coal/cosmotech_api/workspace.py,sha256=0qPf8h23z57AUVxRoSxD9lW_dSvfsq_pawcqFYaaaok,5488
24
- cosmotech/coal/cosmotech_api/dataset/__init__.py,sha256=UqRrDC2AFWImJu-wTAIzj7hIhc826n_57kTtBzCDWp0,972
25
- cosmotech/coal/cosmotech_api/dataset/converters.py,sha256=uTqaEzXMvIH7rTBqQA5if2GML32nsiohEP3eFIVPmLc,6151
26
- cosmotech/coal/cosmotech_api/dataset/upload.py,sha256=Fmwx9zg5FPQcSd72YXeClB6ZCCi6IUlkAgGVs11Fvn4,1458
27
- cosmotech/coal/cosmotech_api/dataset/utils.py,sha256=Pb6Lh3WNZHvbM8P6RdYjVQY8Ymfcxp35YmbBX_KKdWM,4438
28
- cosmotech/coal/cosmotech_api/dataset/download/__init__.py,sha256=kzIe46X9cmdg83y4JBjqAasY624BTXNL1yCADQkoyR4,886
29
- cosmotech/coal/cosmotech_api/dataset/download/adt.py,sha256=PwfkSYTCizkE81Huh5J8BShOXQnrjUvebRsgT-nYZpw,4731
30
- cosmotech/coal/cosmotech_api/dataset/download/common.py,sha256=KuVEzOA_abAwtwbcdry8zmqXAuQvy-NGeg1Ot9_MMuU,5665
31
- cosmotech/coal/cosmotech_api/dataset/download/file.py,sha256=F_Y-O8W0VSXjDkmE9qEaM_fO7FRN_6jCND5dyCZpGz4,8410
32
- cosmotech/coal/cosmotech_api/dataset/download/twingraph.py,sha256=xumPNAGcB3t0ZMztvuUUnSmxoB9t-WUF3lqX8fEtWrY,7383
33
- cosmotech/coal/cosmotech_api/runner/__init__.py,sha256=hEfmv6nF0hxSWMOnWZF0Fy7PJtHexVf-_cXPsxoY7Ho,1081
34
- cosmotech/coal/cosmotech_api/runner/data.py,sha256=BtCRi_c_3aGUSBjaPt5MOajWO6WFsJdBrIrx4GMt1AE,1346
35
- cosmotech/coal/cosmotech_api/runner/datasets.py,sha256=_jrjYGepVfjL5iCNoqg29tKNwweupfbYCUwZkNE_gxo,13555
36
- cosmotech/coal/cosmotech_api/runner/download.py,sha256=G02poKQrKpYQdR_JjKesINjWeqZ95uUyl-TDulWwSN8,5254
37
- cosmotech/coal/cosmotech_api/runner/metadata.py,sha256=eZHtlVMqsQ6WO9F-mDparjIy7tFl7kU-OqguKaI44jE,1432
38
- cosmotech/coal/cosmotech_api/runner/parameters.py,sha256=mKRRBPZOD_tPOTyxUAL2o9nRy_KZ5Z687MkL2uhqvIc,4651
15
+ cosmotech/coal/cosmotech_api/__init__.py,sha256=YlIyLwoqd4CveVnfrA9UDunlhO9HSvrBAfUwzcBay5M,528
16
+ cosmotech/coal/cosmotech_api/apis/__init__.py,sha256=RDe2FXBCrgUoSAONfpf_Udyc3M3w2y9K9mjFzUX_SMY,869
17
+ cosmotech/coal/cosmotech_api/apis/dataset.py,sha256=Ymo5NtIARiZtuzHOciRh4WzDaGMSo92KanxqptfVdzc,3946
18
+ cosmotech/coal/cosmotech_api/apis/meta.py,sha256=lCi5Jo7c_gXewob9B_NgvRyKAmPSJ_bVK7Z_pIVUyP8,1055
19
+ cosmotech/coal/cosmotech_api/apis/organization.py,sha256=JA55Bu4mJL5aE7UNtnPDWyjB6YZktSsnCzOvs9bW34g,1102
20
+ cosmotech/coal/cosmotech_api/apis/run.py,sha256=TOVD0E4QIR26rBZr3MjsWuSbYcKcuTVSJsq_zD4nzD0,1528
21
+ cosmotech/coal/cosmotech_api/apis/runner.py,sha256=ls-xuAfiCFa-2m7DgL_W_iB_y_y1q4JBt1eOOHSMdqE,2834
22
+ cosmotech/coal/cosmotech_api/apis/solution.py,sha256=rGsbtkVkhrt0vAano4HcIYQJFaXGSpWCHLku2TEpEHI,1077
23
+ cosmotech/coal/cosmotech_api/apis/workspace.py,sha256=YeXuODKJnwpdO2g2yaJzg9095VyzAuGlMV1JL1hNMsU,4330
24
+ cosmotech/coal/cosmotech_api/objects/__init__.py,sha256=6cWMAAyhCK478_Je9Opsu0NBv617aTQo3r9mhbV2wwo,556
25
+ cosmotech/coal/cosmotech_api/objects/connection.py,sha256=z0hd2uJNZWZdTyL7NlGuXt_HeZMenSODNJatNdmhk0o,5292
26
+ cosmotech/coal/cosmotech_api/objects/parameters.py,sha256=6T4yJMm_yvularJwMIeg8qoqBYdpWEdbQx0Q2dCieDw,4227
39
27
  cosmotech/coal/csm/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
40
28
  cosmotech/coal/csm/engine/__init__.py,sha256=EG3iArAy3OFGPmClsYXQmb77YYrEkeVC5uwPg_TbAYI,2120
41
29
  cosmotech/coal/postgresql/__init__.py,sha256=766LBZyWCQUI6rlzXDY_C9Yrxg551VX2L4lOuhDiS8w,786
42
- cosmotech/coal/postgresql/runner.py,sha256=lxUDrbwW-YhWgPiEBRMt_I5zR8DfDNJyvbRwzk5ixDU,3647
43
- cosmotech/coal/postgresql/store.py,sha256=uH1MooLfEDmNVO6KNQDrzUpC7YaLGzfdLt3KqBoD_Go,3643
30
+ cosmotech/coal/postgresql/runner.py,sha256=9UVVvX0FX6X-ZwaUGZiGLso410YdR28BF_bb45Urfww,4189
31
+ cosmotech/coal/postgresql/store.py,sha256=bJ-tUjJt8-MFi5GcmkE7sYZFzXDvKkwoo1rbXayvOzo,5371
32
+ cosmotech/coal/postgresql/utils.py,sha256=F-qHUxCS4BXOiOTuA5YVQdkOO10T-BMWTdE70L9UIfM,9642
44
33
  cosmotech/coal/singlestore/__init__.py,sha256=gXQaa8OAHnDcI7iHy-5zzGG7ZqNq-uOtw60qASjwu9I,651
45
34
  cosmotech/coal/singlestore/store.py,sha256=jF3gkg2qW_SrtbVKkBJ-SCtV6ukd2eLnFySOvMZFvMY,3577
46
35
  cosmotech/coal/store/__init__.py,sha256=kWGM9B-Y-M257OYbouJ-1JLES5qecQbGeUBbcgyrsEo,1307
@@ -49,53 +38,47 @@ cosmotech/coal/store/native_python.py,sha256=Mp2U7tkUjH1wT76U4WzekaW4H5nN2VHTrAK
49
38
  cosmotech/coal/store/pandas.py,sha256=A2eWPDsAqI2fTNmIpJa0Qzwrh3vMfvSHPLuoxQ77OPM,906
50
39
  cosmotech/coal/store/pyarrow.py,sha256=i7f7vSd5BtOnR4oleRHTG3ff0o8NaJr4c9fRb7zgYmE,814
51
40
  cosmotech/coal/store/store.py,sha256=a9wSThk7K8tD1r_BJG1Zl2d4pfo23TqKUH82_u9H-tk,3353
41
+ cosmotech/coal/store/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
42
+ cosmotech/coal/store/output/aws_channel.py,sha256=31m--qcnIyyCXpd7G4072tmDaHHx7RzLdVtl1XmaPMQ,2948
43
+ cosmotech/coal/store/output/az_storage_channel.py,sha256=zD8BsjuWsAr9BMq7sLYrlbMOoKDIl792_tqKJwegdsM,1425
44
+ cosmotech/coal/store/output/channel_interface.py,sha256=ojLDvpsBz2xOVhVxgeyQkNx0w-Rw66hcPesp24QQ5TI,670
45
+ cosmotech/coal/store/output/channel_spliter.py,sha256=HU0KssksIPI_cBruVy-jQmS2t8w74mHiy-kYrbCZdEs,2203
46
+ cosmotech/coal/store/output/postgres_channel.py,sha256=DXYVATnvIaCUtjTw3VrKc7EdbvjLOD69WotJESfUC30,1384
52
47
  cosmotech/coal/utils/__init__.py,sha256=N2mPhli0Wj4OesuRVbt2KrzL_pgMELs4fDNIQYb_r-g,830
53
- cosmotech/coal/utils/decorator.py,sha256=j8MGkm4tgYVmBji0Veug4uq3Ed-f4OKe9XANs47_EG0,679
48
+ cosmotech/coal/utils/configuration.py,sha256=0awBiD0d2m0zgMBV4IAm47rTAgHaV9xska2UUa8kVF0,6533
49
+ cosmotech/coal/utils/decorator.py,sha256=p4yqkbln-YTppzeKvi3K4ZZ6AibVYGD6pnJQk0ynWbE,646
54
50
  cosmotech/coal/utils/logger.py,sha256=oYVj2BtDPji4e4J-KgaDyVcbKiewkq0xey_gQPm74Xc,506
55
- cosmotech/coal/utils/postgresql.py,sha256=6gxk3yfkZkaJQbIG-Zxcwo7J3PI29XPLn0VdQimGSH0,8562
56
- cosmotech/coal/utils/semver.py,sha256=VTjixXNBeUe6A02AK6olNxeqToOCWcKih6VZun09E7A,162
57
51
  cosmotech/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
58
52
  cosmotech/csm_data/main.py,sha256=q20Gl1x7DV0gWStnWJUVnhUrvv-TPrUhv3e-5X3TDEM,2424
59
53
  cosmotech/csm_data/commands/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
60
54
  cosmotech/csm_data/commands/adx_send_data.py,sha256=wKt30RmuS1FdnhgW0mJZVXifnI_YLRLaj4w5DYlPl7s,2638
61
55
  cosmotech/csm_data/commands/adx_send_runnerdata.py,sha256=SHZ0RijCnYRrAdCgdbW_YTaN3joihtNtbLzZAICCzJ4,3713
62
56
  cosmotech/csm_data/commands/az_storage_upload.py,sha256=7qXhFxftfil72tG23HTvHiW708t4QkM4rymcFJSPCj4,2344
63
- cosmotech/csm_data/commands/s3_bucket_delete.py,sha256=0k56xKOPp7CRP-OGL5qqETgdCmcSnx6PjiGGE5ha1Bs,3026
64
- cosmotech/csm_data/commands/s3_bucket_download.py,sha256=vyEmKA2EUbb0glBXI1IXx-GKG6Fgoqys1hZONjb_qpI,3347
65
- cosmotech/csm_data/commands/s3_bucket_upload.py,sha256=eKIQd-RDwYKjGADYsoMYuT9-I6jYT2-16BH2rVEMbuM,3563
57
+ cosmotech/csm_data/commands/s3_bucket_delete.py,sha256=2Sx3sLkiQxTd9MNL85zUZCvNTi6bZKxusQmu8xG0EYY,3056
58
+ cosmotech/csm_data/commands/s3_bucket_download.py,sha256=X0IahwMTzPUelTLZvNKIKkV2NMK6W8zw_Qp2hjFSsaA,3367
59
+ cosmotech/csm_data/commands/s3_bucket_upload.py,sha256=MKRFoYS9-MMCdk_jVRDF-Kx3YitjlmqUxX6-kd2CyxQ,3599
66
60
  cosmotech/csm_data/commands/api/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
67
- cosmotech/csm_data/commands/api/api.py,sha256=rzxb7zX_CJCzrOUxIN75mwARmQWhCmha4ogh1r3JAmg,2479
68
- cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py,sha256=ZRcTICk_3DKyl3Gic-L6mqfWeT8hEcU5AvY6c8UQ8_E,4076
69
- cosmotech/csm_data/commands/api/rds_load_csv.py,sha256=9GMK1ViazuZIuwfuRj6SyhqMa39Oc0b0yRtX60_G0mw,2757
70
- cosmotech/csm_data/commands/api/rds_send_csv.py,sha256=_YeKCnKHEtxFluGRjRqZKdUMVn2bO3pAGBJQFYHUj7Y,2310
71
- cosmotech/csm_data/commands/api/rds_send_store.py,sha256=7_pgWtHfmFF-evuO7EOzFX6N7V5d2EsO-KG46B-F13Q,2328
72
- cosmotech/csm_data/commands/api/run_load_data.py,sha256=oRnoIj2EXPG8Q2fcvrzt69eX4LNCVCO612qejsOivBM,3725
73
- cosmotech/csm_data/commands/api/runtemplate_load_handler.py,sha256=qWS0I1fNN6VFx7lHJoOWCv30WGI2q32B9_Zyr5KpBSM,2275
74
- cosmotech/csm_data/commands/api/tdl_load_files.py,sha256=O4FsRNutyEvdDUBYElLxrzEhJcXVyO9hpOdiM6FEQ5E,2394
75
- cosmotech/csm_data/commands/api/tdl_send_files.py,sha256=QXToEHbZvEhdh88xHamXj9mPQo1Hm2UPxaMkgNEspaw,2474
76
- cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=rDu8w4TAP7vmcbs8oh2eZOPVWEVKXsdgAgaYl980lfY,2321
77
- cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=M8pJxf78546onU55a0jKJ4fEnaS9Cyw4x_4NpRps0TE,2201
61
+ cosmotech/csm_data/commands/api/api.py,sha256=v8qBRjI1he8o6hDXmjmptTUdO8F92sgwr9glszsy49o,1698
62
+ cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py,sha256=h_24m7sBrA4Dej7UyIsAzlpJi9mGIM5MBiUHxN8PT08,4235
63
+ cosmotech/csm_data/commands/api/run_load_data.py,sha256=XL_txDSMdNLePa2ScidF8CzUVTasn9SAtKZXE4Gmo8A,2576
64
+ cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=wnNvRY1D7GBE5lQz5KOQ9x5T_wxs-4wIfGPO4jiMcwQ,2103
65
+ cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=AC-Li-2tN5KAtIE5TnElRbSgUTNCa7uWkn90MlOozjg,2045
78
66
  cosmotech/csm_data/commands/store/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
79
67
  cosmotech/csm_data/commands/store/dump_to_azure.py,sha256=5nnK7LUATRTlfX8tmFRLBmkD15bgG37Lyn7fEuPJiNE,3313
80
68
  cosmotech/csm_data/commands/store/dump_to_postgresql.py,sha256=3ZfIwIfLXYeBb6SwBSV6-CiWdid8Udht4ILWeZNdLWM,3596
81
- cosmotech/csm_data/commands/store/dump_to_s3.py,sha256=TUOTtIsx-n8c2hHglQ8yr6diqpS-LYKF41a8RzW_qkc,5248
69
+ cosmotech/csm_data/commands/store/dump_to_s3.py,sha256=6EUwtkP8_k9Hlq65R9h-MYsH58e-UAYmSR95JKoZuvY,5342
82
70
  cosmotech/csm_data/commands/store/list_tables.py,sha256=epgVJAaP8QfSsEuL9n5S8smsfIxK-Cr0bLv88FAnkWw,1860
83
71
  cosmotech/csm_data/commands/store/load_csv_folder.py,sha256=gDKCxWIpXqhtRvlvRMM7SmyJ9_YHQ1UZHuA6NYdPTZA,1717
84
72
  cosmotech/csm_data/commands/store/load_from_singlestore.py,sha256=dWZ4jIKl8wsnOX_6qqtyQmkFQhLgjUAOZWab-sGzQ2g,3104
73
+ cosmotech/csm_data/commands/store/output.py,sha256=RkBQxZ6c-p6F3A0Ci7wW-ODHsrJJO8ivzQCumvjoqXI,1218
85
74
  cosmotech/csm_data/commands/store/reset.py,sha256=3B0E0m0nCy3vwai_D5sWWddMZWuWkWnaL1tvgR6L5Ow,1237
86
- cosmotech/csm_data/commands/store/store.py,sha256=JHfctOCOU2WJd_BpmBsIf805_hGAs2H6XysFt4hxQqI,1693
75
+ cosmotech/csm_data/commands/store/store.py,sha256=oQhTIF1WKKGKdFp1rNm1vuTgblmh27r_jQpPhNFmesE,1663
87
76
  cosmotech/csm_data/utils/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
88
77
  cosmotech/csm_data/utils/click.py,sha256=S_85cbKh3R86-FZVjTK7IXZnmp4ETjKo6K8gbK3HCgs,848
89
78
  cosmotech/csm_data/utils/decorators.py,sha256=dTcPRTYqY18mc8Ql4Qwd3gU7BxbNxfeKskQdVIsE3-g,2504
90
79
  cosmotech/orchestrator_plugins/csm-data/__init__.py,sha256=UPzNgaNBMoBaxSlnofxl988dAswMp_tv8DzaR_IxkiE,519
91
80
  cosmotech/orchestrator_plugins/csm-data/templates/api/postgres_send_runner_metadata.json,sha256=alXzvOa985PvN5nf3uoTsFXgY29EJWeK94CcBjoLFUk,1085
92
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json,sha256=nM2hVFqE6Mja6u6becVStJTD8POcu76Ys0oW-mU6-TU,819
93
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json,sha256=Z8uh4-C1TNVjGxUkuGlHCoSqRPNL4DVn-wVMZXSAOto,790
94
- cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json,sha256=AutPCsUfK0gHGjojDZQdQ0pgHsfC6uLtUtQ80eSHD6o,772
95
81
  cosmotech/orchestrator_plugins/csm-data/templates/api/run_load_data.json,sha256=_ztHPGu5UbbdtQHUCaOuKqwtjB0DrZSZwcUePoOgS40,894
96
- cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json,sha256=aL-4GIUi2dV9jh3vuRI0BkAQJdHzfbf53Sxd91Lfj_U,761
97
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json,sha256=LxrwHU-UxtP0GB7FpNUw2o_HIM6VK2PBH7P928SFk_8,889
98
- cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json,sha256=A9Rt3wiiBly01ugEXGo1Qejp7UV3t8P13tLNLseqgZU,751
99
82
  cosmotech/orchestrator_plugins/csm-data/templates/api/try_api_connection.json,sha256=ApkzR0HQc0J6sxxTh91_DHPRI2cCtx_-qjpYFKGkGVw,215
100
83
  cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_load_file.json,sha256=yX7yUrq7epQTp0PaQElFC-elof8W6jaMtek6uQ0V6Og,1142
101
84
  cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_send_file.json,sha256=7pyYOUPg_lvD4kw3sqd_zBonru7g-DVatQmu4EJw0Ps,1120
@@ -121,7 +104,7 @@ cosmotech/translation/coal/en-US/coal/common/progress.yml,sha256=G_BTYgAYVD_JqU3
121
104
  cosmotech/translation/coal/en-US/coal/common/timing.yml,sha256=YtxIT8EKZDWD4hJ0tJOwkReUsrOY67Wq0MFkEZjzANE,274
122
105
  cosmotech/translation/coal/en-US/coal/common/validation.yml,sha256=OAHeXrvMo2_S-D0z-ESIS--U7bU0z1i0wta4ibaqk3c,528
123
106
  cosmotech/translation/coal/en-US/coal/cosmotech_api/connection.yml,sha256=2Om0DZmOI8u0hzlWWhWnGx-o0mzMk_vgE3hGRBNaWWU,518
124
- cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml,sha256=HogZqb_Mhu4yyJmklBzb44Uk0cDnM5ssb8NZMWQMw6c,84
107
+ cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml,sha256=NGD5bbbS037QosTmHxXOiUJ_Zl3gyIr1iR6hCaSlolY,380
125
108
  cosmotech/translation/coal/en-US/coal/cosmotech_api/run_template.yml,sha256=JIQ0S-9_YhchAPVFkcMpJTVUBlhPo4yS-nBoMysus5o,441
126
109
  cosmotech/translation/coal/en-US/coal/cosmotech_api/runner.yml,sha256=fSoFiJFJcowXwBVgfAkZKL8j3iKzyVDLWm48cFpiYMA,804
127
110
  cosmotech/translation/coal/en-US/coal/cosmotech_api/solution.yml,sha256=dKOTcv8bBYl3ZxyvKsQ8msO--PyLTfEOodb4rHl2bq4,216
@@ -130,20 +113,18 @@ cosmotech/translation/coal/en-US/coal/services/adx.yml,sha256=9yxR0qYKyA5mqgW-pX
130
113
  cosmotech/translation/coal/en-US/coal/services/api.yml,sha256=HPyKJaZrS41Qo9upRvPsLdn29QgS1xw3S-bgipHeO4s,342
131
114
  cosmotech/translation/coal/en-US/coal/services/azure_storage.yml,sha256=p0alxlJA_-NH9MxKXRKGgiLbBvHuQdCROhJKiqvuX0c,595
132
115
  cosmotech/translation/coal/en-US/coal/services/database.yml,sha256=S1pAu4OrVd9nAVb3oYSAurzA4cgD43UDlwK4qWhy2g4,860
133
- cosmotech/translation/coal/en-US/coal/services/dataset.yml,sha256=J-lP802jTb7J6CYBQ5iUXSFF44yd7P230NosQz9tnzk,3752
116
+ cosmotech/translation/coal/en-US/coal/services/dataset.yml,sha256=aaFBSwkoakMbyjMVeljp7RfC-h4wEvclZqMOBD8ynSo,2981
134
117
  cosmotech/translation/coal/en-US/coal/services/postgresql.yml,sha256=tRGNTTayyoxEHVyM-nzudEFdQ5HUeRIHPWIuvFmJGfM,1774
135
118
  cosmotech/translation/coal/en-US/coal/services/s3.yml,sha256=362wt7SdOu-MyMj9MgjQNTNWzIq-RJ9rUqc2VKljGFE,482
119
+ cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml,sha256=4pVJPAXYeIq9Gpmg_0VpSrB17AsGWA9hvk8Qi8-kZvw,85
120
+ cosmotech/translation/coal/en-US/coal/store/output/split.yml,sha256=h89v89sQsK5pGDJP5EE_KNbQbaS8003VPZNIgU2dqWw,265
121
+ cosmotech/translation/coal/en-US/coal/utils/configuration.yml,sha256=q3kEzPTP81SfsdrhAZa2JusHAjbdYqjs0DcvjhewQ-4,96
136
122
  cosmotech/translation/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
137
123
  cosmotech/translation/csm_data/en-US/csm_data/commands/main.yml,sha256=3XbXR5_Uo7PxJOWzm7XRMvPVxK-fzOL4FxWWY7rX3eg,396
138
124
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/api.yml,sha256=eMT4s1H9LuNUh_sAlnEtg7Co4ie3bFfB1vkwnLW28dQ,340
139
125
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml,sha256=avckcrp2aYz7rzGCKEdL81hwHDVbcUn2csQ_krKLcaM,844
140
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml,sha256=jaCmdHY0fJp9Sh2PcOizMyPIdvZU2iTIoSLNdfuLCXE,668
141
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml,sha256=0I9MAGgdCmNHGN3sdRVSWzEobAABFsb9qVG_l0Vk7Sg,580
142
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml,sha256=9vRnrqYYzSYMPNzR-Q7Gw5G7LOWlEd5GWepSyzg49i4,574
143
126
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/run_load_data.yml,sha256=k5J3HTA8vCmI_7CFHvMP51EAbmefubuoEhXZf2V3XDo,765
144
127
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/runtemplate_load_handler.yml,sha256=9jvMa6fb99CPzTMx7nnuoNNzJw7_7rapee7P80Plxjs,353
145
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml,sha256=ko4Nw96w6vBhE-68swd9h4edWgXCWIfMGqYc8Fvh2LM,569
146
- cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml,sha256=13eacNdU6i92XemHyx-bzIdXbQUIs6eg2YiT3SxivyM,836
147
128
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_load_file.yml,sha256=cnR_0PMkbrj8GROgvIUyeLXiB90IBbkcTvM9WJDEM0k,511
148
129
  cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_send_file.yml,sha256=yvsD364GND1jLe_SJxuNXjDgGZFGlJlC5xUTRqV0fr0,632
149
130
  cosmotech/translation/csm_data/en-US/csm_data/commands/storage/adx_send_data.yml,sha256=oNQMHhHMRTl7EfFLuEy5agctzgwGHsyKzAu6hlSUyU4,448
@@ -159,13 +140,14 @@ cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_s3.yml,sha2
159
140
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/list_tables.yml,sha256=-x8zwCjAHILFaJv7VLPskipSoQTwmI7jUsxlx6_OKlc,195
160
141
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_csv_folder.yml,sha256=9HjW9tOq5o31jmADp_QIync7G74c6XGiPqofXh8pvv8,228
161
142
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_from_singlestore.yml,sha256=_OUXlxC_wi-GPgHU7EVdU9g0WIdblVpEA4WIzw4aQcQ,824
143
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml,sha256=odkR9pzeel5l0wW_F4kx4HhNpw_TzEvok-OKyh25ZkQ,232
162
144
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/reset.yml,sha256=JMspZ25Q2HDpH8MYqUaQcveUnCgdH21RmfTCrPf2gW8,139
163
145
  cosmotech/translation/csm_data/en-US/csm_data/commands/store/store.yml,sha256=N1Q8483gqJADaCe30S1M3Rj0tMJiuQiJH70-VK2x2m4,134
164
146
  cosmotech/translation/csm_data/en-US/csm_data/commons/decorators.yml,sha256=Iu59NWMfYlZZf9uUhOiLkIEGa4GY5p0nZ6vG06Xvu7k,51
165
147
  cosmotech/translation/csm_data/en-US/csm_data/commons/version.yml,sha256=7jtCV3O1S6pGjiJa63XpgPDTafjfBS0xmEVRpYNvfDg,86
166
- cosmotech_acceleration_library-1.1.0.dist-info/licenses/LICENSE,sha256=JXKHOQtyObmafNbQlfPYc4HkKjU9FzAP27b2qRTXNM8,1195
167
- cosmotech_acceleration_library-1.1.0.dist-info/METADATA,sha256=0vbzAbv9QAqsAupcJkidNhiKPxpcST4kfalLmzOTm9g,9432
168
- cosmotech_acceleration_library-1.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
169
- cosmotech_acceleration_library-1.1.0.dist-info/entry_points.txt,sha256=HWRqJurKuBUgqFe4jmjIAQrs768Nbb8ZTdRDLbuKM5Q,58
170
- cosmotech_acceleration_library-1.1.0.dist-info/top_level.txt,sha256=t2pzb8mpMUfHTa9l2SjWP0rRB8XVRjBdQK5nLx9XDDo,10
171
- cosmotech_acceleration_library-1.1.0.dist-info/RECORD,,
148
+ cosmotech_acceleration_library-2.0.0.dist-info/licenses/LICENSE,sha256=JXKHOQtyObmafNbQlfPYc4HkKjU9FzAP27b2qRTXNM8,1195
149
+ cosmotech_acceleration_library-2.0.0.dist-info/METADATA,sha256=LVDVBJn0B1rE1z4rKbm3wUG5UKKt3Qj6Jkm2lGcLDxk,9334
150
+ cosmotech_acceleration_library-2.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
151
+ cosmotech_acceleration_library-2.0.0.dist-info/entry_points.txt,sha256=HWRqJurKuBUgqFe4jmjIAQrs768Nbb8ZTdRDLbuKM5Q,58
152
+ cosmotech_acceleration_library-2.0.0.dist-info/top_level.txt,sha256=t2pzb8mpMUfHTa9l2SjWP0rRB8XVRjBdQK5nLx9XDDo,10
153
+ cosmotech_acceleration_library-2.0.0.dist-info/RECORD,,
@@ -1,72 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
- import azure.functions as func
8
- from cosmotech.coal.cosmotech_api.runner.download import download_runner_data
9
- from cosmotech_api.api.runner_api import RunnerApi
10
-
11
- import json
12
- import http
13
- import traceback
14
-
15
-
16
- def generate_main(apply_update, parallel=True):
17
- def main(req: func.HttpRequest) -> func.HttpResponse:
18
- try:
19
- runner_id = req.params.get("scenario-id") # Keep parameter name for backward compatibility
20
- organization_id = req.params.get("organization-id")
21
- workspace_id = req.params.get("workspace-id")
22
- access_token: str = req.headers.get("authorization", None)
23
- if access_token:
24
- access_token = access_token.split(" ")[1]
25
-
26
- if runner_id is None or organization_id is None or workspace_id is None:
27
- return func.HttpResponse(
28
- body=f"Invalid request: organization-id={organization_id}, workspace-id={workspace_id}, scenario-id={runner_id}",
29
- status_code=http.HTTPStatus.BAD_REQUEST,
30
- )
31
-
32
- # Get runner data
33
- result = download_runner_data(
34
- organization_id=organization_id,
35
- workspace_id=workspace_id,
36
- runner_id=runner_id,
37
- parameter_folder=None, # We don't need to save to files
38
- read_files=True,
39
- parallel=parallel,
40
- write_json=False,
41
- write_csv=False,
42
- fetch_dataset=True,
43
- )
44
-
45
- content = {
46
- "datasets": result["datasets"],
47
- "parameters": result["parameters"],
48
- }
49
-
50
- runner_data = result["runner_data"]
51
-
52
- updated_content = apply_update(
53
- content=content, scenario_data=runner_data
54
- ) # Keep parameter name for backward compatibility
55
-
56
- return func.HttpResponse(
57
- body=json.dumps(updated_content),
58
- headers={"Content-Type": "application/json"},
59
- )
60
- except Exception as e:
61
- response = {
62
- "error": getattr(e, "message", str(e)),
63
- "type": type(e).__name__,
64
- "trace": traceback.format_exc(),
65
- }
66
- return func.HttpResponse(
67
- status_code=http.HTTPStatus.INTERNAL_SERVER_ERROR,
68
- body=json.dumps(response),
69
- headers={"Content-Type": "application/json"},
70
- )
71
-
72
- return main
@@ -1,96 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
- import os
8
- import pathlib
9
- import ssl
10
-
11
- import cosmotech_api
12
-
13
- from cosmotech.coal.utils.logger import LOGGER
14
- from cosmotech.orchestrator.utils.translate import T
15
-
16
- api_env_keys = {"CSM_API_KEY", "CSM_API_URL"}
17
- azure_env_keys = {
18
- "AZURE_CLIENT_ID",
19
- "AZURE_CLIENT_SECRET",
20
- "AZURE_TENANT_ID",
21
- "CSM_API_URL",
22
- "CSM_API_SCOPE",
23
- }
24
- keycloak_env_keys = {
25
- "IDP_TENANT_ID",
26
- "IDP_CLIENT_ID",
27
- "IDP_CLIENT_SECRET",
28
- "IDP_BASE_URL",
29
- "CSM_API_URL",
30
- }
31
-
32
-
33
- def get_api_client() -> (cosmotech_api.ApiClient, str):
34
- existing_keys = set(os.environ.keys())
35
- missing_azure_keys = azure_env_keys - existing_keys
36
- missing_api_keys = api_env_keys - existing_keys
37
- missing_keycloak_keys = keycloak_env_keys - existing_keys
38
- if all((missing_api_keys, missing_azure_keys, missing_keycloak_keys)):
39
- LOGGER.error(T("coal.common.errors.no_env_vars"))
40
- LOGGER.error(T("coal.cosmotech_api.connection.existing_sets"))
41
- LOGGER.error(T("coal.cosmotech_api.connection.azure_connection").format(keys=", ".join(azure_env_keys)))
42
- LOGGER.error(T("coal.cosmotech_api.connection.api_key_connection").format(keys=", ".join(api_env_keys)))
43
- LOGGER.error(T("coal.cosmotech_api.connection.keycloak_connection").format(keys=", ".join(keycloak_env_keys)))
44
- raise EnvironmentError(T("coal.common.errors.no_env_vars"))
45
-
46
- if not missing_keycloak_keys:
47
- LOGGER.info(T("coal.cosmotech_api.connection.found_keycloak"))
48
- from keycloak import KeycloakOpenID
49
-
50
- server_url = os.environ.get("IDP_BASE_URL")
51
- if server_url[-1] != "/":
52
- server_url = server_url + "/"
53
- keycloack_parameters = dict(
54
- server_url=server_url,
55
- client_id=os.environ.get("IDP_CLIENT_ID"),
56
- realm_name=os.environ.get("IDP_TENANT_ID"),
57
- client_secret_key=os.environ.get("IDP_CLIENT_SECRET"),
58
- )
59
- if (ca_cert_path := os.environ.get("IDP_CA_CERT")) and pathlib.Path(ca_cert_path).exists():
60
- LOGGER.info(T("coal.cosmotech_api.connection.found_cert_authority"))
61
- keycloack_parameters["verify"] = ca_cert_path
62
- keycloak_openid = KeycloakOpenID(**keycloack_parameters)
63
-
64
- access_token = keycloak_openid.token(grant_type="client_credentials")
65
-
66
- configuration = cosmotech_api.Configuration(
67
- host=os.environ.get("CSM_API_URL"),
68
- access_token=access_token["access_token"],
69
- )
70
- return cosmotech_api.ApiClient(configuration), "Keycloak Connection"
71
-
72
- if not missing_api_keys:
73
- LOGGER.info(T("coal.cosmotech_api.connection.found_api_key"))
74
- configuration = cosmotech_api.Configuration(
75
- host=os.environ.get("CSM_API_URL"),
76
- )
77
- return (
78
- cosmotech_api.ApiClient(
79
- configuration,
80
- os.environ.get("CSM_API_KEY_HEADER", "X-CSM-API-KEY"),
81
- os.environ.get("CSM_API_KEY"),
82
- ),
83
- "Cosmo Tech API Key",
84
- )
85
-
86
- if not missing_azure_keys:
87
- LOGGER.info(T("coal.cosmotech_api.connection.found_azure"))
88
- from azure.identity import EnvironmentCredential
89
-
90
- credentials = EnvironmentCredential()
91
- token = credentials.get_token(os.environ.get("CSM_API_SCOPE"))
92
-
93
- configuration = cosmotech_api.Configuration(host=os.environ.get("CSM_API_URL"), access_token=token.token)
94
- return cosmotech_api.ApiClient(configuration), "Azure Entra Connection"
95
-
96
- raise EnvironmentError(T("coal.common.errors.no_valid_connection"))
@@ -1,26 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- # Re-export all download functions from download submodule
9
- from cosmotech.coal.cosmotech_api.dataset.download import (
10
- download_adt_dataset,
11
- download_twingraph_dataset,
12
- download_legacy_twingraph_dataset,
13
- download_file_dataset,
14
- download_dataset_by_id,
15
- )
16
-
17
- from cosmotech.coal.cosmotech_api.dataset.converters import (
18
- convert_dataset_to_files,
19
- convert_graph_dataset_to_files,
20
- convert_file_dataset_to_files,
21
- )
22
-
23
- from cosmotech.coal.cosmotech_api.dataset.utils import (
24
- get_content_from_twin_graph_data,
25
- sheet_to_header,
26
- )
@@ -1,164 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- import csv
9
- import json
10
- import os
11
- import tempfile
12
- from pathlib import Path
13
- from typing import Dict, List, Any, Optional, Union
14
-
15
- from cosmotech.coal.utils.logger import LOGGER
16
- from cosmotech.orchestrator.utils.translate import T
17
- from cosmotech.coal.cosmotech_api.dataset.utils import sheet_to_header
18
-
19
-
20
- def convert_dataset_to_files(dataset_info: Dict[str, Any], target_folder: Optional[Union[str, Path]] = None) -> Path:
21
- """
22
- Convert dataset info to files.
23
-
24
- Args:
25
- dataset_info: Dataset info dict with type, content, name
26
- target_folder: Optional folder to save files (if None, uses temp dir)
27
-
28
- Returns:
29
- Path to folder containing files
30
- """
31
- dataset_type = dataset_info["type"]
32
- content = dataset_info["content"]
33
- name = dataset_info["name"]
34
-
35
- LOGGER.info(T("coal.services.dataset.converting_to_files").format(dataset_type=dataset_type, dataset_name=name))
36
-
37
- if target_folder is None:
38
- target_folder = Path(tempfile.mkdtemp())
39
- LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
40
- else:
41
- target_folder = Path(target_folder)
42
- target_folder.mkdir(parents=True, exist_ok=True)
43
- LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
44
-
45
- if dataset_type in ["adt", "twincache"]:
46
- return convert_graph_dataset_to_files(content, target_folder)
47
- else:
48
- return convert_file_dataset_to_files(content, target_folder, dataset_type)
49
-
50
-
51
- def convert_graph_dataset_to_files(
52
- content: Dict[str, List[Dict]], target_folder: Optional[Union[str, Path]] = None
53
- ) -> Path:
54
- """
55
- Convert graph dataset content to CSV files.
56
-
57
- Args:
58
- content: Dictionary mapping entity types to lists of entities
59
- target_folder: Folder to save files (if None, uses temp dir)
60
-
61
- Returns:
62
- Path to folder containing files
63
- """
64
- if target_folder is None:
65
- target_folder = Path(tempfile.mkdtemp())
66
- LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
67
- else:
68
- target_folder = Path(target_folder)
69
- target_folder.mkdir(parents=True, exist_ok=True)
70
- LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
71
- file_count = 0
72
-
73
- LOGGER.info(
74
- T("coal.services.dataset.converting_graph_data").format(entity_types=len(content), folder=target_folder)
75
- )
76
-
77
- for entity_type, entities in content.items():
78
- if not entities:
79
- LOGGER.debug(T("coal.services.dataset.skipping_empty_entity").format(entity_type=entity_type))
80
- continue
81
-
82
- file_path = target_folder / f"{entity_type}.csv"
83
- LOGGER.debug(T("coal.services.dataset.writing_csv").format(file_name=file_path.name, count=len(entities)))
84
-
85
- fieldnames = sheet_to_header(entities)
86
-
87
- with open(file_path, "w", newline="") as file:
88
- writer = csv.DictWriter(file, fieldnames=fieldnames, dialect="unix", quoting=csv.QUOTE_MINIMAL)
89
- writer.writeheader()
90
-
91
- for entity in entities:
92
- # Convert values to strings and handle boolean values
93
- row = {
94
- k: str(v).replace("'", '"').replace("True", "true").replace("False", "false")
95
- for k, v in entity.items()
96
- }
97
- writer.writerow(row)
98
-
99
- file_count += 1
100
- LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
101
-
102
- LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
103
-
104
- return target_folder
105
-
106
-
107
- def convert_file_dataset_to_files(
108
- content: Dict[str, Any],
109
- target_folder: Optional[Union[str, Path]] = None,
110
- file_type: str = "",
111
- ) -> Path:
112
- """
113
- Convert file dataset content to files.
114
-
115
- Args:
116
- content: Dictionary mapping file names to content
117
- target_folder: Folder to save files (if None, uses temp dir)
118
- file_type: Type of file (csv, json, etc.)
119
-
120
- Returns:
121
- Path to folder containing files
122
- """
123
- if target_folder is None:
124
- target_folder = Path(tempfile.mkdtemp())
125
- LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
126
- else:
127
- target_folder = Path(target_folder)
128
- target_folder.mkdir(parents=True, exist_ok=True)
129
- LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
130
- file_count = 0
131
-
132
- LOGGER.info(
133
- T("coal.services.dataset.converting_file_data").format(
134
- file_count=len(content), file_type=file_type, folder=target_folder
135
- )
136
- )
137
-
138
- for file_name, file_content in content.items():
139
- file_path = target_folder / file_name
140
-
141
- # Ensure parent directories exist
142
- file_path.parent.mkdir(parents=True, exist_ok=True)
143
-
144
- LOGGER.debug(T("coal.services.dataset.writing_file").format(file_name=file_path.name, file_type=file_type))
145
-
146
- if isinstance(file_content, str):
147
- # Text content
148
- with open(file_path, "w") as file:
149
- file.write(file_content)
150
- elif isinstance(file_content, dict) or isinstance(file_content, list):
151
- # JSON content
152
- with open(file_path, "w") as file:
153
- json.dump(file_content, file, indent=2)
154
- else:
155
- # Other content types
156
- with open(file_path, "w") as file:
157
- file.write(str(file_content))
158
-
159
- file_count += 1
160
- LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
161
-
162
- LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
163
-
164
- return target_folder
@@ -1,19 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- """
9
- Dataset download submodules.
10
- """
11
-
12
- # Re-export all download functions
13
- from cosmotech.coal.cosmotech_api.dataset.download.adt import download_adt_dataset
14
- from cosmotech.coal.cosmotech_api.dataset.download.twingraph import (
15
- download_twingraph_dataset,
16
- download_legacy_twingraph_dataset,
17
- )
18
- from cosmotech.coal.cosmotech_api.dataset.download.file import download_file_dataset
19
- from cosmotech.coal.cosmotech_api.dataset.download.common import download_dataset_by_id