cosmotech-acceleration-library 1.1.0__py3-none-any.whl → 2.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. cosmotech/coal/__init__.py +1 -1
  2. cosmotech/coal/aws/__init__.py +1 -9
  3. cosmotech/coal/aws/s3.py +181 -214
  4. cosmotech/coal/azure/__init__.py +5 -5
  5. cosmotech/coal/azure/adx/__init__.py +24 -10
  6. cosmotech/coal/azure/adx/auth.py +2 -2
  7. cosmotech/coal/azure/adx/ingestion.py +10 -14
  8. cosmotech/coal/azure/adx/query.py +1 -1
  9. cosmotech/coal/azure/adx/runner.py +13 -14
  10. cosmotech/coal/azure/adx/store.py +5 -86
  11. cosmotech/coal/azure/adx/tables.py +2 -2
  12. cosmotech/coal/azure/adx/utils.py +2 -2
  13. cosmotech/coal/azure/blob.py +20 -26
  14. cosmotech/coal/azure/storage.py +3 -3
  15. cosmotech/coal/cosmotech_api/__init__.py +0 -28
  16. cosmotech/coal/cosmotech_api/apis/__init__.py +14 -0
  17. cosmotech/coal/cosmotech_api/apis/dataset.py +222 -0
  18. cosmotech/coal/cosmotech_api/apis/meta.py +25 -0
  19. cosmotech/coal/cosmotech_api/apis/organization.py +24 -0
  20. cosmotech/coal/cosmotech_api/apis/run.py +38 -0
  21. cosmotech/coal/cosmotech_api/apis/runner.py +75 -0
  22. cosmotech/coal/cosmotech_api/apis/solution.py +23 -0
  23. cosmotech/coal/cosmotech_api/apis/workspace.py +108 -0
  24. cosmotech/coal/cosmotech_api/objects/__init__.py +9 -0
  25. cosmotech/coal/cosmotech_api/objects/connection.py +125 -0
  26. cosmotech/coal/cosmotech_api/objects/parameters.py +127 -0
  27. cosmotech/coal/postgresql/runner.py +58 -41
  28. cosmotech/coal/postgresql/store.py +56 -15
  29. cosmotech/coal/postgresql/utils.py +255 -0
  30. cosmotech/coal/singlestore/store.py +3 -2
  31. cosmotech/coal/store/__init__.py +16 -13
  32. cosmotech/coal/store/output/__init__.py +0 -0
  33. cosmotech/coal/store/output/aws_channel.py +74 -0
  34. cosmotech/coal/store/output/az_storage_channel.py +33 -0
  35. cosmotech/coal/store/output/channel_interface.py +38 -0
  36. cosmotech/coal/store/output/channel_spliter.py +61 -0
  37. cosmotech/coal/store/output/postgres_channel.py +37 -0
  38. cosmotech/coal/store/pandas.py +1 -1
  39. cosmotech/coal/store/pyarrow.py +2 -2
  40. cosmotech/coal/store/store.py +4 -7
  41. cosmotech/coal/utils/configuration.py +197 -0
  42. cosmotech/coal/utils/decorator.py +4 -7
  43. cosmotech/csm_data/commands/adx_send_data.py +1 -1
  44. cosmotech/csm_data/commands/adx_send_runnerdata.py +3 -2
  45. cosmotech/csm_data/commands/api/api.py +6 -19
  46. cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py +20 -16
  47. cosmotech/csm_data/commands/api/run_load_data.py +15 -52
  48. cosmotech/csm_data/commands/api/wsf_load_file.py +13 -16
  49. cosmotech/csm_data/commands/api/wsf_send_file.py +11 -14
  50. cosmotech/csm_data/commands/az_storage_upload.py +3 -2
  51. cosmotech/csm_data/commands/s3_bucket_delete.py +16 -15
  52. cosmotech/csm_data/commands/s3_bucket_download.py +16 -16
  53. cosmotech/csm_data/commands/s3_bucket_upload.py +16 -14
  54. cosmotech/csm_data/commands/store/dump_to_azure.py +3 -2
  55. cosmotech/csm_data/commands/store/dump_to_postgresql.py +3 -2
  56. cosmotech/csm_data/commands/store/dump_to_s3.py +18 -16
  57. cosmotech/csm_data/commands/store/list_tables.py +3 -2
  58. cosmotech/csm_data/commands/store/load_csv_folder.py +10 -4
  59. cosmotech/csm_data/commands/store/load_from_singlestore.py +3 -2
  60. cosmotech/csm_data/commands/store/output.py +35 -0
  61. cosmotech/csm_data/commands/store/reset.py +8 -3
  62. cosmotech/csm_data/commands/store/store.py +3 -3
  63. cosmotech/csm_data/main.py +4 -4
  64. cosmotech/csm_data/utils/decorators.py +4 -3
  65. cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml +8 -0
  66. cosmotech/translation/coal/en-US/coal/services/dataset.yml +10 -14
  67. cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml +1 -0
  68. cosmotech/translation/coal/en-US/coal/store/output/split.yml +6 -0
  69. cosmotech/translation/coal/en-US/coal/utils/configuration.yml +2 -0
  70. cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml +7 -0
  71. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/METADATA +29 -33
  72. cosmotech_acceleration_library-2.1.0rc1.dist-info/RECORD +153 -0
  73. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/WHEEL +1 -1
  74. cosmotech/coal/azure/functions.py +0 -72
  75. cosmotech/coal/cosmotech_api/connection.py +0 -96
  76. cosmotech/coal/cosmotech_api/dataset/__init__.py +0 -26
  77. cosmotech/coal/cosmotech_api/dataset/converters.py +0 -164
  78. cosmotech/coal/cosmotech_api/dataset/download/__init__.py +0 -19
  79. cosmotech/coal/cosmotech_api/dataset/download/adt.py +0 -119
  80. cosmotech/coal/cosmotech_api/dataset/download/common.py +0 -140
  81. cosmotech/coal/cosmotech_api/dataset/download/file.py +0 -229
  82. cosmotech/coal/cosmotech_api/dataset/download/twingraph.py +0 -185
  83. cosmotech/coal/cosmotech_api/dataset/upload.py +0 -41
  84. cosmotech/coal/cosmotech_api/dataset/utils.py +0 -132
  85. cosmotech/coal/cosmotech_api/parameters.py +0 -48
  86. cosmotech/coal/cosmotech_api/run.py +0 -25
  87. cosmotech/coal/cosmotech_api/run_data.py +0 -173
  88. cosmotech/coal/cosmotech_api/run_template.py +0 -108
  89. cosmotech/coal/cosmotech_api/runner/__init__.py +0 -28
  90. cosmotech/coal/cosmotech_api/runner/data.py +0 -38
  91. cosmotech/coal/cosmotech_api/runner/datasets.py +0 -416
  92. cosmotech/coal/cosmotech_api/runner/download.py +0 -135
  93. cosmotech/coal/cosmotech_api/runner/metadata.py +0 -42
  94. cosmotech/coal/cosmotech_api/runner/parameters.py +0 -157
  95. cosmotech/coal/cosmotech_api/twin_data_layer.py +0 -512
  96. cosmotech/coal/cosmotech_api/workspace.py +0 -127
  97. cosmotech/coal/utils/postgresql.py +0 -236
  98. cosmotech/coal/utils/semver.py +0 -6
  99. cosmotech/csm_data/commands/api/rds_load_csv.py +0 -90
  100. cosmotech/csm_data/commands/api/rds_send_csv.py +0 -74
  101. cosmotech/csm_data/commands/api/rds_send_store.py +0 -74
  102. cosmotech/csm_data/commands/api/runtemplate_load_handler.py +0 -66
  103. cosmotech/csm_data/commands/api/tdl_load_files.py +0 -76
  104. cosmotech/csm_data/commands/api/tdl_send_files.py +0 -82
  105. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_load_csv.json +0 -27
  106. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_csv.json +0 -27
  107. cosmotech/orchestrator_plugins/csm-data/templates/api/rds_send_store.json +0 -27
  108. cosmotech/orchestrator_plugins/csm-data/templates/api/runtemplate_load_handler.json +0 -27
  109. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_load_files.json +0 -32
  110. cosmotech/orchestrator_plugins/csm-data/templates/api/tdl_send_files.json +0 -27
  111. cosmotech/translation/coal/en-US/coal/cosmotech_api/run_data.yml +0 -2
  112. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml +0 -13
  113. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml +0 -12
  114. cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml +0 -12
  115. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml +0 -14
  116. cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml +0 -18
  117. cosmotech_acceleration_library-1.1.0.dist-info/RECORD +0 -171
  118. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/entry_points.txt +0 -0
  119. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/licenses/LICENSE +0 -0
  120. {cosmotech_acceleration_library-1.1.0.dist-info → cosmotech_acceleration_library-2.1.0rc1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,153 @@
1
+ cosmotech/coal/__init__.py,sha256=-QWX8vGspoMFG2cOvq21Sg4KBpIZUzhd76Kv7-6-6yg,440
2
+ cosmotech/coal/aws/__init__.py,sha256=qZZyxNCwGa1yPpIEc7CS6sVcv00PNcUnUOsKhsHBSdc,614
3
+ cosmotech/coal/aws/s3.py,sha256=5az9BtyUUBlfqFV3BG32V5-9yDpm-6q_O4YV79HcmMA,7522
4
+ cosmotech/coal/azure/__init__.py,sha256=Zp7v4g2Tp6h8j6_-_drMNHdEaq8FqfKhlZOwvfxhiMc,796
5
+ cosmotech/coal/azure/blob.py,sha256=p1iG2_lWA2UDZQrYIkz1TCK1--5IVeaSmyDC0RenFt0,4019
6
+ cosmotech/coal/azure/storage.py,sha256=TItajcil0Ht44kru44T9HIP2cizOvAS_H9uRdSgaBG0,2799
7
+ cosmotech/coal/azure/adx/__init__.py,sha256=-nDfPlb1lC1zy_1llxi82FT0hWVYGfYPMgAaQNa6Dak,1286
8
+ cosmotech/coal/azure/adx/auth.py,sha256=IFNaB8eoc99_j3Ml6rueYfuFK36rze4_8LG7B0gYdIQ,4748
9
+ cosmotech/coal/azure/adx/ingestion.py,sha256=pDksvQzzONSSbFRTcf4EzgNhwe7MkhY3lCkNvlgG8U4,11266
10
+ cosmotech/coal/azure/adx/query.py,sha256=JrTqZaQ6Ody21rbjkEIx4J3lYJ1-gOJxFKJRrbEpdi0,1870
11
+ cosmotech/coal/azure/adx/runner.py,sha256=IXgciGxHjSqkiqm23CuyUk3RgCUgd1G-e0eMyohjygM,8066
12
+ cosmotech/coal/azure/adx/store.py,sha256=7fxG3yVxQ9_6sLaSU1Z4cxmsMcSO58Ni371FD2m_4LA,6214
13
+ cosmotech/coal/azure/adx/tables.py,sha256=y6GBrjXAKAOn7q4Ekf-hV6K5P3ejdV2L57FbFDvp22c,4287
14
+ cosmotech/coal/azure/adx/utils.py,sha256=0MCnIJ-UijovA04byFbSwO4QVUMVn7v3P7dxGpdN_zM,2205
15
+ cosmotech/coal/cosmotech_api/__init__.py,sha256=YlIyLwoqd4CveVnfrA9UDunlhO9HSvrBAfUwzcBay5M,528
16
+ cosmotech/coal/cosmotech_api/apis/__init__.py,sha256=RDe2FXBCrgUoSAONfpf_Udyc3M3w2y9K9mjFzUX_SMY,869
17
+ cosmotech/coal/cosmotech_api/apis/dataset.py,sha256=CtIrRvykUhJ9cFZqz8WEj-gn1trx9NXp9NAfKQklteU,8874
18
+ cosmotech/coal/cosmotech_api/apis/meta.py,sha256=lCi5Jo7c_gXewob9B_NgvRyKAmPSJ_bVK7Z_pIVUyP8,1055
19
+ cosmotech/coal/cosmotech_api/apis/organization.py,sha256=JA55Bu4mJL5aE7UNtnPDWyjB6YZktSsnCzOvs9bW34g,1102
20
+ cosmotech/coal/cosmotech_api/apis/run.py,sha256=TOVD0E4QIR26rBZr3MjsWuSbYcKcuTVSJsq_zD4nzD0,1528
21
+ cosmotech/coal/cosmotech_api/apis/runner.py,sha256=Q9y3DDKFYRMcibMPkU7K0pv6M5z2On5oFG2JrMIHkPY,3115
22
+ cosmotech/coal/cosmotech_api/apis/solution.py,sha256=rGsbtkVkhrt0vAano4HcIYQJFaXGSpWCHLku2TEpEHI,1077
23
+ cosmotech/coal/cosmotech_api/apis/workspace.py,sha256=YeXuODKJnwpdO2g2yaJzg9095VyzAuGlMV1JL1hNMsU,4330
24
+ cosmotech/coal/cosmotech_api/objects/__init__.py,sha256=6cWMAAyhCK478_Je9Opsu0NBv617aTQo3r9mhbV2wwo,556
25
+ cosmotech/coal/cosmotech_api/objects/connection.py,sha256=z0hd2uJNZWZdTyL7NlGuXt_HeZMenSODNJatNdmhk0o,5292
26
+ cosmotech/coal/cosmotech_api/objects/parameters.py,sha256=6T4yJMm_yvularJwMIeg8qoqBYdpWEdbQx0Q2dCieDw,4227
27
+ cosmotech/coal/csm/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
28
+ cosmotech/coal/csm/engine/__init__.py,sha256=EG3iArAy3OFGPmClsYXQmb77YYrEkeVC5uwPg_TbAYI,2120
29
+ cosmotech/coal/postgresql/__init__.py,sha256=766LBZyWCQUI6rlzXDY_C9Yrxg551VX2L4lOuhDiS8w,786
30
+ cosmotech/coal/postgresql/runner.py,sha256=DqXM_GBiVQQ_QcNitEC4t9jthoNxUKsPvuaAWQ3BCQ8,4091
31
+ cosmotech/coal/postgresql/store.py,sha256=dyKwqngpSm40fZ5Fo4vHyU3gnRdC_xdk8USc_Qs06xM,5176
32
+ cosmotech/coal/postgresql/utils.py,sha256=zM69jq2XLiS8KgX6Zgo3W3GOZ8kb8goq-HMIRyx0EBU,9663
33
+ cosmotech/coal/singlestore/__init__.py,sha256=gXQaa8OAHnDcI7iHy-5zzGG7ZqNq-uOtw60qASjwu9I,651
34
+ cosmotech/coal/singlestore/store.py,sha256=C6EbXRAxsZmW1It3FTb9NEOK4H_oMBfNOTvhwPjrE8o,3578
35
+ cosmotech/coal/store/__init__.py,sha256=3UicSottHGQ3yjWv_ib8GBRUYcZ9-cwSq_wvXq7TpxQ,1395
36
+ cosmotech/coal/store/csv.py,sha256=2toydK0aSs1WLhO2_mSKpCJsaMGs7aBlL7RLASNalto,1494
37
+ cosmotech/coal/store/native_python.py,sha256=Mp2U7tkUjH1wT76U4WzekaW4H5nN2VHTrAKnJ4NzB8E,847
38
+ cosmotech/coal/store/pandas.py,sha256=ZvyK1APmcGscdt48401eBfm3zgvX3FKFfARsbV7kWcI,906
39
+ cosmotech/coal/store/pyarrow.py,sha256=tchLfthMRYiEXFrkjLDNXdDOOJYO7l3hSBH35MaNVg0,814
40
+ cosmotech/coal/store/store.py,sha256=1XIIRQL4Af-R1UdJlo-AzmDxtKoIYOtm7KvPdhTWJ-o,3390
41
+ cosmotech/coal/store/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
42
+ cosmotech/coal/store/output/aws_channel.py,sha256=0YZ0OjAEnDGOu48ZAFZP-y2IU_quqY0zSxhFwp509MM,2881
43
+ cosmotech/coal/store/output/az_storage_channel.py,sha256=WYMJD1dovg1SLXJFdaJnBIG86cHaFc9Y4avAX1143hk,842
44
+ cosmotech/coal/store/output/channel_interface.py,sha256=C_rbZnGOf-5YO8rnxTRe_Pd0EZ_2FNwLr3qSiKyUpWw,1241
45
+ cosmotech/coal/store/output/channel_spliter.py,sha256=f7ljfkqcuURLSeok7jr2nW5sqMSnMeuLonQEJTQlrX8,2441
46
+ cosmotech/coal/store/output/postgres_channel.py,sha256=HOqGSlAbRFN6yIneKjyxqAcTri83HmJkZVu74L3l99Y,1184
47
+ cosmotech/coal/utils/__init__.py,sha256=N2mPhli0Wj4OesuRVbt2KrzL_pgMELs4fDNIQYb_r-g,830
48
+ cosmotech/coal/utils/configuration.py,sha256=dFfjJHW1WPLqhg6kBsYOAwvk3h04i-YL2oAuzTAT-Xk,7460
49
+ cosmotech/coal/utils/decorator.py,sha256=p4yqkbln-YTppzeKvi3K4ZZ6AibVYGD6pnJQk0ynWbE,646
50
+ cosmotech/coal/utils/logger.py,sha256=oYVj2BtDPji4e4J-KgaDyVcbKiewkq0xey_gQPm74Xc,506
51
+ cosmotech/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
52
+ cosmotech/csm_data/main.py,sha256=Io-qKkUt7BT9ovA-_rQCATnZHWhgpGLP7n0en7L1SiU,2424
53
+ cosmotech/csm_data/commands/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
54
+ cosmotech/csm_data/commands/adx_send_data.py,sha256=FH8-_DRtM1qccHRCIRFqaZq2YiHz5pWLJUOTfPDSYOI,2638
55
+ cosmotech/csm_data/commands/adx_send_runnerdata.py,sha256=Pg40TQw38W40qcqvqf9B7y9w2uFj8KtH0_9_n0onvG8,3714
56
+ cosmotech/csm_data/commands/az_storage_upload.py,sha256=EvPOLvVL4vteaPbBHbC0gR2drnYbSppP6RlaklmZHXw,2345
57
+ cosmotech/csm_data/commands/s3_bucket_delete.py,sha256=2Sx3sLkiQxTd9MNL85zUZCvNTi6bZKxusQmu8xG0EYY,3056
58
+ cosmotech/csm_data/commands/s3_bucket_download.py,sha256=X0IahwMTzPUelTLZvNKIKkV2NMK6W8zw_Qp2hjFSsaA,3367
59
+ cosmotech/csm_data/commands/s3_bucket_upload.py,sha256=MKRFoYS9-MMCdk_jVRDF-Kx3YitjlmqUxX6-kd2CyxQ,3599
60
+ cosmotech/csm_data/commands/api/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
61
+ cosmotech/csm_data/commands/api/api.py,sha256=v8qBRjI1he8o6hDXmjmptTUdO8F92sgwr9glszsy49o,1698
62
+ cosmotech/csm_data/commands/api/postgres_send_runner_metadata.py,sha256=h_24m7sBrA4Dej7UyIsAzlpJi9mGIM5MBiUHxN8PT08,4235
63
+ cosmotech/csm_data/commands/api/run_load_data.py,sha256=IXCCXb3wKPZTeQybteMqXWreQHggQ3GALfNIw2bphys,2831
64
+ cosmotech/csm_data/commands/api/wsf_load_file.py,sha256=wnNvRY1D7GBE5lQz5KOQ9x5T_wxs-4wIfGPO4jiMcwQ,2103
65
+ cosmotech/csm_data/commands/api/wsf_send_file.py,sha256=AC-Li-2tN5KAtIE5TnElRbSgUTNCa7uWkn90MlOozjg,2045
66
+ cosmotech/csm_data/commands/store/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
67
+ cosmotech/csm_data/commands/store/dump_to_azure.py,sha256=9vavbHHueqdGbSjbt2GLnM2adzm7bDpmt_zUeX9ArZU,3314
68
+ cosmotech/csm_data/commands/store/dump_to_postgresql.py,sha256=7SB9e9TT8Kn3xeK0P4COTww-dsTJdG0vwjHAKkPob18,3597
69
+ cosmotech/csm_data/commands/store/dump_to_s3.py,sha256=6EUwtkP8_k9Hlq65R9h-MYsH58e-UAYmSR95JKoZuvY,5342
70
+ cosmotech/csm_data/commands/store/list_tables.py,sha256=dkrVANYhSD2FD1iny60ZXkaKZP5dPuK_eJ3R0gvlJtE,1861
71
+ cosmotech/csm_data/commands/store/load_csv_folder.py,sha256=aiMWmrC87kqTRJK06g16KlA90eXjPaFGswr2yZXJ2nM,1841
72
+ cosmotech/csm_data/commands/store/load_from_singlestore.py,sha256=cMPHnIfcjl4L7whwhs1sOkHf9eJbsj5s9ltzXO34KcM,3105
73
+ cosmotech/csm_data/commands/store/output.py,sha256=RkBQxZ6c-p6F3A0Ci7wW-ODHsrJJO8ivzQCumvjoqXI,1218
74
+ cosmotech/csm_data/commands/store/reset.py,sha256=At33wH3r_bITlUphp84NkBQNfz_8kW5FRlQDELopsUA,1361
75
+ cosmotech/csm_data/commands/store/store.py,sha256=oQhTIF1WKKGKdFp1rNm1vuTgblmh27r_jQpPhNFmesE,1663
76
+ cosmotech/csm_data/utils/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
77
+ cosmotech/csm_data/utils/click.py,sha256=S_85cbKh3R86-FZVjTK7IXZnmp4ETjKo6K8gbK3HCgs,848
78
+ cosmotech/csm_data/utils/decorators.py,sha256=HlteYuqP8WWIIErKRhvT1wYsUT5ZCZAQNTAXs70Tc8g,2500
79
+ cosmotech/orchestrator_plugins/csm-data/__init__.py,sha256=UPzNgaNBMoBaxSlnofxl988dAswMp_tv8DzaR_IxkiE,519
80
+ cosmotech/orchestrator_plugins/csm-data/templates/api/postgres_send_runner_metadata.json,sha256=alXzvOa985PvN5nf3uoTsFXgY29EJWeK94CcBjoLFUk,1085
81
+ cosmotech/orchestrator_plugins/csm-data/templates/api/run_load_data.json,sha256=_ztHPGu5UbbdtQHUCaOuKqwtjB0DrZSZwcUePoOgS40,894
82
+ cosmotech/orchestrator_plugins/csm-data/templates/api/try_api_connection.json,sha256=ApkzR0HQc0J6sxxTh91_DHPRI2cCtx_-qjpYFKGkGVw,215
83
+ cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_load_file.json,sha256=yX7yUrq7epQTp0PaQElFC-elof8W6jaMtek6uQ0V6Og,1142
84
+ cosmotech/orchestrator_plugins/csm-data/templates/api/wsf_send_file.json,sha256=7pyYOUPg_lvD4kw3sqd_zBonru7g-DVatQmu4EJw0Ps,1120
85
+ cosmotech/orchestrator_plugins/csm-data/templates/main/adx_send_runnerdata.json,sha256=thIyXyOqtnJFh-JWHijPytp_X5CG4194PS3Q_WgMbd4,1009
86
+ cosmotech/orchestrator_plugins/csm-data/templates/main/az_storage_upload.json,sha256=WS0MuXq5AbxkpOPUShmnBS1rHA3clfjXpTsdhooWMfk,804
87
+ cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_delete.json,sha256=iVgLJVDLiI44fLL6ejId-Du3T38bEPnS_j0ZWDSp1OY,1017
88
+ cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_download.json,sha256=aaFO8tWcN_x4ZMugC0peUxSEbarGOHnODh4vbbCXgCA,1131
89
+ cosmotech/orchestrator_plugins/csm-data/templates/main/s3_bucket_upload.json,sha256=pndJLHV-_QWAu0WEvsAs5TJf7G9f_oC-iOLNQsSjYEY,1129
90
+ cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_azure.json,sha256=iO3rSR80YiOb7xH1yl36jGzF65-qOp4NB9hdcYxKsX4,907
91
+ cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_postgresql.json,sha256=g0uG8FfxtLfjadolncbpUw-ZdQVY7uavpEj_DkPzqm8,898
92
+ cosmotech/orchestrator_plugins/csm-data/templates/store/store_dump_to_s3.json,sha256=4r9i4HEGgcAO0r8rlKJq7x6hTtt7dSO2cFeW74-BQbk,1050
93
+ cosmotech/orchestrator_plugins/csm-data/templates/store/store_list_tables.json,sha256=1xbjPqR6B-5o6lEQ6MXSPxuKy-px-85qhnRRYOcUXcQ,349
94
+ cosmotech/orchestrator_plugins/csm-data/templates/store/store_load_csv_folder.json,sha256=Vwgho85XordZV_RhAAhclJDgD727f7t9yQs0gn-MDPs,495
95
+ cosmotech/orchestrator_plugins/csm-data/templates/store/store_load_from_singlestore.json,sha256=AOWnKQ1C6p8hf6HNQ712N59h5xZCVaaqEL2QdhV3vj4,937
96
+ cosmotech/orchestrator_plugins/csm-data/templates/store/store_reset.json,sha256=jWKRDVW9HrUXHPG-MgVU_PljLyJkeYf6Tzi9wmL8yGc,337
97
+ cosmotech/translation/coal/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
98
+ cosmotech/translation/coal/en-US/coal/solution.yml,sha256=gHf26F5XX3BEiHvvV0bUFjje9dwlTZsl9KWm1M0brkU,141
99
+ cosmotech/translation/coal/en-US/coal/web.yml,sha256=I4Hu_zBm55cUsJVSSGQ0sk5KfX6GZPMmaXsLd2rG9u0,78
100
+ cosmotech/translation/coal/en-US/coal/common/data_transfer.yml,sha256=dSoheA2bEpaO5nRXhlg8XSQOJsfa9A7AGHCg6nQ82cM,299
101
+ cosmotech/translation/coal/en-US/coal/common/errors.yml,sha256=l1fiCZdTjff4R55PFLhj3hHx61u52DxHaKzJzc5ddA4,535
102
+ cosmotech/translation/coal/en-US/coal/common/file_operations.yml,sha256=DxggJM431fnFveUemHRM0Y18GPn_Vgv-LRSZXVRNCaI,278
103
+ cosmotech/translation/coal/en-US/coal/common/progress.yml,sha256=G_BTYgAYVD_JqU3rJ5fVTySQa0HtUcLNFoHRjSnBCwk,272
104
+ cosmotech/translation/coal/en-US/coal/common/timing.yml,sha256=YtxIT8EKZDWD4hJ0tJOwkReUsrOY67Wq0MFkEZjzANE,274
105
+ cosmotech/translation/coal/en-US/coal/common/validation.yml,sha256=OAHeXrvMo2_S-D0z-ESIS--U7bU0z1i0wta4ibaqk3c,528
106
+ cosmotech/translation/coal/en-US/coal/cosmotech_api/connection.yml,sha256=2Om0DZmOI8u0hzlWWhWnGx-o0mzMk_vgE3hGRBNaWWU,518
107
+ cosmotech/translation/coal/en-US/coal/cosmotech_api/initialization.yml,sha256=NGD5bbbS037QosTmHxXOiUJ_Zl3gyIr1iR6hCaSlolY,380
108
+ cosmotech/translation/coal/en-US/coal/cosmotech_api/run_template.yml,sha256=JIQ0S-9_YhchAPVFkcMpJTVUBlhPo4yS-nBoMysus5o,441
109
+ cosmotech/translation/coal/en-US/coal/cosmotech_api/runner.yml,sha256=fSoFiJFJcowXwBVgfAkZKL8j3iKzyVDLWm48cFpiYMA,804
110
+ cosmotech/translation/coal/en-US/coal/cosmotech_api/solution.yml,sha256=dKOTcv8bBYl3ZxyvKsQ8msO--PyLTfEOodb4rHl2bq4,216
111
+ cosmotech/translation/coal/en-US/coal/cosmotech_api/workspace.yml,sha256=4RHIHZi_ziSYB_U-YoLxNTm3jD67yN1Cxr14W9XoTok,307
112
+ cosmotech/translation/coal/en-US/coal/services/adx.yml,sha256=9yxR0qYKyA5mqgW-pX2J65JkkMcGFC3cPJChGmikJso,3502
113
+ cosmotech/translation/coal/en-US/coal/services/api.yml,sha256=HPyKJaZrS41Qo9upRvPsLdn29QgS1xw3S-bgipHeO4s,342
114
+ cosmotech/translation/coal/en-US/coal/services/azure_storage.yml,sha256=p0alxlJA_-NH9MxKXRKGgiLbBvHuQdCROhJKiqvuX0c,595
115
+ cosmotech/translation/coal/en-US/coal/services/database.yml,sha256=S1pAu4OrVd9nAVb3oYSAurzA4cgD43UDlwK4qWhy2g4,860
116
+ cosmotech/translation/coal/en-US/coal/services/dataset.yml,sha256=W-MHnp9E3udMGXdPriVZF7xgPSh1ksA3tFc7M7xMCkE,3265
117
+ cosmotech/translation/coal/en-US/coal/services/postgresql.yml,sha256=tRGNTTayyoxEHVyM-nzudEFdQ5HUeRIHPWIuvFmJGfM,1774
118
+ cosmotech/translation/coal/en-US/coal/services/s3.yml,sha256=362wt7SdOu-MyMj9MgjQNTNWzIq-RJ9rUqc2VKljGFE,482
119
+ cosmotech/translation/coal/en-US/coal/store/output/data_interface.yml,sha256=4pVJPAXYeIq9Gpmg_0VpSrB17AsGWA9hvk8Qi8-kZvw,85
120
+ cosmotech/translation/coal/en-US/coal/store/output/split.yml,sha256=h89v89sQsK5pGDJP5EE_KNbQbaS8003VPZNIgU2dqWw,265
121
+ cosmotech/translation/coal/en-US/coal/utils/configuration.yml,sha256=q3kEzPTP81SfsdrhAZa2JusHAjbdYqjs0DcvjhewQ-4,96
122
+ cosmotech/translation/csm_data/__init__.py,sha256=iD-xBT9s4HxXsO6FFIZiwlEmOl3qtsh93zHh9UKkT1g,413
123
+ cosmotech/translation/csm_data/en-US/csm_data/commands/main.yml,sha256=3XbXR5_Uo7PxJOWzm7XRMvPVxK-fzOL4FxWWY7rX3eg,396
124
+ cosmotech/translation/csm_data/en-US/csm_data/commands/api/api.yml,sha256=eMT4s1H9LuNUh_sAlnEtg7Co4ie3bFfB1vkwnLW28dQ,340
125
+ cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml,sha256=avckcrp2aYz7rzGCKEdL81hwHDVbcUn2csQ_krKLcaM,844
126
+ cosmotech/translation/csm_data/en-US/csm_data/commands/api/run_load_data.yml,sha256=k5J3HTA8vCmI_7CFHvMP51EAbmefubuoEhXZf2V3XDo,765
127
+ cosmotech/translation/csm_data/en-US/csm_data/commands/api/runtemplate_load_handler.yml,sha256=9jvMa6fb99CPzTMx7nnuoNNzJw7_7rapee7P80Plxjs,353
128
+ cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_load_file.yml,sha256=cnR_0PMkbrj8GROgvIUyeLXiB90IBbkcTvM9WJDEM0k,511
129
+ cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_send_file.yml,sha256=yvsD364GND1jLe_SJxuNXjDgGZFGlJlC5xUTRqV0fr0,632
130
+ cosmotech/translation/csm_data/en-US/csm_data/commands/storage/adx_send_data.yml,sha256=oNQMHhHMRTl7EfFLuEy5agctzgwGHsyKzAu6hlSUyU4,448
131
+ cosmotech/translation/csm_data/en-US/csm_data/commands/storage/adx_send_runnerdata.yml,sha256=_IAZO1VVc2eVznA2VLb0_o3sEc8cSO-xw3H90pUTVGg,884
132
+ cosmotech/translation/csm_data/en-US/csm_data/commands/storage/az_storage_upload.yml,sha256=KmVGI1RwFfFQnMUPAlRTJ6K3OXAYTy7GAitZfcWKdrw,473
133
+ cosmotech/translation/csm_data/en-US/csm_data/commands/storage/s3_bucket_delete.yml,sha256=GUCNXpiRui-BRwP8A7M4dXGr3LMBJw99z-b1ToFuESY,792
134
+ cosmotech/translation/csm_data/en-US/csm_data/commands/storage/s3_bucket_download.yml,sha256=GQPNFcdB0hIldX0kDBFNzHs11RckPBAA697BAJugZx8,872
135
+ cosmotech/translation/csm_data/en-US/csm_data/commands/storage/s3_bucket_upload.yml,sha256=jIrdisG9FcvTrCM5jvTaCYccKKXtah7huSvXDK8bdEA,1100
136
+ cosmotech/translation/csm_data/en-US/csm_data/commands/storage/storage.yml,sha256=NZrRzfnOpXrX5wBpEqr40CCmNGP5MXJ-aO7ks1YM58s,144
137
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_azure.yml,sha256=JqT9Cu1m4CLmZWvXTHAIS3xS8rK_BeKL041OEliBbn8,1299
138
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_postgresql.yml,sha256=QEQVHuXrSMsT2uOkiF-oMYlir1mINXXL-Irro_Ra3RM,900
139
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_s3.yml,sha256=7pbLObdjZjcKeSfYI_3oAzp9AZOrwT5WiMcLOTj5rCc,1268
140
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/list_tables.yml,sha256=-x8zwCjAHILFaJv7VLPskipSoQTwmI7jUsxlx6_OKlc,195
141
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_csv_folder.yml,sha256=9HjW9tOq5o31jmADp_QIync7G74c6XGiPqofXh8pvv8,228
142
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_from_singlestore.yml,sha256=_OUXlxC_wi-GPgHU7EVdU9g0WIdblVpEA4WIzw4aQcQ,824
143
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/output.yml,sha256=odkR9pzeel5l0wW_F4kx4HhNpw_TzEvok-OKyh25ZkQ,232
144
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/reset.yml,sha256=JMspZ25Q2HDpH8MYqUaQcveUnCgdH21RmfTCrPf2gW8,139
145
+ cosmotech/translation/csm_data/en-US/csm_data/commands/store/store.yml,sha256=N1Q8483gqJADaCe30S1M3Rj0tMJiuQiJH70-VK2x2m4,134
146
+ cosmotech/translation/csm_data/en-US/csm_data/commons/decorators.yml,sha256=Iu59NWMfYlZZf9uUhOiLkIEGa4GY5p0nZ6vG06Xvu7k,51
147
+ cosmotech/translation/csm_data/en-US/csm_data/commons/version.yml,sha256=7jtCV3O1S6pGjiJa63XpgPDTafjfBS0xmEVRpYNvfDg,86
148
+ cosmotech_acceleration_library-2.1.0rc1.dist-info/licenses/LICENSE,sha256=JXKHOQtyObmafNbQlfPYc4HkKjU9FzAP27b2qRTXNM8,1195
149
+ cosmotech_acceleration_library-2.1.0rc1.dist-info/METADATA,sha256=OLnJ8oFZytFr3FZL0R-gPy2lUwoVA4KosLZt5U3Y77c,9217
150
+ cosmotech_acceleration_library-2.1.0rc1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
151
+ cosmotech_acceleration_library-2.1.0rc1.dist-info/entry_points.txt,sha256=HWRqJurKuBUgqFe4jmjIAQrs768Nbb8ZTdRDLbuKM5Q,58
152
+ cosmotech_acceleration_library-2.1.0rc1.dist-info/top_level.txt,sha256=t2pzb8mpMUfHTa9l2SjWP0rRB8XVRjBdQK5nLx9XDDo,10
153
+ cosmotech_acceleration_library-2.1.0rc1.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (80.9.0)
2
+ Generator: setuptools (80.10.2)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -1,72 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
- import azure.functions as func
8
- from cosmotech.coal.cosmotech_api.runner.download import download_runner_data
9
- from cosmotech_api.api.runner_api import RunnerApi
10
-
11
- import json
12
- import http
13
- import traceback
14
-
15
-
16
- def generate_main(apply_update, parallel=True):
17
- def main(req: func.HttpRequest) -> func.HttpResponse:
18
- try:
19
- runner_id = req.params.get("scenario-id") # Keep parameter name for backward compatibility
20
- organization_id = req.params.get("organization-id")
21
- workspace_id = req.params.get("workspace-id")
22
- access_token: str = req.headers.get("authorization", None)
23
- if access_token:
24
- access_token = access_token.split(" ")[1]
25
-
26
- if runner_id is None or organization_id is None or workspace_id is None:
27
- return func.HttpResponse(
28
- body=f"Invalid request: organization-id={organization_id}, workspace-id={workspace_id}, scenario-id={runner_id}",
29
- status_code=http.HTTPStatus.BAD_REQUEST,
30
- )
31
-
32
- # Get runner data
33
- result = download_runner_data(
34
- organization_id=organization_id,
35
- workspace_id=workspace_id,
36
- runner_id=runner_id,
37
- parameter_folder=None, # We don't need to save to files
38
- read_files=True,
39
- parallel=parallel,
40
- write_json=False,
41
- write_csv=False,
42
- fetch_dataset=True,
43
- )
44
-
45
- content = {
46
- "datasets": result["datasets"],
47
- "parameters": result["parameters"],
48
- }
49
-
50
- runner_data = result["runner_data"]
51
-
52
- updated_content = apply_update(
53
- content=content, scenario_data=runner_data
54
- ) # Keep parameter name for backward compatibility
55
-
56
- return func.HttpResponse(
57
- body=json.dumps(updated_content),
58
- headers={"Content-Type": "application/json"},
59
- )
60
- except Exception as e:
61
- response = {
62
- "error": getattr(e, "message", str(e)),
63
- "type": type(e).__name__,
64
- "trace": traceback.format_exc(),
65
- }
66
- return func.HttpResponse(
67
- status_code=http.HTTPStatus.INTERNAL_SERVER_ERROR,
68
- body=json.dumps(response),
69
- headers={"Content-Type": "application/json"},
70
- )
71
-
72
- return main
@@ -1,96 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
- import os
8
- import pathlib
9
- import ssl
10
-
11
- import cosmotech_api
12
-
13
- from cosmotech.coal.utils.logger import LOGGER
14
- from cosmotech.orchestrator.utils.translate import T
15
-
16
- api_env_keys = {"CSM_API_KEY", "CSM_API_URL"}
17
- azure_env_keys = {
18
- "AZURE_CLIENT_ID",
19
- "AZURE_CLIENT_SECRET",
20
- "AZURE_TENANT_ID",
21
- "CSM_API_URL",
22
- "CSM_API_SCOPE",
23
- }
24
- keycloak_env_keys = {
25
- "IDP_TENANT_ID",
26
- "IDP_CLIENT_ID",
27
- "IDP_CLIENT_SECRET",
28
- "IDP_BASE_URL",
29
- "CSM_API_URL",
30
- }
31
-
32
-
33
- def get_api_client() -> (cosmotech_api.ApiClient, str):
34
- existing_keys = set(os.environ.keys())
35
- missing_azure_keys = azure_env_keys - existing_keys
36
- missing_api_keys = api_env_keys - existing_keys
37
- missing_keycloak_keys = keycloak_env_keys - existing_keys
38
- if all((missing_api_keys, missing_azure_keys, missing_keycloak_keys)):
39
- LOGGER.error(T("coal.common.errors.no_env_vars"))
40
- LOGGER.error(T("coal.cosmotech_api.connection.existing_sets"))
41
- LOGGER.error(T("coal.cosmotech_api.connection.azure_connection").format(keys=", ".join(azure_env_keys)))
42
- LOGGER.error(T("coal.cosmotech_api.connection.api_key_connection").format(keys=", ".join(api_env_keys)))
43
- LOGGER.error(T("coal.cosmotech_api.connection.keycloak_connection").format(keys=", ".join(keycloak_env_keys)))
44
- raise EnvironmentError(T("coal.common.errors.no_env_vars"))
45
-
46
- if not missing_keycloak_keys:
47
- LOGGER.info(T("coal.cosmotech_api.connection.found_keycloak"))
48
- from keycloak import KeycloakOpenID
49
-
50
- server_url = os.environ.get("IDP_BASE_URL")
51
- if server_url[-1] != "/":
52
- server_url = server_url + "/"
53
- keycloack_parameters = dict(
54
- server_url=server_url,
55
- client_id=os.environ.get("IDP_CLIENT_ID"),
56
- realm_name=os.environ.get("IDP_TENANT_ID"),
57
- client_secret_key=os.environ.get("IDP_CLIENT_SECRET"),
58
- )
59
- if (ca_cert_path := os.environ.get("IDP_CA_CERT")) and pathlib.Path(ca_cert_path).exists():
60
- LOGGER.info(T("coal.cosmotech_api.connection.found_cert_authority"))
61
- keycloack_parameters["verify"] = ca_cert_path
62
- keycloak_openid = KeycloakOpenID(**keycloack_parameters)
63
-
64
- access_token = keycloak_openid.token(grant_type="client_credentials")
65
-
66
- configuration = cosmotech_api.Configuration(
67
- host=os.environ.get("CSM_API_URL"),
68
- access_token=access_token["access_token"],
69
- )
70
- return cosmotech_api.ApiClient(configuration), "Keycloak Connection"
71
-
72
- if not missing_api_keys:
73
- LOGGER.info(T("coal.cosmotech_api.connection.found_api_key"))
74
- configuration = cosmotech_api.Configuration(
75
- host=os.environ.get("CSM_API_URL"),
76
- )
77
- return (
78
- cosmotech_api.ApiClient(
79
- configuration,
80
- os.environ.get("CSM_API_KEY_HEADER", "X-CSM-API-KEY"),
81
- os.environ.get("CSM_API_KEY"),
82
- ),
83
- "Cosmo Tech API Key",
84
- )
85
-
86
- if not missing_azure_keys:
87
- LOGGER.info(T("coal.cosmotech_api.connection.found_azure"))
88
- from azure.identity import EnvironmentCredential
89
-
90
- credentials = EnvironmentCredential()
91
- token = credentials.get_token(os.environ.get("CSM_API_SCOPE"))
92
-
93
- configuration = cosmotech_api.Configuration(host=os.environ.get("CSM_API_URL"), access_token=token.token)
94
- return cosmotech_api.ApiClient(configuration), "Azure Entra Connection"
95
-
96
- raise EnvironmentError(T("coal.common.errors.no_valid_connection"))
@@ -1,26 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- # Re-export all download functions from download submodule
9
- from cosmotech.coal.cosmotech_api.dataset.download import (
10
- download_adt_dataset,
11
- download_twingraph_dataset,
12
- download_legacy_twingraph_dataset,
13
- download_file_dataset,
14
- download_dataset_by_id,
15
- )
16
-
17
- from cosmotech.coal.cosmotech_api.dataset.converters import (
18
- convert_dataset_to_files,
19
- convert_graph_dataset_to_files,
20
- convert_file_dataset_to_files,
21
- )
22
-
23
- from cosmotech.coal.cosmotech_api.dataset.utils import (
24
- get_content_from_twin_graph_data,
25
- sheet_to_header,
26
- )
@@ -1,164 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- import csv
9
- import json
10
- import os
11
- import tempfile
12
- from pathlib import Path
13
- from typing import Dict, List, Any, Optional, Union
14
-
15
- from cosmotech.coal.utils.logger import LOGGER
16
- from cosmotech.orchestrator.utils.translate import T
17
- from cosmotech.coal.cosmotech_api.dataset.utils import sheet_to_header
18
-
19
-
20
- def convert_dataset_to_files(dataset_info: Dict[str, Any], target_folder: Optional[Union[str, Path]] = None) -> Path:
21
- """
22
- Convert dataset info to files.
23
-
24
- Args:
25
- dataset_info: Dataset info dict with type, content, name
26
- target_folder: Optional folder to save files (if None, uses temp dir)
27
-
28
- Returns:
29
- Path to folder containing files
30
- """
31
- dataset_type = dataset_info["type"]
32
- content = dataset_info["content"]
33
- name = dataset_info["name"]
34
-
35
- LOGGER.info(T("coal.services.dataset.converting_to_files").format(dataset_type=dataset_type, dataset_name=name))
36
-
37
- if target_folder is None:
38
- target_folder = Path(tempfile.mkdtemp())
39
- LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
40
- else:
41
- target_folder = Path(target_folder)
42
- target_folder.mkdir(parents=True, exist_ok=True)
43
- LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
44
-
45
- if dataset_type in ["adt", "twincache"]:
46
- return convert_graph_dataset_to_files(content, target_folder)
47
- else:
48
- return convert_file_dataset_to_files(content, target_folder, dataset_type)
49
-
50
-
51
- def convert_graph_dataset_to_files(
52
- content: Dict[str, List[Dict]], target_folder: Optional[Union[str, Path]] = None
53
- ) -> Path:
54
- """
55
- Convert graph dataset content to CSV files.
56
-
57
- Args:
58
- content: Dictionary mapping entity types to lists of entities
59
- target_folder: Folder to save files (if None, uses temp dir)
60
-
61
- Returns:
62
- Path to folder containing files
63
- """
64
- if target_folder is None:
65
- target_folder = Path(tempfile.mkdtemp())
66
- LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
67
- else:
68
- target_folder = Path(target_folder)
69
- target_folder.mkdir(parents=True, exist_ok=True)
70
- LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
71
- file_count = 0
72
-
73
- LOGGER.info(
74
- T("coal.services.dataset.converting_graph_data").format(entity_types=len(content), folder=target_folder)
75
- )
76
-
77
- for entity_type, entities in content.items():
78
- if not entities:
79
- LOGGER.debug(T("coal.services.dataset.skipping_empty_entity").format(entity_type=entity_type))
80
- continue
81
-
82
- file_path = target_folder / f"{entity_type}.csv"
83
- LOGGER.debug(T("coal.services.dataset.writing_csv").format(file_name=file_path.name, count=len(entities)))
84
-
85
- fieldnames = sheet_to_header(entities)
86
-
87
- with open(file_path, "w", newline="") as file:
88
- writer = csv.DictWriter(file, fieldnames=fieldnames, dialect="unix", quoting=csv.QUOTE_MINIMAL)
89
- writer.writeheader()
90
-
91
- for entity in entities:
92
- # Convert values to strings and handle boolean values
93
- row = {
94
- k: str(v).replace("'", '"').replace("True", "true").replace("False", "false")
95
- for k, v in entity.items()
96
- }
97
- writer.writerow(row)
98
-
99
- file_count += 1
100
- LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
101
-
102
- LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
103
-
104
- return target_folder
105
-
106
-
107
- def convert_file_dataset_to_files(
108
- content: Dict[str, Any],
109
- target_folder: Optional[Union[str, Path]] = None,
110
- file_type: str = "",
111
- ) -> Path:
112
- """
113
- Convert file dataset content to files.
114
-
115
- Args:
116
- content: Dictionary mapping file names to content
117
- target_folder: Folder to save files (if None, uses temp dir)
118
- file_type: Type of file (csv, json, etc.)
119
-
120
- Returns:
121
- Path to folder containing files
122
- """
123
- if target_folder is None:
124
- target_folder = Path(tempfile.mkdtemp())
125
- LOGGER.debug(T("coal.services.dataset.created_temp_folder").format(folder=target_folder))
126
- else:
127
- target_folder = Path(target_folder)
128
- target_folder.mkdir(parents=True, exist_ok=True)
129
- LOGGER.debug(T("coal.services.dataset.using_folder").format(folder=target_folder))
130
- file_count = 0
131
-
132
- LOGGER.info(
133
- T("coal.services.dataset.converting_file_data").format(
134
- file_count=len(content), file_type=file_type, folder=target_folder
135
- )
136
- )
137
-
138
- for file_name, file_content in content.items():
139
- file_path = target_folder / file_name
140
-
141
- # Ensure parent directories exist
142
- file_path.parent.mkdir(parents=True, exist_ok=True)
143
-
144
- LOGGER.debug(T("coal.services.dataset.writing_file").format(file_name=file_path.name, file_type=file_type))
145
-
146
- if isinstance(file_content, str):
147
- # Text content
148
- with open(file_path, "w") as file:
149
- file.write(file_content)
150
- elif isinstance(file_content, dict) or isinstance(file_content, list):
151
- # JSON content
152
- with open(file_path, "w") as file:
153
- json.dump(file_content, file, indent=2)
154
- else:
155
- # Other content types
156
- with open(file_path, "w") as file:
157
- file.write(str(file_content))
158
-
159
- file_count += 1
160
- LOGGER.debug(T("coal.services.dataset.file_written").format(file_path=file_path))
161
-
162
- LOGGER.info(T("coal.services.dataset.files_created").format(count=file_count, folder=target_folder))
163
-
164
- return target_folder
@@ -1,19 +0,0 @@
1
- # Copyright (C) - 2023 - 2025 - Cosmo Tech
2
- # This document and all information contained herein is the exclusive property -
3
- # including all intellectual property rights pertaining thereto - of Cosmo Tech.
4
- # Any use, reproduction, translation, broadcasting, transmission, distribution,
5
- # etc., to any person is prohibited unless it has been previously and
6
- # specifically authorized by written means by Cosmo Tech.
7
-
8
- """
9
- Dataset download submodules.
10
- """
11
-
12
- # Re-export all download functions
13
- from cosmotech.coal.cosmotech_api.dataset.download.adt import download_adt_dataset
14
- from cosmotech.coal.cosmotech_api.dataset.download.twingraph import (
15
- download_twingraph_dataset,
16
- download_legacy_twingraph_dataset,
17
- )
18
- from cosmotech.coal.cosmotech_api.dataset.download.file import download_file_dataset
19
- from cosmotech.coal.cosmotech_api.dataset.download.common import download_dataset_by_id