atlan-application-sdk 0.1.1rc33__py3-none-any.whl → 0.1.1rc35__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. application_sdk/activities/__init__.py +3 -2
  2. application_sdk/activities/common/utils.py +21 -1
  3. application_sdk/activities/metadata_extraction/base.py +104 -0
  4. application_sdk/activities/metadata_extraction/sql.py +13 -12
  5. application_sdk/activities/query_extraction/sql.py +24 -20
  6. application_sdk/application/__init__.py +8 -0
  7. application_sdk/clients/atlan_auth.py +2 -2
  8. application_sdk/clients/base.py +293 -0
  9. application_sdk/clients/temporal.py +6 -10
  10. application_sdk/handlers/base.py +50 -0
  11. application_sdk/inputs/json.py +6 -4
  12. application_sdk/inputs/parquet.py +16 -13
  13. application_sdk/outputs/__init__.py +6 -3
  14. application_sdk/outputs/json.py +9 -6
  15. application_sdk/outputs/parquet.py +10 -36
  16. application_sdk/server/fastapi/__init__.py +4 -5
  17. application_sdk/server/fastapi/models.py +1 -1
  18. application_sdk/services/__init__.py +18 -0
  19. application_sdk/{outputs → services}/atlan_storage.py +64 -16
  20. application_sdk/{outputs → services}/eventstore.py +68 -6
  21. application_sdk/services/objectstore.py +407 -0
  22. application_sdk/services/secretstore.py +344 -0
  23. application_sdk/services/statestore.py +267 -0
  24. application_sdk/version.py +1 -1
  25. application_sdk/worker.py +1 -1
  26. {atlan_application_sdk-0.1.1rc33.dist-info → atlan_application_sdk-0.1.1rc35.dist-info}/METADATA +1 -1
  27. {atlan_application_sdk-0.1.1rc33.dist-info → atlan_application_sdk-0.1.1rc35.dist-info}/RECORD +30 -30
  28. application_sdk/common/credential_utils.py +0 -85
  29. application_sdk/inputs/objectstore.py +0 -238
  30. application_sdk/inputs/secretstore.py +0 -130
  31. application_sdk/inputs/statestore.py +0 -101
  32. application_sdk/outputs/objectstore.py +0 -125
  33. application_sdk/outputs/secretstore.py +0 -38
  34. application_sdk/outputs/statestore.py +0 -113
  35. {atlan_application_sdk-0.1.1rc33.dist-info → atlan_application_sdk-0.1.1rc35.dist-info}/WHEEL +0 -0
  36. {atlan_application_sdk-0.1.1rc33.dist-info → atlan_application_sdk-0.1.1rc35.dist-info}/licenses/LICENSE +0 -0
  37. {atlan_application_sdk-0.1.1rc33.dist-info → atlan_application_sdk-0.1.1rc35.dist-info}/licenses/NOTICE +0 -0
@@ -1,29 +1,30 @@
1
1
  application_sdk/__init__.py,sha256=2e2mvmLJ5dxmJGPELtb33xwP-j6JMdoIuqKycEn7hjg,151
2
2
  application_sdk/constants.py,sha256=_Fmk9PgpM68chPDHHkgrs4Zg2KK4UCqqg7Oj9_u3WVo,9486
3
- application_sdk/version.py,sha256=S2v-GGamis8Ciz___AuEVE_IfiBRwYHL3uV0kM3PWyU,88
4
- application_sdk/worker.py,sha256=dZLxPkAieCSw7XEWxzL-FRk0QAZm7vXQBICjZODy3B4,7488
5
- application_sdk/activities/__init__.py,sha256=EH5VTHcfGykIX7V1HsG0J1Z-1FbJEXTQOET0HdzFDjU,9519
3
+ application_sdk/version.py,sha256=z0Ar6Kx6DAuBs3JjgjdT2lqiLuVvhu2S7idSn3AwbsU,88
4
+ application_sdk/worker.py,sha256=i5f0AeKI39IfsLO05QkwC6uMz0zDPSJqP7B2byri1VI,7489
5
+ application_sdk/activities/__init__.py,sha256=QaXLOBYbb0zPOY5kfDQh56qbXQFaYNXOjJ5PCvatiZ4,9530
6
6
  application_sdk/activities/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
7
  application_sdk/activities/common/models.py,sha256=305WdrZB7EAtCOAU_q9hMw81XowUdCeuFs9zfzb-MHQ,1196
8
- application_sdk/activities/common/utils.py,sha256=CWAj_tQUSQirSs5wwy-9eS8yI_4HoDfXsj2U_Xkb4Bc,6480
8
+ application_sdk/activities/common/utils.py,sha256=F4Fq9Gl_gvUQj_fSdwzTU7obqUnemYL1dgb_yS34vTM,6967
9
9
  application_sdk/activities/metadata_extraction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
+ application_sdk/activities/metadata_extraction/base.py,sha256=ENFojpxqKdN_eVSL4iet3cGfylPOfcl1jnflfo4zhs8,3920
10
11
  application_sdk/activities/metadata_extraction/rest.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
11
- application_sdk/activities/metadata_extraction/sql.py,sha256=3G9_KGKyS0kTBN-nOKdvNSpTqwFv0nUectsqhMewpnU,22594
12
+ application_sdk/activities/metadata_extraction/sql.py,sha256=m_kbWjIMcI-L9Ss8VlHoNzmjfC5I8tZ5MPrfMXkvxH8,22615
12
13
  application_sdk/activities/query_extraction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
13
- application_sdk/activities/query_extraction/sql.py,sha256=xC-dC_so3D9yY88lSL2W8Q8CfDRjiIrF-OHKbITFgd0,21271
14
- application_sdk/application/__init__.py,sha256=sUmng7msiUCPmQu1YY9gRvzXWviVKABVjKT3diPC6GM,7553
14
+ application_sdk/activities/query_extraction/sql.py,sha256=mesGP_kiWzrJ8wboWFVt2jbDuGG_Fl3kQVvVMdH3KWA,21228
15
+ application_sdk/application/__init__.py,sha256=WDWDWP-IQ-ny7okqsrdTwH60cXKgXBRcnlJ1XVYfiNU,7957
15
16
  application_sdk/application/metadata_extraction/sql.py,sha256=ohpV4qZ92uKRlH7I_8G67ocnWkZJAZCU_7XdvqYPiN4,7966
16
17
  application_sdk/clients/__init__.py,sha256=C9T84J7V6ZumcoWJPAxdd3tqSmbyciaGBJn-CaCCny0,1341
17
18
  application_sdk/clients/async_atlan.py,sha256=RTgRbMw6zJWcv1C-7cU4ccaSW5XZsB5dcA1Tlkj32p8,2699
18
19
  application_sdk/clients/atlan.py,sha256=f2-Uk5KiPIDJEhGkfYctA_f3CwoVB_mWNBMVvxeLuY4,2684
19
- application_sdk/clients/atlan_auth.py,sha256=MQznmvVKrlOT_Tp232W4UrOupRrx9Dx9zQm3n1R7kD8,8938
20
+ application_sdk/clients/atlan_auth.py,sha256=D7FuNqv81ohNXLJtdx1AFw_jU6a3g0Pw6149ia4ucFY,8930
21
+ application_sdk/clients/base.py,sha256=TIn3pG89eXUc1XSYf4jk66m1vajWp0WxcCQOOltdazA,14021
20
22
  application_sdk/clients/sql.py,sha256=tW89SHuuWdU5jv8lDUP5AUCEpR2CF_5TyUvYDCBHses,17880
21
- application_sdk/clients/temporal.py,sha256=jyU2MYGZXkTn0Gqy_qvYg0iSc2bKz4snflsU_XcDsfk,23662
23
+ application_sdk/clients/temporal.py,sha256=nzHZMq6mAOYk2E9tpgjzztW2-ztXrFXwV8TiOrSfbNw,23503
22
24
  application_sdk/clients/utils.py,sha256=zLFOJbTr_6TOqnjfVFGY85OtIXZ4FQy_rquzjaydkbY,779
23
25
  application_sdk/clients/workflow.py,sha256=6bSqmA3sNCk9oY68dOjBUDZ9DhNKQxPD75qqE0cfldc,6104
24
26
  application_sdk/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
27
  application_sdk/common/aws_utils.py,sha256=aeL3BTMzv1UWJ4KxfwY5EsfYnxtS1FKNJ4xKdHeoTjc,3438
26
- application_sdk/common/credential_utils.py,sha256=M9oraG2uPeOSbxUAOJlP2IeClsDD79EhNkdow42dFsI,3025
27
28
  application_sdk/common/dapr_utils.py,sha256=0yHqDP6qNb1OT-bX2XRYQPZ5xkGkV13nyRw6GkPlHs8,1136
28
29
  application_sdk/common/dataframe_utils.py,sha256=PId9vT6AUoq3tesiTd4sSUvW7RUhPWdAAEBLuOprks4,1262
29
30
  application_sdk/common/error_codes.py,sha256=uROVfOOMrGPO8JroWB3vs5rIEhr0GfcPqXAK9wdhcVQ,13742
@@ -46,15 +47,13 @@ application_sdk/docgen/parsers/manifest.py,sha256=3NP-dBTpHAUQa477usMIDaKSb_9xfL
46
47
  application_sdk/events/__init__.py,sha256=OcbVWDF4ZKRTJXK9UaFVtYEwu-3DHE77S-Sn6jNafUs,204
47
48
  application_sdk/events/models.py,sha256=7Esqp3WlbriT2EqT4kNiY_sHtRXRPLj27b8SbeC5Sb0,5121
48
49
  application_sdk/handlers/__init__.py,sha256=U7kKwVWK0FZz1uIJ2ANN0C5tD83k_9Nyz0ns6ttr92g,1152
50
+ application_sdk/handlers/base.py,sha256=6GF0a4p9mbdVoBltkzXwt6i0vzOvOKUHcB6fBiqi7v0,1883
49
51
  application_sdk/handlers/sql.py,sha256=oeB-sgWwPYo31xaD87TyMc0h51Sary1F-CmhExt9_Pk,16100
50
52
  application_sdk/inputs/__init__.py,sha256=_d-cUhcDyoJTJR3PdQkC831go6VDw9AM6Bg7-qm3NHI,1900
51
53
  application_sdk/inputs/iceberg.py,sha256=xiv1kNtVx1k0h3ZJbJeXjZwdfBGSy9j9orYP_AyCYlI,2756
52
- application_sdk/inputs/json.py,sha256=J1CVz0YGQHDyq840TyoBHE7Baua2yIFHzsrybiZbeWk,6079
53
- application_sdk/inputs/objectstore.py,sha256=uOJW0uB3FrDmnyHFhAd23QOq3MKrAzhYdKCszdpMeF8,8219
54
- application_sdk/inputs/parquet.py,sha256=j1O5uYZ2ok28iPYFpMoltKzJTDGmht7qoEjRKQ48P8E,6590
55
- application_sdk/inputs/secretstore.py,sha256=0H81CkbKj1KTU_wlPofpsAvPHs9YaOuKbRBmLRdiiNw,4583
54
+ application_sdk/inputs/json.py,sha256=Yv70Y9YuutN2trqK5-z2UNtBL0895ZbdEiBDt9cYM9s,6216
55
+ application_sdk/inputs/parquet.py,sha256=KPcfqXOnK4C2orBCAPIO0DZgw1sYMC69MRNrqXPbCBU,6704
56
56
  application_sdk/inputs/sql_query.py,sha256=1EREgea6kKNaMIyX2HLJgbJ07rtAgLasd9NyvDcdZok,10636
57
- application_sdk/inputs/statestore.py,sha256=ORWnv8ZCqC1wT4vlW4v5EemJT4oQ3t_DlpjKDAgTRTs,3117
58
57
  application_sdk/observability/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
59
58
  application_sdk/observability/logger_adaptor.py,sha256=WTqnNg78W2SRGOQVhELVLn6KMRsurkG1kc7essL08Lk,29529
60
59
  application_sdk/observability/metrics_adaptor.py,sha256=4TYPNn38zLeqxwf7cUbe8wh_zwQlr-nyiXjJsiEhTEM,16445
@@ -62,23 +61,24 @@ application_sdk/observability/observability.py,sha256=DP0I4bHyg3TA4hxCqDFy2IiRmB
62
61
  application_sdk/observability/traces_adaptor.py,sha256=0eQJPN-tYA_dV8D3uEa5ZiX9g12NDuLnPaFuQMVDdL0,18242
63
62
  application_sdk/observability/utils.py,sha256=MKEpT0WYtpATUgLgJDkGQaAP_t-jpDYMUKDfEvr8Phg,2448
64
63
  application_sdk/observability/decorators/observability_decorator.py,sha256=JNrWNXT5W4klmlAc5b8C3_VBjDu0PI64W2ptr7LMzk4,8110
65
- application_sdk/outputs/__init__.py,sha256=HX8VcN22xyrkoRWdjQj8TrC5dEUG7cPzOcvJhlprqAs,8415
66
- application_sdk/outputs/atlan_storage.py,sha256=HQLbuyOZQC-GxYAiCVJakIJizTWy926tdMGOHvaBlD8,6029
67
- application_sdk/outputs/eventstore.py,sha256=1Im6dTQmIqyz6x5tExLFQpwdLrlReYZ1pktHk-tyLd8,4094
64
+ application_sdk/outputs/__init__.py,sha256=-0EGS3EeBMGJ-jbsFn2PlsSAkSdN4uFNvcQFZiTLbEM,8583
68
65
  application_sdk/outputs/iceberg.py,sha256=IGtj5WDgqLu6vzDEvw5DLsKsjm29Krto3AHvWpemr0A,5311
69
- application_sdk/outputs/json.py,sha256=xF-8mY3BZRRejip4s9npIUuFaAxgFmBQVaLMkrI_iCI,14117
70
- application_sdk/outputs/objectstore.py,sha256=TJvgfkJpGRK129ttxY7qRYJ7ASKZA4R6-0BUA3Lk7mc,4450
71
- application_sdk/outputs/parquet.py,sha256=A2EnEx1zWjaXk10u3eJusmWxGxt8WR7CHXDaJgsKpq0,11040
72
- application_sdk/outputs/secretstore.py,sha256=JS9vUzb11leDpcMQSCnLJuE9Ww-9G3wMvCdUKBPaw9I,1342
73
- application_sdk/outputs/statestore.py,sha256=XiEag2e9WW3_D3xbWQGoNrHiFJz9916qcIvhrROX8_8,3999
66
+ application_sdk/outputs/json.py,sha256=YrynTptxtsJksUuVk6jCwUYjr_DTtQGGy_BuFfHA-Ps,14268
67
+ application_sdk/outputs/parquet.py,sha256=vIsrnCQy573sYqxgYAY-Ks4QwDfRr-TQG9d1uLR9wmI,10022
74
68
  application_sdk/server/__init__.py,sha256=KTqE1YPw_3WDVMWatJUuf9OOiobLM2K5SMaBrI62sCo,1568
75
- application_sdk/server/fastapi/__init__.py,sha256=1RNP3170Es_GcxVSweqkl_iz4Sd0Evi8bs6fuVuVTiA,27770
76
- application_sdk/server/fastapi/models.py,sha256=QarZthIq-ileNua-SCh6AldRuEjRCZk-C7QQZ5K6STE,6682
69
+ application_sdk/server/fastapi/__init__.py,sha256=YOdWNE-qqiXfo-exvxPg8T0PSuOxTdeSetUn6-BXxZg,27704
70
+ application_sdk/server/fastapi/models.py,sha256=K6eNl3XXiTXKUvRTpq3oqdGH3jY1-ApobXma04J86fE,6665
77
71
  application_sdk/server/fastapi/utils.py,sha256=2XI4DylhRQsukhX67lpAzRNCHeFCSpbuNd7TlE2IBJA,1164
78
72
  application_sdk/server/fastapi/middleware/logmiddleware.py,sha256=CxcPtDmCbSfSZ8RyI09nIshVIbCokyyA9bByQJ2G_ns,2545
79
73
  application_sdk/server/fastapi/middleware/metrics.py,sha256=5ddHAIg5sT-u9tB_HHMGL3Cfu2g1rm9z7ksienIr9ks,1563
80
74
  application_sdk/server/fastapi/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
81
75
  application_sdk/server/fastapi/routers/server.py,sha256=vfHQwZCysThzfeVFNVW1IjuAdL0c1Cs4fULKTBK2eNo,4209
76
+ application_sdk/services/__init__.py,sha256=H-5HZEPdr53MUfAggyHqHhRXDRLZFZsxvJgWbr257Ds,465
77
+ application_sdk/services/atlan_storage.py,sha256=TKzXxu0yXeUcmZehwp8PcnQTC4A9w9RlZ0Fl-Xp1bLE,8509
78
+ application_sdk/services/eventstore.py,sha256=X03JzodKByXh8w8nOl658rnnZfMFTj0IkmiLVbd6IN8,6729
79
+ application_sdk/services/objectstore.py,sha256=0TO7TYRvZlQHU3OH6PCA6tGO49E2lM7eCvV-fRWLiqU,14098
80
+ application_sdk/services/secretstore.py,sha256=UpyLLcdMia1tqFCpRrn-lE9AnERAt2iGVzET6QqkmqI,13976
81
+ application_sdk/services/statestore.py,sha256=CQuKq4FXPS0ebDH0e0cfTTAjvsIlrA1zz1MpsWCiWnM,9562
82
82
  application_sdk/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
83
83
  application_sdk/test_utils/workflow_monitoring.py,sha256=gqq6CsT62GrMt2GqtNSb1iD_-t4MBffQvpO0BXboZek,3490
84
84
  application_sdk/test_utils/e2e/__init__.py,sha256=qswG5goB5Xzt3yQyZsKGq20EnZMZCmXYTe0CK3Qk3xo,10125
@@ -135,8 +135,8 @@ application_sdk/workflows/metadata_extraction/__init__.py,sha256=jHUe_ZBQ66jx8bg
135
135
  application_sdk/workflows/metadata_extraction/sql.py,sha256=_NhszxIgmcQI6lVpjJoyJRFLwPYvJw1Dyqox_m9K2RA,11947
136
136
  application_sdk/workflows/query_extraction/__init__.py,sha256=n066_CX5RpJz6DIxGMkKS3eGSRg03ilaCtsqfJWQb7Q,117
137
137
  application_sdk/workflows/query_extraction/sql.py,sha256=kT_JQkLCRZ44ZpaC4QvPL6DxnRIIVh8gYHLqRbMI-hA,4826
138
- atlan_application_sdk-0.1.1rc33.dist-info/METADATA,sha256=wXJnZNtP3QrB6mv01MU9HhbVPtdamPBDnNt_oDrACAI,5468
139
- atlan_application_sdk-0.1.1rc33.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
140
- atlan_application_sdk-0.1.1rc33.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
141
- atlan_application_sdk-0.1.1rc33.dist-info/licenses/NOTICE,sha256=A-XVVGt3KOYuuMmvSMIFkg534F1vHiCggEBp4Ez3wGk,1041
142
- atlan_application_sdk-0.1.1rc33.dist-info/RECORD,,
138
+ atlan_application_sdk-0.1.1rc35.dist-info/METADATA,sha256=bQqHjrAoOeWNf2kR6F4CoHgeK-fQb3wRgkDxcvx-6SM,5468
139
+ atlan_application_sdk-0.1.1rc35.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
140
+ atlan_application_sdk-0.1.1rc35.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
141
+ atlan_application_sdk-0.1.1rc35.dist-info/licenses/NOTICE,sha256=A-XVVGt3KOYuuMmvSMIFkg534F1vHiCggEBp4Ez3wGk,1041
142
+ atlan_application_sdk-0.1.1rc35.dist-info/RECORD,,
@@ -1,85 +0,0 @@
1
- """Utilities for credential providers."""
2
-
3
- import asyncio
4
- import copy
5
- from concurrent.futures import ThreadPoolExecutor
6
- from typing import Any, Dict
7
-
8
- from application_sdk.common.error_codes import CommonError
9
- from application_sdk.inputs.secretstore import SecretStoreInput
10
- from application_sdk.inputs.statestore import StateStoreInput, StateType
11
- from application_sdk.observability.logger_adaptor import get_logger
12
-
13
- logger = get_logger(__name__)
14
-
15
-
16
- async def get_credentials(credential_guid: str) -> Dict[str, Any]:
17
- """
18
- Resolve credentials based on credential source.
19
-
20
- Args:
21
- credential_guid: The GUID of the credential to resolve
22
-
23
- Returns:
24
- Dict with resolved credentials
25
-
26
- Raises:
27
- CommonError: If credential resolution fails
28
- """
29
-
30
- def _get_credentials_sync(credential_guid: str) -> Dict[str, Any]:
31
- """Synchronous helper function to perform blocking I/O operations."""
32
- credential_config = StateStoreInput.get_state(
33
- credential_guid, StateType.CREDENTIALS
34
- )
35
-
36
- # Fetch secret data from secret store
37
- secret_key = credential_config.get("secret-path", credential_guid)
38
- secret_data = SecretStoreInput.get_secret(secret_key=secret_key)
39
-
40
- # Resolve credentials
41
- credential_source = credential_config.get("credentialSource", "direct")
42
- if credential_source == "direct":
43
- credential_config.update(secret_data)
44
- return credential_config
45
- else:
46
- return resolve_credentials(credential_config, secret_data)
47
-
48
- try:
49
- # Run blocking I/O operations in a thread pool to avoid blocking the event loop
50
- loop = asyncio.get_running_loop()
51
- with ThreadPoolExecutor() as pool:
52
- return await loop.run_in_executor(
53
- pool, _get_credentials_sync, credential_guid
54
- )
55
- except Exception as e:
56
- logger.error(f"Error resolving credentials: {str(e)}")
57
- raise CommonError(
58
- CommonError.CREDENTIALS_RESOLUTION_ERROR,
59
- f"Failed to resolve credentials: {str(e)}",
60
- )
61
-
62
-
63
- def resolve_credentials(
64
- credential_config: Dict[str, Any], secret_data: Dict[str, Any]
65
- ) -> Dict[str, Any]:
66
- """
67
- Resolve credentials
68
- """
69
- credentials = copy.deepcopy(credential_config)
70
-
71
- # Replace values with secret values
72
- for key, value in list(credentials.items()):
73
- if isinstance(value, str) and value in secret_data:
74
- credentials[key] = secret_data[value]
75
-
76
- # Apply the same substitution to the 'extra' dictionary if it exists
77
- if "extra" in credentials and isinstance(credentials["extra"], dict):
78
- for key, value in list(credentials["extra"].items()):
79
- if isinstance(value, str):
80
- if value in secret_data:
81
- credentials["extra"][key] = secret_data[value]
82
- elif value in secret_data.get("extra", {}):
83
- credentials["extra"][key] = secret_data["extra"][value]
84
-
85
- return credentials
@@ -1,238 +0,0 @@
1
- """Object store interface for the application."""
2
-
3
- import json
4
- import os
5
- from typing import Dict, List, Union
6
-
7
- import orjson
8
- from dapr.clients import DaprClient
9
- from temporalio import activity
10
-
11
- from application_sdk.constants import (
12
- DAPR_MAX_GRPC_MESSAGE_LENGTH,
13
- DEPLOYMENT_OBJECT_STORE_NAME,
14
- )
15
- from application_sdk.observability.logger_adaptor import get_logger
16
-
17
- logger = get_logger(__name__)
18
- activity.logger = logger
19
-
20
-
21
- class ObjectStoreInput:
22
- OBJECT_GET_OPERATION = "get"
23
- OBJECT_LIST_OPERATION = "list"
24
-
25
- @classmethod
26
- def _invoke_dapr_binding(
27
- cls,
28
- operation: str,
29
- metadata: Dict[str, str],
30
- data: Union[bytes, str] = "",
31
- object_store_name: str = DEPLOYMENT_OBJECT_STORE_NAME,
32
- ) -> bytes:
33
- """
34
- Common method to invoke Dapr binding operations.
35
-
36
- Args:
37
- operation (str): The Dapr binding operation to perform
38
- metadata (Dict[str, str]): Metadata for the binding operation
39
- data (Optional[bytes]): Optional data to send with the request
40
-
41
- Returns:
42
- bytes: Response data from the Dapr binding
43
-
44
- Raises:
45
- Exception: If there's an error with the Dapr binding operation
46
- """
47
- try:
48
- with DaprClient(
49
- max_grpc_message_length=DAPR_MAX_GRPC_MESSAGE_LENGTH
50
- ) as client:
51
- response = client.invoke_binding(
52
- binding_name=object_store_name,
53
- operation=operation,
54
- data=data,
55
- binding_metadata=metadata,
56
- )
57
- return response.data
58
- except Exception as e:
59
- logger.error(f"Error in Dapr binding operation '{operation}': {str(e)}")
60
- raise
61
-
62
- @classmethod
63
- def download_files_from_object_store(
64
- cls,
65
- download_file_prefix: str,
66
- file_path: str,
67
- object_store_name: str = DEPLOYMENT_OBJECT_STORE_NAME,
68
- ) -> None:
69
- """
70
- Downloads all files from the object store for a given prefix.
71
-
72
- Args:
73
- download_file_prefix (str): The base path in the object store to download files from.
74
- local_directory (str): The local directory where the files should be downloaded.
75
-
76
- Raises:
77
- Exception: If there's an error downloading any file from the object store.
78
- """
79
- try:
80
- # List all files in the object store path
81
- relative_path = os.path.relpath(file_path, download_file_prefix)
82
- file_list = cls.list_all_files(relative_path, object_store_name)
83
-
84
- logger.info(
85
- f"Found list of files: {file_list} from: {download_file_prefix}"
86
- )
87
-
88
- # Download each file
89
- for relative_path in file_list:
90
- local_file_path = os.path.join(
91
- file_path, os.path.basename(relative_path)
92
- )
93
- cls.download_file_from_object_store(
94
- download_file_prefix, local_file_path
95
- )
96
-
97
- logger.info(
98
- f"Successfully downloaded all files from: {download_file_prefix}"
99
- )
100
- except Exception as e:
101
- logger.warning(f"Failed to download files from object store: {str(e)}")
102
- raise
103
-
104
- @classmethod
105
- def download_file_from_object_store(
106
- cls,
107
- download_file_prefix: str,
108
- file_path: str,
109
- object_store_name: str = DEPLOYMENT_OBJECT_STORE_NAME,
110
- ) -> None:
111
- """Downloads a single file from the object store.
112
-
113
- Args:
114
- download_file_prefix (str): The base path to calculate relative paths from.
115
- example: /tmp/output
116
- file_path (str): The full path to where the file should be downloaded.
117
- example: /tmp/output/persistent-artifacts/apps/myapp/data/wf-123/state.json
118
-
119
- Raises:
120
- Exception: If there's an error downloading the file from the object store.
121
- """
122
- if not os.path.exists(os.path.dirname(file_path)):
123
- os.makedirs(os.path.dirname(file_path), exist_ok=True)
124
-
125
- relative_path = os.path.relpath(file_path, download_file_prefix)
126
-
127
- try:
128
- # Use get_file_data to retrieve the file bytes
129
- response_data = cls.get_file_data(relative_path, object_store_name)
130
-
131
- # Write the bytes to the local file
132
- with open(file_path, "wb") as f:
133
- f.write(response_data)
134
-
135
- logger.info(f"Successfully downloaded file: {relative_path}")
136
- except Exception as e:
137
- logger.warning(
138
- f"Failed to download file {relative_path} from object store: {str(e)}"
139
- )
140
- raise e
141
-
142
- @classmethod
143
- def list_all_files(
144
- cls, prefix: str = "", object_store_name: str = DEPLOYMENT_OBJECT_STORE_NAME
145
- ) -> List[str]:
146
- """
147
- List all files in the object store under a given prefix.
148
-
149
- Args:
150
- prefix (str): The prefix to filter files. Empty string returns all files.
151
-
152
- Returns:
153
- List[str]: List of file paths in the object store
154
-
155
- Raises:
156
- Exception: If there's an error listing files from the object store.
157
- """
158
- try:
159
- # this takes care of listing from all type of storage - local as well as object stores
160
- metadata = {"prefix": prefix, "fileName": prefix} if prefix else {}
161
- data = json.dumps({"prefix": prefix}).encode("utf-8") if prefix else ""
162
-
163
- response_data = cls._invoke_dapr_binding(
164
- operation=cls.OBJECT_LIST_OPERATION,
165
- metadata=metadata,
166
- data=data,
167
- object_store_name=object_store_name,
168
- )
169
-
170
- if not response_data:
171
- return []
172
-
173
- file_list = orjson.loads(response_data.decode("utf-8"))
174
-
175
- # Extract paths based on response type
176
- if isinstance(file_list, list):
177
- paths = file_list
178
- elif isinstance(file_list, dict) and "Contents" in file_list:
179
- paths = [item["Key"] for item in file_list["Contents"] if "Key" in item]
180
- elif isinstance(file_list, dict):
181
- paths = file_list.get("files") or file_list.get("keys") or []
182
- else:
183
- return []
184
-
185
- valid_list = []
186
- for path in paths:
187
- if not isinstance(path, str):
188
- logger.warning(f"Skipping non-string path: {path}")
189
- continue
190
- valid_list.append(
191
- path[path.find(prefix) :]
192
- if prefix and prefix in path
193
- else os.path.basename(path)
194
- if prefix
195
- else path
196
- )
197
-
198
- return valid_list
199
-
200
- except Exception as e:
201
- logger.error(f"Error listing files with prefix {prefix}: {str(e)}")
202
- raise e
203
-
204
- @classmethod
205
- def get_file_data(
206
- cls, file_path: str, object_store_name: str = DEPLOYMENT_OBJECT_STORE_NAME
207
- ) -> bytes:
208
- """
209
- Get raw file data from the object store.
210
-
211
- Args:
212
- file_path (str): The full path of the file in the object store
213
-
214
- Returns:
215
- bytes: The raw file data
216
-
217
- Raises:
218
- Exception: If there's an error getting the file from the object store.
219
- """
220
- try:
221
- metadata = {"key": file_path, "fileName": file_path}
222
- data = json.dumps({"key": file_path}).encode("utf-8") if file_path else ""
223
-
224
- response_data = cls._invoke_dapr_binding(
225
- operation=cls.OBJECT_GET_OPERATION,
226
- metadata=metadata,
227
- data=data,
228
- object_store_name=object_store_name,
229
- )
230
- if not response_data:
231
- raise Exception(f"No data received for file: {file_path}")
232
-
233
- logger.debug(f"Successfully retrieved file data: {file_path}")
234
- return response_data
235
-
236
- except Exception as e:
237
- logger.error(f"Error getting file data for {file_path}: {str(e)}")
238
- raise e
@@ -1,130 +0,0 @@
1
- """Secret store for the application."""
2
-
3
- import collections.abc
4
- import copy
5
- import json
6
- from typing import Any, Dict
7
-
8
- from dapr.clients import DaprClient
9
-
10
- from application_sdk.common.dapr_utils import is_component_registered
11
- from application_sdk.constants import (
12
- DEPLOYMENT_NAME,
13
- DEPLOYMENT_SECRET_PATH,
14
- DEPLOYMENT_SECRET_STORE_NAME,
15
- LOCAL_ENVIRONMENT,
16
- SECRET_STORE_NAME,
17
- )
18
- from application_sdk.observability.logger_adaptor import get_logger
19
-
20
- logger = get_logger(__name__)
21
-
22
-
23
- class SecretStoreInput:
24
- @classmethod
25
- def get_deployment_secret(cls) -> Dict[str, Any]:
26
- """Get deployment config from the deployment secret store.
27
-
28
- Validates that the deployment secret store component is registered
29
- before attempting to fetch secrets to prevent errors.
30
-
31
- Returns:
32
- Dict[str, Any]: Deployment configuration data, or empty dict if
33
- component is unavailable or fetch fails.
34
- """
35
- if not is_component_registered(DEPLOYMENT_SECRET_STORE_NAME):
36
- logger.warning(
37
- f"Deployment secret store component '{DEPLOYMENT_SECRET_STORE_NAME}' is not registered"
38
- )
39
- return {}
40
-
41
- try:
42
- return cls.get_secret(DEPLOYMENT_SECRET_PATH, DEPLOYMENT_SECRET_STORE_NAME)
43
- except Exception as e:
44
- logger.error(f"Failed to fetch deployment config: {e}")
45
- return {}
46
-
47
- @classmethod
48
- def get_secret(
49
- cls, secret_key: str, component_name: str = SECRET_STORE_NAME
50
- ) -> Dict[str, Any]:
51
- """Get secret from the Dapr component.
52
-
53
- Args:
54
- secret_key: Key of the secret to fetch
55
- component_name: Name of the Dapr component to fetch from
56
-
57
- Returns:
58
- Dict with processed secret data
59
- """
60
- if DEPLOYMENT_NAME == LOCAL_ENVIRONMENT:
61
- return {}
62
-
63
- try:
64
- with DaprClient() as client:
65
- dapr_secret_object = client.get_secret(
66
- store_name=component_name, key=secret_key
67
- )
68
- return cls._process_secret_data(dapr_secret_object.secret)
69
- except Exception as e:
70
- logger.error(
71
- f"Failed to fetch secret using component {component_name}: {str(e)}"
72
- )
73
- raise
74
-
75
- @classmethod
76
- def _process_secret_data(cls, secret_data: Any) -> Dict[str, Any]:
77
- """Process raw secret data into a standardized dictionary format.
78
-
79
- Args:
80
- secret_data: Raw secret data from various sources.
81
-
82
- Returns:
83
- Dict[str, Any]: Processed secret data as a dictionary.
84
- """
85
- # Convert ScalarMapContainer to dict if needed
86
- if isinstance(secret_data, collections.abc.Mapping):
87
- secret_data = dict(secret_data)
88
-
89
- # If the dict has a single key and its value is a JSON string, parse it
90
- if len(secret_data) == 1 and isinstance(next(iter(secret_data.values())), str):
91
- try:
92
- parsed = json.loads(next(iter(secret_data.values())))
93
- if isinstance(parsed, dict):
94
- secret_data = parsed
95
- except Exception as e:
96
- logger.error(f"Failed to parse secret data: {e}")
97
- pass
98
-
99
- return secret_data
100
-
101
- @classmethod
102
- def apply_secret_values(
103
- cls, source_data: Dict[str, Any], secret_data: Dict[str, Any]
104
- ) -> Dict[str, Any]:
105
- """Apply secret values to source data by substituting references.
106
-
107
- This function replaces values in the source data with values
108
- from the secret data when the source value exists as a key in the secrets.
109
-
110
- Args:
111
- source_data: Original data with potential references to secrets
112
- secret_data: Secret data containing actual values
113
-
114
- Returns:
115
- Dict[str, Any]: Data with secret values applied
116
- """
117
- result_data = copy.deepcopy(source_data)
118
-
119
- # Replace values with secret values
120
- for key, value in list(result_data.items()):
121
- if isinstance(value, str) and value in secret_data:
122
- result_data[key] = secret_data[value]
123
-
124
- # Apply the same substitution to the 'extra' dictionary if it exists
125
- if "extra" in result_data and isinstance(result_data["extra"], dict):
126
- for key, value in list(result_data["extra"].items()):
127
- if isinstance(value, str) and value in secret_data:
128
- result_data["extra"][key] = secret_data[value]
129
-
130
- return result_data
@@ -1,101 +0,0 @@
1
- """State store for the application."""
2
-
3
- import json
4
- import os
5
- from enum import Enum
6
- from typing import Any, Dict
7
-
8
- from temporalio import activity
9
-
10
- from application_sdk.constants import (
11
- APPLICATION_NAME,
12
- STATE_STORE_PATH_TEMPLATE,
13
- TEMPORARY_PATH,
14
- UPSTREAM_OBJECT_STORE_NAME,
15
- )
16
- from application_sdk.inputs.objectstore import ObjectStoreInput
17
- from application_sdk.observability.logger_adaptor import get_logger
18
-
19
- logger = get_logger(__name__)
20
- activity.logger = logger
21
-
22
-
23
- class StateType(Enum):
24
- WORKFLOWS = "workflows"
25
- CREDENTIALS = "credentials"
26
-
27
- @classmethod
28
- def is_member(cls, type: str) -> bool:
29
- return type in cls._value2member_map_
30
-
31
-
32
- def build_state_store_path(id: str, state_type: StateType) -> str:
33
- """Build the state file path for the given id and type.
34
-
35
- Args:
36
- id: The unique identifier for the state.
37
- state_type: The type of state (workflows, credentials, etc.).
38
-
39
- Returns:
40
- str: The constructed state file path.
41
-
42
- Example:
43
- >>> build_state_store_path("wf-123", "workflows")
44
- 'persistent-artifacts/apps/my-app/workflows/wf-123/config.json'
45
- """
46
- return STATE_STORE_PATH_TEMPLATE.format(
47
- application_name=APPLICATION_NAME, state_type=state_type.value, id=id
48
- )
49
-
50
-
51
- class StateStoreInput:
52
- @classmethod
53
- def get_state(cls, id: str, type: StateType) -> Dict[str, Any]:
54
- """Get state from the store.
55
-
56
- Args:
57
- id: The key to retrieve the state for.
58
- type: The type of state to retrieve.
59
-
60
- Returns:
61
- Dict[str, Any]: The retrieved state data.
62
-
63
- Raises:
64
- ValueError: If no state is found for the given key.
65
- IOError: If there's an error with the Dapr client operations.
66
-
67
- Example:
68
- >>> from application_sdk.inputs.statestore import StateStoreInput
69
-
70
- >>> state = StateStoreInput.get_state("wf-123")
71
- >>> print(state)
72
- {'test': 'test'}
73
- """
74
-
75
- state_file_path = build_state_store_path(id, type)
76
- state = {}
77
-
78
- try:
79
- logger.info(f"Trying to download state object for {id} with type {type}")
80
- local_state_file_path = os.path.join(TEMPORARY_PATH, state_file_path)
81
- ObjectStoreInput.download_file_from_object_store(
82
- download_file_prefix=TEMPORARY_PATH,
83
- file_path=local_state_file_path,
84
- object_store_name=UPSTREAM_OBJECT_STORE_NAME,
85
- )
86
-
87
- with open(local_state_file_path, "r") as file:
88
- state = json.load(file)
89
-
90
- logger.info(f"State object downloaded for {id} with type {type}")
91
- except Exception as e:
92
- # local error message is "file not found", while in object store it is "object not found"
93
- if "not found" in str(e).lower():
94
- logger.info(
95
- f"No state found for {type.value} with id '{id}', returning empty dict"
96
- )
97
- else:
98
- logger.error(f"Failed to extract state: {str(e)}")
99
- raise
100
-
101
- return state