atlan-application-sdk 1.1.1__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- application_sdk/activities/common/sql_utils.py +308 -0
- application_sdk/activities/common/utils.py +1 -45
- application_sdk/activities/metadata_extraction/sql.py +110 -353
- application_sdk/activities/query_extraction/sql.py +12 -11
- application_sdk/application/__init__.py +1 -1
- application_sdk/clients/sql.py +167 -1
- application_sdk/clients/temporal.py +6 -6
- application_sdk/common/types.py +8 -0
- application_sdk/common/utils.py +1 -8
- application_sdk/constants.py +1 -1
- application_sdk/handlers/sql.py +10 -25
- application_sdk/interceptors/events.py +1 -1
- application_sdk/io/__init__.py +654 -0
- application_sdk/io/json.py +429 -0
- application_sdk/{outputs → io}/parquet.py +358 -47
- application_sdk/io/utils.py +307 -0
- application_sdk/observability/observability.py +23 -12
- application_sdk/server/fastapi/middleware/logmiddleware.py +23 -17
- application_sdk/server/fastapi/middleware/metrics.py +27 -24
- application_sdk/server/fastapi/models.py +1 -1
- application_sdk/server/fastapi/routers/server.py +1 -1
- application_sdk/server/fastapi/utils.py +10 -0
- application_sdk/services/eventstore.py +4 -4
- application_sdk/services/secretstore.py +1 -1
- application_sdk/test_utils/hypothesis/strategies/outputs/json_output.py +0 -1
- application_sdk/test_utils/hypothesis/strategies/server/fastapi/__init__.py +1 -1
- application_sdk/version.py +1 -1
- application_sdk/worker.py +1 -1
- {atlan_application_sdk-1.1.1.dist-info → atlan_application_sdk-2.0.0.dist-info}/METADATA +9 -11
- {atlan_application_sdk-1.1.1.dist-info → atlan_application_sdk-2.0.0.dist-info}/RECORD +35 -42
- application_sdk/common/dataframe_utils.py +0 -42
- application_sdk/events/__init__.py +0 -5
- application_sdk/inputs/.cursor/BUGBOT.md +0 -250
- application_sdk/inputs/__init__.py +0 -168
- application_sdk/inputs/iceberg.py +0 -75
- application_sdk/inputs/json.py +0 -136
- application_sdk/inputs/parquet.py +0 -272
- application_sdk/inputs/sql_query.py +0 -271
- application_sdk/outputs/.cursor/BUGBOT.md +0 -295
- application_sdk/outputs/__init__.py +0 -453
- application_sdk/outputs/iceberg.py +0 -139
- application_sdk/outputs/json.py +0 -268
- /application_sdk/{events → interceptors}/models.py +0 -0
- /application_sdk/{common/dapr_utils.py → services/_utils.py} +0 -0
- {atlan_application_sdk-1.1.1.dist-info → atlan_application_sdk-2.0.0.dist-info}/WHEEL +0 -0
- {atlan_application_sdk-1.1.1.dist-info → atlan_application_sdk-2.0.0.dist-info}/licenses/LICENSE +0 -0
- {atlan_application_sdk-1.1.1.dist-info → atlan_application_sdk-2.0.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: atlan-application-sdk
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: Atlan Application SDK is a Python library for developing applications on the Atlan Platform
|
|
5
5
|
Project-URL: Repository, https://github.com/atlanhq/application-sdk
|
|
6
6
|
Project-URL: Documentation, https://github.com/atlanhq/application-sdk/README.md
|
|
@@ -22,29 +22,27 @@ Requires-Python: >=3.11
|
|
|
22
22
|
Requires-Dist: aiohttp<3.14.0,>=3.10.0
|
|
23
23
|
Requires-Dist: duckdb-engine<0.18.0,>=0.17.0
|
|
24
24
|
Requires-Dist: duckdb<1.5.0,>=1.1.3
|
|
25
|
-
Requires-Dist: fastapi[standard]==0.
|
|
25
|
+
Requires-Dist: fastapi[standard]==0.127.0
|
|
26
26
|
Requires-Dist: loguru<0.8.0,>=0.7.3
|
|
27
|
-
Requires-Dist: opentelemetry-exporter-otlp==1.39.
|
|
28
|
-
Requires-Dist: psutil<7.
|
|
27
|
+
Requires-Dist: opentelemetry-exporter-otlp==1.39.1
|
|
28
|
+
Requires-Dist: psutil<7.3.0,>=7.0.0
|
|
29
29
|
Requires-Dist: pyatlan<8.5.0,>=8.0.2
|
|
30
30
|
Requires-Dist: pydantic<2.13.0,>=2.10.6
|
|
31
31
|
Requires-Dist: python-dotenv<1.3.0,>=1.1.0
|
|
32
32
|
Requires-Dist: uvloop<0.23.0,>=0.21.0; sys_platform != 'win32'
|
|
33
33
|
Provides-Extra: daft
|
|
34
|
-
Requires-Dist: daft<0.
|
|
34
|
+
Requires-Dist: daft<0.8.0,>=0.4.12; extra == 'daft'
|
|
35
35
|
Provides-Extra: distributed-lock
|
|
36
36
|
Requires-Dist: redis[hiredis]<7.2.0,>=5.2.0; extra == 'distributed-lock'
|
|
37
37
|
Provides-Extra: iam-auth
|
|
38
38
|
Requires-Dist: boto3<1.43.0,>=1.38.6; extra == 'iam-auth'
|
|
39
|
-
Provides-Extra: iceberg
|
|
40
|
-
Requires-Dist: pyiceberg<0.11.0,>=0.8.1; extra == 'iceberg'
|
|
41
39
|
Provides-Extra: mcp
|
|
42
|
-
Requires-Dist: fastmcp<2.
|
|
40
|
+
Requires-Dist: fastmcp<2.15.0,>=2.12.3; extra == 'mcp'
|
|
43
41
|
Provides-Extra: pandas
|
|
44
42
|
Requires-Dist: pandas<2.4.0,>=2.2.3; extra == 'pandas'
|
|
45
43
|
Provides-Extra: scale-data-generator
|
|
46
|
-
Requires-Dist: faker<
|
|
47
|
-
Requires-Dist: numpy<2.
|
|
44
|
+
Requires-Dist: faker<39.1.0,>=37.1.0; extra == 'scale-data-generator'
|
|
45
|
+
Requires-Dist: numpy<2.5.0,>=1.23.5; extra == 'scale-data-generator'
|
|
48
46
|
Requires-Dist: pandas<2.4.0,>=2.2.3; extra == 'scale-data-generator'
|
|
49
47
|
Requires-Dist: pyarrow<23.0.0,>=20.0.0; extra == 'scale-data-generator'
|
|
50
48
|
Requires-Dist: pyyaml<6.1.0,>=6.0.2; extra == 'scale-data-generator'
|
|
@@ -57,7 +55,7 @@ Requires-Dist: pytest-order<1.4.0,>=1.3.0; extra == 'tests'
|
|
|
57
55
|
Provides-Extra: workflows
|
|
58
56
|
Requires-Dist: dapr==1.16.0; extra == 'workflows'
|
|
59
57
|
Requires-Dist: orjson<3.12.0,>=3.10.18; extra == 'workflows'
|
|
60
|
-
Requires-Dist: temporalio<1.
|
|
58
|
+
Requires-Dist: temporalio<1.22.0,>=1.7.1; extra == 'workflows'
|
|
61
59
|
Description-Content-Type: text/markdown
|
|
62
60
|
|
|
63
61
|
# Atlan Application SDK
|
|
@@ -1,20 +1,21 @@
|
|
|
1
1
|
application_sdk/__init__.py,sha256=2e2mvmLJ5dxmJGPELtb33xwP-j6JMdoIuqKycEn7hjg,151
|
|
2
|
-
application_sdk/constants.py,sha256=
|
|
3
|
-
application_sdk/version.py,sha256=
|
|
4
|
-
application_sdk/worker.py,sha256=
|
|
2
|
+
application_sdk/constants.py,sha256=TvdmKQShVWBNQZdVF2y-fxuE31FmeraTnqQ9jT_n5XY,11567
|
|
3
|
+
application_sdk/version.py,sha256=TY0SZFUH9N2qGF6tlbmJIww-CY7x-myMGotNEhFJko4,84
|
|
4
|
+
application_sdk/worker.py,sha256=DLMocpHvvwpdAopyXhxwM7ftaNlKvZMQfkgy1MFyiik,7561
|
|
5
5
|
application_sdk/activities/__init__.py,sha256=6SiefuOPUDGfN3z6oPY4RkQLiUmkHpoDy5xadzpDzAw,11588
|
|
6
6
|
application_sdk/activities/lock_management.py,sha256=6Wdf3jMKitoarHQP91PIJOoGFz4aaOLS_40c7n1yAOA,3902
|
|
7
7
|
application_sdk/activities/.cursor/BUGBOT.md,sha256=FNykX5aMkdOhzgpiGqstOnSp9JN63iR2XP3onU4AGh8,15843
|
|
8
8
|
application_sdk/activities/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
9
|
application_sdk/activities/common/models.py,sha256=43MF_w0EzEQiJvGIqF_FNet4X6MEmwqYd3YAsHdQn08,1362
|
|
10
|
-
application_sdk/activities/common/
|
|
10
|
+
application_sdk/activities/common/sql_utils.py,sha256=csLM8H9L5NY5_rhxHBFo-jkMoOKjxhB9xaFbnLbkBGg,10177
|
|
11
|
+
application_sdk/activities/common/utils.py,sha256=ngyFmiZnMCAQtyu6vGeAlkzwNkM29MD_gBU5pWqOxJ4,8392
|
|
11
12
|
application_sdk/activities/metadata_extraction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
13
|
application_sdk/activities/metadata_extraction/base.py,sha256=ENFojpxqKdN_eVSL4iet3cGfylPOfcl1jnflfo4zhs8,3920
|
|
13
14
|
application_sdk/activities/metadata_extraction/rest.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
-
application_sdk/activities/metadata_extraction/sql.py,sha256=
|
|
15
|
+
application_sdk/activities/metadata_extraction/sql.py,sha256=IkI1ZhOKyoSwosRT-g8c8IDBuFBq7mwyHLpDvwYO_B4,25451
|
|
15
16
|
application_sdk/activities/query_extraction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
-
application_sdk/activities/query_extraction/sql.py,sha256=
|
|
17
|
-
application_sdk/application/__init__.py,sha256=
|
|
17
|
+
application_sdk/activities/query_extraction/sql.py,sha256=Gsa79R8CYY0uyt3rA2nLMfQs8-C4_zg1pJ_yYSF2cZw,21193
|
|
18
|
+
application_sdk/application/__init__.py,sha256=vcrQsqlfmGvKcCZuOtHHaNRqHSGdXlEDftkb8Tv_shI,9867
|
|
18
19
|
application_sdk/application/metadata_extraction/sql.py,sha256=rOd06Wodr4GyzupCYxVSCsNcuNar1rJM66ej9vocNHw,8138
|
|
19
20
|
application_sdk/clients/__init__.py,sha256=C9T84J7V6ZumcoWJPAxdd3tqSmbyciaGBJn-CaCCny0,1341
|
|
20
21
|
application_sdk/clients/atlan.py,sha256=l6yV39fr1006SJFwkOTNDQlbSFlHCZQaUPfdUlzdVEg,5053
|
|
@@ -22,18 +23,17 @@ application_sdk/clients/atlan_auth.py,sha256=_MykgutI-Ill1t8ERgc1a7QrfaxnrtZjD48
|
|
|
22
23
|
application_sdk/clients/base.py,sha256=TIn3pG89eXUc1XSYf4jk66m1vajWp0WxcCQOOltdazA,14021
|
|
23
24
|
application_sdk/clients/models.py,sha256=iZOTyH6LO64kozdiUPCFCN0NgLhd_Gtv0lH7ZIPdo8w,1800
|
|
24
25
|
application_sdk/clients/redis.py,sha256=IfAD32vLp88BCvsDTaQtxFHxzHlEx4V7TK7h1HwDDBg,15917
|
|
25
|
-
application_sdk/clients/sql.py,sha256=
|
|
26
|
-
application_sdk/clients/temporal.py,sha256=
|
|
26
|
+
application_sdk/clients/sql.py,sha256=J43FCxLW2YbnH2MlSm5hCTRFOMOEBtHFqi4ZTTul4JQ,26300
|
|
27
|
+
application_sdk/clients/temporal.py,sha256=7ZkQSwSSZTFkBkhwFlqmLBFxlD2-jLS8QWpSiFTX7V4,20024
|
|
27
28
|
application_sdk/clients/utils.py,sha256=zLFOJbTr_6TOqnjfVFGY85OtIXZ4FQy_rquzjaydkbY,779
|
|
28
29
|
application_sdk/clients/workflow.py,sha256=6bSqmA3sNCk9oY68dOjBUDZ9DhNKQxPD75qqE0cfldc,6104
|
|
29
30
|
application_sdk/clients/.cursor/BUGBOT.md,sha256=7nEDUqWBEMI_uU6eK1jCSZGeXoQtLQcKwOrDn8AIDWo,10595
|
|
30
31
|
application_sdk/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
31
32
|
application_sdk/common/aws_utils.py,sha256=xlSMIQyjvQ-CydEXaxXrnPUygv7AAbCLsxhZ2wtKnzg,11219
|
|
32
|
-
application_sdk/common/dapr_utils.py,sha256=0yHqDP6qNb1OT-bX2XRYQPZ5xkGkV13nyRw6GkPlHs8,1136
|
|
33
|
-
application_sdk/common/dataframe_utils.py,sha256=PId9vT6AUoq3tesiTd4sSUvW7RUhPWdAAEBLuOprks4,1262
|
|
34
33
|
application_sdk/common/error_codes.py,sha256=bxgvugN_0H5b8VXfJw-44mybgX5I9lRJbRdYjtPjqDI,14561
|
|
35
34
|
application_sdk/common/file_converter.py,sha256=ta0PVh7uIEGJg0BTPUJnSjj55ve2iVAOkqwAeg96_-g,3079
|
|
36
|
-
application_sdk/common/
|
|
35
|
+
application_sdk/common/types.py,sha256=qkVr3SAR1zn5_0w9hFt18vGtcnaPLKlMJLSBhTSKePU,134
|
|
36
|
+
application_sdk/common/utils.py,sha256=czcWvqoe2PawDvHOahk_AI88Zqth-CM3KzdGmehHQJ4,19286
|
|
37
37
|
application_sdk/common/.cursor/BUGBOT.md,sha256=OkB5TMAEJFzaBfbNb3g9ZDPW2r1krQE_KEuJbytMPuI,12176
|
|
38
38
|
application_sdk/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
39
39
|
application_sdk/decorators/locks.py,sha256=-cdbICCMns3lkqZ4CCQabW1du8cEu9XSWlwzWTTbIPk,1411
|
|
@@ -54,53 +54,46 @@ application_sdk/docgen/models/manifest/section.py,sha256=4-cqJjSojQ31QiNXI-3koHc
|
|
|
54
54
|
application_sdk/docgen/parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
55
55
|
application_sdk/docgen/parsers/directory.py,sha256=8Kk2sjb-0l2wLO_njdlcuHjv5akoNgmf-FmaDSaE4WM,7751
|
|
56
56
|
application_sdk/docgen/parsers/manifest.py,sha256=3NP-dBTpHAUQa477usMIDaKSb_9xfLE8G3RX0T1Bq2s,3318
|
|
57
|
-
application_sdk/events/__init__.py,sha256=OcbVWDF4ZKRTJXK9UaFVtYEwu-3DHE77S-Sn6jNafUs,204
|
|
58
|
-
application_sdk/events/models.py,sha256=kEzJKvb-G1M7aKrLPgAmsukJXLXeh8hIJKwEkOiaY28,6115
|
|
59
57
|
application_sdk/handlers/__init__.py,sha256=3Wf7jCVFR2nYOyHZEc9jj8BQUnHCylFqoezp70J2Df0,1329
|
|
60
58
|
application_sdk/handlers/base.py,sha256=ieWFbv8Gm7vfrrpS-mdMSm-mHGuQY02qiAVX2qPdj3w,2467
|
|
61
|
-
application_sdk/handlers/sql.py,sha256=
|
|
62
|
-
application_sdk/inputs/__init__.py,sha256=_O5lK2A5EYyqwid8txKNEds3pHkoHGKrSTTWnQ-UzRA,6022
|
|
63
|
-
application_sdk/inputs/iceberg.py,sha256=xiv1kNtVx1k0h3ZJbJeXjZwdfBGSy9j9orYP_AyCYlI,2756
|
|
64
|
-
application_sdk/inputs/json.py,sha256=ZOgB3tuZSsb2m_KxiAdnbUQgU5ythCs-Mq-n4pPfeHA,4905
|
|
65
|
-
application_sdk/inputs/parquet.py,sha256=9OzbrLZfkWdabqyLvHklwt4bONDp0WvNS5PHn2D0bnA,9519
|
|
66
|
-
application_sdk/inputs/sql_query.py,sha256=9deGGI5Wob8mDuq-vRjYgQvVrfuU_IjLkiYufZ2NqTo,10632
|
|
67
|
-
application_sdk/inputs/.cursor/BUGBOT.md,sha256=hwKGDbopv3NU0bpC_ElpAPDFcS59GWS3TunObGC6eLQ,9731
|
|
59
|
+
application_sdk/handlers/sql.py,sha256=WkFSo61SLiY3FoHO35Gqxx6YXRTftVJGrWBdLRELwQY,17273
|
|
68
60
|
application_sdk/interceptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
69
61
|
application_sdk/interceptors/cleanup.py,sha256=JlFcM_2Y5AIEfGTSNe0aoon7eoE68MIXI0rA3LHsSeY,5966
|
|
70
62
|
application_sdk/interceptors/correlation_context.py,sha256=j486uXZbLA85eTBJ8hxXS13vA8Ucf3_Nt5KiyA_cJVY,5864
|
|
71
|
-
application_sdk/interceptors/events.py,sha256=
|
|
63
|
+
application_sdk/interceptors/events.py,sha256=e0O6uK9_aCTmOORaTGN9RbcTg9_KNaakq-Meh8l2lsI,6477
|
|
72
64
|
application_sdk/interceptors/lock.py,sha256=5ETm20zrTaH2b9fepN4Ckp1tGJV-uINqDrno_5RW3aw,6169
|
|
65
|
+
application_sdk/interceptors/models.py,sha256=kEzJKvb-G1M7aKrLPgAmsukJXLXeh8hIJKwEkOiaY28,6115
|
|
73
66
|
application_sdk/interceptors/.cursor/BUGBOT.md,sha256=pxmUF2c7dtaXAX8yAa1-LBa6FCrj_uw7aQcHrppjf1A,14570
|
|
67
|
+
application_sdk/io/__init__.py,sha256=i5EbFW0IInErum4DTRAhWBjwT2EwiUHTpdYBHNmI-I4,24317
|
|
68
|
+
application_sdk/io/json.py,sha256=yjgCMzbz4Ltyt0PCylVRrUqGl2FhQL_wfWW1KtSi-II,17435
|
|
69
|
+
application_sdk/io/parquet.py,sha256=1J_C5c_LwDfuugIpiQqYgmU2uVcGearF5Kkrx-_34_I,31740
|
|
70
|
+
application_sdk/io/utils.py,sha256=sn_8Q6HgjeC8uyZp2XGMAfqdJ8XzkIllOEVYXIH54DY,10724
|
|
74
71
|
application_sdk/observability/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
75
72
|
application_sdk/observability/context.py,sha256=lJjpfxEjMY_hrdSDqq519YaWcztgc_1nM4d-mGV5shs,634
|
|
76
73
|
application_sdk/observability/logger_adaptor.py,sha256=Fq5OE579ozr0EzsNYEh2H0q3POVAxtlWfJ-PSwWDGLM,30194
|
|
77
74
|
application_sdk/observability/metrics_adaptor.py,sha256=5Oz02lUED60duryoVDF9mbD11fpxhbXi7P1609n_15Y,16446
|
|
78
|
-
application_sdk/observability/observability.py,sha256=
|
|
75
|
+
application_sdk/observability/observability.py,sha256=m5OgD_akc3YrkF5mCme2HcRaY7ysjLbVSaEs2C-K2Fs,24062
|
|
79
76
|
application_sdk/observability/traces_adaptor.py,sha256=0eQJPN-tYA_dV8D3uEa5ZiX9g12NDuLnPaFuQMVDdL0,18242
|
|
80
77
|
application_sdk/observability/utils.py,sha256=-02GAFom8Bg4SNyCTNYySmen2dzvLfTu43bqsNq1AH0,3096
|
|
81
78
|
application_sdk/observability/decorators/observability_decorator.py,sha256=yd6qfrg1MmH5KcZ5Ydzb0RaBzmxx5FrmiI9qwvZx3EU,8963
|
|
82
|
-
application_sdk/outputs/__init__.py,sha256=PRBc0UEyLzNfuKDRstCdI--6tWe4TUxpW_par9tWyiU,17204
|
|
83
|
-
application_sdk/outputs/iceberg.py,sha256=TdppOMEMfojMhGyBmhWeu1AJQexRyHM-huAYeJmhjdY,5533
|
|
84
|
-
application_sdk/outputs/json.py,sha256=gYDDNOVb8EFxxeOkb6zKWZWjTEVgZLoapFM97_roK4A,10883
|
|
85
|
-
application_sdk/outputs/parquet.py,sha256=DxcKh1IXPdiXNQJS1HIn6-JRdLkmN4At8uF1zppiZX0,20762
|
|
86
|
-
application_sdk/outputs/.cursor/BUGBOT.md,sha256=KxEC3CIyRSK1YftZou5BgKc6PRXT3qQmBNFJp-HSyYE,11496
|
|
87
79
|
application_sdk/server/__init__.py,sha256=KTqE1YPw_3WDVMWatJUuf9OOiobLM2K5SMaBrI62sCo,1568
|
|
88
80
|
application_sdk/server/.cursor/BUGBOT.md,sha256=p_MMoWUW5G1894WfOKYReZKWCuyJT_OJz3rL5g21NbI,16566
|
|
89
81
|
application_sdk/server/fastapi/__init__.py,sha256=BVqf63z1hxEdpJqLU4LXpFTbk5q8dVkjEbWbu_vbW_Y,29578
|
|
90
|
-
application_sdk/server/fastapi/models.py,sha256=
|
|
91
|
-
application_sdk/server/fastapi/utils.py,sha256=
|
|
92
|
-
application_sdk/server/fastapi/middleware/logmiddleware.py,sha256=
|
|
93
|
-
application_sdk/server/fastapi/middleware/metrics.py,sha256=
|
|
82
|
+
application_sdk/server/fastapi/models.py,sha256=h0hMtMg_p5G0Ug2MBkmBcT94W025VKCLMFyh0FciNoQ,7559
|
|
83
|
+
application_sdk/server/fastapi/utils.py,sha256=WoDGDmq6E1kwS2FN5pjIuzygKNTHpA-tg2SQJZgJOOI,1415
|
|
84
|
+
application_sdk/server/fastapi/middleware/logmiddleware.py,sha256=sKKi-ysI9XQDT0uKW3sfw2r3XQq2iJT4_XrNGXZWdjI,2907
|
|
85
|
+
application_sdk/server/fastapi/middleware/metrics.py,sha256=F_EhayzNgEn9KJvITv8VFxwXhBZVaOQkDptyfa9nQK4,1820
|
|
94
86
|
application_sdk/server/fastapi/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
95
|
-
application_sdk/server/fastapi/routers/server.py,sha256=
|
|
87
|
+
application_sdk/server/fastapi/routers/server.py,sha256=_0WHtpmI6UGuLavoUmyHDgQK8xy1gNAErpu5ZH5O_JM,4210
|
|
96
88
|
application_sdk/server/mcp/__init__.py,sha256=29HI-GkNn9wOPsb86Litmv0MhXWFBNVBOivJcNFWZas,154
|
|
97
89
|
application_sdk/server/mcp/models.py,sha256=3jX_3wXBuFs6XX9OG-xoag8VkKYWO_Y5r5-CHcLU5vg,236
|
|
98
90
|
application_sdk/server/mcp/server.py,sha256=HG8tFmcc-f9Wj3vZzs2oRoNJzN1s5hwjnKykSdTXgCQ,3450
|
|
99
91
|
application_sdk/services/__init__.py,sha256=H-5HZEPdr53MUfAggyHqHhRXDRLZFZsxvJgWbr257Ds,465
|
|
92
|
+
application_sdk/services/_utils.py,sha256=0yHqDP6qNb1OT-bX2XRYQPZ5xkGkV13nyRw6GkPlHs8,1136
|
|
100
93
|
application_sdk/services/atlan_storage.py,sha256=TKzXxu0yXeUcmZehwp8PcnQTC4A9w9RlZ0Fl-Xp1bLE,8509
|
|
101
|
-
application_sdk/services/eventstore.py,sha256=
|
|
94
|
+
application_sdk/services/eventstore.py,sha256=wCT921KRzUe3fAWKC-bbM6_OtIJTKpSQrOutPQzMEgs,6745
|
|
102
95
|
application_sdk/services/objectstore.py,sha256=JgQrL9z_6aG5micVd7I2N6l3RA4EUJh4T2BCuC_ATwQ,20161
|
|
103
|
-
application_sdk/services/secretstore.py,sha256=
|
|
96
|
+
application_sdk/services/secretstore.py,sha256=Pmn1WlmHmgaDhWz5OXBB5_rKXQQMyLMzadwZSNKwc6Q,19070
|
|
104
97
|
application_sdk/services/statestore.py,sha256=3-afiM3Vsoe1XDYRokdGTB5I5CwOKyieuX5RwIZf77o,9413
|
|
105
98
|
application_sdk/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
106
99
|
application_sdk/test_utils/workflow_monitoring.py,sha256=gqq6CsT62GrMt2GqtNSb1iD_-t4MBffQvpO0BXboZek,3490
|
|
@@ -125,10 +118,10 @@ application_sdk/test_utils/hypothesis/strategies/inputs/__init__.py,sha256=47DEQ
|
|
|
125
118
|
application_sdk/test_utils/hypothesis/strategies/inputs/json_input.py,sha256=HfdCZnXIZFJiRuORpnmioXh8qHls9sWNSFDysy8il-o,913
|
|
126
119
|
application_sdk/test_utils/hypothesis/strategies/inputs/parquet_input.py,sha256=agjRA9agpak_GmWiIt9bi_oLGvLM_eunxXfxcNHK3MQ,1081
|
|
127
120
|
application_sdk/test_utils/hypothesis/strategies/outputs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
128
|
-
application_sdk/test_utils/hypothesis/strategies/outputs/json_output.py,sha256=
|
|
121
|
+
application_sdk/test_utils/hypothesis/strategies/outputs/json_output.py,sha256=X4I-s9VDGpEWXEwLRdXQiLik33lubXbiHJzjS05f9SE,1760
|
|
129
122
|
application_sdk/test_utils/hypothesis/strategies/outputs/statestore.py,sha256=gmYBwePNoSI_pl2WTXOClgkruzRwkOX_1SmBaUTha0c,2903
|
|
130
123
|
application_sdk/test_utils/hypothesis/strategies/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
131
|
-
application_sdk/test_utils/hypothesis/strategies/server/fastapi/__init__.py,sha256=
|
|
124
|
+
application_sdk/test_utils/hypothesis/strategies/server/fastapi/__init__.py,sha256=lZrDZ92UAoXepv3bcllviFwjjUwmndVFJjEFxLMBnIs,2654
|
|
132
125
|
application_sdk/test_utils/scale_data_generator/README.md,sha256=s3oNDuKOKNUT9WxM-r0Ve1VPBl0cRM3Po7I3Vi-PGxM,1693
|
|
133
126
|
application_sdk/test_utils/scale_data_generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
134
127
|
application_sdk/test_utils/scale_data_generator/config_loader.py,sha256=ISk-fEIDlAJozhoNWwnl3Gp3IYnDDrRS-5SdiLYta24,2528
|
|
@@ -159,8 +152,8 @@ application_sdk/workflows/metadata_extraction/__init__.py,sha256=jHUe_ZBQ66jx8bg
|
|
|
159
152
|
application_sdk/workflows/metadata_extraction/sql.py,sha256=6ZaVt84n-8U2ZvR9GR7uIJKv5v8CuyQjhlnoRJvDszc,12435
|
|
160
153
|
application_sdk/workflows/query_extraction/__init__.py,sha256=n066_CX5RpJz6DIxGMkKS3eGSRg03ilaCtsqfJWQb7Q,117
|
|
161
154
|
application_sdk/workflows/query_extraction/sql.py,sha256=kT_JQkLCRZ44ZpaC4QvPL6DxnRIIVh8gYHLqRbMI-hA,4826
|
|
162
|
-
atlan_application_sdk-
|
|
163
|
-
atlan_application_sdk-
|
|
164
|
-
atlan_application_sdk-
|
|
165
|
-
atlan_application_sdk-
|
|
166
|
-
atlan_application_sdk-
|
|
155
|
+
atlan_application_sdk-2.0.0.dist-info/METADATA,sha256=B9-UVeLlDGDuxko6Nvb6Y8zrLNueaATYaJZMESikYZU,5806
|
|
156
|
+
atlan_application_sdk-2.0.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
157
|
+
atlan_application_sdk-2.0.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
158
|
+
atlan_application_sdk-2.0.0.dist-info/licenses/NOTICE,sha256=A-XVVGt3KOYuuMmvSMIFkg534F1vHiCggEBp4Ez3wGk,1041
|
|
159
|
+
atlan_application_sdk-2.0.0.dist-info/RECORD,,
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
from typing import TYPE_CHECKING, Union
|
|
2
|
-
|
|
3
|
-
from application_sdk.observability.logger_adaptor import get_logger
|
|
4
|
-
|
|
5
|
-
if TYPE_CHECKING:
|
|
6
|
-
import daft
|
|
7
|
-
import pandas as pd
|
|
8
|
-
|
|
9
|
-
logger = get_logger(__name__)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def is_empty_dataframe(dataframe: Union["pd.DataFrame", "daft.DataFrame"]) -> bool: # noqa: F821
|
|
13
|
-
"""Check if a DataFrame is empty.
|
|
14
|
-
|
|
15
|
-
This function determines whether a DataFrame has any rows, supporting both
|
|
16
|
-
pandas and daft DataFrame types. For pandas DataFrames, it uses the `empty`
|
|
17
|
-
property, and for daft DataFrames, it checks if the row count is 0.
|
|
18
|
-
|
|
19
|
-
Args:
|
|
20
|
-
dataframe (Union[pd.DataFrame, daft.DataFrame]): The DataFrame to check,
|
|
21
|
-
can be either a pandas DataFrame or a daft DataFrame.
|
|
22
|
-
|
|
23
|
-
Returns:
|
|
24
|
-
bool: True if the DataFrame has no rows, False otherwise.
|
|
25
|
-
|
|
26
|
-
Note:
|
|
27
|
-
If daft is not available and a daft DataFrame is passed, the function
|
|
28
|
-
will log a warning and return True.
|
|
29
|
-
"""
|
|
30
|
-
import pandas as pd
|
|
31
|
-
|
|
32
|
-
if isinstance(dataframe, pd.DataFrame):
|
|
33
|
-
return dataframe.empty
|
|
34
|
-
|
|
35
|
-
try:
|
|
36
|
-
import daft
|
|
37
|
-
|
|
38
|
-
if isinstance(dataframe, daft.DataFrame):
|
|
39
|
-
return dataframe.count_rows() == 0
|
|
40
|
-
except Exception:
|
|
41
|
-
logger.warning("Module daft not found")
|
|
42
|
-
return True
|
|
@@ -1,250 +0,0 @@
|
|
|
1
|
-
# Input Code Review Guidelines - Data Input Processing
|
|
2
|
-
|
|
3
|
-
## Context-Specific Patterns
|
|
4
|
-
|
|
5
|
-
This directory contains input processing implementations for various data formats (JSON, Parquet, SQL). Input processors must handle data efficiently while maintaining data integrity and performance.
|
|
6
|
-
|
|
7
|
-
### Phase 1: Critical Input Safety Issues
|
|
8
|
-
|
|
9
|
-
**Object Store Path Management:**
|
|
10
|
-
|
|
11
|
-
- **Correct path calculation**: Source paths must use the actual object store prefix, not derived local paths
|
|
12
|
-
- **Path validation**: Verify that object store keys are valid and within constraints
|
|
13
|
-
- **User-provided prefixes**: Respect user-configured input prefixes and download paths
|
|
14
|
-
- **Path consistency**: Ensure downloaded files match the expected object store locations
|
|
15
|
-
|
|
16
|
-
**Data Validation and Security:**
|
|
17
|
-
|
|
18
|
-
- All input data must be validated before processing
|
|
19
|
-
- File size limits must be enforced to prevent resource exhaustion
|
|
20
|
-
- File type validation required for uploaded/downloaded files
|
|
21
|
-
- Malicious file content detection for executable or script files
|
|
22
|
-
- Input path traversal prevention
|
|
23
|
-
|
|
24
|
-
```python
|
|
25
|
-
# ✅ DO: Proper object store path handling
|
|
26
|
-
class JsonInput:
|
|
27
|
-
async def download_from_object_store(
|
|
28
|
-
self,
|
|
29
|
-
input_prefix: str, # User-provided prefix
|
|
30
|
-
local_destination: str
|
|
31
|
-
) -> List[str]:
|
|
32
|
-
"""Download files with correct path handling."""
|
|
33
|
-
|
|
34
|
-
# Use the actual input prefix, not derived local path
|
|
35
|
-
object_store_source = input_prefix # Keep user's intended source
|
|
36
|
-
|
|
37
|
-
downloaded_files = await self.object_store.download_files(
|
|
38
|
-
source=object_store_source,
|
|
39
|
-
destination=local_destination
|
|
40
|
-
)
|
|
41
|
-
|
|
42
|
-
return downloaded_files
|
|
43
|
-
|
|
44
|
-
# ❌ REJECT: Incorrect path handling
|
|
45
|
-
class BadJsonInput:
|
|
46
|
-
async def download_from_object_store(
|
|
47
|
-
self,
|
|
48
|
-
input_prefix: str,
|
|
49
|
-
local_destination: str
|
|
50
|
-
) -> List[str]:
|
|
51
|
-
# Wrong: derives object store path from local path
|
|
52
|
-
object_store_source = get_object_store_prefix(local_destination)
|
|
53
|
-
# This ignores the user's actual input_prefix!
|
|
54
|
-
|
|
55
|
-
return await self.object_store.download_files(
|
|
56
|
-
source=object_store_source, # Wrong source!
|
|
57
|
-
destination=local_destination
|
|
58
|
-
)
|
|
59
|
-
```
|
|
60
|
-
|
|
61
|
-
### Phase 2: Input Architecture Patterns
|
|
62
|
-
|
|
63
|
-
**Performance Optimization Requirements:**
|
|
64
|
-
|
|
65
|
-
- **Parallelization opportunities**: Flag sequential file operations that could be parallelized
|
|
66
|
-
- **Batch processing**: Group related operations to reduce overhead
|
|
67
|
-
- **Memory efficiency**: Process large files in chunks, not all at once
|
|
68
|
-
- **Connection reuse**: Optimize object store connections across operations
|
|
69
|
-
|
|
70
|
-
**Resource Management:**
|
|
71
|
-
|
|
72
|
-
- Use proper connection pooling for object store operations
|
|
73
|
-
- Implement timeout handling for download operations
|
|
74
|
-
- Clean up temporary files after processing
|
|
75
|
-
- Handle partial download failures gracefully
|
|
76
|
-
- Monitor memory usage during large file processing
|
|
77
|
-
|
|
78
|
-
```python
|
|
79
|
-
# ✅ DO: Parallelized file processing
|
|
80
|
-
async def download_multiple_files_parallel(
|
|
81
|
-
self,
|
|
82
|
-
file_paths: List[str],
|
|
83
|
-
destination_dir: str
|
|
84
|
-
) -> List[str]:
|
|
85
|
-
"""Download multiple files in parallel for better performance."""
|
|
86
|
-
|
|
87
|
-
async def download_single_file(file_path: str) -> str:
|
|
88
|
-
"""Download a single file with error handling."""
|
|
89
|
-
try:
|
|
90
|
-
return await self.object_store.download_file(
|
|
91
|
-
source=file_path,
|
|
92
|
-
destination=os.path.join(destination_dir, os.path.basename(file_path))
|
|
93
|
-
)
|
|
94
|
-
except Exception as e:
|
|
95
|
-
logger.error(f"Failed to download {file_path}: {e}")
|
|
96
|
-
raise
|
|
97
|
-
|
|
98
|
-
# Parallel processing with controlled concurrency
|
|
99
|
-
semaphore = asyncio.Semaphore(10) # Limit concurrent downloads
|
|
100
|
-
|
|
101
|
-
async def download_with_semaphore(file_path: str) -> str:
|
|
102
|
-
async with semaphore:
|
|
103
|
-
return await download_single_file(file_path)
|
|
104
|
-
|
|
105
|
-
tasks = [download_with_semaphore(path) for path in file_paths]
|
|
106
|
-
return await asyncio.gather(*tasks)
|
|
107
|
-
|
|
108
|
-
# ❌ REJECT: Sequential processing
|
|
109
|
-
async def download_multiple_files_sequential(self, file_paths: List[str]) -> List[str]:
|
|
110
|
-
"""Sequential download - should be flagged for parallelization."""
|
|
111
|
-
downloaded = []
|
|
112
|
-
for file_path in file_paths: # FLAG: Could be parallelized
|
|
113
|
-
result = await self.object_store.download_file(file_path)
|
|
114
|
-
downloaded.append(result)
|
|
115
|
-
return downloaded
|
|
116
|
-
```
|
|
117
|
-
|
|
118
|
-
### Phase 3: Input Testing Requirements
|
|
119
|
-
|
|
120
|
-
**Data Input Testing:**
|
|
121
|
-
|
|
122
|
-
- Test with various file formats and sizes
|
|
123
|
-
- Test malformed data handling
|
|
124
|
-
- Test partial download/upload scenarios
|
|
125
|
-
- Mock object store operations in unit tests
|
|
126
|
-
- Include integration tests with real object store
|
|
127
|
-
- Test error recovery and retry logic
|
|
128
|
-
|
|
129
|
-
**Performance Testing:**
|
|
130
|
-
|
|
131
|
-
- Include tests for large file processing
|
|
132
|
-
- Test memory usage with different chunk sizes
|
|
133
|
-
- Test concurrent download/upload operations
|
|
134
|
-
- Verify timeout handling works correctly
|
|
135
|
-
- Test connection pool behavior
|
|
136
|
-
|
|
137
|
-
### Phase 4: Performance and Scalability
|
|
138
|
-
|
|
139
|
-
**Data Processing Efficiency:**
|
|
140
|
-
|
|
141
|
-
- Use streaming for large files instead of loading entirely into memory
|
|
142
|
-
- Implement proper chunking for batch operations
|
|
143
|
-
- Use async generators for memory-efficient data processing
|
|
144
|
-
- Monitor memory usage and processing time
|
|
145
|
-
- Optimize file I/O operations
|
|
146
|
-
|
|
147
|
-
**Object Store Optimization:**
|
|
148
|
-
|
|
149
|
-
- Use connection pooling for object store clients
|
|
150
|
-
- Implement proper retry logic for transient failures
|
|
151
|
-
- Use parallel operations where appropriate
|
|
152
|
-
- Cache frequently accessed metadata
|
|
153
|
-
- Monitor object store operation metrics
|
|
154
|
-
|
|
155
|
-
### Phase 5: Input Data Maintainability
|
|
156
|
-
|
|
157
|
-
**Error Handling and Recovery:**
|
|
158
|
-
|
|
159
|
-
- Implement comprehensive error handling for all input operations
|
|
160
|
-
- Provide meaningful error messages with context
|
|
161
|
-
- Handle partial failures gracefully (some files fail, others succeed)
|
|
162
|
-
- Implement proper retry logic for transient failures
|
|
163
|
-
- Log all input operations with sufficient context
|
|
164
|
-
|
|
165
|
-
**Configuration Management:**
|
|
166
|
-
|
|
167
|
-
- Externalize all input-related configuration
|
|
168
|
-
- Support different input sources and formats
|
|
169
|
-
- Validate input configuration before processing
|
|
170
|
-
- Document all supported input parameters
|
|
171
|
-
- Handle environment-specific input requirements
|
|
172
|
-
|
|
173
|
-
---
|
|
174
|
-
|
|
175
|
-
## Input-Specific Anti-Patterns
|
|
176
|
-
|
|
177
|
-
**Always Reject:**
|
|
178
|
-
|
|
179
|
-
- **Path calculation errors**: Using local paths to derive object store paths
|
|
180
|
-
- **Sequential processing**: Processing multiple files sequentially when parallel processing is possible
|
|
181
|
-
- **Memory inefficiency**: Loading large files entirely into memory
|
|
182
|
-
- **Missing error handling**: Input operations without proper try-catch blocks
|
|
183
|
-
- **Poor path validation**: Not validating object store keys or file paths
|
|
184
|
-
- **Resource leaks**: Not cleaning up temporary files or connections
|
|
185
|
-
|
|
186
|
-
**Object Store Anti-Patterns:**
|
|
187
|
-
|
|
188
|
-
```python
|
|
189
|
-
# ❌ REJECT: Incorrect object store usage
|
|
190
|
-
class BadInputProcessor:
|
|
191
|
-
async def process_files(self, local_files: List[str]):
|
|
192
|
-
# Wrong: derives object store path from local path
|
|
193
|
-
for local_file in local_files:
|
|
194
|
-
object_store_key = get_object_store_prefix(local_file) # Incorrect!
|
|
195
|
-
await self.object_store.download_file(object_store_key, local_file)
|
|
196
|
-
|
|
197
|
-
# ✅ REQUIRE: Correct object store usage
|
|
198
|
-
class GoodInputProcessor:
|
|
199
|
-
async def process_files(
|
|
200
|
-
self,
|
|
201
|
-
object_store_paths: List[str], # Actual object store paths
|
|
202
|
-
local_destination_dir: str
|
|
203
|
-
):
|
|
204
|
-
# Use actual object store paths, not derived ones
|
|
205
|
-
for object_store_path in object_store_paths:
|
|
206
|
-
local_file = os.path.join(
|
|
207
|
-
local_destination_dir,
|
|
208
|
-
os.path.basename(object_store_path)
|
|
209
|
-
)
|
|
210
|
-
await self.object_store.download_file(object_store_path, local_file)
|
|
211
|
-
```
|
|
212
|
-
|
|
213
|
-
**Performance Anti-Patterns:**
|
|
214
|
-
|
|
215
|
-
```python
|
|
216
|
-
# ❌ REJECT: Sequential file processing
|
|
217
|
-
async def process_files_sequential(file_list: List[str]):
|
|
218
|
-
results = []
|
|
219
|
-
for file_path in file_list: # Should be parallelized
|
|
220
|
-
result = await process_single_file(file_path)
|
|
221
|
-
results.append(result)
|
|
222
|
-
return results
|
|
223
|
-
|
|
224
|
-
# ✅ REQUIRE: Parallel file processing
|
|
225
|
-
async def process_files_parallel(file_list: List[str], max_concurrency: int = 10):
|
|
226
|
-
semaphore = asyncio.Semaphore(max_concurrency)
|
|
227
|
-
|
|
228
|
-
async def process_with_semaphore(file_path: str):
|
|
229
|
-
async with semaphore:
|
|
230
|
-
return await process_single_file(file_path)
|
|
231
|
-
|
|
232
|
-
tasks = [process_with_semaphore(path) for path in file_list]
|
|
233
|
-
return await asyncio.gather(*tasks, return_exceptions=True)
|
|
234
|
-
```
|
|
235
|
-
|
|
236
|
-
## Educational Context for Input Reviews
|
|
237
|
-
|
|
238
|
-
When reviewing input code, emphasize:
|
|
239
|
-
|
|
240
|
-
1. **Data Integrity Impact**: "Incorrect object store path handling can cause data loss or corruption. Files uploaded to wrong locations become inaccessible, breaking data processing pipelines."
|
|
241
|
-
|
|
242
|
-
2. **Performance Impact**: "Sequential file processing creates unnecessary bottlenecks. For enterprise datasets with hundreds of files, parallelization can reduce processing time from hours to minutes."
|
|
243
|
-
|
|
244
|
-
3. **Resource Impact**: "Poor memory management in input processing can cause out-of-memory errors with large datasets. Streaming and chunking are essential for enterprise-scale data processing."
|
|
245
|
-
|
|
246
|
-
4. **User Experience Impact**: "Input path handling errors are often silent until runtime, causing difficult-to-debug failures. Proper validation and clear error messages save hours of troubleshooting."
|
|
247
|
-
|
|
248
|
-
5. **Scalability Impact**: "Input processing patterns that work for small datasets can fail catastrophically at enterprise scale. Always design for the largest expected dataset size."
|
|
249
|
-
|
|
250
|
-
6. **Reliability Impact**: "Input operations are often the first point of failure in data pipelines. Robust error handling and retry logic in input processing prevents entire workflows from failing due to transient issues."
|