atlan-application-sdk 0.1.1rc59__py3-none-any.whl → 0.1.1rc61__py3-none-any.whl

application_sdk/constants.py

@@ -51,7 +51,9 @@ SQL_SERVER_MIN_VERSION = os.getenv("ATLAN_SQL_SERVER_MIN_VERSION")
 SQL_QUERIES_PATH = os.getenv("ATLAN_SQL_QUERIES_PATH", "app/sql")
 
 # Output Path Constants
-#: Output path format for workflows (example: objectstore://bucket/artifacts/apps/{application_name}/workflows/{workflow_id}/{workflow_run_id})
+#: Output path format for workflows.
+#:
+#: Example: objectstore://bucket/artifacts/apps/{application_name}/workflows/{workflow_id}/{workflow_run_id}
 WORKFLOW_OUTPUT_PATH_TEMPLATE = (
     "artifacts/apps/{application_name}/workflows/{workflow_id}/{run_id}"
 )
@@ -68,7 +70,9 @@ CLEANUP_BASE_PATHS = [
 ]
 
 # State Store Constants
-#: Path template for state store files (example: objectstore://bucket/persistent-artifacts/apps/{application_name}/{state_type}/{id}/config.json)
+#: Path template for state store files.
+#:
+#: Example: objectstore://bucket/persistent-artifacts/apps/{application_name}/{state_type}/{id}/config.json
 STATE_STORE_PATH_TEMPLATE = (
     "persistent-artifacts/apps/{application_name}/{state_type}/{id}/config.json"
 )
@@ -147,7 +151,9 @@ WORKER_START_EVENT_VERSION = "v1"
 #: Whether to enable Atlan storage upload
 ENABLE_ATLAN_UPLOAD = os.getenv("ENABLE_ATLAN_UPLOAD", "false").lower() == "true"
 # Dapr Client Configuration
-#: Maximum gRPC message length in bytes for Dapr client (default: 100MB)
+#: Maximum gRPC message length in bytes for Dapr client.
+#:
+#: Default: 100MB
 DAPR_MAX_GRPC_MESSAGE_LENGTH = int(
     os.getenv("DAPR_MAX_GRPC_MESSAGE_LENGTH", "104857600")
 )
@@ -250,9 +256,9 @@ REDIS_HOST = os.getenv("REDIS_HOST", "")
 REDIS_PORT = os.getenv("REDIS_PORT", "")
 #: Redis password (required for authenticated Redis instances)
 REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
-#: Redis Sentinel service name (default: mymaster)
+#: Redis Sentinel service name. Default: mymaster
 REDIS_SENTINEL_SERVICE_NAME = os.getenv("REDIS_SENTINEL_SERVICE_NAME", "mymaster")
-#: Redis Sentinel hosts (comma-separated host:port pairs)
+#: Redis Sentinel hosts as comma-separated host:port pairs
 REDIS_SENTINEL_HOSTS = os.getenv("REDIS_SENTINEL_HOSTS", "")
 #: Whether to enable strict locking
 IS_LOCKING_DISABLED = os.getenv("IS_LOCKING_DISABLED", "true").lower() == "true"

application_sdk/observability/observability.py

@@ -10,7 +10,6 @@ from pathlib import Path
 from time import time
 from typing import Any, Dict, Generic, List, TypeVar
 
-import daft
 import duckdb
 import pandas as pd
 from dapr.clients import DaprClient
@@ -422,28 +421,14 @@ class AtlanObservability(Generic[T], ABC):
                     # Lazy import and instantiation of ParquetOutput
                     from application_sdk.outputs.parquet import ParquetOutput
 
-                    parquet_output = ParquetOutput(output_path=partition_path)
-                    logging.info(
-                        f"Successfully instantiated ParquetOutput for partition: {partition_path}"
-                    )
-
-                    # Use write_daft_dataframe with the DataFrame we have
-                    from application_sdk.outputs.parquet import WriteMode
-
-                    daft_df = daft.from_pandas(df)
-                    await parquet_output.write_daft_dataframe(
-                        dataframe=daft_df,
-                        write_mode=WriteMode.APPEND,  # Append mode to merge with existing data
-                    )
-
-                    logging.info(
-                        f"Successfully wrote {len(df)} records to partition: {partition_path}"
-                    )
-
-                except Exception as partition_error:
-                    logging.error(
-                        f"Error processing partition {partition_path}: {str(partition_error)}"
+                    parquet_output = ParquetOutput(
+                        output_path=partition_path,
+                        chunk_start=0,
+                        chunk_part=int(time()),
                     )
+                    await parquet_output.write_dataframe(dataframe=df)
+                except Exception as e:
+                    print(f"Error writing records to partition: {str(e)}")
 
             # Clean up old records if enabled
             if self._cleanup_enabled:

application_sdk/outputs/__init__.py

@@ -64,6 +64,7 @@ class Output(ABC):
     output_prefix: str
     total_record_count: int
     chunk_count: int
+    chunk_part: int
     buffer_size: int
     max_file_size_bytes: int
     current_buffer_size: int

application_sdk/outputs/parquet.py

@@ -65,6 +65,7 @@ class ParquetOutput(Output):
         buffer_size: int = 5000,
         total_record_count: int = 0,
         chunk_count: int = 0,
+        chunk_part: int = 0,
         chunk_start: Optional[int] = None,
         start_marker: Optional[str] = None,
         end_marker: Optional[str] = None,
@@ -105,7 +106,7 @@ class ParquetOutput(Output):
             DAPR_MAX_GRPC_MESSAGE_LENGTH * 0.75
         )  # 75% of DAPR limit as safety buffer
         self.chunk_start = chunk_start
-        self.chunk_part = 0
+        self.chunk_part = chunk_part
         self.start_marker = start_marker
         self.end_marker = end_marker
         self.partitions = []

application_sdk/version.py

@@ -2,4 +2,4 @@
 Version information for the application_sdk package.
 """
 
-__version__ = "0.1.1rc59"
+__version__ = "0.1.1rc61"

{atlan_application_sdk-0.1.1rc59.dist-info → atlan_application_sdk-0.1.1rc61.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlan-application-sdk
-Version: 0.1.1rc59
+Version: 0.1.1rc61
 Summary: Atlan Application SDK is a Python library for developing applications on the Atlan Platform
 Project-URL: Repository, https://github.com/atlanhq/application-sdk
 Project-URL: Documentation, https://github.com/atlanhq/application-sdk/README.md
@@ -22,7 +22,7 @@ Requires-Python: >=3.11
 Requires-Dist: aiohttp>=3.10.0
 Requires-Dist: duckdb-engine>=0.17.0
 Requires-Dist: duckdb>=1.1.3
-Requires-Dist: fastapi[standard]>=0.115.0
+Requires-Dist: fastapi[standard]==0.120.2
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: psutil>=7.0.0

{atlan_application_sdk-0.1.1rc59.dist-info → atlan_application_sdk-0.1.1rc61.dist-info}/RECORD

@@ -1,6 +1,6 @@
 application_sdk/__init__.py,sha256=2e2mvmLJ5dxmJGPELtb33xwP-j6JMdoIuqKycEn7hjg,151
-application_sdk/constants.py,sha256=ySrjME6CSoiyjVLgQt0s4dIdgJ0JWIMmMs7WBZBVUpA,11013
-application_sdk/version.py,sha256=sXMApD_x-2esC_2hKV1Jo0L6_WWdwAw9zFGgtDsWblQ,88
+application_sdk/constants.py,sha256=XCRJRiug2GXgeeVZu3Mp1QavRw54bWtzxIZ2l1bIkkA,11028
+application_sdk/version.py,sha256=CEakpB1d62-VZEzrOEEq4LSV4f0w3c23I9GOWtTnui8,88
 application_sdk/worker.py,sha256=i5f0AeKI39IfsLO05QkwC6uMz0zDPSJqP7B2byri1VI,7489
 application_sdk/activities/__init__.py,sha256=L5WXkTwOwGtjWAlXrUJRCKGwyIyp3z8fBv8BZVCRFQI,11175
 application_sdk/activities/lock_management.py,sha256=6Wdf3jMKitoarHQP91PIJOoGFz4aaOLS_40c7n1yAOA,3902
@@ -73,14 +73,14 @@ application_sdk/interceptors/.cursor/BUGBOT.md,sha256=pxmUF2c7dtaXAX8yAa1-LBa6FC
 application_sdk/observability/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 application_sdk/observability/logger_adaptor.py,sha256=00c0F7maDkp1xrHttW6VQbWFDGr6NkXeDPjmf97ojlY,29989
 application_sdk/observability/metrics_adaptor.py,sha256=5Oz02lUED60duryoVDF9mbD11fpxhbXi7P1609n_15Y,16446
-application_sdk/observability/observability.py,sha256=7gRws1Lg1bM6WWL5Y-ZJOxDMHDx1tKqo8Qw-xxSkDEs,24312
+application_sdk/observability/observability.py,sha256=MGxNFPx6pOdpWrpNXZp44NPk3SG4xjA9cKrTeZ1ENK8,23681
 application_sdk/observability/traces_adaptor.py,sha256=0eQJPN-tYA_dV8D3uEa5ZiX9g12NDuLnPaFuQMVDdL0,18242
 application_sdk/observability/utils.py,sha256=JoHEA68cjkXTnAXHzgiULYOzRTk8rG4kPZRvFYah3aU,2505
 application_sdk/observability/decorators/observability_decorator.py,sha256=yd6qfrg1MmH5KcZ5Ydzb0RaBzmxx5FrmiI9qwvZx3EU,8963
-application_sdk/outputs/__init__.py,sha256=hrOPw0xuG9xP720Bt309TfbY2Qq_i51R8Xt3ZjwWDUY,15906
+application_sdk/outputs/__init__.py,sha256=dekaEqJEVAmXQYSy_AohXOHNNI56OXG3Xn27FFRmoPQ,15926
 application_sdk/outputs/iceberg.py,sha256=TdppOMEMfojMhGyBmhWeu1AJQexRyHM-huAYeJmhjdY,5533
 application_sdk/outputs/json.py,sha256=gYDDNOVb8EFxxeOkb6zKWZWjTEVgZLoapFM97_roK4A,10883
-application_sdk/outputs/parquet.py,sha256=bAv-IrWLNFqF-j8D7jSDi0ug1qV_qBTA3dDxK2eiFcA,20724
+application_sdk/outputs/parquet.py,sha256=DxcKh1IXPdiXNQJS1HIn6-JRdLkmN4At8uF1zppiZX0,20762
 application_sdk/outputs/.cursor/BUGBOT.md,sha256=KxEC3CIyRSK1YftZou5BgKc6PRXT3qQmBNFJp-HSyYE,11496
 application_sdk/server/__init__.py,sha256=KTqE1YPw_3WDVMWatJUuf9OOiobLM2K5SMaBrI62sCo,1568
 application_sdk/server/.cursor/BUGBOT.md,sha256=p_MMoWUW5G1894WfOKYReZKWCuyJT_OJz3rL5g21NbI,16566
@@ -157,8 +157,8 @@ application_sdk/workflows/metadata_extraction/__init__.py,sha256=jHUe_ZBQ66jx8bg
 application_sdk/workflows/metadata_extraction/sql.py,sha256=6ZaVt84n-8U2ZvR9GR7uIJKv5v8CuyQjhlnoRJvDszc,12435
 application_sdk/workflows/query_extraction/__init__.py,sha256=n066_CX5RpJz6DIxGMkKS3eGSRg03ilaCtsqfJWQb7Q,117
 application_sdk/workflows/query_extraction/sql.py,sha256=kT_JQkLCRZ44ZpaC4QvPL6DxnRIIVh8gYHLqRbMI-hA,4826
-atlan_application_sdk-0.1.1rc59.dist-info/METADATA,sha256=f39Mtuyd4ASvIaEwAGtaGyWlSx0JqbFFfa2fNA20P_Y,5730
-atlan_application_sdk-0.1.1rc59.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-atlan_application_sdk-0.1.1rc59.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-atlan_application_sdk-0.1.1rc59.dist-info/licenses/NOTICE,sha256=A-XVVGt3KOYuuMmvSMIFkg534F1vHiCggEBp4Ez3wGk,1041
-atlan_application_sdk-0.1.1rc59.dist-info/RECORD,,
+atlan_application_sdk-0.1.1rc61.dist-info/METADATA,sha256=sRo4wQ9dNcScMKLwS0nqbceTkr-pbE5rr8voxSMRoqo,5730
+atlan_application_sdk-0.1.1rc61.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+atlan_application_sdk-0.1.1rc61.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+atlan_application_sdk-0.1.1rc61.dist-info/licenses/NOTICE,sha256=A-XVVGt3KOYuuMmvSMIFkg534F1vHiCggEBp4Ez3wGk,1041
+atlan_application_sdk-0.1.1rc61.dist-info/RECORD,,