atlan-application-sdk 0.1.1rc59__py3-none-any.whl → 0.1.1rc60__py3-none-any.whl

application_sdk/observability/observability.py

@@ -10,7 +10,6 @@ from pathlib import Path
 from time import time
 from typing import Any, Dict, Generic, List, TypeVar
 
-import daft
 import duckdb
 import pandas as pd
 from dapr.clients import DaprClient
@@ -422,28 +421,14 @@ class AtlanObservability(Generic[T], ABC):
                     # Lazy import and instantiation of ParquetOutput
                     from application_sdk.outputs.parquet import ParquetOutput
 
-                    parquet_output = ParquetOutput(output_path=partition_path)
-                    logging.info(
-                        f"Successfully instantiated ParquetOutput for partition: {partition_path}"
-                    )
-
-                    # Use write_daft_dataframe with the DataFrame we have
-                    from application_sdk.outputs.parquet import WriteMode
-
-                    daft_df = daft.from_pandas(df)
-                    await parquet_output.write_daft_dataframe(
-                        dataframe=daft_df,
-                        write_mode=WriteMode.APPEND,  # Append mode to merge with existing data
-                    )
-
-                    logging.info(
-                        f"Successfully wrote {len(df)} records to partition: {partition_path}"
-                    )
-
-                except Exception as partition_error:
-                    logging.error(
-                        f"Error processing partition {partition_path}: {str(partition_error)}"
+                    parquet_output = ParquetOutput(
+                        output_path=partition_path,
+                        chunk_start=0,
+                        chunk_part=int(time()),
                     )
+                    await parquet_output.write_dataframe(dataframe=df)
+                except Exception as e:
+                    print(f"Error writing records to partition: {str(e)}")
 
             # Clean up old records if enabled
             if self._cleanup_enabled:

application_sdk/outputs/__init__.py

@@ -64,6 +64,7 @@ class Output(ABC):
     output_prefix: str
     total_record_count: int
     chunk_count: int
+    chunk_part: int
     buffer_size: int
     max_file_size_bytes: int
     current_buffer_size: int

application_sdk/outputs/parquet.py

@@ -65,6 +65,7 @@ class ParquetOutput(Output):
         buffer_size: int = 5000,
         total_record_count: int = 0,
         chunk_count: int = 0,
+        chunk_part: int = 0,
         chunk_start: Optional[int] = None,
         start_marker: Optional[str] = None,
         end_marker: Optional[str] = None,
@@ -105,7 +106,7 @@ class ParquetOutput(Output):
             DAPR_MAX_GRPC_MESSAGE_LENGTH * 0.75
         )  # 75% of DAPR limit as safety buffer
         self.chunk_start = chunk_start
-        self.chunk_part = 0
+        self.chunk_part = chunk_part
         self.start_marker = start_marker
         self.end_marker = end_marker
         self.partitions = []

application_sdk/version.py

@@ -2,4 +2,4 @@
 Version information for the application_sdk package.
 """
 
-__version__ = "0.1.1rc59"
+__version__ = "0.1.1rc60"

{atlan_application_sdk-0.1.1rc59.dist-info → atlan_application_sdk-0.1.1rc60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlan-application-sdk
-Version: 0.1.1rc59
+Version: 0.1.1rc60
 Summary: Atlan Application SDK is a Python library for developing applications on the Atlan Platform
 Project-URL: Repository, https://github.com/atlanhq/application-sdk
 Project-URL: Documentation, https://github.com/atlanhq/application-sdk/README.md

{atlan_application_sdk-0.1.1rc59.dist-info → atlan_application_sdk-0.1.1rc60.dist-info}/RECORD

@@ -1,6 +1,6 @@
 application_sdk/__init__.py,sha256=2e2mvmLJ5dxmJGPELtb33xwP-j6JMdoIuqKycEn7hjg,151
 application_sdk/constants.py,sha256=ySrjME6CSoiyjVLgQt0s4dIdgJ0JWIMmMs7WBZBVUpA,11013
-application_sdk/version.py,sha256=sXMApD_x-2esC_2hKV1Jo0L6_WWdwAw9zFGgtDsWblQ,88
+application_sdk/version.py,sha256=2aOWC-Y08sXYBsPMJehGpskX5CTH1TNdtzY-QR-Ub8o,88
 application_sdk/worker.py,sha256=i5f0AeKI39IfsLO05QkwC6uMz0zDPSJqP7B2byri1VI,7489
 application_sdk/activities/__init__.py,sha256=L5WXkTwOwGtjWAlXrUJRCKGwyIyp3z8fBv8BZVCRFQI,11175
 application_sdk/activities/lock_management.py,sha256=6Wdf3jMKitoarHQP91PIJOoGFz4aaOLS_40c7n1yAOA,3902
@@ -73,14 +73,14 @@ application_sdk/interceptors/.cursor/BUGBOT.md,sha256=pxmUF2c7dtaXAX8yAa1-LBa6FC
 application_sdk/observability/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 application_sdk/observability/logger_adaptor.py,sha256=00c0F7maDkp1xrHttW6VQbWFDGr6NkXeDPjmf97ojlY,29989
 application_sdk/observability/metrics_adaptor.py,sha256=5Oz02lUED60duryoVDF9mbD11fpxhbXi7P1609n_15Y,16446
-application_sdk/observability/observability.py,sha256=7gRws1Lg1bM6WWL5Y-ZJOxDMHDx1tKqo8Qw-xxSkDEs,24312
+application_sdk/observability/observability.py,sha256=MGxNFPx6pOdpWrpNXZp44NPk3SG4xjA9cKrTeZ1ENK8,23681
 application_sdk/observability/traces_adaptor.py,sha256=0eQJPN-tYA_dV8D3uEa5ZiX9g12NDuLnPaFuQMVDdL0,18242
 application_sdk/observability/utils.py,sha256=JoHEA68cjkXTnAXHzgiULYOzRTk8rG4kPZRvFYah3aU,2505
 application_sdk/observability/decorators/observability_decorator.py,sha256=yd6qfrg1MmH5KcZ5Ydzb0RaBzmxx5FrmiI9qwvZx3EU,8963
-application_sdk/outputs/__init__.py,sha256=hrOPw0xuG9xP720Bt309TfbY2Qq_i51R8Xt3ZjwWDUY,15906
+application_sdk/outputs/__init__.py,sha256=dekaEqJEVAmXQYSy_AohXOHNNI56OXG3Xn27FFRmoPQ,15926
 application_sdk/outputs/iceberg.py,sha256=TdppOMEMfojMhGyBmhWeu1AJQexRyHM-huAYeJmhjdY,5533
 application_sdk/outputs/json.py,sha256=gYDDNOVb8EFxxeOkb6zKWZWjTEVgZLoapFM97_roK4A,10883
-application_sdk/outputs/parquet.py,sha256=bAv-IrWLNFqF-j8D7jSDi0ug1qV_qBTA3dDxK2eiFcA,20724
+application_sdk/outputs/parquet.py,sha256=DxcKh1IXPdiXNQJS1HIn6-JRdLkmN4At8uF1zppiZX0,20762
 application_sdk/outputs/.cursor/BUGBOT.md,sha256=KxEC3CIyRSK1YftZou5BgKc6PRXT3qQmBNFJp-HSyYE,11496
 application_sdk/server/__init__.py,sha256=KTqE1YPw_3WDVMWatJUuf9OOiobLM2K5SMaBrI62sCo,1568
 application_sdk/server/.cursor/BUGBOT.md,sha256=p_MMoWUW5G1894WfOKYReZKWCuyJT_OJz3rL5g21NbI,16566
@@ -157,8 +157,8 @@ application_sdk/workflows/metadata_extraction/__init__.py,sha256=jHUe_ZBQ66jx8bg
 application_sdk/workflows/metadata_extraction/sql.py,sha256=6ZaVt84n-8U2ZvR9GR7uIJKv5v8CuyQjhlnoRJvDszc,12435
 application_sdk/workflows/query_extraction/__init__.py,sha256=n066_CX5RpJz6DIxGMkKS3eGSRg03ilaCtsqfJWQb7Q,117
 application_sdk/workflows/query_extraction/sql.py,sha256=kT_JQkLCRZ44ZpaC4QvPL6DxnRIIVh8gYHLqRbMI-hA,4826
-atlan_application_sdk-0.1.1rc59.dist-info/METADATA,sha256=f39Mtuyd4ASvIaEwAGtaGyWlSx0JqbFFfa2fNA20P_Y,5730
-atlan_application_sdk-0.1.1rc59.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-atlan_application_sdk-0.1.1rc59.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-atlan_application_sdk-0.1.1rc59.dist-info/licenses/NOTICE,sha256=A-XVVGt3KOYuuMmvSMIFkg534F1vHiCggEBp4Ez3wGk,1041
-atlan_application_sdk-0.1.1rc59.dist-info/RECORD,,
+atlan_application_sdk-0.1.1rc60.dist-info/METADATA,sha256=vbNpysrGHIBYdJGSe-s8V3km-Amr5xeXIbGunIDHnnI,5730
+atlan_application_sdk-0.1.1rc60.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+atlan_application_sdk-0.1.1rc60.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+atlan_application_sdk-0.1.1rc60.dist-info/licenses/NOTICE,sha256=A-XVVGt3KOYuuMmvSMIFkg534F1vHiCggEBp4Ez3wGk,1041
+atlan_application_sdk-0.1.1rc60.dist-info/RECORD,,