atlan-application-sdk 0.1.1rc58__py3-none-any.whl → 0.1.1rc60__py3-none-any.whl

application_sdk/constants.py

@@ -75,7 +75,7 @@ STATE_STORE_PATH_TEMPLATE = (
 
 # Observability Constants
 #: Directory for storing observability data
-OBSERVABILITY_DIR = "artifacts/apps/{application_name}/observability"
+OBSERVABILITY_DIR = "artifacts/apps/{application_name}/{deployment_name}/observability"
 
 # Workflow Client Constants
 #: Host address for the Temporal server
@@ -88,6 +88,7 @@ WORKFLOW_NAMESPACE = os.getenv("ATLAN_WORKFLOW_NAMESPACE", "default")
 WORKFLOW_UI_HOST = os.getenv("ATLAN_WORKFLOW_UI_HOST", "localhost")
 #: Port number for the Temporal UI
 WORKFLOW_UI_PORT = os.getenv("ATLAN_WORKFLOW_UI_PORT", "8233")
+
 #: Maximum timeout duration for workflows
 WORKFLOW_MAX_TIMEOUT_HOURS = timedelta(
     hours=int(os.getenv("ATLAN_WORKFLOW_MAX_TIMEOUT_HOURS", "1"))

application_sdk/observability/observability.py

@@ -16,13 +16,11 @@ from dapr.clients import DaprClient
 from pydantic import BaseModel
 
 from application_sdk.constants import (
-    DAPR_BINDING_OPERATION_CREATE,
     DEPLOYMENT_OBJECT_STORE_NAME,
     ENABLE_OBSERVABILITY_DAPR_SINK,
     LOG_FILE_NAME,
     METRICS_FILE_NAME,
     STATE_STORE_NAME,
-    TEMPORARY_PATH,
     TRACES_FILE_NAME,
 )
 from application_sdk.observability.utils import get_observability_dir
@@ -365,16 +363,23 @@ class AtlanObservability(Generic[T], ABC):
             logging.error(f"Error buffering log: {e}")
 
     async def _flush_records(self, records: List[Dict[str, Any]]):
-        """Flush records to parquet file and object store.
+        """Flush records to parquet file and object store using ParquetOutput abstraction.
 
         Args:
             records: List of records to flush
 
         This method:
-        - Groups records by partition
-        - Writes records to each partition
-        - Uploads to object store if enabled
+        - Groups records by partition (year/month/day)
+        - Uses ParquetOutput abstraction for efficient writing
+        - Automatically handles chunking, compression, and dual upload
+        - Provides robust error handling per partition
         - Cleans up old records if enabled
+
+        Features:
+        - Automatic chunking for large datasets
+        - Dual upload support (primary + upstream if enabled)
+        - Advanced consolidation for optimal performance
+        - Fault-tolerant processing (continues on partition errors)
         """
         if not ENABLE_OBSERVABILITY_DAPR_SINK:
             return
@@ -390,30 +395,15 @@ class AtlanObservability(Generic[T], ABC):
                     partition_records[partition_path] = []
                 partition_records[partition_path].append(record)
 
-            # Write records to each partition
+            # Write records to each partition using ParquetOutput abstraction
             for partition_path, partition_data in partition_records.items():
-                os.makedirs(partition_path, exist_ok=True)
-                # Use a consistent file name for each partition
-                parquet_path = os.path.join(partition_path, "data.parquet")
-
-                # Read existing data if any
-                existing_df = None
-                if os.path.exists(parquet_path):
-                    try:
-                        # Read the entire parquet file without excluding any columns
-                        existing_df = pd.read_parquet(parquet_path)
-                    except Exception as e:
-                        logging.error(f"Error reading existing parquet file: {e}")
-                        # If there's an error reading the existing file, we'll overwrite it
-                        existing_df = None
-
                 # Create new dataframe from current records
                 new_df = pd.DataFrame(partition_data)
 
-                # Extract partition values from path
-                partition_parts = os.path.basename(os.path.dirname(parquet_path)).split(
-                    os.sep
-                )
+                # Extract partition values from path and add to dataframe
+                partition_parts = os.path.basename(
+                    os.path.dirname(partition_path)
+                ).split(os.sep)
                 for part in partition_parts:
                     if part.startswith("year="):
                         new_df["year"] = int(part.split("=")[1])
@@ -422,38 +412,23 @@ class AtlanObservability(Generic[T], ABC):
                     elif part.startswith("day="):
                         new_df["day"] = int(part.split("=")[1])
 
-                # Merge with existing data if any
-                if existing_df is not None:
-                    df = pd.concat([existing_df, new_df], ignore_index=True)
-                else:
-                    df = new_df
-
-                # Sort by timestamp to maintain order
-                df = df.sort_values("timestamp")
+                # Use new data directly - let ParquetOutput handle consolidation and merging
+                df = new_df
 
-                # Write to parquet file
-                df.to_parquet(
-                    parquet_path,
-                    compression="snappy",
-                    index=False,
-                )
+                # Use ParquetOutput abstraction for efficient writing and uploading
+                # Set the output path for this partition
+                try:
+                    # Lazy import and instantiation of ParquetOutput
+                    from application_sdk.outputs.parquet import ParquetOutput
 
-                # Upload to object store
-                with open(parquet_path, "rb") as f:
-                    file_content = f.read()
-                    relative_path = os.path.relpath(parquet_path, TEMPORARY_PATH)
-                    metadata = {
-                        "key": relative_path,
-                        "blobName": relative_path,
-                        "fileName": relative_path,
-                    }
-                    with DaprClient() as client:
-                        client.invoke_binding(
-                            binding_name=DEPLOYMENT_OBJECT_STORE_NAME,
-                            operation=DAPR_BINDING_OPERATION_CREATE,
-                            data=file_content,
-                            binding_metadata=metadata,
-                        )
+                    parquet_output = ParquetOutput(
+                        output_path=partition_path,
+                        chunk_start=0,
+                        chunk_part=int(time()),
+                    )
+                    await parquet_output.write_dataframe(dataframe=df)
+                except Exception as e:
+                    print(f"Error writing records to partition: {str(e)}")
 
             # Clean up old records if enabled
             if self._cleanup_enabled:

application_sdk/observability/utils.py

@@ -5,6 +5,7 @@ from temporalio import activity, workflow
 
 from application_sdk.constants import (
     APPLICATION_NAME,
+    DEPLOYMENT_NAME,
     OBSERVABILITY_DIR,
     TEMPORARY_PATH,
 )
@@ -26,16 +27,16 @@ class WorkflowContext(BaseModel):
 
 
 def get_observability_dir() -> str:
-    """Build the observability path.
-
-    Args:
-        path: The path to build the observability path from.
+    """Build the observability path using deployment name.
 
     Returns:
-        str: The built observability path.
+        str: The built observability path using deployment name.
     """
     return os.path.join(
-        TEMPORARY_PATH, OBSERVABILITY_DIR.format(application_name=APPLICATION_NAME)
+        TEMPORARY_PATH,
+        OBSERVABILITY_DIR.format(
+            application_name=APPLICATION_NAME, deployment_name=DEPLOYMENT_NAME
+        ),
     )
 
 

application_sdk/outputs/__init__.py

@@ -64,6 +64,7 @@ class Output(ABC):
     output_prefix: str
     total_record_count: int
     chunk_count: int
+    chunk_part: int
     buffer_size: int
     max_file_size_bytes: int
     current_buffer_size: int

application_sdk/outputs/parquet.py

@@ -8,7 +8,11 @@ from temporalio import activity
 
 from application_sdk.activities.common.utils import get_object_store_prefix
 from application_sdk.common.dataframe_utils import is_empty_dataframe
-from application_sdk.constants import DAPR_MAX_GRPC_MESSAGE_LENGTH
+from application_sdk.constants import (
+    DAPR_MAX_GRPC_MESSAGE_LENGTH,
+    ENABLE_ATLAN_UPLOAD,
+    UPSTREAM_OBJECT_STORE_NAME,
+)
 from application_sdk.observability.logger_adaptor import get_logger
 from application_sdk.observability.metrics_adaptor import MetricType, get_metrics
 from application_sdk.outputs import Output
@@ -61,6 +65,7 @@ class ParquetOutput(Output):
         buffer_size: int = 5000,
         total_record_count: int = 0,
         chunk_count: int = 0,
+        chunk_part: int = 0,
         chunk_start: Optional[int] = None,
         start_marker: Optional[str] = None,
         end_marker: Optional[str] = None,
@@ -101,7 +106,7 @@ class ParquetOutput(Output):
             DAPR_MAX_GRPC_MESSAGE_LENGTH * 0.75
         )  # 75% of DAPR limit as safety buffer
         self.chunk_start = chunk_start
-        self.chunk_part = 0
+        self.chunk_part = chunk_part
         self.start_marker = start_marker
         self.end_marker = end_marker
         self.partitions = []
@@ -269,6 +274,13 @@ class ParquetOutput(Output):
                         f"No files found under prefix {get_object_store_prefix(self.output_path)}: {str(e)}"
                     )
             for path in file_paths:
+                if ENABLE_ATLAN_UPLOAD:
+                    await ObjectStore.upload_file(
+                        source=path,
+                        store_name=UPSTREAM_OBJECT_STORE_NAME,
+                        destination=get_object_store_prefix(path),
+                        retain_local_copy=True,
+                    )
                 await ObjectStore.upload_file(
                     source=path,
                     destination=get_object_store_prefix(path),

application_sdk/version.py

@@ -2,4 +2,4 @@
 Version information for the application_sdk package.
 """
 
-__version__ = "0.1.1rc58"
+__version__ = "0.1.1rc60"

{atlan_application_sdk-0.1.1rc58.dist-info → atlan_application_sdk-0.1.1rc60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlan-application-sdk
-Version: 0.1.1rc58
+Version: 0.1.1rc60
 Summary: Atlan Application SDK is a Python library for developing applications on the Atlan Platform
 Project-URL: Repository, https://github.com/atlanhq/application-sdk
 Project-URL: Documentation, https://github.com/atlanhq/application-sdk/README.md

{atlan_application_sdk-0.1.1rc58.dist-info → atlan_application_sdk-0.1.1rc60.dist-info}/RECORD

@@ -1,6 +1,6 @@
 application_sdk/__init__.py,sha256=2e2mvmLJ5dxmJGPELtb33xwP-j6JMdoIuqKycEn7hjg,151
-application_sdk/constants.py,sha256=WDw0I9u_OoXIE79IFDFBHJYmIS9KAsghMeGOxfshnpg,10994
-application_sdk/version.py,sha256=GfLIvCy7o6Bup3dK-TlbQNu-KhQHzePlpsQ45bXgCqg,88
+application_sdk/constants.py,sha256=ySrjME6CSoiyjVLgQt0s4dIdgJ0JWIMmMs7WBZBVUpA,11013
+application_sdk/version.py,sha256=2aOWC-Y08sXYBsPMJehGpskX5CTH1TNdtzY-QR-Ub8o,88
 application_sdk/worker.py,sha256=i5f0AeKI39IfsLO05QkwC6uMz0zDPSJqP7B2byri1VI,7489
 application_sdk/activities/__init__.py,sha256=L5WXkTwOwGtjWAlXrUJRCKGwyIyp3z8fBv8BZVCRFQI,11175
 application_sdk/activities/lock_management.py,sha256=6Wdf3jMKitoarHQP91PIJOoGFz4aaOLS_40c7n1yAOA,3902
@@ -73,14 +73,14 @@ application_sdk/interceptors/.cursor/BUGBOT.md,sha256=pxmUF2c7dtaXAX8yAa1-LBa6FC
 application_sdk/observability/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 application_sdk/observability/logger_adaptor.py,sha256=00c0F7maDkp1xrHttW6VQbWFDGr6NkXeDPjmf97ojlY,29989
 application_sdk/observability/metrics_adaptor.py,sha256=5Oz02lUED60duryoVDF9mbD11fpxhbXi7P1609n_15Y,16446
-application_sdk/observability/observability.py,sha256=DP0I4bHyg3TA4hxCqDFy2IiRmBGOpZ7449m7BUoc_RA,24530
+application_sdk/observability/observability.py,sha256=MGxNFPx6pOdpWrpNXZp44NPk3SG4xjA9cKrTeZ1ENK8,23681
 application_sdk/observability/traces_adaptor.py,sha256=0eQJPN-tYA_dV8D3uEa5ZiX9g12NDuLnPaFuQMVDdL0,18242
-application_sdk/observability/utils.py,sha256=MKEpT0WYtpATUgLgJDkGQaAP_t-jpDYMUKDfEvr8Phg,2448
+application_sdk/observability/utils.py,sha256=JoHEA68cjkXTnAXHzgiULYOzRTk8rG4kPZRvFYah3aU,2505
 application_sdk/observability/decorators/observability_decorator.py,sha256=yd6qfrg1MmH5KcZ5Ydzb0RaBzmxx5FrmiI9qwvZx3EU,8963
-application_sdk/outputs/__init__.py,sha256=hrOPw0xuG9xP720Bt309TfbY2Qq_i51R8Xt3ZjwWDUY,15906
+application_sdk/outputs/__init__.py,sha256=dekaEqJEVAmXQYSy_AohXOHNNI56OXG3Xn27FFRmoPQ,15926
 application_sdk/outputs/iceberg.py,sha256=TdppOMEMfojMhGyBmhWeu1AJQexRyHM-huAYeJmhjdY,5533
 application_sdk/outputs/json.py,sha256=gYDDNOVb8EFxxeOkb6zKWZWjTEVgZLoapFM97_roK4A,10883
-application_sdk/outputs/parquet.py,sha256=lEUosbdLfLhZZPS9lZJxnQHEbLc5e1O7JeiDX958frw,20330
+application_sdk/outputs/parquet.py,sha256=DxcKh1IXPdiXNQJS1HIn6-JRdLkmN4At8uF1zppiZX0,20762
 application_sdk/outputs/.cursor/BUGBOT.md,sha256=KxEC3CIyRSK1YftZou5BgKc6PRXT3qQmBNFJp-HSyYE,11496
 application_sdk/server/__init__.py,sha256=KTqE1YPw_3WDVMWatJUuf9OOiobLM2K5SMaBrI62sCo,1568
 application_sdk/server/.cursor/BUGBOT.md,sha256=p_MMoWUW5G1894WfOKYReZKWCuyJT_OJz3rL5g21NbI,16566
@@ -157,8 +157,8 @@ application_sdk/workflows/metadata_extraction/__init__.py,sha256=jHUe_ZBQ66jx8bg
 application_sdk/workflows/metadata_extraction/sql.py,sha256=6ZaVt84n-8U2ZvR9GR7uIJKv5v8CuyQjhlnoRJvDszc,12435
 application_sdk/workflows/query_extraction/__init__.py,sha256=n066_CX5RpJz6DIxGMkKS3eGSRg03ilaCtsqfJWQb7Q,117
 application_sdk/workflows/query_extraction/sql.py,sha256=kT_JQkLCRZ44ZpaC4QvPL6DxnRIIVh8gYHLqRbMI-hA,4826
-atlan_application_sdk-0.1.1rc58.dist-info/METADATA,sha256=sLmqBn4MVdNZQluPCk4z_iItSspSM4W_W1RZGqyPL1c,5730
-atlan_application_sdk-0.1.1rc58.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-atlan_application_sdk-0.1.1rc58.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-atlan_application_sdk-0.1.1rc58.dist-info/licenses/NOTICE,sha256=A-XVVGt3KOYuuMmvSMIFkg534F1vHiCggEBp4Ez3wGk,1041
-atlan_application_sdk-0.1.1rc58.dist-info/RECORD,,
+atlan_application_sdk-0.1.1rc60.dist-info/METADATA,sha256=vbNpysrGHIBYdJGSe-s8V3km-Amr5xeXIbGunIDHnnI,5730
+atlan_application_sdk-0.1.1rc60.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+atlan_application_sdk-0.1.1rc60.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+atlan_application_sdk-0.1.1rc60.dist-info/licenses/NOTICE,sha256=A-XVVGt3KOYuuMmvSMIFkg534F1vHiCggEBp4Ez3wGk,1041
+atlan_application_sdk-0.1.1rc60.dist-info/RECORD,,