atlan-application-sdk 0.1.1rc58__tar.gz → 0.1.1rc59__tar.gz

atlan_application_sdk-0.1.1rc59/.github/release_notes.md

@@ -0,0 +1,7 @@
+## v0.1.1rc59 (October 25, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc58...v0.1.1rc59
+
+### Features
+
+- Adding support to push logs decoupled for workflow (#786) (by @Garavitey in [2b291ff](https://github.com/atlanhq/application-sdk/commit/2b291ff))

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc59}/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## v0.1.1rc59 (October 25, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc58...v0.1.1rc59
+
+### Features
+
+- Adding support to push logs decoupled for workflow (#786) (by @Garavitey in [2b291ff](https://github.com/atlanhq/application-sdk/commit/2b291ff))
+
+
 ## v0.1.1rc58 (October 22, 2025)
 
 Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc57...v0.1.1rc58

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlan-application-sdk
-Version: 0.1.1rc58
+Version: 0.1.1rc59
 Summary: Atlan Application SDK is a Python library for developing applications on the Atlan Platform
 Project-URL: Repository, https://github.com/atlanhq/application-sdk
 Project-URL: Documentation, https://github.com/atlanhq/application-sdk/README.md

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc59}/application_sdk/constants.py

@@ -75,7 +75,7 @@ STATE_STORE_PATH_TEMPLATE = (
 
 # Observability Constants
 #: Directory for storing observability data
-OBSERVABILITY_DIR = "artifacts/apps/{application_name}/observability"
+OBSERVABILITY_DIR = "artifacts/apps/{application_name}/{deployment_name}/observability"
 
 # Workflow Client Constants
 #: Host address for the Temporal server
@@ -88,6 +88,7 @@ WORKFLOW_NAMESPACE = os.getenv("ATLAN_WORKFLOW_NAMESPACE", "default")
 WORKFLOW_UI_HOST = os.getenv("ATLAN_WORKFLOW_UI_HOST", "localhost")
 #: Port number for the Temporal UI
 WORKFLOW_UI_PORT = os.getenv("ATLAN_WORKFLOW_UI_PORT", "8233")
+
 #: Maximum timeout duration for workflows
 WORKFLOW_MAX_TIMEOUT_HOURS = timedelta(
     hours=int(os.getenv("ATLAN_WORKFLOW_MAX_TIMEOUT_HOURS", "1"))

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc59}/application_sdk/observability/observability.py

@@ -10,19 +10,18 @@ from pathlib import Path
 from time import time
 from typing import Any, Dict, Generic, List, TypeVar
 
+import daft
 import duckdb
 import pandas as pd
 from dapr.clients import DaprClient
 from pydantic import BaseModel
 
 from application_sdk.constants import (
-    DAPR_BINDING_OPERATION_CREATE,
     DEPLOYMENT_OBJECT_STORE_NAME,
     ENABLE_OBSERVABILITY_DAPR_SINK,
     LOG_FILE_NAME,
     METRICS_FILE_NAME,
     STATE_STORE_NAME,
-    TEMPORARY_PATH,
     TRACES_FILE_NAME,
 )
 from application_sdk.observability.utils import get_observability_dir
@@ -365,16 +364,23 @@ class AtlanObservability(Generic[T], ABC):
             logging.error(f"Error buffering log: {e}")
 
     async def _flush_records(self, records: List[Dict[str, Any]]):
-        """Flush records to parquet file and object store.
+        """Flush records to parquet file and object store using ParquetOutput abstraction.
 
         Args:
             records: List of records to flush
 
         This method:
-        - Groups records by partition
-        - Writes records to each partition
-        - Uploads to object store if enabled
+        - Groups records by partition (year/month/day)
+        - Uses ParquetOutput abstraction for efficient writing
+        - Automatically handles chunking, compression, and dual upload
+        - Provides robust error handling per partition
         - Cleans up old records if enabled
+
+        Features:
+        - Automatic chunking for large datasets
+        - Dual upload support (primary + upstream if enabled)
+        - Advanced consolidation for optimal performance
+        - Fault-tolerant processing (continues on partition errors)
         """
         if not ENABLE_OBSERVABILITY_DAPR_SINK:
             return
@@ -390,30 +396,15 @@ class AtlanObservability(Generic[T], ABC):
                     partition_records[partition_path] = []
                 partition_records[partition_path].append(record)
 
-            # Write records to each partition
+            # Write records to each partition using ParquetOutput abstraction
             for partition_path, partition_data in partition_records.items():
-                os.makedirs(partition_path, exist_ok=True)
-                # Use a consistent file name for each partition
-                parquet_path = os.path.join(partition_path, "data.parquet")
-
-                # Read existing data if any
-                existing_df = None
-                if os.path.exists(parquet_path):
-                    try:
-                        # Read the entire parquet file without excluding any columns
-                        existing_df = pd.read_parquet(parquet_path)
-                    except Exception as e:
-                        logging.error(f"Error reading existing parquet file: {e}")
-                        # If there's an error reading the existing file, we'll overwrite it
-                        existing_df = None
-
                 # Create new dataframe from current records
                 new_df = pd.DataFrame(partition_data)
 
-                # Extract partition values from path
-                partition_parts = os.path.basename(os.path.dirname(parquet_path)).split(
-                    os.sep
-                )
+                # Extract partition values from path and add to dataframe
+                partition_parts = os.path.basename(
+                    os.path.dirname(partition_path)
+                ).split(os.sep)
                 for part in partition_parts:
                     if part.startswith("year="):
                         new_df["year"] = int(part.split("=")[1])
@@ -422,38 +413,37 @@ class AtlanObservability(Generic[T], ABC):
                     elif part.startswith("day="):
                         new_df["day"] = int(part.split("=")[1])
 
-                # Merge with existing data if any
-                if existing_df is not None:
-                    df = pd.concat([existing_df, new_df], ignore_index=True)
-                else:
-                    df = new_df
+                # Use new data directly - let ParquetOutput handle consolidation and merging
+                df = new_df
 
-                # Sort by timestamp to maintain order
-                df = df.sort_values("timestamp")
+                # Use ParquetOutput abstraction for efficient writing and uploading
+                # Set the output path for this partition
+                try:
+                    # Lazy import and instantiation of ParquetOutput
+                    from application_sdk.outputs.parquet import ParquetOutput
 
-                # Write to parquet file
-                df.to_parquet(
-                    parquet_path,
-                    compression="snappy",
-                    index=False,
-                )
+                    parquet_output = ParquetOutput(output_path=partition_path)
+                    logging.info(
+                        f"Successfully instantiated ParquetOutput for partition: {partition_path}"
+                    )
 
-                # Upload to object store
-                with open(parquet_path, "rb") as f:
-                    file_content = f.read()
-                    relative_path = os.path.relpath(parquet_path, TEMPORARY_PATH)
-                    metadata = {
-                        "key": relative_path,
-                        "blobName": relative_path,
-                        "fileName": relative_path,
-                    }
-                    with DaprClient() as client:
-                        client.invoke_binding(
-                            binding_name=DEPLOYMENT_OBJECT_STORE_NAME,
-                            operation=DAPR_BINDING_OPERATION_CREATE,
-                            data=file_content,
-                            binding_metadata=metadata,
-                        )
+                    # Use write_daft_dataframe with the DataFrame we have
+                    from application_sdk.outputs.parquet import WriteMode
+
+                    daft_df = daft.from_pandas(df)
+                    await parquet_output.write_daft_dataframe(
+                        dataframe=daft_df,
+                        write_mode=WriteMode.APPEND,  # Append mode to merge with existing data
+                    )
+
+                    logging.info(
+                        f"Successfully wrote {len(df)} records to partition: {partition_path}"
+                    )
+
+                except Exception as partition_error:
+                    logging.error(
+                        f"Error processing partition {partition_path}: {str(partition_error)}"
+                    )
 
             # Clean up old records if enabled
             if self._cleanup_enabled:

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc59}/application_sdk/observability/utils.py

@@ -5,6 +5,7 @@ from temporalio import activity, workflow
 
 from application_sdk.constants import (
     APPLICATION_NAME,
+    DEPLOYMENT_NAME,
     OBSERVABILITY_DIR,
     TEMPORARY_PATH,
 )
@@ -26,16 +27,16 @@ class WorkflowContext(BaseModel):
 
 
 def get_observability_dir() -> str:
-    """Build the observability path.
-
-    Args:
-        path: The path to build the observability path from.
+    """Build the observability path using deployment name.
 
     Returns:
-        str: The built observability path.
+        str: The built observability path using deployment name.
     """
     return os.path.join(
-        TEMPORARY_PATH, OBSERVABILITY_DIR.format(application_name=APPLICATION_NAME)
+        TEMPORARY_PATH,
+        OBSERVABILITY_DIR.format(
+            application_name=APPLICATION_NAME, deployment_name=DEPLOYMENT_NAME
+        ),
     )
 
 

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc59}/application_sdk/outputs/parquet.py

@@ -8,7 +8,11 @@ from temporalio import activity
 
 from application_sdk.activities.common.utils import get_object_store_prefix
 from application_sdk.common.dataframe_utils import is_empty_dataframe
-from application_sdk.constants import DAPR_MAX_GRPC_MESSAGE_LENGTH
+from application_sdk.constants import (
+    DAPR_MAX_GRPC_MESSAGE_LENGTH,
+    ENABLE_ATLAN_UPLOAD,
+    UPSTREAM_OBJECT_STORE_NAME,
+)
 from application_sdk.observability.logger_adaptor import get_logger
 from application_sdk.observability.metrics_adaptor import MetricType, get_metrics
 from application_sdk.outputs import Output
@@ -269,6 +273,13 @@ class ParquetOutput(Output):
                         f"No files found under prefix {get_object_store_prefix(self.output_path)}: {str(e)}"
                     )
             for path in file_paths:
+                if ENABLE_ATLAN_UPLOAD:
+                    await ObjectStore.upload_file(
+                        source=path,
+                        store_name=UPSTREAM_OBJECT_STORE_NAME,
+                        destination=get_object_store_prefix(path),
+                        retain_local_copy=True,
+                    )
                 await ObjectStore.upload_file(
                     source=path,
                     destination=get_object_store_prefix(path),

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc59}/application_sdk/version.py

@@ -2,4 +2,4 @@
 Version information for the application_sdk package.
 """
 
-__version__ = "0.1.1rc58"
+__version__ = "0.1.1rc59"

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc59}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atlan-application-sdk"
-version = "0.1.1rc58"
+version = "0.1.1rc59"
 description = "Atlan Application SDK is a Python library for developing applications on the Atlan Platform"
 license = "Apache-2.0"
 authors = [

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc59}/uv.lock

@@ -140,7 +140,7 @@ wheels = [
 
 [[package]]
 name = "atlan-application-sdk"
-version = "0.1.1rc58"
+version = "0.1.1rc59"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },

atlan_application_sdk-0.1.1rc58/.github/release_notes.md

@@ -1,7 +0,0 @@
-## v0.1.1rc58 (October 22, 2025)
-
-Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc57...v0.1.1rc58
-
-### Bug Fixes
-
-- Updated lockname to not use APPLICATION_NAME (#797) (by @SanilK2108 in [151b811](https://github.com/atlanhq/application-sdk/commit/151b811))