atlan-application-sdk 0.1.1rc58__tar.gz → 0.1.1rc60__tar.gz

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/.cursor/rules/setup.mdc

@@ -10,7 +10,7 @@ The Atlan Python Application SDK requires the following components regardless of
 1. Python 3.11.10
 2. uv 0.7.3
 3. Temporal CLI
-4. DAPR CLI 1.16.0
+4. DAPR CLI 1.16.2
 5. Project dependencies
 
 ## OS-Specific Setup Guides

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/.github/actions/e2e-examples/action.yaml

@@ -45,14 +45,14 @@ runs:
       if: runner.os != 'Windows'
       shell: bash
       run: |
-        wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash -s 1.16.0
+        wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash -s 1.16.2
         dapr init --runtime-version 1.16.0 --slim
 
     - name: Install Dapr CLI (Windows)
       if: runner.os == 'Windows'
       shell: pwsh
       run: |
-        $script=iwr -useb https://raw.githubusercontent.com/dapr/cli/master/install/install.ps1; $block=[ScriptBlock]::Create($script); invoke-command -ScriptBlock $block -ArgumentList 1.16.0, "$env:USERPROFILE\.dapr\bin\"
+        $script=iwr -useb https://raw.githubusercontent.com/dapr/cli/master/install/install.ps1; $block=[ScriptBlock]::Create($script); invoke-command -ScriptBlock $block -ArgumentList 1.16.2, "$env:USERPROFILE\.dapr\bin\"
         $env:Path += ";$env:USERPROFILE\.dapr\bin\"
         dapr init --runtime-version 1.16.0 --slim
 

atlan_application_sdk-0.1.1rc60/.github/release_notes.md

@@ -0,0 +1,11 @@
+## v0.1.1rc60 (October 29, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc59...v0.1.1rc60
+
+### Features
+
+- Fixing the daft import issue (#803) (by @Garavitey in [0f27353](https://github.com/atlanhq/application-sdk/commit/0f27353))
+
+### Bug Fixes
+
+- bump Dapr cli to v1.16.2, helm security patch (#804) (by @inishchith in [d36a4e7](https://github.com/atlanhq/application-sdk/commit/d36a4e7))

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/.github/workflows/scale-tests.yaml

@@ -47,7 +47,7 @@ jobs:
       # Install Dapr
       - name: Install Dapr CLI
         run: |
-          wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash -s 1.16.0
+          wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash -s 1.16.2
           dapr init --runtime-version 1.16.0 --slim
 
       # Install Temporal

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## v0.1.1rc60 (October 29, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc59...v0.1.1rc60
+
+### Features
+
+- Fixing the daft import issue (#803) (by @Garavitey in [0f27353](https://github.com/atlanhq/application-sdk/commit/0f27353))
+
+### Bug Fixes
+
+- bump Dapr cli to v1.16.2, helm security patch (#804) (by @inishchith in [d36a4e7](https://github.com/atlanhq/application-sdk/commit/d36a4e7))
+
+
+## v0.1.1rc59 (October 25, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc58...v0.1.1rc59
+
+### Features
+
+- Adding support to push logs decoupled for workflow (#786) (by @Garavitey in [2b291ff](https://github.com/atlanhq/application-sdk/commit/2b291ff))
+
+
 ## v0.1.1rc58 (October 22, 2025)
 
 Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc57...v0.1.1rc58

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlan-application-sdk
-Version: 0.1.1rc58
+Version: 0.1.1rc60
 Summary: Atlan Application SDK is a Python library for developing applications on the Atlan Platform
 Project-URL: Repository, https://github.com/atlanhq/application-sdk
 Project-URL: Documentation, https://github.com/atlanhq/application-sdk/README.md

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/application_sdk/constants.py

@@ -75,7 +75,7 @@ STATE_STORE_PATH_TEMPLATE = (
 
 # Observability Constants
 #: Directory for storing observability data
-OBSERVABILITY_DIR = "artifacts/apps/{application_name}/observability"
+OBSERVABILITY_DIR = "artifacts/apps/{application_name}/{deployment_name}/observability"
 
 # Workflow Client Constants
 #: Host address for the Temporal server
@@ -88,6 +88,7 @@ WORKFLOW_NAMESPACE = os.getenv("ATLAN_WORKFLOW_NAMESPACE", "default")
 WORKFLOW_UI_HOST = os.getenv("ATLAN_WORKFLOW_UI_HOST", "localhost")
 #: Port number for the Temporal UI
 WORKFLOW_UI_PORT = os.getenv("ATLAN_WORKFLOW_UI_PORT", "8233")
+
 #: Maximum timeout duration for workflows
 WORKFLOW_MAX_TIMEOUT_HOURS = timedelta(
     hours=int(os.getenv("ATLAN_WORKFLOW_MAX_TIMEOUT_HOURS", "1"))

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/application_sdk/observability/observability.py

@@ -16,13 +16,11 @@ from dapr.clients import DaprClient
 from pydantic import BaseModel
 
 from application_sdk.constants import (
-    DAPR_BINDING_OPERATION_CREATE,
     DEPLOYMENT_OBJECT_STORE_NAME,
     ENABLE_OBSERVABILITY_DAPR_SINK,
     LOG_FILE_NAME,
     METRICS_FILE_NAME,
     STATE_STORE_NAME,
-    TEMPORARY_PATH,
     TRACES_FILE_NAME,
 )
 from application_sdk.observability.utils import get_observability_dir
@@ -365,16 +363,23 @@ class AtlanObservability(Generic[T], ABC):
             logging.error(f"Error buffering log: {e}")
 
     async def _flush_records(self, records: List[Dict[str, Any]]):
-        """Flush records to parquet file and object store.
+        """Flush records to parquet file and object store using ParquetOutput abstraction.
 
         Args:
             records: List of records to flush
 
         This method:
-        - Groups records by partition
-        - Writes records to each partition
-        - Uploads to object store if enabled
+        - Groups records by partition (year/month/day)
+        - Uses ParquetOutput abstraction for efficient writing
+        - Automatically handles chunking, compression, and dual upload
+        - Provides robust error handling per partition
         - Cleans up old records if enabled
+
+        Features:
+        - Automatic chunking for large datasets
+        - Dual upload support (primary + upstream if enabled)
+        - Advanced consolidation for optimal performance
+        - Fault-tolerant processing (continues on partition errors)
         """
         if not ENABLE_OBSERVABILITY_DAPR_SINK:
             return
@@ -390,30 +395,15 @@ class AtlanObservability(Generic[T], ABC):
                     partition_records[partition_path] = []
                 partition_records[partition_path].append(record)
 
-            # Write records to each partition
+            # Write records to each partition using ParquetOutput abstraction
             for partition_path, partition_data in partition_records.items():
-                os.makedirs(partition_path, exist_ok=True)
-                # Use a consistent file name for each partition
-                parquet_path = os.path.join(partition_path, "data.parquet")
-
-                # Read existing data if any
-                existing_df = None
-                if os.path.exists(parquet_path):
-                    try:
-                        # Read the entire parquet file without excluding any columns
-                        existing_df = pd.read_parquet(parquet_path)
-                    except Exception as e:
-                        logging.error(f"Error reading existing parquet file: {e}")
-                        # If there's an error reading the existing file, we'll overwrite it
-                        existing_df = None
-
                 # Create new dataframe from current records
                 new_df = pd.DataFrame(partition_data)
 
-                # Extract partition values from path
-                partition_parts = os.path.basename(os.path.dirname(parquet_path)).split(
-                    os.sep
-                )
+                # Extract partition values from path and add to dataframe
+                partition_parts = os.path.basename(
+                    os.path.dirname(partition_path)
+                ).split(os.sep)
                 for part in partition_parts:
                     if part.startswith("year="):
                         new_df["year"] = int(part.split("=")[1])
@@ -422,38 +412,23 @@ class AtlanObservability(Generic[T], ABC):
                     elif part.startswith("day="):
                         new_df["day"] = int(part.split("=")[1])
 
-                # Merge with existing data if any
-                if existing_df is not None:
-                    df = pd.concat([existing_df, new_df], ignore_index=True)
-                else:
-                    df = new_df
-
-                # Sort by timestamp to maintain order
-                df = df.sort_values("timestamp")
+                # Use new data directly - let ParquetOutput handle consolidation and merging
+                df = new_df
 
-                # Write to parquet file
-                df.to_parquet(
-                    parquet_path,
-                    compression="snappy",
-                    index=False,
-                )
+                # Use ParquetOutput abstraction for efficient writing and uploading
+                # Set the output path for this partition
+                try:
+                    # Lazy import and instantiation of ParquetOutput
+                    from application_sdk.outputs.parquet import ParquetOutput
 
-                # Upload to object store
-                with open(parquet_path, "rb") as f:
-                    file_content = f.read()
-                    relative_path = os.path.relpath(parquet_path, TEMPORARY_PATH)
-                    metadata = {
-                        "key": relative_path,
-                        "blobName": relative_path,
-                        "fileName": relative_path,
-                    }
-                    with DaprClient() as client:
-                        client.invoke_binding(
-                            binding_name=DEPLOYMENT_OBJECT_STORE_NAME,
-                            operation=DAPR_BINDING_OPERATION_CREATE,
-                            data=file_content,
-                            binding_metadata=metadata,
-                        )
+                    parquet_output = ParquetOutput(
+                        output_path=partition_path,
+                        chunk_start=0,
+                        chunk_part=int(time()),
+                    )
+                    await parquet_output.write_dataframe(dataframe=df)
+                except Exception as e:
+                    print(f"Error writing records to partition: {str(e)}")
 
             # Clean up old records if enabled
             if self._cleanup_enabled:

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/application_sdk/observability/utils.py

@@ -5,6 +5,7 @@ from temporalio import activity, workflow
 
 from application_sdk.constants import (
     APPLICATION_NAME,
+    DEPLOYMENT_NAME,
     OBSERVABILITY_DIR,
     TEMPORARY_PATH,
 )
@@ -26,16 +27,16 @@ class WorkflowContext(BaseModel):
 
 
 def get_observability_dir() -> str:
-    """Build the observability path.
-
-    Args:
-        path: The path to build the observability path from.
+    """Build the observability path using deployment name.
 
     Returns:
-        str: The built observability path.
+        str: The built observability path using deployment name.
     """
     return os.path.join(
-        TEMPORARY_PATH, OBSERVABILITY_DIR.format(application_name=APPLICATION_NAME)
+        TEMPORARY_PATH,
+        OBSERVABILITY_DIR.format(
+            application_name=APPLICATION_NAME, deployment_name=DEPLOYMENT_NAME
+        ),
     )
 
 

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/application_sdk/outputs/__init__.py

@@ -64,6 +64,7 @@ class Output(ABC):
     output_prefix: str
     total_record_count: int
     chunk_count: int
+    chunk_part: int
     buffer_size: int
     max_file_size_bytes: int
     current_buffer_size: int

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/application_sdk/outputs/parquet.py

@@ -8,7 +8,11 @@ from temporalio import activity
 
 from application_sdk.activities.common.utils import get_object_store_prefix
 from application_sdk.common.dataframe_utils import is_empty_dataframe
-from application_sdk.constants import DAPR_MAX_GRPC_MESSAGE_LENGTH
+from application_sdk.constants import (
+    DAPR_MAX_GRPC_MESSAGE_LENGTH,
+    ENABLE_ATLAN_UPLOAD,
+    UPSTREAM_OBJECT_STORE_NAME,
+)
 from application_sdk.observability.logger_adaptor import get_logger
 from application_sdk.observability.metrics_adaptor import MetricType, get_metrics
 from application_sdk.outputs import Output
@@ -61,6 +65,7 @@ class ParquetOutput(Output):
         buffer_size: int = 5000,
         total_record_count: int = 0,
         chunk_count: int = 0,
+        chunk_part: int = 0,
         chunk_start: Optional[int] = None,
         start_marker: Optional[str] = None,
         end_marker: Optional[str] = None,
@@ -101,7 +106,7 @@ class ParquetOutput(Output):
             DAPR_MAX_GRPC_MESSAGE_LENGTH * 0.75
         )  # 75% of DAPR limit as safety buffer
         self.chunk_start = chunk_start
-        self.chunk_part = 0
+        self.chunk_part = chunk_part
         self.start_marker = start_marker
         self.end_marker = end_marker
         self.partitions = []
@@ -269,6 +274,13 @@ class ParquetOutput(Output):
                         f"No files found under prefix {get_object_store_prefix(self.output_path)}: {str(e)}"
                     )
             for path in file_paths:
+                if ENABLE_ATLAN_UPLOAD:
+                    await ObjectStore.upload_file(
+                        source=path,
+                        store_name=UPSTREAM_OBJECT_STORE_NAME,
+                        destination=get_object_store_prefix(path),
+                        retain_local_copy=True,
+                    )
                 await ObjectStore.upload_file(
                     source=path,
                     destination=get_object_store_prefix(path),

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/application_sdk/version.py

@@ -2,4 +2,4 @@
 Version information for the application_sdk package.
 """
 
-__version__ = "0.1.1rc58"
+__version__ = "0.1.1rc60"

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/docs/docs/setup/LINUX.md

@@ -57,7 +57,7 @@ Install DAPR using the following commands:
 
 ```bash
 # Install DAPR CLI
-wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash -s 1.16.0
+wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash -s 1.16.2
 
 # Initialize DAPR (slim mode)
 dapr init --runtime-version 1.16.0 --slim

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/docs/docs/setup/MAC.md

@@ -52,7 +52,7 @@ brew install temporal
 DAPR (Distributed Application Runtime) simplifies microservice development:
 
 ```bash
-curl -fsSL https://raw.githubusercontent.com/dapr/cli/master/install/install.sh | /bin/bash -s 1.16.0
+curl -fsSL https://raw.githubusercontent.com/dapr/cli/master/install/install.sh | /bin/bash -s 1.16.2
 dapr init --runtime-version 1.16.0 --slim
 ```
 

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/docs/docs/setup/WINDOWS.md

@@ -59,7 +59,7 @@ Install DAPR using PowerShell:
 Set-ExecutionPolicy RemoteSigned -scope CurrentUser
 
 # Install DAPR CLI
-$script=iwr -useb https://raw.githubusercontent.com/dapr/cli/master/install/install.ps1; $block=[ScriptBlock]::Create($script); invoke-command -ScriptBlock $block -ArgumentList 1.16.0, "$env:USERPROFILE\.dapr\bin\"
+$script=iwr -useb https://raw.githubusercontent.com/dapr/cli/master/install/install.ps1; $block=[ScriptBlock]::Create($script); invoke-command -ScriptBlock $block -ArgumentList 1.16.2, "$env:USERPROFILE\.dapr\bin\"
 
 # Add to PATH
 $env:Path += ";$env:USERPROFILE\.dapr\bin\"

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atlan-application-sdk"
-version = "0.1.1rc58"
+version = "0.1.1rc60"
 description = "Atlan Application SDK is a Python library for developing applications on the Atlan Platform"
 license = "Apache-2.0"
 authors = [

{atlan_application_sdk-0.1.1rc58 → atlan_application_sdk-0.1.1rc60}/uv.lock

@@ -140,7 +140,7 @@ wheels = [
 
 [[package]]
 name = "atlan-application-sdk"
-version = "0.1.1rc58"
+version = "0.1.1rc60"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },

atlan_application_sdk-0.1.1rc58/.github/release_notes.md

@@ -1,7 +0,0 @@
-## v0.1.1rc58 (October 22, 2025)
-
-Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc57...v0.1.1rc58
-
-### Bug Fixes
-
-- Updated lockname to not use APPLICATION_NAME (#797) (by @SanilK2108 in [151b811](https://github.com/atlanhq/application-sdk/commit/151b811))