atlan-application-sdk 0.1.1rc34__tar.gz → 0.1.1rc36__tar.gz

atlan_application_sdk-0.1.1rc36/.github/release_notes.md

@@ -0,0 +1,9 @@
+## v0.1.1rc36 (September 01, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc35...v0.1.1rc36
+
+### Features
+
+- Update to pyatlan 8.0.2 to get fix for AsyncWorkflowSearchResponse async iterator (#680) (by @Ernest Hill in [a33bc4c](https://github.com/atlanhq/application-sdk/commit/a33bc4c))
+- implement Redis client with sync/async support and distributed locking (#621) (by @Mustafa in [e384ceb](https://github.com/atlanhq/application-sdk/commit/e384ceb))
+- implement abstract methods in BaseHandler (by @Mustafa in [c7c1937](https://github.com/atlanhq/application-sdk/commit/c7c1937))

atlan_application_sdk-0.1.1rc36/.github/workflows/verify-snyk-status.yml

@@ -0,0 +1,17 @@
+name: 'Verify Snyk PR Checks'
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches:
+      - 'main'
+      - 'master'
+
+jobs:
+  snyk_scan_results:
+    # This 'name' is what appears in the pull request's check list.
+    name: Snyk Scan Results
+    # Calls the reusable workflow from your central .github repository.
+    uses: atlanhq/.github/.github/workflows/reusable-verify-snyk-scans.yml@main
+    # Allows the reusable workflow to use the GITHUB_TOKEN from this repository.
+    secrets: inherit

{atlan_application_sdk-0.1.1rc34 → atlan_application_sdk-0.1.1rc36}/CHANGELOG.md

@@ -1,6 +1,26 @@
 # Changelog
 
 
+## v0.1.1rc36 (September 01, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc35...v0.1.1rc36
+
+### Features
+
+- Update to pyatlan 8.0.2 to get fix for AsyncWorkflowSearchResponse async iterator (#680) (by @Ernest Hill in [a33bc4c](https://github.com/atlanhq/application-sdk/commit/a33bc4c))
+- implement Redis client with sync/async support and distributed locking (#621) (by @Mustafa in [e384ceb](https://github.com/atlanhq/application-sdk/commit/e384ceb))
+- implement abstract methods in BaseHandler (by @Mustafa in [c7c1937](https://github.com/atlanhq/application-sdk/commit/c7c1937))
+
+## v0.1.1rc35 (August 28, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc34...v0.1.1rc35
+
+### Features
+
+- improve services abstraction (#658) (by @Nishchith Shetty in [f5a4c4e](https://github.com/atlanhq/application-sdk/commit/f5a4c4e))
+
+
+
 ## v0.1.1rc34 (August 27, 2025)
 
 Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc33...v0.1.1rc34

{atlan_application_sdk-0.1.1rc34 → atlan_application_sdk-0.1.1rc36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlan-application-sdk
-Version: 0.1.1rc34
+Version: 0.1.1rc36
 Summary: Atlan Application SDK is a Python library for developing applications on the Atlan Platform
 Project-URL: Repository, https://github.com/atlanhq/application-sdk
 Project-URL: Documentation, https://github.com/atlanhq/application-sdk/README.md
@@ -26,12 +26,14 @@ Requires-Dist: fastapi[standard]>=0.115.0
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: opentelemetry-exporter-otlp>=1.27.0
 Requires-Dist: psutil>=7.0.0
-Requires-Dist: pyatlan>=8.0.0
+Requires-Dist: pyatlan>=8.0.2
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: uvloop>=0.21.0; sys_platform != 'win32'
 Provides-Extra: daft
 Requires-Dist: daft>=0.4.12; extra == 'daft'
+Provides-Extra: distributed-lock
+Requires-Dist: redis[hiredis]>=5.2.0; extra == 'distributed-lock'
 Provides-Extra: iam-auth
 Requires-Dist: boto3>=1.38.6; extra == 'iam-auth'
 Provides-Extra: iceberg

{atlan_application_sdk-0.1.1rc34 → atlan_application_sdk-0.1.1rc36}/application_sdk/activities/__init__.py

@@ -29,7 +29,6 @@ from application_sdk.activities.common.utils import (
 from application_sdk.common.error_codes import OrchestratorError
 from application_sdk.constants import TEMPORARY_PATH
 from application_sdk.handlers import HandlerInterface
-from application_sdk.inputs.statestore import StateStoreInput, StateType
 from application_sdk.observability.logger_adaptor import get_logger
 
 logger = get_logger(__name__)
@@ -190,7 +189,9 @@ class ActivitiesInterface(ABC, Generic[ActivitiesStateType]):
 
         try:
             # This already handles the Dapr call internally
-            workflow_args = StateStoreInput.get_state(workflow_id, StateType.WORKFLOWS)
+            from application_sdk.services.statestore import StateStore, StateType
+
+            workflow_args = await StateStore.get_state(workflow_id, StateType.WORKFLOWS)
             workflow_args["output_prefix"] = workflow_args.get(
                 "output_prefix", TEMPORARY_PATH
             )

{atlan_application_sdk-0.1.1rc34 → atlan_application_sdk-0.1.1rc36}/application_sdk/activities/common/utils.py

@@ -5,13 +5,18 @@ including workflow ID retrieval, automatic heartbeating, and periodic heartbeat
 """
 
 import asyncio
+import os
 from datetime import timedelta
 from functools import wraps
 from typing import Any, Awaitable, Callable, Optional, TypeVar, cast
 
 from temporalio import activity
 
-from application_sdk.constants import APPLICATION_NAME, WORKFLOW_OUTPUT_PATH_TEMPLATE
+from application_sdk.constants import (
+    APPLICATION_NAME,
+    TEMPORARY_PATH,
+    WORKFLOW_OUTPUT_PATH_TEMPLATE,
+)
 from application_sdk.observability.logger_adaptor import get_logger
 
 logger = get_logger(__name__)
@@ -72,6 +77,21 @@ def build_output_path() -> str:
     )
 
 
+def get_object_store_prefix(path: str) -> str:
+    """Get the object store prefix for the path.
+    Args:
+        path: The path to the output directory.
+
+    Returns:
+        The object store prefix for the path.
+
+    Example:
+        >>> get_object_store_prefix("./local/tmp/artifacts/apps/appName/workflows/wf-123/run-456")
+        "artifacts/apps/appName/workflows/wf-123/run-456"
+    """
+    return os.path.relpath(path, TEMPORARY_PATH)
+
+
 def auto_heartbeater(fn: F) -> F:
     """Decorator that automatically sends heartbeats during activity execution.
 

atlan_application_sdk-0.1.1rc36/application_sdk/activities/lock_management.py

@@ -0,0 +1,110 @@
+"""Lock management activities for distributed locking.
+
+These activities handle the actual Redis lock acquisition and release,
+allowing the workflow to orchestrate locking without hitting Temporal's
+deadlock timeout.
+"""
+
+import asyncio
+import random
+from typing import Any, Dict
+
+from temporalio import activity
+
+from application_sdk.clients.redis import RedisClientAsync
+from application_sdk.common.error_codes import ActivityError
+from application_sdk.constants import APPLICATION_NAME, LOCK_RETRY_INTERVAL
+from application_sdk.observability.logger_adaptor import get_logger
+
+logger = get_logger(__name__)
+
+
+@activity.defn
+async def acquire_distributed_lock(
+    lock_name: str,
+    max_locks: int,
+    ttl_seconds: int = 100,
+    owner_id: str = "default_owner",
+) -> Dict[str, Any]:
+    """Acquire a distributed lock with retry logic.
+
+    Args:
+        lock_name (str): Name of the resource to lock.
+        max_locks (int): Maximum number of concurrent locks allowed; must be > 0.
+        ttl_seconds (int): Time-to-live for the lock in seconds.
+        owner_id (str): Unique identifier for the lock owner.
+
+    Returns:
+        dict[str, Any]: Lock information with the following keys:
+            - slot_id (int): Allocated lock slot.
+            - resource_id (str): Fully qualified lock resource ID.
+            - owner_id (str): Owner identifier.
+
+    Raises:
+        ActivityError: If lock acquisition fails due to Redis errors or invalid parameters.
+    """
+    # Input validation
+    if max_locks <= 0:
+        raise ActivityError(
+            f"{ActivityError.LOCK_ACQUISITION_ERROR}: max_locks must be greater than 0, got {max_locks}"
+        )
+
+    async with RedisClientAsync() as redis_client:
+        while True:
+            slot = random.randint(0, max_locks - 1)
+            resource_id = f"{APPLICATION_NAME}:{lock_name}:{slot}"
+
+            try:
+                # Acquire lock - connection will stay open until context exits
+                acquired = await redis_client._acquire_lock(
+                    resource_id, owner_id, ttl_seconds
+                )
+                if acquired:
+                    logger.info(
+                        f"Lock acquired for slot {slot}, resource: {resource_id}"
+                    )
+                    return {
+                        "slot_id": slot,
+                        "resource_id": resource_id,
+                        "owner_id": owner_id,
+                    }
+                # If not acquired, continue retrying (lock held by another owner)
+
+            except Exception as e:
+                # Redis connection or operation failed - propagate as activity error
+                logger.error(f"Redis error during lock acquisition: {e}")
+                raise ActivityError(
+                    f"{ActivityError.LOCK_ACQUISITION_ERROR}: Redis error during lock acquisition for {resource_id}"
+                )
+
+            # Wait before retrying
+            await asyncio.sleep(LOCK_RETRY_INTERVAL)
+
+
+@activity.defn
+async def release_distributed_lock(
+    resource_id: str, owner_id: str = "default_owner"
+) -> bool:
+    """Release a distributed lock.
+
+    Args:
+        resource_id: Full resource identifier for the lock
+        owner_id: Unique identifier for the lock owner
+
+    Returns:
+        True if lock was released successfully, False otherwise
+    """
+    async with RedisClientAsync() as redis_client:
+        try:
+            released, result = await redis_client._release_lock(resource_id, owner_id)
+            if released:
+                logger.info(
+                    f"Lock released successfully: {resource_id}, result: {result.value}"
+                )
+            return released
+
+        except Exception as e:
+            logger.error(f"Redis error during lock release for {resource_id}: {e}")
+            # Don't raise exception for lock release failures - log and return False
+            # Lock release is best-effort and shouldn't fail the workflow
+            return False

{atlan_application_sdk-0.1.1rc34 → atlan_application_sdk-0.1.1rc36}/application_sdk/activities/metadata_extraction/base.py

@@ -5,10 +5,10 @@ from temporalio import activity
 from application_sdk.activities import ActivitiesInterface, ActivitiesState
 from application_sdk.activities.common.utils import get_workflow_id
 from application_sdk.clients.base import BaseClient
-from application_sdk.common.credential_utils import get_credentials
 from application_sdk.constants import APP_TENANT_ID, APPLICATION_NAME
 from application_sdk.handlers.base import BaseHandler
 from application_sdk.observability.logger_adaptor import get_logger
+from application_sdk.services.secretstore import SecretStore
 from application_sdk.transformers import TransformerInterface
 
 logger = get_logger(__name__)
@@ -76,7 +76,9 @@ class BaseMetadataExtractionActivities(ActivitiesInterface):
                 f"Retrieving credentials for credential_guid: {workflow_args['credential_guid']}"
             )
             try:
-                credentials = await get_credentials(workflow_args["credential_guid"])
+                credentials = await SecretStore.get_credentials(
+                    workflow_args["credential_guid"]
+                )
                 logger.info(
                     f"Successfully retrieved credentials with keys: {list(credentials.keys())}"
                 )

{atlan_application_sdk-0.1.1rc34 → atlan_application_sdk-0.1.1rc36}/application_sdk/activities/metadata_extraction/sql.py

@@ -5,25 +5,24 @@ from temporalio import activity
 
 from application_sdk.activities import ActivitiesInterface, ActivitiesState
 from application_sdk.activities.common.models import ActivityStatistics
-from application_sdk.activities.common.utils import auto_heartbeater, get_workflow_id
+from application_sdk.activities.common.utils import (
+    auto_heartbeater,
+    get_object_store_prefix,
+    get_workflow_id,
+)
 from application_sdk.clients.sql import BaseSQLClient
-from application_sdk.common.credential_utils import get_credentials
 from application_sdk.common.dataframe_utils import is_empty_dataframe
 from application_sdk.common.error_codes import ActivityError
 from application_sdk.common.utils import prepare_query, read_sql_files
-from application_sdk.constants import (
-    APP_TENANT_ID,
-    APPLICATION_NAME,
-    SQL_QUERIES_PATH,
-    TEMPORARY_PATH,
-)
+from application_sdk.constants import APP_TENANT_ID, APPLICATION_NAME, SQL_QUERIES_PATH
 from application_sdk.handlers.sql import BaseSQLHandler
 from application_sdk.inputs.parquet import ParquetInput
 from application_sdk.inputs.sql_query import SQLQueryInput
 from application_sdk.observability.logger_adaptor import get_logger
-from application_sdk.outputs.atlan_storage import AtlanStorageOutput
 from application_sdk.outputs.json import JsonOutput
 from application_sdk.outputs.parquet import ParquetOutput
+from application_sdk.services.atlan_storage import AtlanStorage
+from application_sdk.services.secretstore import SecretStore
 from application_sdk.transformers import TransformerInterface
 from application_sdk.transformers.query import QueryBasedTransformer
 
@@ -144,7 +143,9 @@ class BaseSQLMetadataExtractionActivities(ActivitiesInterface):
         self._state[workflow_id].handler = handler
 
         if "credential_guid" in workflow_args:
-            credentials = await get_credentials(workflow_args["credential_guid"])
+            credentials = await SecretStore.get_credentials(
+                workflow_args["credential_guid"]
+            )
             await sql_client.load(credentials)
 
         self._state[workflow_id].sql_client = sql_client
@@ -536,11 +537,11 @@ class BaseSQLMetadataExtractionActivities(ActivitiesInterface):
 
         # Upload data from object store to Atlan storage
         # Use workflow_id/workflow_run_id as the prefix to migrate specific data
-        migration_prefix = os.path.relpath(workflow_args["output_path"], TEMPORARY_PATH)
+        migration_prefix = get_object_store_prefix(workflow_args["output_path"])
         logger.info(
             f"Starting migration from object store with prefix: {migration_prefix}"
         )
-        upload_stats = await AtlanStorageOutput.migrate_from_objectstore_to_atlan(
+        upload_stats = await AtlanStorage.migrate_from_objectstore_to_atlan(
             prefix=migration_prefix
         )
 

{atlan_application_sdk-0.1.1rc34 → atlan_application_sdk-0.1.1rc36}/application_sdk/activities/query_extraction/sql.py

@@ -7,17 +7,20 @@ from pydantic import BaseModel, Field
 from temporalio import activity
 
 from application_sdk.activities import ActivitiesInterface, ActivitiesState
-from application_sdk.activities.common.utils import auto_heartbeater, get_workflow_id
+from application_sdk.activities.common.utils import (
+    auto_heartbeater,
+    get_object_store_prefix,
+    get_workflow_id,
+)
 from application_sdk.clients.sql import BaseSQLClient
-from application_sdk.common.credential_utils import get_credentials
 from application_sdk.constants import UPSTREAM_OBJECT_STORE_NAME
 from application_sdk.handlers import HandlerInterface
 from application_sdk.handlers.sql import BaseSQLHandler
-from application_sdk.inputs.objectstore import ObjectStoreInput
 from application_sdk.inputs.sql_query import SQLQueryInput
 from application_sdk.observability.logger_adaptor import get_logger
-from application_sdk.outputs.objectstore import ObjectStoreOutput
 from application_sdk.outputs.parquet import ParquetOutput
+from application_sdk.services.objectstore import ObjectStore
+from application_sdk.services.secretstore import SecretStore
 from application_sdk.transformers import TransformerInterface
 from application_sdk.transformers.atlas import AtlasTransformer
 
@@ -129,7 +132,9 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
         workflow_id = get_workflow_id()
         sql_client = self.sql_client_class()
         if "credential_guid" in workflow_args:
-            credentials = await get_credentials(workflow_args["credential_guid"])
+            credentials = await SecretStore.get_credentials(
+                workflow_args["credential_guid"]
+            )
             await sql_client.load(credentials)
 
         handler = self.handler_class(sql_client)
@@ -412,14 +417,14 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
             f.write(last_marker)
 
         logger.info(f"Last marker: {last_marker}")
-        await ObjectStoreOutput.push_file_to_object_store(
-            workflow_args["output_prefix"],
-            marker_file_path,
-            object_store_name=UPSTREAM_OBJECT_STORE_NAME,
+        await ObjectStore.upload_file(
+            source=marker_file_path,
+            destination=get_object_store_prefix(marker_file_path),
+            store_name=UPSTREAM_OBJECT_STORE_NAME,
         )
         logger.info(f"Marker file written to {marker_file_path}")
 
-    def read_marker(self, workflow_args: Dict[str, Any]) -> Optional[int]:
+    async def read_marker(self, workflow_args: Dict[str, Any]) -> Optional[int]:
         """Read the marker from the output path.
 
         This method reads the current marker value from a marker file to determine the
@@ -442,15 +447,12 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
             marker_file_path = os.path.join(output_path, "markerfile")
             logger.info(f"Downloading marker file from {marker_file_path}")
 
-            os.makedirs(workflow_args["output_prefix"], exist_ok=True)
-
-            ObjectStoreInput.download_file_from_object_store(
-                workflow_args["output_prefix"],
-                marker_file_path,
-                object_store_name=UPSTREAM_OBJECT_STORE_NAME,
+            await ObjectStore.download_file(
+                source=get_object_store_prefix(marker_file_path),
+                destination=marker_file_path,
+                store_name=UPSTREAM_OBJECT_STORE_NAME,
             )
 
-            logger.info(f"Output prefix: {workflow_args['output_prefix']}")
             logger.info(f"Marker file downloaded to {marker_file_path}")
             if not os.path.exists(marker_file_path):
                 logger.warning(f"Marker file does not exist at {marker_file_path}")
@@ -487,7 +489,7 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
 
         miner_args = MinerArgs(**workflow_args.get("miner_args", {}))
 
-        current_marker = self.read_marker(workflow_args)
+        current_marker = await self.read_marker(workflow_args)
         if current_marker:
             miner_args.current_marker = current_marker
 
@@ -522,8 +524,10 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
         with open(metadata_file_path, "w") as f:
             f.write(json.dumps(parallel_markers))
 
-        await ObjectStoreOutput.push_file_to_object_store(
-            workflow_args["output_prefix"], metadata_file_path
+        await ObjectStore.upload_file(
+            source=metadata_file_path,
+            destination=get_object_store_prefix(metadata_file_path),
+            store_name=UPSTREAM_OBJECT_STORE_NAME,
         )
 
         try:

{atlan_application_sdk-0.1.1rc34 → atlan_application_sdk-0.1.1rc36}/application_sdk/clients/atlan_auth.py

@@ -13,8 +13,8 @@ from application_sdk.constants import (
     WORKFLOW_AUTH_ENABLED,
     WORKFLOW_AUTH_URL_KEY,
 )
-from application_sdk.inputs.secretstore import SecretStoreInput
 from application_sdk.observability.logger_adaptor import get_logger
+from application_sdk.services.secretstore import SecretStore
 
 logger = get_logger(__name__)
 
@@ -39,7 +39,7 @@ class AtlanAuthClient:
         (environment variables, AWS Secrets Manager, Azure Key Vault, etc.)
         """
         self.application_name = APPLICATION_NAME
-        self.auth_config: Dict[str, Any] = SecretStoreInput.get_deployment_secret()
+        self.auth_config: Dict[str, Any] = SecretStore.get_deployment_secret()
         self.auth_enabled: bool = WORKFLOW_AUTH_ENABLED
         self.auth_url: Optional[str] = None