atlan-application-sdk 0.1.1rc41__tar.gz → 0.1.1rc42__tar.gz

{atlan_application_sdk-0.1.1rc41 → atlan_application_sdk-0.1.1rc42}/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -1,13 +1,13 @@
-name: "🐛 Bug Report"
+name: "Bug Report"
 description: File a bug report for the Application SDK.
-title: "🐛 [BUG] - <title>"
+title: "<title>"
 labels: ["bug"]
 
 body:
   - type: textarea
     id: description
     attributes:
-      label: "🐞 Description"
+      label: "Description"
       description: Clear and concise description of the issue.
       placeholder: What went wrong?
     validations:
@@ -16,7 +16,7 @@ body:
   - type: input
     id: reprod-url
     attributes:
-      label: "🔗 Reproduction Repo/Script (if any)"
+      label: "Reproduction Repo/Script (if any)"
       description: Link to a minimal repo or script that reproduces the issue.
       placeholder: https://github.com/your/repo
     validations:
@@ -25,7 +25,7 @@ body:
   - type: textarea
     id: reprod
     attributes:
-      label: "📋 Reproduction Steps"
+      label: "Reproduction Steps"
       description: Steps to reproduce the bug.
       value: |
         1. Import SDK
@@ -38,7 +38,7 @@ body:
   - type: textarea
     id: logs
     attributes:
-      label: "🪵 Logs / Tracebacks"
+      label: "Logs / Tracebacks"
       description: Paste any relevant logs or error messages here (no need for backticks).
       render: bash
     validations:
@@ -47,7 +47,7 @@ body:
   - type: textarea
     id: expected
     attributes:
-      label: "✅ Expected vs 🧨 Actual"
+      label: "Expected vs Actual"
       description: What was the expected behavior and what actually happened?
     validations:
       required: true
@@ -55,7 +55,7 @@ body:
   - type: dropdown
     id: environment
     attributes:
-      label: "💻 Environment"
+      label: "Environment"
       description: What environment are you running this in?
       options:
         - MacOS
@@ -68,7 +68,7 @@ body:
   - type: input
     id: sdk-version
     attributes:
-      label: "📦 SDK Version"
+      label: "SDK Version"
       description: What version of the SDK are you using?
       placeholder: e.g. 0.1.0
     validations:

{atlan_application_sdk-0.1.1rc41 → atlan_application_sdk-0.1.1rc42}/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -1,13 +1,13 @@
-name: "💡 Feature Request"
+name: "Feature Request"
 description: Suggest a new feature or enhancement for the Application SDK.
-title: "💡 [REQUEST] - <title>"
+title: "<title>"
 labels: ["enhancement"]
 
 body:
   - type: textarea
     id: summary
     attributes:
-      label: "📝 Summary"
+      label: "Summary"
       description: Briefly describe the proposed feature.
       placeholder: What would you like to see added or improved?
     validations:
@@ -16,7 +16,7 @@ body:
   - type: textarea
     id: example
     attributes:
-      label: "💡 Basic Example"
+      label: "Basic Example"
       description: Provide a simple code snippet or example of how this feature would work.
       placeholder: Share a code example or expected usage.
     validations:
@@ -25,7 +25,7 @@ body:
   - type: textarea
     id: motivation
     attributes:
-      label: "🎯 Motivation"
+      label: "Motivation"
       description: What problem does this feature solve or what use case does it support?
       placeholder: Why is this feature needed?
     validations:
@@ -34,7 +34,7 @@ body:
   - type: textarea
     id: drawbacks
     attributes:
-      label: "⚠️ Drawbacks"
+      label: "Drawbacks"
       description: Any potential downsides, risks, or performance impacts?
       placeholder: Be honest and neutral about what this might affect.
     validations:
@@ -43,7 +43,7 @@ body:
   - type: textarea
     id: unresolved
     attributes:
-      label: "❓ Unresolved Questions"
+      label: "Unresolved Questions"
       description: Are there any open questions or decisions to be made?
       placeholder: Anything unclear about how to approach this?
     validations:
@@ -52,7 +52,7 @@ body:
   - type: textarea
     id: references
     attributes:
-      label: "🔗 Reference Issues / PRs"
+      label: "Reference Issues / PRs"
       description: Link to any related issues or pull requests.
       placeholder: e.g. #123, #456
     validations:

atlan_application_sdk-0.1.1rc42/.github/release_notes.md

@@ -0,0 +1,7 @@
+## v0.1.1rc42 (September 15, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc41...v0.1.1rc42
+
+### Features
+
+- enhance file handling in Input classes (#697) (by @Mustafa in [a792fa6](https://github.com/atlanhq/application-sdk/commit/a792fa6))

{atlan_application_sdk-0.1.1rc41 → atlan_application_sdk-0.1.1rc42}/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+
+## v0.1.1rc42 (September 15, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc41...v0.1.1rc42
+
+### Features
+
+- enhance file handling in Input classes (#697) (by @Mustafa in [a792fa6](https://github.com/atlanhq/application-sdk/commit/a792fa6))
+
 ## v0.1.1rc41 (September 11, 2025)
 
 Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc40...v0.1.1rc41

{atlan_application_sdk-0.1.1rc41 → atlan_application_sdk-0.1.1rc42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlan-application-sdk
-Version: 0.1.1rc41
+Version: 0.1.1rc42
 Summary: Atlan Application SDK is a Python library for developing applications on the Atlan Platform
 Project-URL: Repository, https://github.com/atlanhq/application-sdk
 Project-URL: Documentation, https://github.com/atlanhq/application-sdk/README.md

{atlan_application_sdk-0.1.1rc41 → atlan_application_sdk-0.1.1rc42}/application_sdk/activities/common/utils.py

@@ -5,10 +5,11 @@ including workflow ID retrieval, automatic heartbeating, and periodic heartbeat
 """
 
 import asyncio
+import glob
 import os
 from datetime import timedelta
 from functools import wraps
-from typing import Any, Awaitable, Callable, Optional, TypeVar, cast
+from typing import Any, Awaitable, Callable, List, Optional, TypeVar, cast
 
 from temporalio import activity
 
@@ -79,17 +80,47 @@ def build_output_path() -> str:
 
 def get_object_store_prefix(path: str) -> str:
     """Get the object store prefix for the path.
+
+    This function handles two types of paths:
+    1. Paths under TEMPORARY_PATH - converts them to relative object store paths
+    2. User-provided paths - returns them as-is (already relative object store paths)
+
     Args:
-        path: The path to the output directory.
+        path: The path to convert to object store prefix.
 
     Returns:
         The object store prefix for the path.
 
-    Example:
+    Examples:
+        >>> # Temporary path case
         >>> get_object_store_prefix("./local/tmp/artifacts/apps/appName/workflows/wf-123/run-456")
         "artifacts/apps/appName/workflows/wf-123/run-456"
+
+        >>> # User-provided path case
+        >>> get_object_store_prefix("datasets/sales/2024/")
+        "datasets/sales/2024"
     """
-    return os.path.relpath(path, TEMPORARY_PATH)
+    # Normalize paths for comparison
+    abs_path = os.path.abspath(path)
+    abs_temp_path = os.path.abspath(TEMPORARY_PATH)
+
+    # Check if path is under TEMPORARY_PATH
+    try:
+        # Use os.path.commonpath to properly check if path is under temp directory
+        # This prevents false positives like '/tmp/local123' matching '/tmp/local'
+        common_path = os.path.commonpath([abs_path, abs_temp_path])
+        if common_path == abs_temp_path:
+            # Path is under temp directory, convert to relative object store path
+            relative_path = os.path.relpath(abs_path, abs_temp_path)
+            # Normalize path separators to forward slashes for object store
+            return relative_path.replace(os.path.sep, "/")
+        else:
+            # Path is already a relative object store path, return as-is
+            return path.strip("/")
+    except ValueError:
+        # os.path.commonpath or os.path.relpath can raise ValueError on Windows with different drives
+        # In this case, treat as user-provided path, return as-is
+        return path.strip("/")
 
 
 def auto_heartbeater(fn: F) -> F:
@@ -199,3 +230,46 @@ async def send_periodic_heartbeat(delay: float, *details: Any) -> None:
     while True:
         await asyncio.sleep(delay)
         activity.heartbeat(*details)
+
+
+def find_local_files_by_extension(
+    path: str,
+    extension: str,
+    file_names: Optional[List[str]] = None,
+) -> List[str]:
+    """Find local files at the specified local path, optionally filtering by file names.
+
+    Args:
+        path (str): Local path to search in (file or directory)
+        extension (str): File extension to filter by (e.g., '.parquet', '.json')
+        file_names (Optional[List[str]]): List of file names (basenames) to filter by, paths are not supported
+
+    Returns:
+        List[str]: List of matching file paths
+
+    Example:
+        >>> find_local_files_by_extension("/data", ".parquet", ["file1.parquet", "file2.parquet"])
+        ['file1.parquet', 'file2.parquet']
+
+        >>> find_local_files_by_extension("/data/single.json", ".json")
+        ['single.json']
+    """
+    if os.path.isfile(path) and path.endswith(extension):
+        # Single file - return it directly
+        return [path]
+
+    elif os.path.isdir(path):
+        # Directory - find all files in directory
+        all_files = glob.glob(
+            os.path.join(path, "**", f"*{extension}"),
+            recursive=True,
+        )
+
+        # Filter by file names if specified
+        if file_names:
+            file_names_set = set(file_names)  # Convert to set for O(1) lookup
+            return [f for f in all_files if os.path.basename(f) in file_names_set]
+        else:
+            return all_files
+
+    return []

{atlan_application_sdk-0.1.1rc41 → atlan_application_sdk-0.1.1rc42}/application_sdk/activities/metadata_extraction/sql.py

@@ -860,9 +860,7 @@ class BaseSQLMetadataExtractionActivities(ActivitiesInterface):
 
         raw_input = ParquetInput(
             path=os.path.join(output_path, "raw"),
-            input_prefix=output_prefix,
             file_names=workflow_args.get("file_names"),
-            chunk_size=None,
         )
         raw_input = raw_input.get_batched_daft_dataframe()
         transformed_output = JsonOutput(

atlan_application_sdk-0.1.1rc42/application_sdk/inputs/__init__.py

@@ -0,0 +1,168 @@
+import os
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, AsyncIterator, Iterator, List, Union
+
+from application_sdk.activities.common.utils import (
+    find_local_files_by_extension,
+    get_object_store_prefix,
+)
+from application_sdk.common.error_codes import IOError
+from application_sdk.constants import TEMPORARY_PATH
+from application_sdk.observability.logger_adaptor import get_logger
+from application_sdk.services.objectstore import ObjectStore
+
+logger = get_logger(__name__)
+
+if TYPE_CHECKING:
+    import daft
+    import pandas as pd
+
+
+class Input(ABC):
+    """
+    Abstract base class for input data sources.
+    """
+
+    async def download_files(self) -> List[str]:
+        """Download files from object store if not available locally.
+
+        Flow:
+        1. Check if files exist locally at self.path
+        2. If not, try to download from object store
+        3. Filter by self.file_names if provided
+        4. Return list of file paths for logging purposes
+
+        Returns:
+            List[str]: List of file paths
+
+        Raises:
+            AttributeError: When the input class doesn't support file operations or _extension
+            IOError: When no files found locally or in object store
+        """
+        # Step 1: Check if files exist locally
+        local_files = find_local_files_by_extension(
+            self.path, self._EXTENSION, self.file_names
+        )
+        if local_files:
+            logger.info(
+                f"Found {len(local_files)} {self._EXTENSION} files locally at: {self.path}"
+            )
+            return local_files
+
+        # Step 2: Try to download from object store
+        logger.info(
+            f"No local {self._EXTENSION} files found at {self.path}, checking object store..."
+        )
+
+        try:
+            # Determine what to download based on path type and filters
+            downloaded_paths = []
+
+            if self.path.endswith(self._EXTENSION):
+                # Single file case (file_names validation already ensures this is valid)
+                source_path = get_object_store_prefix(self.path)
+                destination_path = os.path.join(TEMPORARY_PATH, source_path)
+                await ObjectStore.download_file(
+                    source=source_path,
+                    destination=destination_path,
+                )
+                downloaded_paths.append(destination_path)
+
+            elif self.file_names:
+                # Directory with specific files - download each file individually
+                for file_name in self.file_names:
+                    file_path = os.path.join(self.path, file_name)
+                    source_path = get_object_store_prefix(file_path)
+                    destination_path = os.path.join(TEMPORARY_PATH, source_path)
+                    await ObjectStore.download_file(
+                        source=source_path,
+                        destination=destination_path,
+                    )
+                    downloaded_paths.append(destination_path)
+            else:
+                # Download entire directory
+                source_path = get_object_store_prefix(self.path)
+                destination_path = os.path.join(TEMPORARY_PATH, source_path)
+                await ObjectStore.download_prefix(
+                    source=source_path,
+                    destination=destination_path,
+                )
+                # Find the actual files in the downloaded directory
+                found_files = find_local_files_by_extension(
+                    destination_path, self._EXTENSION, getattr(self, "file_names", None)
+                )
+                downloaded_paths.extend(found_files)
+
+            # Check results
+            if downloaded_paths:
+                logger.info(
+                    f"Successfully downloaded {len(downloaded_paths)} {self._EXTENSION} files from object store"
+                )
+                return downloaded_paths
+            else:
+                raise IOError(
+                    f"{IOError.OBJECT_STORE_READ_ERROR}: Downloaded from object store but no {self._EXTENSION} files found"
+                )
+
+        except Exception as e:
+            logger.error(f"Failed to download from object store: {str(e)}")
+            raise IOError(
+                f"{IOError.OBJECT_STORE_DOWNLOAD_ERROR}: No {self._EXTENSION} files found locally at '{self.path}' and failed to download from object store. "
+                f"Error: {str(e)}"
+            )
+
+    @abstractmethod
+    async def get_batched_dataframe(
+        self,
+    ) -> Union[Iterator["pd.DataFrame"], AsyncIterator["pd.DataFrame"]]:
+        """
+        Get an iterator of batched pandas DataFrames.
+
+        Returns:
+            Iterator["pd.DataFrame"]: An iterator of batched pandas DataFrames.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def get_dataframe(self) -> "pd.DataFrame":
+        """
+        Get a single pandas DataFrame.
+
+        Returns:
+            "pd.DataFrame": A pandas DataFrame.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def get_batched_daft_dataframe(
+        self,
+    ) -> Union[Iterator["daft.DataFrame"], AsyncIterator["daft.DataFrame"]]:  # noqa: F821
+        """
+        Get an iterator of batched daft DataFrames.
+
+        Returns:
+            Iterator[daft.DataFrame]: An iterator of batched daft DataFrames.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def get_daft_dataframe(self) -> "daft.DataFrame":  # noqa: F821
+        """
+        Get a single daft DataFrame.
+
+        Returns:
+            daft.DataFrame: A daft DataFrame.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+        raise NotImplementedError

atlan_application_sdk-0.1.1rc42/application_sdk/inputs/json.py

@@ -0,0 +1,136 @@
+from typing import TYPE_CHECKING, AsyncIterator, Iterator, List, Optional, Union
+
+from application_sdk.inputs import Input
+from application_sdk.observability.logger_adaptor import get_logger
+
+if TYPE_CHECKING:
+    import daft
+    import pandas as pd
+
+logger = get_logger(__name__)
+
+
+class JsonInput(Input):
+    """
+    JSON Input class to read data from JSON files using daft and pandas.
+    Supports reading both single files and directories containing multiple JSON files.
+    """
+
+    _EXTENSION = ".json"
+
+    def __init__(
+        self,
+        path: str,
+        file_names: Optional[List[str]] = None,
+        chunk_size: int = 100000,
+    ):
+        """Initialize the JsonInput class.
+
+        Args:
+            path (str): Path to JSON file or directory containing JSON files.
+                It accepts both types of paths:
+                local path or object store path
+                Wildcards are not supported.
+            file_names (Optional[List[str]]): List of specific file names to read. Defaults to None.
+            chunk_size (int): Number of rows per batch. Defaults to 100000.
+
+        Raises:
+            ValueError: When path is not provided or when single file path is combined with file_names
+        """
+
+        # Validate that single file path and file_names are not both specified
+        if path.endswith(self._EXTENSION) and file_names:
+            raise ValueError(
+                f"Cannot specify both a single file path ('{path}') and file_names filter. "
+                f"Either provide a directory path with file_names, or specify the exact file path without file_names."
+            )
+
+        self.path = path
+        self.chunk_size = chunk_size
+        self.file_names = file_names
+
+    async def get_batched_dataframe(
+        self,
+    ) -> Union[AsyncIterator["pd.DataFrame"], Iterator["pd.DataFrame"]]:
+        """
+        Method to read the data from the json files in the path
+        and return as a batched pandas dataframe
+        """
+        try:
+            import pandas as pd
+
+            # Ensure files are available (local or downloaded)
+            json_files = await self.download_files()
+            logger.info(f"Reading {len(json_files)} JSON files in batches")
+
+            for json_file in json_files:
+                json_reader_obj = pd.read_json(
+                    json_file,
+                    chunksize=self.chunk_size,
+                    lines=True,
+                )
+                for chunk in json_reader_obj:
+                    yield chunk
+        except Exception as e:
+            logger.error(f"Error reading batched data from JSON: {str(e)}")
+            raise
+
+    async def get_dataframe(self) -> "pd.DataFrame":
+        """
+        Method to read the data from the json files in the path
+        and return as a single combined pandas dataframe
+        """
+        try:
+            import pandas as pd
+
+            # Ensure files are available (local or downloaded)
+            json_files = await self.download_files()
+            logger.info(f"Reading {len(json_files)} JSON files as pandas dataframe")
+
+            return pd.concat(
+                (pd.read_json(json_file, lines=True) for json_file in json_files),
+                ignore_index=True,
+            )
+
+        except Exception as e:
+            logger.error(f"Error reading data from JSON: {str(e)}")
+            raise
+
+    async def get_batched_daft_dataframe(
+        self,
+    ) -> Union[AsyncIterator["daft.DataFrame"], Iterator["daft.DataFrame"]]:  # noqa: F821
+        """
+        Method to read the data from the json files in the path
+        and return as a batched daft dataframe
+        """
+        try:
+            import daft
+
+            # Ensure files are available (local or downloaded)
+            json_files = await self.download_files()
+            logger.info(f"Reading {len(json_files)} JSON files as daft batches")
+
+            # Yield each discovered file as separate batch with chunking
+            for json_file in json_files:
+                yield daft.read_json(json_file, _chunk_size=self.chunk_size)
+        except Exception as e:
+            logger.error(f"Error reading batched data from JSON using daft: {str(e)}")
+            raise
+
+    async def get_daft_dataframe(self) -> "daft.DataFrame":  # noqa: F821
+        """
+        Method to read the data from the json files in the path
+        and return as a single combined daft dataframe
+        """
+        try:
+            import daft
+
+            # Ensure files are available (local or downloaded)
+            json_files = await self.download_files()
+            logger.info(f"Reading {len(json_files)} JSON files with daft")
+
+            # Use the discovered/downloaded files directly
+            return daft.read_json(json_files)
+        except Exception as e:
+            logger.error(f"Error reading data from JSON using daft: {str(e)}")
+            raise