atlan-application-sdk 0.1.1rc22__tar.gz → 0.1.1rc24__tar.gz

atlan_application_sdk-0.1.1rc24/.github/release_notes.md

@@ -0,0 +1,12 @@
+## v0.1.1rc24 (August 01, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc23...v0.1.1rc24
+
+### Features
+
+- Add utility function to fetch include/exclude databases from include-exclude regex (#623) (by @Abhishek Agrawal in [0c6cc62](https://github.com/atlanhq/application-sdk/commit/0c6cc62))
+
+### Bug Fixes
+
+- miner output path (#626) (by @Onkar Ravgan in [cabe7eb](https://github.com/atlanhq/application-sdk/commit/cabe7eb))
+- db name regex pattern (#630) (by @Onkar Ravgan in [bb26992](https://github.com/atlanhq/application-sdk/commit/bb26992))

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/CHANGELOG.md

@@ -1,6 +1,30 @@
 # Changelog
 
 
+## v0.1.1rc24 (August 01, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc23...v0.1.1rc24
+
+### Features
+
+- Add utility function to fetch include/exclude databases from include-exclude regex (#623) (by @Abhishek Agrawal in [0c6cc62](https://github.com/atlanhq/application-sdk/commit/0c6cc62))
+
+### Bug Fixes
+
+- miner output path (#626) (by @Onkar Ravgan in [cabe7eb](https://github.com/atlanhq/application-sdk/commit/cabe7eb))
+- db name regex pattern (#630) (by @Onkar Ravgan in [bb26992](https://github.com/atlanhq/application-sdk/commit/bb26992))
+
+## v0.1.1rc23 (July 29, 2025)
+
+Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc22...v0.1.1rc23
+
+### Bug Fixes
+
+- marker file being saved at incorrect location (#620) (by @Abhishek Agrawal in [d3de56f](https://github.com/atlanhq/application-sdk/commit/d3de56f))
+- issues with inferring workflow_run_id (#624) (by @Nishchith Shetty in [61e7a4d](https://github.com/atlanhq/application-sdk/commit/61e7a4d))
+
+
+
 ## v0.1.1rc22 (July 21, 2025)
 
 Full Changelog: https://github.com/atlanhq/application-sdk/compare/v0.1.1rc21...v0.1.1rc22

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlan-application-sdk
-Version: 0.1.1rc22
+Version: 0.1.1rc24
 Summary: Atlan Application SDK is a Python library for developing applications on the Atlan Platform
 Project-URL: Repository, https://github.com/atlanhq/application-sdk
 Project-URL: Documentation, https://github.com/atlanhq/application-sdk/README.md

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/application_sdk/activities/__init__.py

@@ -24,6 +24,7 @@ from application_sdk.activities.common.utils import (
     auto_heartbeater,
     build_output_path,
     get_workflow_id,
+    get_workflow_run_id,
 )
 from application_sdk.common.error_codes import OrchestratorError
 from application_sdk.constants import TEMPORARY_PATH
@@ -183,7 +184,7 @@ class ActivitiesInterface(ABC, Generic[ActivitiesStateType]):
         Raises:
             IOError: If configuration cannot be retrieved from state store
         """
-        workflow_id = workflow_config.get("workflow_id")
+        workflow_id = workflow_config.get("workflow_id", get_workflow_id())
         if not workflow_id:
             raise ValueError("workflow_id is required in workflow_config")
 
@@ -196,6 +197,8 @@ class ActivitiesInterface(ABC, Generic[ActivitiesStateType]):
             workflow_args["output_path"] = os.path.join(
                 workflow_args["output_prefix"], build_output_path()
             )
+            workflow_args["workflow_id"] = workflow_id
+            workflow_args["workflow_run_id"] = get_workflow_run_id()
             return workflow_args
 
         except Exception as e:

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/application_sdk/activities/query_extraction/sql.py

@@ -201,8 +201,9 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
             sql_input = SQLQueryInput(
                 engine=state.sql_client.engine,
                 query=self.get_formatted_query(self.fetch_queries_sql, workflow_args),
+                chunk_size=None,
             )
-            sql_input = await sql_input.get_daft_dataframe()
+            sql_input = await sql_input.get_dataframe()
 
             raw_output = ParquetOutput(
                 output_prefix=workflow_args["output_prefix"],
@@ -212,7 +213,7 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
                 start_marker=workflow_args["start_marker"],
                 end_marker=workflow_args["end_marker"],
             )
-            await raw_output.write_daft_dataframe(sql_input)
+            await raw_output.write_dataframe(sql_input)
 
             logger.info(
                 f"Query fetch completed, {raw_output.total_record_count} records processed",
@@ -398,7 +399,7 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
         Raises:
             Exception: If marker file writing or object store upload fails
         """
-        output_path = workflow_args["output_path"].rsplit("/", 2)[0]
+        output_path = workflow_args["output_path"].rsplit("/", 1)[0]
         logger.info(f"Writing marker file to {output_path}")
         marker_file_path = os.path.join(output_path, "markerfile")
 
@@ -436,7 +437,7 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
             Exception: If marker file reading fails (logged as warning, not re-raised)
         """
         try:
-            output_path = workflow_args["output_path"].rsplit("/", 2)[0]
+            output_path = workflow_args["output_path"].rsplit("/", 1)[0]
             marker_file_path = os.path.join(output_path, "markerfile")
             logger.info(f"Downloading marker file from {marker_file_path}")
 
@@ -485,7 +486,7 @@ class SQLQueryExtractionActivities(ActivitiesInterface):
 
         current_marker = self.read_marker(workflow_args)
         if current_marker:
-            miner_args.miner_start_time_epoch = current_marker
+            miner_args.current_marker = current_marker
 
         queries_sql_query = self.fetch_queries_sql.format(
             database_name_cleaned=miner_args.database_name_cleaned,

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/application_sdk/common/utils.py

@@ -25,35 +25,29 @@ logger = get_logger(__name__)
 F = TypeVar("F", bound=Callable[..., Awaitable[Any]])
 
 
-def extract_database_names_from_regex(normalized_regex: str) -> str:
+def extract_database_names_from_regex_common(
+    normalized_regex: str,
+    empty_default: str,
+    require_wildcard_schema: bool,
+) -> str:
     """
-    Extract database names from normalized regex patterns and return a regex string suitable for SQL queries.
-
-    This function parses regex patterns like 'dev\\.external_schema$|wide_world_importers\\.bronze_sales$'
-    or 'dev\\.*|wide_world_importers\\.*' to extract the database names, and returns a regex string
-    like '^(dev|wide_world_importers)$' for use in SQL queries.
+    Common implementation for extracting database names from regex patterns.
 
     Args:
         normalized_regex (str): The normalized regex pattern containing database.schema patterns
+        empty_default (str): Default value to return for empty/null inputs
+        require_wildcard_schema (bool): Whether to only extract database names for wildcard schemas
 
     Returns:
-        str: A regex string in the format ^(name1|name2|...)$ or '^$' if no names are found.
-
-    Examples:
-        >>> extract_database_names_from_regex('dev\\.external_schema$|wide_world_importers\\.bronze_sales$')
-        '^(dev|wide_world_importers)$'
-        >>> extract_database_names_from_regex('dev\\.*|wide_world_importers\\.*')
-        '^(dev|wide_world_importers)$'
-        >>> extract_database_names_from_regex('^$')
-        '^$'
-
-    Raises:
-        CommonError: If the input is invalid or processing fails
+        str: A regex string in the format ^(name1|name2|...)$ or default values
     """
     try:
-        if not normalized_regex or not isinstance(normalized_regex, str):
-            logger.warning("Invalid normalized_regex input: empty or non-string value")
-            return "'^$'"
+        # Handle special cases based on regex type
+        if not normalized_regex or normalized_regex == "^$":
+            return empty_default
+
+        if normalized_regex == ".*":
+            return "'.*'"
 
         database_names: Set[str] = set()
 
@@ -66,32 +60,50 @@ def extract_database_names_from_regex(normalized_regex: str) -> str:
                 if not pattern or not pattern.strip():
                     continue
 
-                # Split by \\. to get database name (first part)
+                # Split by \\. to get database name and schema part
                 # The \\. represents an escaped dot in the regex
                 parts = pattern.split("\\.")
-                if parts:
-                    db_name = parts[0].strip()
-                    if db_name and db_name not in (".*", "^$"):
-                        # Validate database name format
-                        if re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", db_name):
-                            database_names.add(db_name)
-                        else:
-                            logger.warning(f"Invalid database name format: {db_name}")
 
+                # Handle different validation requirements
+                if require_wildcard_schema:
+                    # For exclude regex, we need at least 2 parts and schema must be wildcard
+                    if len(parts) < 2:
+                        logger.warning(f"Invalid database name format: {pattern}")
+                        continue
+                    db_name = parts[0].strip()
+                    schema_part = parts[1].strip()
+                    # Only extract database name if the schema part is a wildcard (*)
+                    if not (
+                        db_name and db_name not in (".*", "^$") and schema_part == "*"
+                    ):
+                        continue
+                else:
+                    # For include regex, we just need the database name
+                    if not parts:
+                        continue
+                    db_name = parts[0].strip()
+                    if not (db_name and db_name not in (".*", "^$")):
+                        continue
+
+                # Validate database name format
+                if re.match(r"^[a-zA-Z_][a-zA-Z0-9_$-]*$", db_name):
+                    database_names.add(db_name)
+                else:
+                    logger.warning(f"Invalid database name format: {db_name}")
             except Exception as e:
                 logger.warning(f"Error processing pattern '{pattern}': {str(e)}")
                 continue
 
         if not database_names:
-            return "'^$'"
+            return empty_default
         return f"'^({'|'.join(sorted(database_names))})$'"
 
     except Exception as e:
         logger.error(
             f"Error extracting database names from regex '{normalized_regex}': {str(e)}"
         )
-        # Return a safe default that excludes everything
-        return "'^$'"
+        # Return appropriate default based on regex type
+        return empty_default
 
 
 def prepare_query(
@@ -147,12 +159,16 @@ def prepare_query(
         )
 
         # Extract database names from the normalized regex patterns
-        include_databases = extract_database_names_from_regex(normalized_include_regex)
-        exclude_databases = extract_database_names_from_regex(normalized_exclude_regex)
-
-        if include_databases == "'^$'" and exclude_databases == "'^$'":
-            include_databases = "'.*'"
-            exclude_databases = "'^$'"
+        include_databases = extract_database_names_from_regex_common(
+            normalized_regex=normalized_include_regex,
+            empty_default="'.*'",
+            require_wildcard_schema=False,
+        )
+        exclude_databases = extract_database_names_from_regex_common(
+            normalized_regex=normalized_exclude_regex,
+            empty_default="'^$'",
+            require_wildcard_schema=True,
+        )
 
         # Use sets directly for SQL query formatting
         exclude_empty_tables = workflow_args.get("metadata", {}).get(

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/application_sdk/outputs/parquet.py

@@ -188,7 +188,10 @@ class ParquetOutput(Output):
             self.total_record_count += row_count
 
             # Generate file path using path_gen function
-            file_path = f"{self.output_path}/{self.path_gen(self.chunk_start, self.chunk_count, self.start_marker, self.end_marker)}"
+            if self.start_marker and self.end_marker:
+                file_path = self.output_path
+            else:
+                file_path = f"{self.output_path}/{self.path_gen(self.chunk_start, self.chunk_count, self.start_marker, self.end_marker)}"
 
             # Write the dataframe to parquet using daft
             dataframe.write_parquet(

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/application_sdk/version.py

@@ -2,4 +2,4 @@
 Version information for the application_sdk package.
 """
 
-__version__ = "0.1.1rc22"
+__version__ = "0.1.1rc24"

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/application_sdk/workflows/__init__.py

@@ -85,9 +85,6 @@ class WorkflowInterface(ABC, Generic[ActivitiesInterfaceType]):
         logger.info("Starting workflow execution")
 
         try:
-            workflow_run_id = workflow.info().run_id
-            workflow_args["workflow_run_id"] = workflow_run_id
-
             retry_policy = RetryPolicy(maximum_attempts=2, backoff_coefficient=2)
 
             await workflow.execute_activity_method(

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/application_sdk/workflows/metadata_extraction/sql.py

@@ -201,9 +201,6 @@ class BaseSQLMetadataExtractionWorkflow(MetadataExtractionWorkflow):
                 heartbeat_timeout=self.default_heartbeat_timeout,
             )
 
-            workflow_run_id = workflow.info().run_id
-            workflow_args["workflow_run_id"] = workflow_run_id
-
             logger.info(f"Starting extraction workflow for {workflow_id}")
             retry_policy = RetryPolicy(
                 maximum_attempts=6,

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/docs/docs/concepts/workflows.md

@@ -129,7 +129,6 @@ While most customization typically happens in `Activities`, `Handlers`, or `Clie
         @workflow.run
         async def run(self, workflow_config: Dict[str, Any]) -> str: # Return type example
             # 1. Load full arguments (Essential step)
-            workflow_id = workflow_config["workflow_id"]
             workflow_args: Dict[str, Any] = await workflow.execute_activity_method(
                 self.activities_cls.get_workflow_args,
                 workflow_config,  # Pass the whole config containing workflow_id
@@ -137,8 +136,6 @@ While most customization typically happens in `Activities`, `Handlers`, or `Clie
                 start_to_close_timeout=self.default_start_to_close_timeout,
                 heartbeat_timeout=self.default_heartbeat_timeout,
             )
-            workflow.logger.info(f"Starting custom run logic for {workflow_id}")
-            workflow_args["workflow_run_id"] = workflow.info().run_id # Add run ID
 
             # Define a standard retry policy for activities
             retry_policy = RetryPolicy(maximum_attempts=3)

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atlan-application-sdk"
-version = "0.1.1rc22"
+version = "0.1.1rc24"
 description = "Atlan Application SDK is a Python library for developing applications on the Atlan Platform"
 license = "Apache-2.0"
 authors = [

{atlan_application_sdk-0.1.1rc22 → atlan_application_sdk-0.1.1rc24}/tests/unit/activities/test_activities.py

@@ -178,14 +178,16 @@ class TestActivitiesInterface:
 class TestActivitiesInterfaceActivities:
     """Test cases for ActivitiesInterface activity methods."""
 
-    @patch("application_sdk.activities.common.utils.get_workflow_run_id")
-    @patch("application_sdk.activities.common.utils.get_workflow_id")
+    @patch("application_sdk.activities.build_output_path")
+    @patch("application_sdk.activities.get_workflow_run_id")
+    @patch("application_sdk.activities.get_workflow_id")
     @patch("application_sdk.inputs.statestore.StateStoreInput.get_state")
     async def test_get_workflow_args_success(
         self,
         mock_get_state,
         mock_get_workflow_id,
         mock_get_workflow_run_id,
+        mock_build_output_path,
         mock_activities,
     ):
         """Test successful get_workflow_args activity."""
@@ -194,6 +196,7 @@ class TestActivitiesInterfaceActivities:
         mock_get_state.return_value = expected_config
         mock_get_workflow_id.return_value = "test-123"
         mock_get_workflow_run_id.return_value = "run-456"
+        mock_build_output_path.return_value = "test/output/path"
 
         result = await mock_activities.get_workflow_args(workflow_config)
 
@@ -207,20 +210,41 @@ class TestActivitiesInterfaceActivities:
 
         mock_get_state.assert_called_once_with("test-123", StateType.WORKFLOWS)
 
-    async def test_get_workflow_args_missing_workflow_id(self, mock_activities):
+    @patch("application_sdk.activities.build_output_path")
+    @patch("application_sdk.activities.get_workflow_run_id")
+    @patch("application_sdk.activities.get_workflow_id")
+    async def test_get_workflow_args_missing_workflow_id(
+        self,
+        mock_get_workflow_id,
+        mock_get_workflow_run_id,
+        mock_build_output_path,
+        mock_activities,
+    ):
         """Test get_workflow_args with missing workflow_id."""
         workflow_config = {"other_param": "value"}
+        mock_get_workflow_id.side_effect = Exception("Failed to get workflow id")
 
-        with pytest.raises(ValueError, match="workflow_id is required"):
+        with pytest.raises(Exception, match="Failed to get workflow id"):
             await mock_activities.get_workflow_args(workflow_config)
 
+    @patch("application_sdk.activities.build_output_path")
+    @patch("application_sdk.activities.get_workflow_run_id")
+    @patch("application_sdk.activities.get_workflow_id")
     @patch("application_sdk.inputs.statestore.StateStoreInput.get_state")
     async def test_get_workflow_args_extraction_error(
-        self, mock_get_state, mock_activities
+        self,
+        mock_get_state,
+        mock_get_workflow_id,
+        mock_get_workflow_run_id,
+        mock_build_output_path,
+        mock_activities,
     ):
         """Test get_workflow_args when extraction fails."""
         workflow_config = {"workflow_id": "test-123"}
         mock_get_state.side_effect = Exception("Extraction failed")
+        mock_get_workflow_id.return_value = "test-123"
+        mock_get_workflow_run_id.return_value = "run-456"
+        mock_build_output_path.return_value = "test/output/path"
 
         with pytest.raises(Exception, match="Extraction failed"):
             await mock_activities.get_workflow_args(workflow_config)