ethyca-fides 2.67.0rc0__py2.py3-none-any.whl → 2.67.1b0__py2.py3-none-any.whl

fides/api/graph/execution.py

@@ -4,6 +4,7 @@ from fideslang.validation import FidesKey
 from loguru import logger
 
 from fides.api.graph.config import (
+    TERMINATOR_ADDRESS,
     Collection,
     CollectionAddress,
     Field,
@@ -157,3 +158,18 @@ class ExecutionNode(Contextualizable):  # pylint: disable=too-many-instance-attr
             )
 
         return out
+
+    def has_outgoing_dependencies(self) -> bool:
+        """
+        Check if this node has outgoing edges to collections other than the terminal node.
+
+        Returns:
+            bool: True if the node has dependencies on other collections, False otherwise
+        """
+
+        for edge in self.outgoing_edges:
+            # Check if the outgoing edge points to a collection other than the terminal node
+            target_collection = edge.f2.collection_address()
+            if target_collection != TERMINATOR_ADDRESS:
+                return True
+        return False

fides/api/models/privacy_request/privacy_request.py

@@ -1009,6 +1009,38 @@ class PrivacyRequest(
                 request_task_celery_ids.append(request_task_id)
         return request_task_celery_ids
 
+    def cancel_celery_tasks(self) -> None:
+        """Cancel all Celery tasks associated with this privacy request.
+
+        This includes both the main privacy request task and any sub-tasks (Request Tasks).
+        """
+        task_ids: List[str] = []
+
+        # Add the main privacy request task ID
+        parent_task_id = self.get_cached_task_id()
+        if parent_task_id:
+            task_ids.append(parent_task_id)
+
+        # Add all request task IDs
+        request_task_celery_ids = self.get_request_task_celery_task_ids()
+        task_ids.extend(request_task_celery_ids)
+
+        if not task_ids:
+            return
+
+        # Revoke all Celery tasks in batch
+        logger.info(f"Revoking {len(task_ids)} tasks for privacy request {self.id}")
+        try:
+            # Use terminate=False to allow graceful shutdown if already running
+            celery_app.control.revoke(task_ids, terminate=False)
+            logger.info(
+                f"Successfully revoked {len(task_ids)} tasks for privacy request {self.id}"
+            )
+        except Exception as exc:
+            logger.warning(
+                f"Failed to revoke {len(task_ids)} tasks for privacy request {self.id}: {exc}"
+            )
+
     def cancel_processing(self, db: Session, cancel_reason: Optional[str]) -> None:
         """Cancels a privacy request.  Currently should only cancel 'pending' tasks
 
@@ -1021,19 +1053,7 @@ class PrivacyRequest(
             self.canceled_at = datetime.utcnow()
             self.save(db)
 
-            task_ids: List[str] = (
-                self.get_request_task_celery_task_ids()
-            )  # Celery tasks for sub tasks (DSR 3.0 Request Tasks)
-            parent_task_id = (
-                self.get_cached_task_id()
-            )  # Celery task for current Privacy Request
-            if parent_task_id:
-                task_ids.append(parent_task_id)
-
-            for celery_task_id in task_ids:
-                logger.info("Revoking task {} for request {}", celery_task_id, self.id)
-                # Only revokes if execution is not already in progress.
-                celery_app.control.revoke(celery_task_id, terminate=False)
+            self.cancel_celery_tasks()
 
     def error_processing(self, db: Session) -> None:
         """Mark privacy request as errored, and note time processing was finished"""

fides/api/schemas/application_config.py

@@ -70,6 +70,7 @@ class ExecutionApplicationConfig(FidesSchema):
     subject_identity_verification_required: Optional[bool] = None
     disable_consent_identity_verification: Optional[bool] = None
     require_manual_request_approval: Optional[bool] = None
+    memory_watchdog_enabled: Optional[bool] = None
     sql_dry_run: Optional[SqlDryRunMode] = None
 
     model_config = ConfigDict(use_enum_values=True, extra="forbid")

fides/api/schemas/connection_configuration/connection_secrets_datahub.py

@@ -16,6 +16,15 @@ class PeriodicIntegrationFrequency(Enum):
     daily = "daily"
     weekly = "weekly"
     monthly = "monthly"
+    not_scheduled = "not scheduled"
+
+
+PERIODIC_INTEGRATION_FREQUENCY_TO_DAYS = {
+    PeriodicIntegrationFrequency.daily.value: 1,
+    PeriodicIntegrationFrequency.weekly.value: 7,
+    PeriodicIntegrationFrequency.monthly.value: 30,
+    PeriodicIntegrationFrequency.not_scheduled.value: -1,  # negative value to indicate that the integration is not scheduled
+}
 
 
 class DatahubSchema(ConnectionConfigSecretsSchema):
@@ -30,7 +39,7 @@ class DatahubSchema(ConnectionConfigSecretsSchema):
     )
     frequency: PeriodicIntegrationFrequency = Field(
         title="Frequency",
-        description="The frequency at which the integration should run. Available options are daily, weekly, and monthly.",
+        description="The frequency at which the integration should run. Available options are daily, weekly, monthly, and not scheduled.",
     )
     glossary_node: str = Field(
         title="Glossary node",

fides/api/service/connectors/base_connector.py

@@ -144,3 +144,17 @@ class BaseConnector(Generic[DB_CONNECTOR_TYPE], ABC):
         # Defaulting to true for now so we can keep the default behavior and
         # incrementally determine the need for primary keys across all connectors
         return True
+
+    def get_qualified_table_name(self, node: ExecutionNode) -> str:
+        """
+        Get the fully qualified table name for the given execution node.
+        """
+        raise NotImplementedError(
+            "get_qualified_table_name is not implemented by this connector"
+        )
+
+    def table_exists(self, qualified_table_name: str) -> bool:
+        """
+        Check if a table exists in the datastore.
+        """
+        raise NotImplementedError("table_exists is not implemented by this connector")

fides/api/service/connectors/bigquery_connector.py

@@ -70,6 +70,11 @@ class BigQueryConnector(SQLConnector):
             node, SQLConnector.get_namespace_meta(db, node.address.dataset)
         )
 
+    def get_qualified_table_name(self, node: ExecutionNode) -> str:
+        """Get fully qualified BigQuery table name using existing query config logic"""
+        query_config = self.query_config(node)
+        return query_config.generate_table_name()
+
     def partitioned_retrieval(
         self,
         query_config: SQLQueryConfig,

fides/api/service/connectors/query_configs/bigquery_query_config.py

@@ -93,7 +93,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
 
         return where_clauses
 
-    def _generate_table_name(self) -> str:
+    def generate_table_name(self) -> str:
         """
         Prepends the dataset ID and project ID to the base table name
         if the BigQuery namespace meta is provided.
@@ -116,7 +116,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
         Returns a query string with backtick formatting for tables that have the same names as
         BigQuery reserved words.
         """
-        return f'SELECT {field_list} FROM `{self._generate_table_name()}` WHERE ({" OR ".join(clauses)})'
+        return f'SELECT {field_list} FROM `{self.generate_table_name()}` WHERE ({" OR ".join(clauses)})'
 
     def generate_masking_stmt(
         self,
@@ -197,7 +197,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             )
             return []
 
-        table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
+        table = Table(self.generate_table_name(), MetaData(bind=client), autoload=True)
         where_clauses: List[ColumnElement] = [
             table.c[k] == v for k, v in non_empty_reference_field_keys.items()
         ]
@@ -256,7 +256,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             )
             return []
 
-        table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
+        table = Table(self.generate_table_name(), MetaData(bind=client), autoload=True)
 
         # Build individual reference clauses
         where_clauses: List[ColumnElement] = []

fides/api/service/connectors/query_configs/snowflake_query_config.py

@@ -30,7 +30,7 @@ class SnowflakeQueryConfig(SQLQueryConfig):
         """Returns field names in clauses surrounded by quotation marks as required by Snowflake syntax."""
         return f'"{string_path}" {operator} (:{operand})'
 
-    def _generate_table_name(self) -> str:
+    def generate_table_name(self) -> str:
         """
         Prepends the dataset name and schema to the base table name
         if the Snowflake namespace meta is provided.
@@ -57,7 +57,7 @@ class SnowflakeQueryConfig(SQLQueryConfig):
         clauses: List[str],
     ) -> str:
         """Returns a query string with double quotation mark formatting as required by Snowflake syntax."""
-        return f'SELECT {field_list} FROM {self._generate_table_name()} WHERE ({" OR ".join(clauses)})'
+        return f'SELECT {field_list} FROM {self.generate_table_name()} WHERE ({" OR ".join(clauses)})'
 
     def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]:
         """Adds the appropriate formatting for update statements in this datastore."""
@@ -69,4 +69,4 @@ class SnowflakeQueryConfig(SQLQueryConfig):
         where_clauses: List[str],
     ) -> str:
         """Returns a parameterized update statement in Snowflake dialect."""
-        return f'UPDATE {self._generate_table_name()} SET {", ".join(update_clauses)} WHERE {" AND ".join(where_clauses)}'
+        return f'UPDATE {self.generate_table_name()} SET {", ".join(update_clauses)} WHERE {" AND ".join(where_clauses)}'

fides/api/service/connectors/snowflake_connector.py

@@ -3,11 +3,11 @@ from typing import Any, Dict, Union
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from snowflake.sqlalchemy import URL as Snowflake_URL
+from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from fides.api.graph.execution import ExecutionNode
 from fides.api.schemas.connection_configuration import SnowflakeSchema
-from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
 from fides.api.service.connectors.query_configs.snowflake_query_config import (
     SnowflakeQueryConfig,
 )
@@ -69,10 +69,63 @@ class SnowflakeConnector(SQLConnector):
                 connect_args["private_key"] = private_key
         return connect_args
 
-    def query_config(self, node: ExecutionNode) -> SQLQueryConfig:
+    def query_config(self, node: ExecutionNode) -> SnowflakeQueryConfig:
         """Query wrapper corresponding to the input execution_node."""
 
         db: Session = Session.object_session(self.configuration)
         return SnowflakeQueryConfig(
             node, SQLConnector.get_namespace_meta(db, node.address.dataset)
         )
+
+    def get_qualified_table_name(self, node: ExecutionNode) -> str:
+        """Get fully qualified Snowflake table name using existing query config logic"""
+        query_config = self.query_config(node)
+        return query_config.generate_table_name()
+
+    def table_exists(self, qualified_table_name: str) -> bool:
+        """
+        Check if table exists in Snowflake using the proper three-part naming convention.
+
+        Snowflake supports database.schema.table naming, and the generic SQLConnector
+        table_exists method doesn't handle quoted identifiers properly.
+        """
+        try:
+            client = self.create_client()
+            with client.connect() as connection:
+                # Remove quotes and split the parts
+                clean_name = qualified_table_name.replace('"', "")
+                parts = clean_name.split(".")
+
+                if len(parts) == 1:
+                    # Simple table name - use current schema
+                    table_name = parts[0]
+                    result = connection.execute(text(f'DESC TABLE "{table_name}"'))
+                elif len(parts) == 2:
+                    # schema.table format
+                    schema_name, table_name = parts
+                    result = connection.execute(
+                        text(f'DESC TABLE "{schema_name}"."{table_name}"')
+                    )
+                elif len(parts) >= 3:
+                    # database.schema.table format
+                    database_name, schema_name, table_name = (
+                        parts[-3],
+                        parts[-2],
+                        parts[-1],
+                    )
+                    # Use the database.schema.table format
+                    result = connection.execute(
+                        text(
+                            f'DESC TABLE "{database_name}"."{schema_name}"."{table_name}"'
+                        )
+                    )
+                else:
+                    return False
+
+                # If we get here without an exception, the table exists
+                result.close()
+                return True
+
+        except Exception:
+            # Table doesn't exist or other error
+            return False

fides/api/service/connectors/sql_connector.py

@@ -6,7 +6,7 @@ import paramiko
 import sshtunnel  # type: ignore
 from aiohttp.client_exceptions import ClientResponseError
 from loguru import logger
-from sqlalchemy import Column, select
+from sqlalchemy import Column, inspect, select
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.engine import (  # type: ignore
     Connection,
@@ -22,6 +22,7 @@ from sqlalchemy.sql.elements import TextClause
 from fides.api.common_exceptions import (
     ConnectionException,
     SSHTunnelConfigNotFoundException,
+    TableNotFound,
 )
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
@@ -189,14 +190,28 @@ class SQLConnector(BaseConnector[Engine]):
 
         logger.info("Starting data retrieval for {}", node.address)
         with client.connect() as connection:
-            self.set_schema(connection)
-            if (
-                query_config.partitioning
-            ):  # only BigQuery supports partitioning, for now
-                return self.partitioned_retrieval(query_config, connection, stmt)
-
-            results = connection.execute(stmt)
-            return self.cursor_result_to_rows(results)
+            try:
+                self.set_schema(connection)
+                if (
+                    query_config.partitioning
+                ):  # only BigQuery supports partitioning, for now
+                    return self.partitioned_retrieval(query_config, connection, stmt)
+
+                results = connection.execute(stmt)
+                return self.cursor_result_to_rows(results)
+            except Exception as exc:
+                # Check if table exists using qualified table name
+                qualified_table_name = self.get_qualified_table_name(node)
+                if not self.table_exists(qualified_table_name):
+                    # Central decision point - will raise TableNotFound or ConnectionException
+                    self.handle_table_not_found(
+                        node=node,
+                        table_name=qualified_table_name,
+                        operation_context="data retrieval",
+                        original_exception=exc,
+                    )
+                # Table exists or can't check - re-raise original exception
+                raise
 
     def mask_data(
         self,
@@ -290,3 +305,86 @@ class SQLConnector(BaseConnector[Engine]):
         raise NotImplementedError(
             "Partitioned retrieval is only supported for BigQuery currently!"
         )
+
+    def get_qualified_table_name(self, node: ExecutionNode) -> str:
+        """
+        Get the fully qualified table name for this database.
+
+        Default: Returns the simple collection name
+        Override: Database-specific connectors can implement namespace resolution
+        """
+        return node.collection.name
+
+    def table_exists(self, qualified_table_name: str) -> bool:
+        """
+        Check if table exists using SQLAlchemy introspection.
+
+        This is a generic implementation that should work for most SQL databases.
+        Override: Connectors can implement database-specific table existence checking
+        """
+        try:
+            client = self.create_client()
+            with client.connect() as connection:
+                inspector = inspect(connection)
+
+                # For simple table names
+                if "." not in qualified_table_name:
+                    return inspector.has_table(qualified_table_name)
+
+                # For qualified names like schema.table or database.schema.table
+                parts = qualified_table_name.split(".")
+
+                if len(parts) == 2:
+                    # schema.table format
+                    schema_name, table_name = parts
+                    return inspector.has_table(table_name, schema=schema_name)
+
+                if len(parts) >= 3:
+                    # database.schema.table format (use schema.table)
+                    schema_name, table_name = parts[-2], parts[-1]
+                    return inspector.has_table(table_name, schema=schema_name)
+
+                # Fallback for unexpected format
+                return inspector.has_table(qualified_table_name)
+
+        except Exception as exc:
+            # Graceful fallback - if we can't check, assume table exists
+            # to preserve existing behavior for connectors that don't implement this
+            logger.error("Unable to check if table exists, assuming it does: {}", exc)
+            return True
+
+    def handle_table_not_found(
+        self,
+        node: ExecutionNode,
+        table_name: str,
+        operation_context: str,
+        original_exception: Optional[Exception] = None,
+    ) -> None:
+        """
+        Central decision point for table-not-found scenarios.
+
+        Raises TableNotFound (for collection skipping) or ConnectionException (for hard errors).
+        The raised exception will be caught by the @retry decorator in graph_task.py.
+
+        Args:
+            node: The ExecutionNode being processed
+            table_name: Name of the missing table
+            operation_context: Context like "data retrieval" or "data masking"
+            original_exception: The original exception that triggered this check
+        """
+        if node.has_outgoing_dependencies():
+            # Collection has dependencies - cannot skip safely
+            error_msg = (
+                f"Table '{table_name}' did not exist during {operation_context}. "
+                f"Cannot skip collection '{node.address}' because other collections depend on it."
+            )
+            if original_exception:
+                raise ConnectionException(error_msg) from original_exception
+            raise ConnectionException(error_msg)
+
+        # Safe to skip - raise TableNotFound for @retry decorator to catch
+        skip_msg = f"Table '{table_name}' did not exist during {operation_context}."
+        if original_exception:
+            raise TableNotFound(skip_msg) from original_exception
+
+        raise TableNotFound(skip_msg)

fides/api/service/privacy_request/request_runner_service.py

@@ -80,6 +80,7 @@ from fides.api.util.cache import get_all_masking_secret_keys
 from fides.api.util.collection_util import Row
 from fides.api.util.logger import Pii, _log_exception, _log_warning
 from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
+from fides.api.util.memory_watchdog import memory_limiter
 from fides.common.api.v1.urn_registry import (
     PRIVACY_REQUEST_TRANSFER_TO_PARENT,
     V1_URL_PREFIX,
@@ -358,8 +359,8 @@ def upload_and_save_access_results(  # pylint: disable=R0912
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
-# TODO: Add log_context back in, this is just for some temporary testing
-# @log_context(capture_args={"privacy_request_id": LoggerContextKeys.privacy_request_id})
+@memory_limiter
+@log_context(capture_args={"privacy_request_id": LoggerContextKeys.privacy_request_id})
 def run_privacy_request(
     self: DatabaseTask,
     privacy_request_id: str,