ethyca-fides 2.66.3b0__py2.py3-none-any.whl → 2.67.0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-07-29T12:14:04-0400",
+ "date": "2025-08-01T17:34:13-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "9e0a314647173d6c1527fa8e2567d8058075bd15",
- "version": "2.66.3b0"
+ "full-revisionid": "fd2fea83c304d1b8f1d9fa0c01172ccead8d45a3",
+ "version": "2.67.0"
 }
 '''  # END VERSION_JSON
 

fides/api/common_exceptions.py

@@ -151,6 +151,10 @@ class NotSupportedForCollection(BaseException):
     """The given action is not supported for this type of collection"""
 
 
+class TableNotFound(BaseException):
+    """Table or collection does not exist in the database/system"""
+
+
 class PrivacyRequestExit(BaseException):
     """Privacy request exiting processing waiting on subtasks to complete"""
 

fides/api/graph/execution.py

@@ -4,6 +4,7 @@ from fideslang.validation import FidesKey
 from loguru import logger
 
 from fides.api.graph.config import (
+    TERMINATOR_ADDRESS,
     Collection,
     CollectionAddress,
     Field,
@@ -157,3 +158,18 @@ class ExecutionNode(Contextualizable):  # pylint: disable=too-many-instance-attr
             )
 
         return out
+
+    def has_outgoing_dependencies(self) -> bool:
+        """
+        Check if this node has outgoing edges to collections other than the terminal node.
+
+        Returns:
+            bool: True if the node has dependencies on other collections, False otherwise
+        """
+
+        for edge in self.outgoing_edges:
+            # Check if the outgoing edge points to a collection other than the terminal node
+            target_collection = edge.f2.collection_address()
+            if target_collection != TERMINATOR_ADDRESS:
+                return True
+        return False

fides/api/graph/graph.py

@@ -180,8 +180,19 @@ class DatasetGraph:
         this is a destructive operation on the input datasets, as it
         will alter references within them"""
 
-        # build nodes
-        nodes = [Node(dr, ds) for dr in datasets for ds in dr.collections]
+        # build nodes, filtering out collections with skip_processing=True
+        nodes = []
+        for dr in datasets:
+            for ds in dr.collections:
+                if ds.skip_processing:
+                    logger.debug(
+                        "Skipping collection {} on dataset {} marked with skip_processing",
+                        ds.name,
+                        dr.name,
+                    )
+                    continue
+                nodes.append(Node(dr, ds))
+
         self.nodes: dict[CollectionAddress, Node] = {
             node.address: node for node in nodes
         }

fides/api/graph/traversal.py

@@ -1,6 +1,8 @@
+# pylint: disable=too-many-instance-attributes, too-many-branches, too-many-statements
 from __future__ import annotations
 
 import json
+from collections import defaultdict
 from itertools import chain
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, cast
 
@@ -99,20 +101,51 @@ class BaseTraversal:
         self.traversal_node_dict = {k: TraversalNode(v) for k, v in graph.nodes.items()}
         self.edges: Set[Edge] = graph.edges.copy()
         self.root_node = artificial_traversal_node(ROOT_COLLECTION_ADDRESS)
+
+        # Pre-index edges by node address for O(1) lookup
+        self.edges_by_node: Dict[CollectionAddress, List[Edge]] = defaultdict(list)
+        for edge in self.edges:
+            self.edges_by_node[edge.f1.collection_address()].append(edge)
+            self.edges_by_node[edge.f2.collection_address()].append(edge)
+
+        # Pre-compute string versions of node dependencies
+        # This avoids expensive hash operations during traversal
+        self.node_after_str: Dict[str, Set[str]] = {}
+        self.dataset_after_str: Dict[str, Set[str]] = {}
+
+        for addr, traversal_node in self.traversal_node_dict.items():
+            # Collection-level after dependencies
+            self.node_after_str[addr.value] = {
+                dep.value for dep in traversal_node.node.collection.after
+            }
+            # Dataset-level after dependencies (need to find all collections in those datasets)
+            dataset_deps = set()
+            for dataset_name in traversal_node.node.dataset.after:
+                for other_addr in self.traversal_node_dict.keys():
+                    if other_addr.dataset == dataset_name:
+                        dataset_deps.add(other_addr.value)
+            self.dataset_after_str[addr.value] = dataset_deps
+
+        # Add root node to the pre-computed dependencies (it has no dependencies)
+        self.node_after_str[ROOT_COLLECTION_ADDRESS.value] = set()
+        self.dataset_after_str[ROOT_COLLECTION_ADDRESS.value] = set()
+
         for (
             start_field_address,
             seed_key,
         ) in self.extract_seed_field_addresses().items():
-            self.edges.add(
-                Edge(
-                    FieldAddress(
-                        ROOT_COLLECTION_ADDRESS.dataset,
-                        ROOT_COLLECTION_ADDRESS.collection,
-                        seed_key,
-                    ),
-                    start_field_address,
-                )
+            edge = Edge(
+                FieldAddress(
+                    ROOT_COLLECTION_ADDRESS.dataset,
+                    ROOT_COLLECTION_ADDRESS.collection,
+                    seed_key,
+                ),
+                start_field_address,
             )
+            self.edges.add(edge)
+            # Add to edge index
+            self.edges_by_node[ROOT_COLLECTION_ADDRESS].append(edge)
+            self.edges_by_node[start_field_address.collection_address()].append(edge)
 
         # Ensure manual_task collections execute right after ROOT
         from fides.api.task.manual.manual_task_address import ManualTaskAddress
@@ -120,16 +153,18 @@ class BaseTraversal:
         for addr in self.traversal_node_dict.keys():
             if ManualTaskAddress.is_manual_task_address(addr):
                 # Add a simple synthetic edge ROOT.id -> manual_data.id
-                self.edges.add(
-                    Edge(
-                        FieldAddress(
-                            ROOT_COLLECTION_ADDRESS.dataset,
-                            ROOT_COLLECTION_ADDRESS.collection,
-                            "id",
-                        ),
-                        addr.field_address(FieldPath("id")),
-                    )
+                edge = Edge(
+                    FieldAddress(
+                        ROOT_COLLECTION_ADDRESS.dataset,
+                        ROOT_COLLECTION_ADDRESS.collection,
+                        "id",
+                    ),
+                    addr.field_address(FieldPath("id")),
                 )
+                self.edges.add(edge)
+                # Add to edge index
+                self.edges_by_node[ROOT_COLLECTION_ADDRESS].append(edge)
+                self.edges_by_node[addr].append(edge)
 
         self._verify_traversal()
 
@@ -204,28 +239,52 @@ class BaseTraversal:
             logger.info(
                 "Starting traversal",
             )
-        remaining_node_keys: Set[CollectionAddress] = set(
-            self.traversal_node_dict.keys()
-        )
+
+        # Use string sets instead of CollectionAddress sets
+        # This avoids expensive hash operations
+        remaining_node_keys_str: Set[str] = {
+            addr.value for addr in self.traversal_node_dict.keys()
+        }
         finished_nodes: dict[CollectionAddress, TraversalNode] = {}
         running_node_queue: MatchingQueue[TraversalNode] = MatchingQueue(self.root_node)
 
-        remaining_edges: Set[Edge] = self.edges.copy()
+        # Instead of copying entire edge set, use a more efficient approach
+        # We'll simulate Edge.delete_edges behavior without the expensive set operations
+        deleted_edges_tracker: Dict[Edge, bool] = {}
+
         while not running_node_queue.is_empty():
             # this is to support the "run traversal_node A AFTER traversal_node B functionality:"
             n = running_node_queue.pop_first_match(
-                lambda x: x.can_run_given(remaining_node_keys)
+                lambda x: x.can_run_given_str(
+                    remaining_node_keys_str, self.node_after_str, self.dataset_after_str
+                )
             )
 
             if n:
                 node_run_fn(n, environment)
                 # delete all edges between the traversal_node that's just run and any completed nodes
                 for finished_node_address, finished_node in finished_nodes.items():
-                    completed_edges: Set[Edge] = Edge.delete_edges(
-                        remaining_edges,
-                        finished_node_address,
-                        cast(TraversalNode, n).address,  # type: ignore[redundant-cast]
+                    # Find edges to delete manually instead of using Edge.delete_edges
+                    completed_edges: Set[Edge] = set()
+
+                    # Only check edges connected to the relevant nodes
+                    relevant_edges = set()
+                    relevant_edges.update(
+                        self.edges_by_node.get(finished_node_address, [])
                     )
+                    relevant_edges.update(self.edges_by_node.get(n.address, []))
+
+                    for edge in relevant_edges:
+                        # Skip if already deleted
+                        if deleted_edges_tracker.get(edge, False):
+                            continue
+
+                        # Check if this edge spans between the two nodes (bidirectional check)
+                        if edge.spans(
+                            finished_node_address, cast(TraversalNode, n).address  # type: ignore[redundant-cast]
+                        ):
+                            completed_edges.add(edge)
+                            deleted_edges_tracker[edge] = True
 
                     def edge_ends_with_collection(_edge: Edge) -> bool:
                         # append edges that end in this traversal_node
@@ -236,14 +295,16 @@ class BaseTraversal:
                     for edge in filter(edge_ends_with_collection, completed_edges):
                         # note, this will not work for self-reference
                         finished_node.add_child(n, edge)
+
                 # next edges = take all edges including n that are _not_ in edges_from_completed_nodes
                 # in the form (field_address_this, field_address_foreign)
 
+                # Use pre-indexed edges instead of iterating through all edges
                 edges_to_children = pydash.collections.filter_(
                     [
                         e.split_by_address(cast(TraversalNode, n).address)  # type: ignore[redundant-cast]
-                        for e in remaining_edges
-                        if e.contains(n.address)
+                        for e in self.edges_by_node[n.address]
+                        if not deleted_edges_tracker.get(e, False)
                     ]
                 )
                 if not edges_to_children:
@@ -259,7 +320,7 @@ class BaseTraversal:
                         self.traversal_node_dict[nxt_address]
                     )
                 finished_nodes[n.address] = n
-                remaining_node_keys.difference_update({n.address})
+                remaining_node_keys_str.discard(n.address.value)  # Use string value
             else:
                 # traversal traversal_node dict diff finished nodes
                 logger.error(
@@ -271,12 +332,17 @@ class BaseTraversal:
                     [{', '.join([str(tn.address) for tn in running_node_queue.data])}]""",
                 )
 
+        # Convert back to CollectionAddress set for filtering
         remaining_node_keys = {
             key
-            for key in remaining_node_keys
-            if not self.should_exclude_node(self.traversal_node_dict[key])
+            for key in self.traversal_node_dict.keys()
+            if key.value in remaining_node_keys_str
+            and not self.should_exclude_node(self.traversal_node_dict[key])
         }
 
+        # Update string set after filtering
+        remaining_node_keys_str = {key.value for key in remaining_node_keys}
+
         # error if there are nodes that have not been visited
         if remaining_node_keys:
             logger.error(
@@ -289,12 +355,16 @@ class BaseTraversal:
             )
 
         # filter out remaining_edges if the nodes they link are allowed to remain unreachable
-        remaining_edges = {
-            edge
-            for edge in remaining_edges
-            if edge.f1.collection_address() in remaining_node_keys
-            and edge.f2.collection_address() in remaining_node_keys
-        }
+        remaining_edges = set()
+        for node_key in remaining_node_keys:
+            for edge in self.edges_by_node.get(node_key, []):
+                if not deleted_edges_tracker.get(edge, False):
+                    # Check if both ends of the edge are in remaining nodes
+                    if (
+                        edge.f1.collection_address() in remaining_node_keys
+                        and edge.f2.collection_address() in remaining_node_keys
+                    ):
+                        remaining_edges.add(edge)
 
         # error if there are edges that have not been visited
         if remaining_edges:
@@ -478,6 +548,23 @@ class TraversalNode(Contextualizable):
             return False
         return True
 
+    def can_run_given_str(
+        self,
+        remaining_node_keys_str: Set[str],
+        node_after_str: Dict[str, Set[str]],
+        dataset_after_str: Dict[str, Set[str]],
+    ) -> bool:
+        """Optimized version using pre-computed string sets to avoid expensive hash operations."""
+        # Check collection-level dependencies
+        node_deps = node_after_str.get(self.address.value, set())
+        if node_deps & remaining_node_keys_str:
+            return False
+        # Check dataset-level dependencies
+        dataset_deps = dataset_after_str.get(self.address.value, set())
+        if dataset_deps & remaining_node_keys_str:
+            return False
+        return True
+
     def is_root_node(self) -> bool:
         """This traversal_node is the defined traversal start"""
         return self.address == ROOT_COLLECTION_ADDRESS

fides/api/models/privacy_request/privacy_request.py

@@ -1009,6 +1009,38 @@ class PrivacyRequest(
                 request_task_celery_ids.append(request_task_id)
         return request_task_celery_ids
 
+    def cancel_celery_tasks(self) -> None:
+        """Cancel all Celery tasks associated with this privacy request.
+
+        This includes both the main privacy request task and any sub-tasks (Request Tasks).
+        """
+        task_ids: List[str] = []
+
+        # Add the main privacy request task ID
+        parent_task_id = self.get_cached_task_id()
+        if parent_task_id:
+            task_ids.append(parent_task_id)
+
+        # Add all request task IDs
+        request_task_celery_ids = self.get_request_task_celery_task_ids()
+        task_ids.extend(request_task_celery_ids)
+
+        if not task_ids:
+            return
+
+        # Revoke all Celery tasks in batch
+        logger.info(f"Revoking {len(task_ids)} tasks for privacy request {self.id}")
+        try:
+            # Use terminate=False to allow graceful shutdown if already running
+            celery_app.control.revoke(task_ids, terminate=False)
+            logger.info(
+                f"Successfully revoked {len(task_ids)} tasks for privacy request {self.id}"
+            )
+        except Exception as exc:
+            logger.warning(
+                f"Failed to revoke {len(task_ids)} tasks for privacy request {self.id}: {exc}"
+            )
+
     def cancel_processing(self, db: Session, cancel_reason: Optional[str]) -> None:
         """Cancels a privacy request.  Currently should only cancel 'pending' tasks
 
@@ -1021,19 +1053,7 @@ class PrivacyRequest(
             self.canceled_at = datetime.utcnow()
             self.save(db)
 
-            task_ids: List[str] = (
-                self.get_request_task_celery_task_ids()
-            )  # Celery tasks for sub tasks (DSR 3.0 Request Tasks)
-            parent_task_id = (
-                self.get_cached_task_id()
-            )  # Celery task for current Privacy Request
-            if parent_task_id:
-                task_ids.append(parent_task_id)
-
-            for celery_task_id in task_ids:
-                logger.info("Revoking task {} for request {}", celery_task_id, self.id)
-                # Only revokes if execution is not already in progress.
-                celery_app.control.revoke(celery_task_id, terminate=False)
+            self.cancel_celery_tasks()
 
     def error_processing(self, db: Session) -> None:
         """Mark privacy request as errored, and note time processing was finished"""

fides/api/schemas/application_config.py

@@ -70,6 +70,7 @@ class ExecutionApplicationConfig(FidesSchema):
     subject_identity_verification_required: Optional[bool] = None
     disable_consent_identity_verification: Optional[bool] = None
     require_manual_request_approval: Optional[bool] = None
+    memory_watchdog_enabled: Optional[bool] = None
     sql_dry_run: Optional[SqlDryRunMode] = None
 
     model_config = ConfigDict(use_enum_values=True, extra="forbid")

fides/api/schemas/connection_configuration/connection_secrets_datahub.py

@@ -16,6 +16,15 @@ class PeriodicIntegrationFrequency(Enum):
     daily = "daily"
     weekly = "weekly"
     monthly = "monthly"
+    not_scheduled = "not scheduled"
+
+
+PERIODIC_INTEGRATION_FREQUENCY_TO_DAYS = {
+    PeriodicIntegrationFrequency.daily.value: 1,
+    PeriodicIntegrationFrequency.weekly.value: 7,
+    PeriodicIntegrationFrequency.monthly.value: 30,
+    PeriodicIntegrationFrequency.not_scheduled.value: -1,  # negative value to indicate that the integration is not scheduled
+}
 
 
 class DatahubSchema(ConnectionConfigSecretsSchema):
@@ -30,7 +39,7 @@ class DatahubSchema(ConnectionConfigSecretsSchema):
     )
     frequency: PeriodicIntegrationFrequency = Field(
         title="Frequency",
-        description="The frequency at which the integration should run. Available options are daily, weekly, and monthly.",
+        description="The frequency at which the integration should run. Available options are daily, weekly, monthly, and not scheduled.",
     )
     glossary_node: str = Field(
         title="Glossary node",

fides/api/service/connectors/base_connector.py

@@ -144,3 +144,17 @@ class BaseConnector(Generic[DB_CONNECTOR_TYPE], ABC):
         # Defaulting to true for now so we can keep the default behavior and
         # incrementally determine the need for primary keys across all connectors
         return True
+
+    def get_qualified_table_name(self, node: ExecutionNode) -> str:
+        """
+        Get the fully qualified table name for the given execution node.
+        """
+        raise NotImplementedError(
+            "get_qualified_table_name is not implemented by this connector"
+        )
+
+    def table_exists(self, qualified_table_name: str) -> bool:
+        """
+        Check if a table exists in the datastore.
+        """
+        raise NotImplementedError("table_exists is not implemented by this connector")

fides/api/service/connectors/bigquery_connector.py

@@ -70,6 +70,11 @@ class BigQueryConnector(SQLConnector):
             node, SQLConnector.get_namespace_meta(db, node.address.dataset)
         )
 
+    def get_qualified_table_name(self, node: ExecutionNode) -> str:
+        """Get fully qualified BigQuery table name using existing query config logic"""
+        query_config = self.query_config(node)
+        return query_config.generate_table_name()
+
     def partitioned_retrieval(
         self,
         query_config: SQLQueryConfig,

fides/api/service/connectors/query_configs/bigquery_query_config.py

@@ -93,7 +93,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
 
         return where_clauses
 
-    def _generate_table_name(self) -> str:
+    def generate_table_name(self) -> str:
         """
         Prepends the dataset ID and project ID to the base table name
         if the BigQuery namespace meta is provided.
@@ -116,7 +116,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
         Returns a query string with backtick formatting for tables that have the same names as
         BigQuery reserved words.
         """
-        return f'SELECT {field_list} FROM `{self._generate_table_name()}` WHERE ({" OR ".join(clauses)})'
+        return f'SELECT {field_list} FROM `{self.generate_table_name()}` WHERE ({" OR ".join(clauses)})'
 
     def generate_masking_stmt(
         self,
@@ -197,7 +197,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             )
             return []
 
-        table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
+        table = Table(self.generate_table_name(), MetaData(bind=client), autoload=True)
         where_clauses: List[ColumnElement] = [
             table.c[k] == v for k, v in non_empty_reference_field_keys.items()
         ]
@@ -256,7 +256,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             )
             return []
 
-        table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
+        table = Table(self.generate_table_name(), MetaData(bind=client), autoload=True)
 
         # Build individual reference clauses
         where_clauses: List[ColumnElement] = []

fides/api/service/connectors/query_configs/snowflake_query_config.py

@@ -30,7 +30,7 @@ class SnowflakeQueryConfig(SQLQueryConfig):
         """Returns field names in clauses surrounded by quotation marks as required by Snowflake syntax."""
         return f'"{string_path}" {operator} (:{operand})'
 
-    def _generate_table_name(self) -> str:
+    def generate_table_name(self) -> str:
         """
         Prepends the dataset name and schema to the base table name
         if the Snowflake namespace meta is provided.
@@ -57,7 +57,7 @@ class SnowflakeQueryConfig(SQLQueryConfig):
         clauses: List[str],
     ) -> str:
         """Returns a query string with double quotation mark formatting as required by Snowflake syntax."""
-        return f'SELECT {field_list} FROM {self._generate_table_name()} WHERE ({" OR ".join(clauses)})'
+        return f'SELECT {field_list} FROM {self.generate_table_name()} WHERE ({" OR ".join(clauses)})'
 
     def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]:
         """Adds the appropriate formatting for update statements in this datastore."""
@@ -69,4 +69,4 @@ class SnowflakeQueryConfig(SQLQueryConfig):
         where_clauses: List[str],
     ) -> str:
         """Returns a parameterized update statement in Snowflake dialect."""
-        return f'UPDATE {self._generate_table_name()} SET {", ".join(update_clauses)} WHERE {" AND ".join(where_clauses)}'
+        return f'UPDATE {self.generate_table_name()} SET {", ".join(update_clauses)} WHERE {" AND ".join(where_clauses)}'

fides/api/service/connectors/snowflake_connector.py

@@ -3,11 +3,11 @@ from typing import Any, Dict, Union
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from snowflake.sqlalchemy import URL as Snowflake_URL
+from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from fides.api.graph.execution import ExecutionNode
 from fides.api.schemas.connection_configuration import SnowflakeSchema
-from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
 from fides.api.service.connectors.query_configs.snowflake_query_config import (
     SnowflakeQueryConfig,
 )
@@ -69,10 +69,63 @@ class SnowflakeConnector(SQLConnector):
                 connect_args["private_key"] = private_key
         return connect_args
 
-    def query_config(self, node: ExecutionNode) -> SQLQueryConfig:
+    def query_config(self, node: ExecutionNode) -> SnowflakeQueryConfig:
         """Query wrapper corresponding to the input execution_node."""
 
         db: Session = Session.object_session(self.configuration)
         return SnowflakeQueryConfig(
             node, SQLConnector.get_namespace_meta(db, node.address.dataset)
         )
+
+    def get_qualified_table_name(self, node: ExecutionNode) -> str:
+        """Get fully qualified Snowflake table name using existing query config logic"""
+        query_config = self.query_config(node)
+        return query_config.generate_table_name()
+
+    def table_exists(self, qualified_table_name: str) -> bool:
+        """
+        Check if table exists in Snowflake using the proper three-part naming convention.
+
+        Snowflake supports database.schema.table naming, and the generic SQLConnector
+        table_exists method doesn't handle quoted identifiers properly.
+        """
+        try:
+            client = self.create_client()
+            with client.connect() as connection:
+                # Remove quotes and split the parts
+                clean_name = qualified_table_name.replace('"', "")
+                parts = clean_name.split(".")
+
+                if len(parts) == 1:
+                    # Simple table name - use current schema
+                    table_name = parts[0]
+                    result = connection.execute(text(f'DESC TABLE "{table_name}"'))
+                elif len(parts) == 2:
+                    # schema.table format
+                    schema_name, table_name = parts
+                    result = connection.execute(
+                        text(f'DESC TABLE "{schema_name}"."{table_name}"')
+                    )
+                elif len(parts) >= 3:
+                    # database.schema.table format
+                    database_name, schema_name, table_name = (
+                        parts[-3],
+                        parts[-2],
+                        parts[-1],
+                    )
+                    # Use the database.schema.table format
+                    result = connection.execute(
+                        text(
+                            f'DESC TABLE "{database_name}"."{schema_name}"."{table_name}"'
+                        )
+                    )
+                else:
+                    return False
+
+                # If we get here without an exception, the table exists
+                result.close()
+                return True
+
+        except Exception:
+            # Table doesn't exist or other error
+            return False