ethyca-fides 2.66.0rc1__py2.py3-none-any.whl → 2.66.1b1__py2.py3-none-any.whl

fides/api/service/connectors/bigquery_connector.py

@@ -1,13 +1,8 @@
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
 from loguru import logger
 from sqlalchemy import text
-from sqlalchemy.engine import (  # type: ignore
-    Connection,
-    Engine,
-    LegacyCursorResult,
-    create_engine,
-)
+from sqlalchemy.engine import Connection, Engine, create_engine  # type: ignore
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import Executable  # type: ignore
 from sqlalchemy.sql.elements import TextClause
@@ -17,6 +12,7 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.schemas.application_config import SqlDryRunMode
 from fides.api.schemas.connection_configuration.connection_secrets_bigquery import (
     BigQuerySchema,
 )
@@ -99,6 +95,16 @@ class BigQueryConnector(SQLConnector):
         logger.info(
             f"Executing {len(partition_clauses)} partition queries for node '{query_config.node.address}' in DSR execution"
         )
+
+        if self.should_dry_run(SqlDryRunMode.access):
+            for partition_clause in partition_clauses:
+                existing_bind_params = stmt.compile().params
+                partitioned_stmt = text(
+                    f"{stmt} AND ({text(partition_clause)})"
+                ).params(existing_bind_params)
+                logger.warning(f"SQL DRY RUN - Would execute SQL: {partitioned_stmt}")
+            return []
+
         rows = []
         for partition_clause in partition_clauses:
             logger.debug(
@@ -135,6 +141,39 @@ class BigQueryConnector(SQLConnector):
             logger.exception(f"Error testing connection to remote BigQuery {str(e)}")
             raise ConnectionException(f"Connection error: {e}")
 
+    def _execute_statements_with_sql_dry_run(
+        self,
+        statements: List[Executable],
+        sql_dry_run_enabled: bool,
+        client: Engine,
+    ) -> int:
+        """
+        Execute SQL statements with sql_dry_run support.
+
+        Args:
+            statements: List of SQL statements to execute
+            sql_dry_run_enabled: Whether sql_dry_run mode is enabled
+            client: Database engine
+
+        Returns:
+            int: Number of affected rows (0 in sql_dry_run mode)
+        """
+        if not statements:
+            return 0
+
+        if sql_dry_run_enabled:
+            for stmt in statements:
+                logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
+            return 0
+
+        row_count = 0
+        with client.connect() as connection:
+            for stmt in statements:
+                results = connection.execute(stmt)
+                logger.debug(f"Affected {results.rowcount} rows")
+                row_count += results.rowcount
+        return row_count
+
     def mask_data(
         self,
         node: ExecutionNode,
@@ -142,22 +181,31 @@ class BigQueryConnector(SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Returns the number of records updated or deleted"""
+
         query_config = self.query_config(node)
         update_or_delete_ct = 0
         client = self.client()
-        for row in rows:
-            update_or_delete_stmts: List[Executable] = (
-                query_config.generate_masking_stmt(
-                    node, row, policy, privacy_request, client
-                )
+
+        if query_config.uses_delete_masking_strategy():
+            delete_stmts = query_config.generate_delete(client, input_data or {})
+            logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
+            update_or_delete_ct += self._execute_statements_with_sql_dry_run(
+                delete_stmts, self.should_dry_run(SqlDryRunMode.erasure), client
             )
-            if update_or_delete_stmts:
-                with client.connect() as connection:
-                    for update_or_delete_stmt in update_or_delete_stmts:
-                        results: LegacyCursorResult = connection.execute(
-                            update_or_delete_stmt
-                        )
-                        update_or_delete_ct = update_or_delete_ct + results.rowcount
+        else:
+            for row in rows:
+                update_or_delete_stmts: List[Executable] = query_config.generate_update(
+                    row, policy, privacy_request, client
+                )
+                logger.debug(
+                    f"Generated {len(update_or_delete_stmts)} UPDATE statements"
+                )
+                update_or_delete_ct += self._execute_statements_with_sql_dry_run(
+                    update_or_delete_stmts,
+                    self.should_dry_run(SqlDryRunMode.erasure),
+                    client,
+                )
         return update_or_delete_ct

fides/api/service/connectors/dynamodb_connector.py

@@ -140,8 +140,9 @@ class DynamoDBConnector(BaseConnector[Any]):  # type: ignore
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
-        """Execute a masking requestfor DynamoDB"""
+        """Execute a masking request for DynamoDB"""
 
         query_config = self.query_config(node)
         collection_name = node.address.collection

fides/api/service/connectors/fides_connector.py

@@ -142,6 +142,7 @@ class FidesConnector(BaseConnector[FidesClient]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute an erasure request on remote fides"""
         identity_data = {

fides/api/service/connectors/http_connector.py

@@ -90,6 +90,7 @@ class HTTPSConnector(BaseConnector[None]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Currently not supported as webhooks are not called at the collection level"""
         raise NotImplementedError(

fides/api/service/connectors/manual_task_connector.py

@@ -80,6 +80,7 @@ class ManualTaskConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """
         Manual tasks don't support erasure operations.

fides/api/service/connectors/manual_webhook_connector.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
@@ -53,6 +53,7 @@ class ManualWebhookConnector(BaseConnector[None]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[List[List[Row]]] = None,
     ) -> None:
         """
         Not applicable for a manual webhook.  Manual webhooks are not called as part of the traversal.

fides/api/service/connectors/mongodb_connector.py

@@ -126,6 +126,7 @@ class MongoDBConnector(BaseConnector[MongoClient]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request"""
         query_config = self.query_config(node)

fides/api/service/connectors/okta_connector.py

@@ -82,6 +82,7 @@ class OktaConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not supported for Okta connector"""
         return 0

fides/api/service/connectors/query_configs/bigquery_query_config.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Union, cast
 import pydash
 from fideslang.models import MaskingStrategies
 from loguru import logger
-from sqlalchemy import MetaData, Table, text
+from sqlalchemy import MetaData, Table, or_, text
 from sqlalchemy.engine import Engine
 from sqlalchemy.sql import Delete, Update
 from sqlalchemy.sql.elements import ColumnElement, TextClause
@@ -125,6 +125,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
         policy: Policy,
         request: PrivacyRequest,
         client: Engine,
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> Union[List[Update], List[Delete]]:
         """
         Generate a masking statement for BigQuery.
@@ -137,7 +138,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             logger.info(
                 f"Masking override detected for collection {node.address.value}: {masking_override.strategy.value}"
             )
-            return self.generate_delete(row, client)
+            return self.generate_delete(client, input_data or {})
         return self.generate_update(row, policy, request, client)
 
     def generate_update(
@@ -218,27 +219,30 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
 
         return [table.update().where(*where_clauses).values(**final_update_map)]
 
-    def generate_delete(self, row: Row, client: Engine) -> List[Delete]:
-        """Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
+    def generate_delete(
+        self,
+        client: Engine,
+        input_data: Optional[Dict[str, List[Any]]] = None,
+    ) -> List[Delete]:
+        """
+        Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
 
         Used when a collection-level masking override is present and the masking strategy is DELETE.
 
         A List of multiple DELETE statements are returned for partitioned tables; for a non-partitioned table,
         a single DELETE statement is returned in a List for consistent typing.
-
-        TODO: DRY up this method and `generate_update` a bit
         """
 
-        non_empty_reference_field_keys: Dict[str, Field] = filter_nonempty_values(
-            {
-                fpath.string_path: fld.cast(row[fpath.string_path])
-                for fpath, fld in self.reference_field_paths.items()
-                if fpath.string_path in row
-            }
-        )
+        if not input_data:
+            logger.warning(
+                "No input data provided for node '{}', skipping DELETE statement generation",
+                self.node.address,
+            )
+            return []
 
-        valid = len(non_empty_reference_field_keys) > 0
-        if not valid:
+        filtered_data = self.node.typed_filtered_values(input_data)
+
+        if not filtered_data:
             logger.warning(
                 "There is not enough data to generate a valid DELETE statement for {}",
                 self.node.address,
@@ -246,9 +250,17 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             return []
 
         table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
-        where_clauses: List[ColumnElement] = [
-            getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items()
-        ]
+
+        # Build individual reference clauses
+        where_clauses: List[ColumnElement] = []
+        for column_name, values in filtered_data.items():
+            if len(values) == 1:
+                where_clauses.append(getattr(table.c, column_name) == values[0])
+            else:
+                where_clauses.append(getattr(table.c, column_name).in_(values))
+
+        # Combine reference clauses with OR instead of AND
+        combined_reference_clause = or_(*where_clauses)
 
         if self.partitioning:
             partition_clauses = self.get_partition_clauses()
@@ -259,12 +271,25 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
 
             for partition_clause in partition_clauses:
                 partitioned_queries.append(
-                    table.delete().where(*(where_clauses + [text(partition_clause)]))
+                    table.delete()
+                    .where(combined_reference_clause)
+                    .where(text(partition_clause))
                 )
 
             return partitioned_queries
 
-        return [table.delete().where(*where_clauses)]
+        return [table.delete().where(combined_reference_clause)]
+
+    def uses_delete_masking_strategy(self) -> bool:
+        """Check if this collection uses DELETE masking strategy.
+
+        Returns True if masking override is present and strategy is DELETE.
+        """
+        masking_override = self.node.collection.masking_strategy_override
+        return (
+            masking_override is not None
+            and masking_override.strategy == MaskingStrategies.DELETE
+        )
 
     def format_fields_for_query(
         self,

fides/api/service/connectors/rds_mysql_connector.py

@@ -158,6 +158,7 @@ class RDSMySQLConnector(RDSConnectorMixin, SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for RDS MySQL"""
         return 0

fides/api/service/connectors/rds_postgres_connector.py

@@ -147,6 +147,7 @@ class RDSPostgresConnector(RDSConnectorMixin, SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for RDS Postgres"""
         return 0

fides/api/service/connectors/s3_connector.py

@@ -66,6 +66,7 @@ class S3Connector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for S3"""
         return 0

fides/api/service/connectors/saas_connector.py

@@ -517,6 +517,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Return the number of rows that have been updated."""
         self.set_privacy_request_state(privacy_request, node, request_task)

fides/api/service/connectors/scylla_connector.py

@@ -150,6 +150,7 @@ class ScyllaConnector(BaseConnector[Cluster]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request"""
         query_config = self.query_config(node)

fides/api/service/connectors/sql_connector.py

@@ -27,11 +27,13 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.schemas.application_config import SqlDryRunMode
 from fides.api.schemas.connection_configuration import ConnectionConfigSecretsSchema
 from fides.api.service.connectors.base_connector import BaseConnector
 from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
 from fides.api.util.collection_util import Row
 from fides.config import get_config
+from fides.config.config_proxy import ConfigProxy
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
     Dataset as CtlDataset,
@@ -58,6 +60,23 @@ class SQLConnector(BaseConnector[Engine]):
             )
         self.ssh_server: sshtunnel._ForwardServer = None
 
+    def should_dry_run(self, mode_to_check: SqlDryRunMode) -> bool:
+        """
+        Check if SQL dry run is enabled for the specified mode.
+
+        Args:
+            mode_to_check: The SqlDryRunMode to check for
+
+        Returns:
+            bool: True if the current mode matches the mode to check
+        """
+        from fides.api.api.deps import get_autoclose_db_session as get_db
+
+        with get_db() as db:
+            config_proxy = ConfigProxy(db)
+            current_mode = getattr(config_proxy.execution, "sql_dry_run", None)
+            return current_mode == mode_to_check
+
     @staticmethod
     def cursor_result_to_rows(results: CursorResult) -> List[Row]:
         """Convert SQLAlchemy results to a list of dictionaries"""
@@ -140,6 +159,10 @@ class SQLConnector(BaseConnector[Engine]):
         if query is None:
             return []
 
+        if self.should_dry_run(SqlDryRunMode.access):
+            logger.warning(f"SQL DRY RUN - Would execute SQL: {query}")
+            return []
+
         with client.connect() as connection:
             self.set_schema(connection)
             results = connection.execute(query)
@@ -160,6 +183,10 @@ class SQLConnector(BaseConnector[Engine]):
         if stmt is None:
             return []
 
+        if self.should_dry_run(SqlDryRunMode.access):
+            logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
+            return []
+
         logger.info("Starting data retrieval for {}", node.address)
         with client.connect() as connection:
             self.set_schema(connection)
@@ -178,20 +205,25 @@ class SQLConnector(BaseConnector[Engine]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Returns the number of records masked"""
         query_config = self.query_config(node)
         update_ct = 0
         client = self.client()
+
         for row in rows:
             update_stmt: Optional[TextClause] = query_config.generate_update_stmt(
                 row, policy, privacy_request
             )
             if update_stmt is not None:
-                with client.connect() as connection:
-                    self.set_schema(connection)
-                    results: LegacyCursorResult = connection.execute(update_stmt)
-                    update_ct = update_ct + results.rowcount
+                if self.should_dry_run(SqlDryRunMode.erasure):
+                    logger.warning(f"SQL DRY RUN - Would execute SQL: {update_stmt}")
+                else:
+                    with client.connect() as connection:
+                        self.set_schema(connection)
+                        results: LegacyCursorResult = connection.execute(update_stmt)
+                        update_ct = update_ct + results.rowcount
         return update_ct
 
     def close(self) -> None:

fides/api/service/connectors/website_connector.py

@@ -75,6 +75,7 @@ class WebsiteConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not supported for website connector"""
         return 0

fides/api/task/conditional_dependencies/evaluator.py

@@ -0,0 +1,109 @@
+import operator as py_operator
+from typing import Any, Union
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.graph.config import FieldPath
+from fides.api.task.conditional_dependencies.schemas import (
+    Condition,
+    ConditionGroup,
+    ConditionLeaf,
+    GroupOperator,
+    Operator,
+)
+
+operator_methods = {
+    Operator.exists: lambda a, _: a is not None,
+    Operator.not_exists: lambda a, _: a is None,
+    Operator.eq: py_operator.eq,
+    Operator.neq: py_operator.ne,
+    Operator.lt: lambda a, b: a < b if a is not None else False,
+    Operator.lte: lambda a, b: a <= b if a is not None else False,
+    Operator.gt: lambda a, b: a > b if a is not None else False,
+    Operator.gte: lambda a, b: a >= b if a is not None else False,
+    Operator.list_contains: lambda a, b: b in a if isinstance(a, list) else False,
+    Operator.not_in_list: lambda a, b: a not in b if isinstance(b, list) else True,
+}
+
+
+class ConditionEvaluator:
+    """Evaluates nested conditions for manual task creation"""
+
+    def __init__(self, db: Session):
+        self.db = db
+
+    def evaluate_rule(self, rule: Condition, data: Union[dict, Any]) -> bool:
+        """Evaluate a nested condition rule against input data"""
+        if isinstance(rule, ConditionLeaf):
+            return self._evaluate_leaf_condition(rule, data)
+        # ConditionGroup
+        return self._evaluate_group_condition(rule, data)
+
+    def _evaluate_leaf_condition(
+        self, condition: ConditionLeaf, data: Union[dict, Any]
+    ) -> bool:
+        """Evaluate a leaf condition against input data"""
+        actual_value = self._get_nested_value(data, condition.field_address.split("."))
+        # Apply operator and return result
+        return self._apply_operator(actual_value, condition.operator, condition.value)
+
+    def _evaluate_group_condition(
+        self, group: ConditionGroup, data: Union[dict, Any]
+    ) -> bool:
+        """Evaluate a group condition against input data"""
+        results = [
+            self.evaluate_rule(condition, data) for condition in group.conditions
+        ]
+
+        logical_operators = {GroupOperator.and_: all, GroupOperator.or_: any}
+        operator_func = logical_operators.get(group.logical_operator)
+
+        if operator_func is None:
+            logger.warning(f"Unknown logical operator: {group.logical_operator}")
+            return False
+
+        return operator_func(results)
+
+    def _get_nested_value(self, data: Union[dict, Any], keys: list[str]) -> Any:
+        """Get nested value from data using dot notation
+
+        Supports both simple dictionary access and Fides reference structures:
+        - Simple dict: data["user"]["name"]
+        - Fides FieldAddress: data.get_field_value(FieldAddress("dataset", "collection", "field_address"))
+        - Fides Collection: data.get_field_value(FieldPath("field_address", "subfield"))
+        """
+        if not keys:
+            return data
+
+        current = data
+
+        # Try Fides reference structures first
+        if hasattr(current, "get_field_value"):
+            try:
+                field_path = FieldPath(*keys) if len(keys) > 1 else FieldPath(keys[0])
+                return current.get_field_value(field_path)
+            except (AttributeError, ValueError):
+                pass
+
+        # Fall back to dictionary access
+        for key in keys:
+            if not isinstance(current, dict):
+                current = current.get(key, {}) if hasattr(current, "get") else None
+            else:
+                current = current.get(key, {})
+
+        return current if current != {} else None
+
+    def _apply_operator(
+        self, actual_value: Any, operator: Operator, expected_value: Any
+    ) -> bool:
+        """Apply operator to actual and expected values"""
+
+        # Get the method for the operator and execute it
+        operator_method = operator_methods.get(operator)
+        if operator_method is None:
+            logger.warning(f"Unknown operator: {operator}")
+            return False
+
+        return operator_method(actual_value, expected_value)

fides/api/task/conditional_dependencies/schemas.py

@@ -0,0 +1,54 @@
+from enum import Enum
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class Operator(str, Enum):
+    eq = "eq"
+    neq = "neq"
+    lt = "lt"
+    lte = "lte"
+    gt = "gt"
+    gte = "gte"
+    exists = "exists"
+    not_exists = "not_exists"
+    list_contains = "list_contains"
+    not_in_list = "not_in_list"
+
+
+class GroupOperator(str, Enum):
+    and_ = "and"
+    or_ = "or"
+
+
+# Leaf condition (e.g., "user.name exists", "user.created_at >= 2024-01-01")
+class ConditionLeaf(BaseModel):
+    field_address: str = Field(
+        description="Field path to check (e.g., 'user.name', 'billing.subscription.status')"
+    )
+    operator: Operator = Field(description="Operator to apply")
+    value: Optional[
+        Union[str, int, float, bool, List[Union[str, int, float, bool]]]
+    ] = Field(default=None, description="Expected value for comparison")
+
+
+# Nested logical condition (AND, OR)
+class ConditionGroup(BaseModel):
+    logical_operator: GroupOperator = Field(
+        description="Logical operator: 'and' or 'or'"
+    )
+    conditions: list["Condition"] = Field(
+        description="List of conditions or nested groups"
+    )
+
+    @model_validator(mode="after")
+    def validate_conditions(self) -> "ConditionGroup":
+        if not self.conditions:
+            raise ValueError("conditions list cannot be empty")
+        return self
+
+
+# Use model_rebuild to allow recursive nesting
+Condition = Union[ConditionLeaf, ConditionGroup]
+ConditionGroup.model_rebuild()