ethyca-fides 2.66.2b2__py2.py3-none-any.whl → 2.66.2rc0__py2.py3-none-any.whl

fides/api/service/connectors/bigquery_connector.py

@@ -1,8 +1,13 @@
-from typing import Any, Dict, List, Optional
+from typing import List, Optional
 
 from loguru import logger
 from sqlalchemy import text
-from sqlalchemy.engine import Connection, Engine, create_engine  # type: ignore
+from sqlalchemy.engine import (  # type: ignore
+    Connection,
+    Engine,
+    LegacyCursorResult,
+    create_engine,
+)
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import Executable  # type: ignore
 from sqlalchemy.sql.elements import TextClause
@@ -12,7 +17,6 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
-from fides.api.schemas.application_config import SqlDryRunMode
 from fides.api.schemas.connection_configuration.connection_secrets_bigquery import (
     BigQuerySchema,
 )
@@ -95,16 +99,6 @@ class BigQueryConnector(SQLConnector):
         logger.info(
             f"Executing {len(partition_clauses)} partition queries for node '{query_config.node.address}' in DSR execution"
         )
-
-        if self.should_dry_run(SqlDryRunMode.access):
-            for partition_clause in partition_clauses:
-                existing_bind_params = stmt.compile().params
-                partitioned_stmt = text(
-                    f"{stmt} AND ({text(partition_clause)})"
-                ).params(existing_bind_params)
-                logger.warning(f"SQL DRY RUN - Would execute SQL: {partitioned_stmt}")
-            return []
-
         rows = []
         for partition_clause in partition_clauses:
             logger.debug(
@@ -141,39 +135,6 @@ class BigQueryConnector(SQLConnector):
             logger.exception(f"Error testing connection to remote BigQuery {str(e)}")
             raise ConnectionException(f"Connection error: {e}")
 
-    def _execute_statements_with_sql_dry_run(
-        self,
-        statements: List[Executable],
-        sql_dry_run_enabled: bool,
-        client: Engine,
-    ) -> int:
-        """
-        Execute SQL statements with sql_dry_run support.
-
-        Args:
-            statements: List of SQL statements to execute
-            sql_dry_run_enabled: Whether sql_dry_run mode is enabled
-            client: Database engine
-
-        Returns:
-            int: Number of affected rows (0 in sql_dry_run mode)
-        """
-        if not statements:
-            return 0
-
-        if sql_dry_run_enabled:
-            for stmt in statements:
-                logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
-            return 0
-
-        row_count = 0
-        with client.connect() as connection:
-            for stmt in statements:
-                results = connection.execute(stmt)
-                logger.debug(f"Affected {results.rowcount} rows")
-                row_count += results.rowcount
-        return row_count
-
     def mask_data(
         self,
         node: ExecutionNode,
@@ -181,31 +142,22 @@ class BigQueryConnector(SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Returns the number of records updated or deleted"""
-
         query_config = self.query_config(node)
         update_or_delete_ct = 0
         client = self.client()
-
-        if query_config.uses_delete_masking_strategy():
-            delete_stmts = query_config.generate_delete(client, input_data or {})
-            logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
-            update_or_delete_ct += self._execute_statements_with_sql_dry_run(
-                delete_stmts, self.should_dry_run(SqlDryRunMode.erasure), client
-            )
-        else:
-            for row in rows:
-                update_or_delete_stmts: List[Executable] = query_config.generate_update(
-                    row, policy, privacy_request, client
-                )
-                logger.debug(
-                    f"Generated {len(update_or_delete_stmts)} UPDATE statements"
-                )
-                update_or_delete_ct += self._execute_statements_with_sql_dry_run(
-                    update_or_delete_stmts,
-                    self.should_dry_run(SqlDryRunMode.erasure),
-                    client,
+        for row in rows:
+            update_or_delete_stmts: List[Executable] = (
+                query_config.generate_masking_stmt(
+                    node, row, policy, privacy_request, client
                 )
+            )
+            if update_or_delete_stmts:
+                with client.connect() as connection:
+                    for update_or_delete_stmt in update_or_delete_stmts:
+                        results: LegacyCursorResult = connection.execute(
+                            update_or_delete_stmt
+                        )
+                        update_or_delete_ct = update_or_delete_ct + results.rowcount
         return update_or_delete_ct

fides/api/service/connectors/dynamodb_connector.py

@@ -140,9 +140,8 @@ class DynamoDBConnector(BaseConnector[Any]):  # type: ignore
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
-        """Execute a masking request for DynamoDB"""
+        """Execute a masking requestfor DynamoDB"""
 
         query_config = self.query_config(node)
         collection_name = node.address.collection

fides/api/service/connectors/fides_connector.py

@@ -142,7 +142,6 @@ class FidesConnector(BaseConnector[FidesClient]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute an erasure request on remote fides"""
         identity_data = {

fides/api/service/connectors/http_connector.py

@@ -90,7 +90,6 @@ class HTTPSConnector(BaseConnector[None]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Currently not supported as webhooks are not called at the collection level"""
         raise NotImplementedError(

fides/api/service/connectors/manual_task_connector.py

@@ -80,7 +80,6 @@ class ManualTaskConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """
         Manual tasks don't support erasure operations.

fides/api/service/connectors/manual_webhook_connector.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
@@ -53,7 +53,6 @@ class ManualWebhookConnector(BaseConnector[None]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[List[List[Row]]] = None,
     ) -> None:
         """
         Not applicable for a manual webhook.  Manual webhooks are not called as part of the traversal.

fides/api/service/connectors/mongodb_connector.py

@@ -126,7 +126,6 @@ class MongoDBConnector(BaseConnector[MongoClient]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request"""
         query_config = self.query_config(node)

fides/api/service/connectors/okta_connector.py

@@ -82,7 +82,6 @@ class OktaConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not supported for Okta connector"""
         return 0

fides/api/service/connectors/query_configs/bigquery_query_config.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Union, cast
 import pydash
 from fideslang.models import MaskingStrategies
 from loguru import logger
-from sqlalchemy import MetaData, Table, or_, text
+from sqlalchemy import MetaData, Table, text
 from sqlalchemy.engine import Engine
 from sqlalchemy.sql import Delete, Update
 from sqlalchemy.sql.elements import ColumnElement, TextClause
@@ -125,7 +125,6 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
         policy: Policy,
         request: PrivacyRequest,
         client: Engine,
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> Union[List[Update], List[Delete]]:
         """
         Generate a masking statement for BigQuery.
@@ -138,7 +137,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             logger.info(
                 f"Masking override detected for collection {node.address.value}: {masking_override.strategy.value}"
             )
-            return self.generate_delete(client, input_data or {})
+            return self.generate_delete(row, client)
         return self.generate_update(row, policy, request, client)
 
     def generate_update(
@@ -199,16 +198,9 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
 
         table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
         where_clauses: List[ColumnElement] = [
-            table.c[k] == v for k, v in non_empty_reference_field_keys.items()
+            getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items()
         ]
 
-        # Create update values using Column objects as keys to handle column names with spaces
-        update_values = {}
-        for column_name, value in final_update_map.items():
-            # Use bracket notation to access columns with spaces in their names
-            column = table.c[column_name]
-            update_values[column] = value
-
         if self.partitioning:
             partition_clauses = self.get_partition_clauses()
             partitioned_queries = []
@@ -219,37 +211,34 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
                 partitioned_queries.append(
                     table.update()
                     .where(*(where_clauses + [text(partition_clause)]))
-                    .values(update_values)
+                    .values(**final_update_map)
                 )
 
             return partitioned_queries
 
-        return [table.update().where(*where_clauses).values(update_values)]
+        return [table.update().where(*where_clauses).values(**final_update_map)]
 
-    def generate_delete(
-        self,
-        client: Engine,
-        input_data: Optional[Dict[str, List[Any]]] = None,
-    ) -> List[Delete]:
-        """
-        Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
+    def generate_delete(self, row: Row, client: Engine) -> List[Delete]:
+        """Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
 
         Used when a collection-level masking override is present and the masking strategy is DELETE.
 
         A List of multiple DELETE statements are returned for partitioned tables; for a non-partitioned table,
         a single DELETE statement is returned in a List for consistent typing.
-        """
 
-        if not input_data:
-            logger.warning(
-                "No input data provided for node '{}', skipping DELETE statement generation",
-                self.node.address,
-            )
-            return []
+        TODO: DRY up this method and `generate_update` a bit
+        """
 
-        filtered_data = self.node.typed_filtered_values(input_data)
+        non_empty_reference_field_keys: Dict[str, Field] = filter_nonempty_values(
+            {
+                fpath.string_path: fld.cast(row[fpath.string_path])
+                for fpath, fld in self.reference_field_paths.items()
+                if fpath.string_path in row
+            }
+        )
 
-        if not filtered_data:
+        valid = len(non_empty_reference_field_keys) > 0
+        if not valid:
             logger.warning(
                 "There is not enough data to generate a valid DELETE statement for {}",
                 self.node.address,
@@ -257,17 +246,9 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             return []
 
         table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
-
-        # Build individual reference clauses
-        where_clauses: List[ColumnElement] = []
-        for column_name, values in filtered_data.items():
-            if len(values) == 1:
-                where_clauses.append(table.c[column_name] == values[0])
-            else:
-                where_clauses.append(table.c[column_name].in_(values))
-
-        # Combine reference clauses with OR instead of AND
-        combined_reference_clause = or_(*where_clauses)
+        where_clauses: List[ColumnElement] = [
+            getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items()
+        ]
 
         if self.partitioning:
             partition_clauses = self.get_partition_clauses()
@@ -278,25 +259,12 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
 
             for partition_clause in partition_clauses:
                 partitioned_queries.append(
-                    table.delete()
-                    .where(combined_reference_clause)
-                    .where(text(partition_clause))
+                    table.delete().where(*(where_clauses + [text(partition_clause)]))
                 )
 
             return partitioned_queries
 
-        return [table.delete().where(combined_reference_clause)]
-
-    def uses_delete_masking_strategy(self) -> bool:
-        """Check if this collection uses DELETE masking strategy.
-
-        Returns True if masking override is present and strategy is DELETE.
-        """
-        masking_override = self.node.collection.masking_strategy_override
-        return (
-            masking_override is not None
-            and masking_override.strategy == MaskingStrategies.DELETE
-        )
+        return [table.delete().where(*where_clauses)]
 
     def format_fields_for_query(
         self,
@@ -313,25 +281,6 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
                 formatted_fields.append(field_path.levels[0])
         return formatted_fields
 
-    def format_clause_for_query(
-        self, string_path: str, operator: str, operand: str
-    ) -> str:
-        """
-        Returns clauses with proper BigQuery backtick escaping for column names.
-        Handles column names with spaces and nested fields (dot-separated) by escaping each part individually.
-        """
-        # For nested fields (containing dots), escape each part individually
-        if "." in string_path:
-            parts = string_path.split(".")
-            escaped_field = ".".join(f"`{part}`" for part in parts)
-        else:
-            # For simple fields, wrap the entire name in backticks
-            escaped_field = f"`{string_path}`"
-
-        if operator == "IN":
-            return f"{escaped_field} IN ({operand})"
-        return f"{escaped_field} {operator} :{operand}"
-
     def generate_raw_query_without_tuples(
         self, field_list: List[str], filters: Dict[str, List[Any]]
     ) -> Optional[TextClause]:
@@ -341,35 +290,27 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
 
         This is an override of the base class method that supports nested fields for BigQuery.
 
-        Examples with field names containing dots and spaces, notice these are replaced with underscores in the parameter bindings:
+        Examples with dot-delimited field names, notice the periods are replaced with underscores in the parameter bindings:
 
         1. Single value filter:
            field_list = ["id", "name", "email"]
            filters = {"user.id": [123]}
 
-           Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`user`.`id` = :user_id)
+           Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (user.id = :user_id)
            With parameter binding: user_id = 123
 
-        2. Field with spaces:
-           field_list = ["id", "custom id", "email"]
-           filters = {"custom id": ["abc123"]}
-
-           Generates: SELECT id, `custom id`, email FROM `project_id.dataset_id.table_name` WHERE (`custom id` = :custom_id)
-           With parameter binding: custom_id = "abc123"
-
-        3. Multiple value filter with nested field:
+        2. Multiple value filter:
            field_list = ["id", "name", "email"]
-           filters = {"contact_info.primary_email": ["active", "pending"]}
+           filters = {"user.status": ["active", "pending"]}
 
-           Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`contact_info`.`primary_email` IN (:contact_info_primary_email_in_stmt_generated_0, :contact_info_primary_email_in_stmt_generated_1))
-           With parameter bindings: contact_info_primary_email_in_stmt_generated_0 = "active", contact_info_primary_email_in_stmt_generated_1 = "pending"
+           Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (user.status IN (:user_status_in_stmt_generated_0, :user_status_in_stmt_generated_1))
+           With parameter bindings: user_status_in_stmt_generated_0 = "active", user_status_in_stmt_generated_1 = "pending"
         """
         clauses = []
         query_data = {}
         for field_name, field_value in filters.items():
-            # Replace dots and spaces with underscores in field names for parameter binding
-            # SQLAlchemy parameter names cannot contain spaces or special characters
-            field_binding_name = field_name.replace(".", "_").replace(" ", "_")
+            # Replace dots with underscores in field names for parameter binding
+            field_binding_name = field_name.replace(".", "_")
             data = set(field_value)
             if len(data) == 1:
                 clauses.append(
@@ -392,7 +333,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
                 clauses.append(self.format_clause_for_query(field_name, "IN", operand))
 
         if len(clauses) > 0:
-            formatted_fields = ", ".join([f"`{field}`" for field in field_list])
+            formatted_fields = ", ".join(field_list)
             query_str = self.get_formatted_query_string(formatted_fields, clauses)
             return text(query_str).params(query_data)
 

fides/api/service/connectors/rds_mysql_connector.py

@@ -158,7 +158,6 @@ class RDSMySQLConnector(RDSConnectorMixin, SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for RDS MySQL"""
         return 0

fides/api/service/connectors/rds_postgres_connector.py

@@ -147,7 +147,6 @@ class RDSPostgresConnector(RDSConnectorMixin, SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for RDS Postgres"""
         return 0

fides/api/service/connectors/s3_connector.py

@@ -66,7 +66,6 @@ class S3Connector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for S3"""
         return 0

fides/api/service/connectors/saas_connector.py

@@ -517,7 +517,6 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Return the number of rows that have been updated."""
         self.set_privacy_request_state(privacy_request, node, request_task)

fides/api/service/connectors/scylla_connector.py

@@ -150,7 +150,6 @@ class ScyllaConnector(BaseConnector[Cluster]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request"""
         query_config = self.query_config(node)

fides/api/service/connectors/sql_connector.py

@@ -27,13 +27,11 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
-from fides.api.schemas.application_config import SqlDryRunMode
 from fides.api.schemas.connection_configuration import ConnectionConfigSecretsSchema
 from fides.api.service.connectors.base_connector import BaseConnector
 from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
 from fides.api.util.collection_util import Row
 from fides.config import get_config
-from fides.config.config_proxy import ConfigProxy
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
     Dataset as CtlDataset,
@@ -60,23 +58,6 @@ class SQLConnector(BaseConnector[Engine]):
             )
         self.ssh_server: sshtunnel._ForwardServer = None
 
-    def should_dry_run(self, mode_to_check: SqlDryRunMode) -> bool:
-        """
-        Check if SQL dry run is enabled for the specified mode.
-
-        Args:
-            mode_to_check: The SqlDryRunMode to check for
-
-        Returns:
-            bool: True if the current mode matches the mode to check
-        """
-        from fides.api.api.deps import get_autoclose_db_session as get_db
-
-        with get_db() as db:
-            config_proxy = ConfigProxy(db)
-            current_mode = getattr(config_proxy.execution, "sql_dry_run", None)
-            return current_mode == mode_to_check
-
     @staticmethod
     def cursor_result_to_rows(results: CursorResult) -> List[Row]:
         """Convert SQLAlchemy results to a list of dictionaries"""
@@ -159,10 +140,6 @@ class SQLConnector(BaseConnector[Engine]):
         if query is None:
             return []
 
-        if self.should_dry_run(SqlDryRunMode.access):
-            logger.warning(f"SQL DRY RUN - Would execute SQL: {query}")
-            return []
-
         with client.connect() as connection:
             self.set_schema(connection)
             results = connection.execute(query)
@@ -183,10 +160,6 @@ class SQLConnector(BaseConnector[Engine]):
         if stmt is None:
             return []
 
-        if self.should_dry_run(SqlDryRunMode.access):
-            logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
-            return []
-
         logger.info("Starting data retrieval for {}", node.address)
         with client.connect() as connection:
             self.set_schema(connection)
@@ -205,25 +178,20 @@ class SQLConnector(BaseConnector[Engine]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Returns the number of records masked"""
         query_config = self.query_config(node)
         update_ct = 0
         client = self.client()
-
         for row in rows:
             update_stmt: Optional[TextClause] = query_config.generate_update_stmt(
                 row, policy, privacy_request
             )
             if update_stmt is not None:
-                if self.should_dry_run(SqlDryRunMode.erasure):
-                    logger.warning(f"SQL DRY RUN - Would execute SQL: {update_stmt}")
-                else:
-                    with client.connect() as connection:
-                        self.set_schema(connection)
-                        results: LegacyCursorResult = connection.execute(update_stmt)
-                        update_ct = update_ct + results.rowcount
+                with client.connect() as connection:
+                    self.set_schema(connection)
+                    results: LegacyCursorResult = connection.execute(update_stmt)
+                    update_ct = update_ct + results.rowcount
         return update_ct
 
     def close(self) -> None:

fides/api/service/connectors/website_connector.py

@@ -75,7 +75,6 @@ class WebsiteConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
-        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not supported for website connector"""
         return 0