ethyca-fides 2.66.1b0__py2.py3-none-any.whl → 2.66.1b1__py2.py3-none-any.whl

fides/api/service/connectors/sql_connector.py

@@ -27,11 +27,13 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.schemas.application_config import SqlDryRunMode
 from fides.api.schemas.connection_configuration import ConnectionConfigSecretsSchema
 from fides.api.service.connectors.base_connector import BaseConnector
 from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
 from fides.api.util.collection_util import Row
 from fides.config import get_config
+from fides.config.config_proxy import ConfigProxy
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
     Dataset as CtlDataset,
@@ -58,6 +60,23 @@ class SQLConnector(BaseConnector[Engine]):
             )
         self.ssh_server: sshtunnel._ForwardServer = None
 
+    def should_dry_run(self, mode_to_check: SqlDryRunMode) -> bool:
+        """
+        Check if SQL dry run is enabled for the specified mode.
+
+        Args:
+            mode_to_check: The SqlDryRunMode to check for
+
+        Returns:
+            bool: True if the current mode matches the mode to check
+        """
+        from fides.api.api.deps import get_autoclose_db_session as get_db
+
+        with get_db() as db:
+            config_proxy = ConfigProxy(db)
+            current_mode = getattr(config_proxy.execution, "sql_dry_run", None)
+            return current_mode == mode_to_check
+
     @staticmethod
     def cursor_result_to_rows(results: CursorResult) -> List[Row]:
         """Convert SQLAlchemy results to a list of dictionaries"""
@@ -140,6 +159,10 @@ class SQLConnector(BaseConnector[Engine]):
         if query is None:
             return []
 
+        if self.should_dry_run(SqlDryRunMode.access):
+            logger.warning(f"SQL DRY RUN - Would execute SQL: {query}")
+            return []
+
         with client.connect() as connection:
             self.set_schema(connection)
             results = connection.execute(query)
@@ -160,6 +183,10 @@ class SQLConnector(BaseConnector[Engine]):
         if stmt is None:
             return []
 
+        if self.should_dry_run(SqlDryRunMode.access):
+            logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
+            return []
+
         logger.info("Starting data retrieval for {}", node.address)
         with client.connect() as connection:
             self.set_schema(connection)
@@ -178,20 +205,25 @@ class SQLConnector(BaseConnector[Engine]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Returns the number of records masked"""
         query_config = self.query_config(node)
         update_ct = 0
         client = self.client()
+
         for row in rows:
             update_stmt: Optional[TextClause] = query_config.generate_update_stmt(
                 row, policy, privacy_request
             )
             if update_stmt is not None:
-                with client.connect() as connection:
-                    self.set_schema(connection)
-                    results: LegacyCursorResult = connection.execute(update_stmt)
-                    update_ct = update_ct + results.rowcount
+                if self.should_dry_run(SqlDryRunMode.erasure):
+                    logger.warning(f"SQL DRY RUN - Would execute SQL: {update_stmt}")
+                else:
+                    with client.connect() as connection:
+                        self.set_schema(connection)
+                        results: LegacyCursorResult = connection.execute(update_stmt)
+                        update_ct = update_ct + results.rowcount
         return update_ct
 
     def close(self) -> None:

fides/api/service/connectors/website_connector.py

@@ -75,6 +75,7 @@ class WebsiteConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not supported for website connector"""
         return 0

fides/api/task/deprecated_graph_task.py

@@ -1,4 +1,5 @@
 # pylint: disable=too-many-lines
+from functools import partial
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
 import dask
@@ -252,16 +253,33 @@ def run_erasure_request_deprecated(  # pylint: disable = too-many-arguments
 
         access_request_data[ROOT_COLLECTION_ADDRESS.value] = [identity]
 
-        dsk: Dict[CollectionAddress, Any] = {
-            k: (
-                t.erasure_request,
+        # Build the dask task graph for erasure, ensuring we also pass along the
+        # upstream access data that may be required by the connector (e.g. BigQuery
+        # delete statements).  We accomplish this by partially applying the
+        # `inputs` kwarg on each task's `erasure_request` method.  The resulting
+        # callable accepts the original positional arguments expected by Dask.
+
+        dsk: Dict[CollectionAddress, Any] = {}
+        for k, t in env.items():
+            # Collect upstream access data in the same order as the input keys
+            upstream_access_data: List[List[Row]] = [
+                access_request_data.get(str(input_key), [])
+                for input_key in t.execution_node.input_keys
+            ]
+
+            # Bind the `inputs` keyword argument so it is supplied when the task executes
+            erasure_fn_with_inputs = partial(
+                t.erasure_request, inputs=upstream_access_data
+            )
+
+            # Build the task tuple: (callable, retrieved_data, *prereqs)
+            dsk[k] = (
+                erasure_fn_with_inputs,
                 access_request_data.get(
                     str(k), []
-                ),  # Pass in the results of the access request for this collection
+                ),  # Data retrieved for this collection
                 *_evaluate_erasure_dependencies(t, erasure_end_nodes),
             )
-            for k, t in env.items()
-        }
 
         # root node returns 0 to be consistent with the output of the other erasure tasks
         dsk[ROOT_COLLECTION_ADDRESS] = 0

fides/api/task/execute_request_tasks.py

@@ -298,17 +298,11 @@ def run_access_node(
                         )
                         # Currently, upstream tasks and "input keys" (which are built by data dependencies)
                         # are the same, but they may not be the same in the future.
-                        ordered_upstream_tasks: List[Optional[RequestTask]] = (
-                            _order_tasks_by_input_key(
+                        upstream_access_data: List[List[Row]] = (
+                            _build_upstream_access_data(
                                 graph_task.execution_node.input_keys, upstream_results
                             )
                         )
-                        # Pass in access data dependencies in the same order as the input keys.
-                        # If we don't have access data for an upstream node, pass in an empty list
-                        upstream_access_data: List[List[Row]] = [
-                            upstream.get_access_data() if upstream else []
-                            for upstream in ordered_upstream_tasks
-                        ]
                         # Run the main access function
                         graph_task.access_request(*upstream_access_data)
 
@@ -359,7 +353,7 @@ def run_erasure_node(
                     session,
                 ) as resources:
                     # Build GraphTask resource to facilitate execution
-                    graph_task: GraphTask = create_graph_task(
+                    erasure_graph_task: GraphTask = create_graph_task(
                         session, request_task, resources
                     )
                     # Get access data that was saved in the erasure format that was collected from the
@@ -368,8 +362,24 @@ def run_erasure_node(
                         request_task.get_data_for_erasures() or []
                     )
 
-                    # Run the main erasure function!
-                    graph_task.erasure_request(retrieved_data)
+                    upstream_access_data: List[List[Row]] = []
+
+                    try:
+                        upstream_access_data = (
+                            get_upstream_access_data_for_erasure_task(
+                                request_task, session, resources
+                            )
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Unable to get upstream access data for erasure task {request_task.collection_address}: {e}"
+                        )
+
+                    # Run the main erasure function, passing along the upstream access data.
+                    # The extra data is currently only needed for generating BigQuery delete statements.
+                    erasure_graph_task.erasure_request(
+                        retrieved_data, inputs=upstream_access_data
+                    )
 
     queue_downstream_tasks_with_retries(
         self,
@@ -487,6 +497,73 @@ def _order_tasks_by_input_key(
     return tasks
 
 
+def get_upstream_access_data_for_erasure_task(
+    erasure_request_task: RequestTask,
+    session: Session,
+    resources: TaskResources,
+) -> List[List[Row]]:
+    """
+    Retrieves upstream access data for a given erasure request task.
+
+    This function finds the corresponding access task for the erasure task,
+    creates a GraphTask to extract input keys, and builds the upstream access data
+    needed for erasure operations (particularly for BigQuery delete statements).
+
+    Args:
+        erasure_request_task: The erasure task that needs upstream access data
+        session: Database session for querying
+        resources: Task resources for creating GraphTask
+
+    Returns:
+        List[List[Row]]: Upstream access data ordered by input keys
+
+    Raises:
+        Exception: If the corresponding access task cannot be found
+    """
+    # Get the corresponding access task for the current erasure task
+    access_request_task = (
+        session.query(RequestTask)
+        .filter(
+            RequestTask.privacy_request_id == erasure_request_task.privacy_request_id,
+            RequestTask.collection_address == erasure_request_task.collection_address,
+            RequestTask.action_type == ActionType.access,
+        )
+        .first()
+    )
+
+    if not access_request_task:
+        raise Exception(
+            f"Unable to find access request task for erasure task {erasure_request_task.collection_address}"
+        )
+
+    # Convert the request task to a GraphTask to get the input_keys
+    access_graph_task: GraphTask = create_graph_task(
+        session, access_request_task, resources
+    )
+
+    # Build and return the upstream access data
+    return _build_upstream_access_data(
+        access_graph_task.execution_node.input_keys,
+        access_request_task.upstream_tasks_objects(session),
+    )
+
+
+def _build_upstream_access_data(
+    input_keys: List[CollectionAddress],
+    upstream_tasks: Query,
+) -> List[List[Row]]:
+    """
+    Helper function to build the access data for the current node.
+    The access data is passed in the same order as the input keys.
+    If we don't have access data for an upstream node, return an empty list.
+    """
+
+    ordered_upstream: List[Optional[RequestTask]] = _order_tasks_by_input_key(
+        input_keys, upstream_tasks
+    )
+    return [task.get_access_data() if task else [] for task in ordered_upstream]
+
+
 mapping = {
     ActionType.access.value: run_access_node,
     ActionType.erasure.value: run_erasure_node,

fides/api/task/graph_task.py

@@ -645,9 +645,15 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
         self,
         retrieved_data: List[Row],
         *erasure_prereqs: int,  # TODO Remove when we stop support for DSR 2.0. DSR 3.0 enforces with downstream_tasks.
+        inputs: Optional[
+            List[List[Row]]
+        ] = None,  # Upstream data from corresponding access task
     ) -> int:
         """Run erasure request"""
 
+        if inputs is None:
+            inputs = []
+
         # if there is no primary key specified in the graph node configuration
         # note this in the execution log and perform no erasures on this node
         if (
@@ -694,6 +700,10 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
             )
             return 0
 
+        formatted_input_data: NodeInput = self.pre_process_input_data(
+            *inputs, group_dependent_fields=True
+        )
+
         # Use execution context to capture postprocessor messages
         with collect_execution_log_messages() as messages:
             output = self.connector.mask_data(
@@ -702,6 +712,7 @@ class GraphTask(ABC):  # pylint: disable=too-many-instance-attributes
                 self.resources.request,
                 self.resources.privacy_request_task,
                 retrieved_data,
+                formatted_input_data,
             )
 
         if self.request_task.id:

fides/api/task/manual/manual_task_graph_task.py

@@ -268,6 +268,7 @@ class ManualTaskGraphTask(GraphTask):
         self,
         retrieved_data: List[Row],
         *erasure_prereqs: int,  # noqa: D401, pylint: disable=unused-argument
+        inputs: Optional[List[List[Row]]] = None,
     ) -> int:
         """Execute manual-task-driven erasure logic.
 

fides/common/api/scope_registry.py

@@ -51,6 +51,7 @@ PRIVACY_REQUEST_ACCESS_RESULTS = "privacy-request-access-results"
 PRIVACY_REQUEST_EMAIL_INTEGRATIONS = "privacy-request-email-integrations"
 PRIVACY_REQUEST_NOTIFICATIONS = "privacy-request-notifications"
 READ = "read"
+READ_OWN = "read-own"
 REGISTER = "register"
 RESET = "reset"
 RESPOND = "respond"
@@ -226,6 +227,7 @@ TAXONOMY_DELETE = f"{TAXONOMY}:{DELETE}"
 USER_CREATE = f"{USER}:{CREATE}"
 USER_DELETE = f"{USER}:{DELETE}"
 USER_READ = f"{USER}:{READ}"
+USER_READ_OWN = f"{USER}:{READ_OWN}"
 USER_UPDATE = f"{USER}:{UPDATE}"
 USER_PASSWORD_RESET = f"{USER}:{PASSWORD_RESET}"
 
@@ -354,6 +356,7 @@ SCOPE_DOCS = {
     USER_UPDATE: "Update users",
     USER_DELETE: "Remove users",
     USER_READ: "View users",
+    USER_READ_OWN: "View own user",
     USER_PASSWORD_RESET: "Reset another user's password",
     USER_PERMISSION_CREATE: "Create user permissions",
     USER_PERMISSION_UPDATE: "Update user permissions",

fides/config/utils.py

@@ -61,6 +61,7 @@ CONFIG_KEY_ALLOWLIST = {
         "task_retry_backoff",
         "require_manual_request_approval",
         "subject_identity_verification_required",
+        "sql_dry_run",
     ],
     "storage": [
         "active_default_storage_type",

fides/ui-build/static/admin/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/b81194f2c3930152.css" as="style"/><link rel="stylesheet" href="/_next/static/css/b81194f2c3930152.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-ff4c22c9f0840531.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-f66151e613766714.js" defer=""></script><script src="/_next/static/chunks/pages/404-2d803dab6a00f353.js" defer=""></script><script src="/_next/static/jeMhTiDcCKPk_H4S0nSQq/_buildManifest.js" defer=""></script><script src="/_next/static/jeMhTiDcCKPk_H4S0nSQq/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"jeMhTiDcCKPk_H4S0nSQq","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/b81194f2c3930152.css" as="style"/><link rel="stylesheet" href="/_next/static/css/b81194f2c3930152.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-ff4c22c9f0840531.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-090643377c8254e6.js" defer=""></script><script src="/_next/static/chunks/pages/_app-a152f52351c84ef4.js" defer=""></script><script src="/_next/static/chunks/pages/404-2d803dab6a00f353.js" defer=""></script><script src="/_next/static/cUz9aQNEfv77_K6F0m_Ja/_buildManifest.js" defer=""></script><script src="/_next/static/cUz9aQNEfv77_K6F0m_Ja/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"cUz9aQNEfv77_K6F0m_Ja","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>