ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0__py2.py3-none-any.whl

fides/api/models/privacy_request/privacy_request.py

@@ -50,6 +50,7 @@ from fides.api.models.comment import Comment, CommentReference, CommentReference
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.field_types import EncryptedLargeDataDescriptor
 from fides.api.models.manual_webhook import AccessManualWebhook
+from fides.api.models.masking_secret import MaskingSecret
 from fides.api.models.policy import (
     Policy,
     PolicyPreWebhook,
@@ -98,7 +99,6 @@ from fides.api.util.cache import (
     get_drp_request_body_cache_key,
     get_encryption_cache_key,
     get_identity_cache_key,
-    get_masking_secret_cache_key,
 )
 from fides.api.util.collection_util import Row, extract_key_for_address
 from fides.api.util.constants import API_DATE_FORMAT
@@ -145,6 +145,12 @@ class PrivacyRequest(
         ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
         nullable=True,
     )
+    finalized_at = Column(DateTime(timezone=True), nullable=True)
+    finalized_by = Column(
+        String,
+        ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
+        nullable=True,
+    )
     submitted_by = Column(
         String,
         ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
@@ -289,6 +295,13 @@ class PrivacyRequest(
         order_by="RequestTask.created_at",
     )
 
+    masking_secrets: "RelationshipProperty[List[MaskingSecret]]" = relationship(
+        "MaskingSecret",
+        back_populates="privacy_request",
+        uselist=True,
+        passive_deletes="all",
+    )
+
     @property
     def days_left(self: PrivacyRequest) -> Union[int, None]:
         if self.due_date is None:
@@ -566,19 +579,24 @@ class PrivacyRequest(
             encryption_key,
         )
 
-    def cache_masking_secret(self, masking_secret: MaskingSecretCache) -> None:
-        """Sets masking encryption secrets in the Fides app cache if provided"""
-        if not masking_secret:
+    def persist_masking_secrets(
+        self, masking_secrets: List[MaskingSecretCache]
+    ) -> None:
+        """Persists masking encryption secrets to database."""
+        if not masking_secrets:
             return
-        cache: FidesopsRedis = get_cache()
-        cache.set_with_autoexpire(
-            get_masking_secret_cache_key(
-                self.id,
-                masking_strategy=masking_secret.masking_strategy,
-                secret_type=masking_secret.secret_type,
-            ),
-            FidesopsRedis.encode_obj(masking_secret.secret),
-        )
+
+        session = Session.object_session(self)
+        for masking_secret in masking_secrets:
+            MaskingSecret.create(
+                db=session,
+                data={
+                    "privacy_request_id": self.id,
+                    "secret": masking_secret.secret,
+                    "masking_strategy": masking_secret.masking_strategy,
+                    "secret_type": masking_secret.secret_type,
+                },
+            )
 
     def get_cached_identity_data(self) -> Dict[str, Any]:
         """Retrieves any identity data pertaining to this request from the cache"""
@@ -991,6 +1009,38 @@ class PrivacyRequest(
                 request_task_celery_ids.append(request_task_id)
         return request_task_celery_ids
 
+    def cancel_celery_tasks(self) -> None:
+        """Cancel all Celery tasks associated with this privacy request.
+
+        This includes both the main privacy request task and any sub-tasks (Request Tasks).
+        """
+        task_ids: List[str] = []
+
+        # Add the main privacy request task ID
+        parent_task_id = self.get_cached_task_id()
+        if parent_task_id:
+            task_ids.append(parent_task_id)
+
+        # Add all request task IDs
+        request_task_celery_ids = self.get_request_task_celery_task_ids()
+        task_ids.extend(request_task_celery_ids)
+
+        if not task_ids:
+            return
+
+        # Revoke all Celery tasks in batch
+        logger.info(f"Revoking {len(task_ids)} tasks for privacy request {self.id}")
+        try:
+            # Use terminate=False to allow graceful shutdown if already running
+            celery_app.control.revoke(task_ids, terminate=False)
+            logger.info(
+                f"Successfully revoked {len(task_ids)} tasks for privacy request {self.id}"
+            )
+        except Exception as exc:
+            logger.warning(
+                f"Failed to revoke {len(task_ids)} tasks for privacy request {self.id}: {exc}"
+            )
+
     def cancel_processing(self, db: Session, cancel_reason: Optional[str]) -> None:
         """Cancels a privacy request.  Currently should only cancel 'pending' tasks
 
@@ -1003,19 +1053,7 @@ class PrivacyRequest(
             self.canceled_at = datetime.utcnow()
             self.save(db)
 
-            task_ids: List[str] = (
-                self.get_request_task_celery_task_ids()
-            )  # Celery tasks for sub tasks (DSR 3.0 Request Tasks)
-            parent_task_id = (
-                self.get_cached_task_id()
-            )  # Celery task for current Privacy Request
-            if parent_task_id:
-                task_ids.append(parent_task_id)
-
-            for celery_task_id in task_ids:
-                logger.info("Revoking task {} for request {}", celery_task_id, self.id)
-                # Only revokes if execution is not already in progress.
-                celery_app.control.revoke(celery_task_id, terminate=False)
+            self.cancel_celery_tasks()
 
     def error_processing(self, db: Session) -> None:
         """Mark privacy request as errored, and note time processing was finished"""

fides/api/oauth/roles.py

@@ -47,6 +47,7 @@ from fides.common.api.scope_registry import (
     SYSTEM_READ,
     USER_PERMISSION_ASSIGN_OWNERS,
     USER_READ,
+    USER_READ_OWN,
     WEBHOOK_READ,
 )
 
@@ -126,6 +127,7 @@ viewer_scopes = [  # Intentionally omitted USER_PERMISSION_READ and PRIVACY_REQU
 
 respondent_scopes = [
     PRIVACY_REQUEST_MANUAL_STEPS_RESPOND,  # allows respondents to respond to assigned manual steps
+    USER_READ_OWN,
 ]
 
 external_respondent_scopes = [

fides/api/schemas/application_config.py

@@ -11,6 +11,14 @@ from fides.api.schemas.messaging.messaging import MessagingServiceType
 from fides.config.admin_ui_settings import ErrorNotificationMode
 
 
+class SqlDryRunMode(str, Enum):
+    """SQL dry run mode for controlling execution of SQL statements in privacy requests"""
+
+    none = "none"
+    access = "access"
+    erasure = "erasure"
+
+
 class StorageTypeApiAccepted(Enum):
     """Enum for storage destination types accepted in API updates"""
 
@@ -62,7 +70,10 @@ class ExecutionApplicationConfig(FidesSchema):
     subject_identity_verification_required: Optional[bool] = None
     disable_consent_identity_verification: Optional[bool] = None
     require_manual_request_approval: Optional[bool] = None
-    model_config = ConfigDict(extra="forbid")
+    memory_watchdog_enabled: Optional[bool] = None
+    sql_dry_run: Optional[SqlDryRunMode] = None
+
+    model_config = ConfigDict(use_enum_values=True, extra="forbid")
 
 
 class AdminUIConfig(FidesSchema):

fides/api/schemas/connection_configuration/connection_secrets_datahub.py

@@ -16,6 +16,15 @@ class PeriodicIntegrationFrequency(Enum):
     daily = "daily"
     weekly = "weekly"
     monthly = "monthly"
+    not_scheduled = "not scheduled"
+
+
+PERIODIC_INTEGRATION_FREQUENCY_TO_DAYS = {
+    PeriodicIntegrationFrequency.daily.value: 1,
+    PeriodicIntegrationFrequency.weekly.value: 7,
+    PeriodicIntegrationFrequency.monthly.value: 30,
+    PeriodicIntegrationFrequency.not_scheduled.value: -1,  # negative value to indicate that the integration is not scheduled
+}
 
 
 class DatahubSchema(ConnectionConfigSecretsSchema):
@@ -30,7 +39,7 @@ class DatahubSchema(ConnectionConfigSecretsSchema):
     )
     frequency: PeriodicIntegrationFrequency = Field(
         title="Frequency",
-        description="The frequency at which the integration should run. Available options are daily, weekly, and monthly.",
+        description="The frequency at which the integration should run. Available options are daily, weekly, monthly, and not scheduled.",
     )
     glossary_node: str = Field(
         title="Glossary node",

fides/api/schemas/masking/masking_secrets.py

@@ -5,7 +5,7 @@ from typing import Callable, Generic, TypeVar
 T = TypeVar("T")
 
 
-class SecretType(Enum):
+class SecretType(str, Enum):
     """Enum that holds all possible types of secrets across all masking strategies"""
 
     key = "key"

fides/api/schemas/policy.py

@@ -28,6 +28,7 @@ class CurrentStep(EnumType):
     finalize_consent = "finalize_consent"
     email_post_send = "email_post_send"
     post_webhooks = "post_webhooks"
+    finalization = "finalization"
 
 
 # action types we actively support in policies/requests

fides/api/schemas/privacy_request.py

@@ -124,6 +124,8 @@ class PrivacyRequestResubmit(PrivacyRequestCreate):
     identity_verified_at: Optional[datetime] = None
     custom_privacy_request_fields_approved_at: Optional[datetime] = None
     custom_privacy_request_fields_approved_by: Optional[str] = None
+    finalized_at: Optional[datetime] = None
+    finalized_by: Optional[str] = None
 
 
 class ConsentRequestCreate(FidesSchema):
@@ -287,6 +289,7 @@ class PrivacyRequestStatus(str, EnumType):
     complete = "complete"
     paused = "paused"
     awaiting_email_send = "awaiting_email_send"
+    requires_manual_finalization = "requires_manual_finalization"
     canceled = "canceled"
     error = "error"
 
@@ -320,6 +323,8 @@ class PrivacyRequestResponse(FidesSchema):
     source: Optional[PrivacyRequestSource] = None
     deleted_at: Optional[datetime] = None
     deleted_by: Optional[str] = None
+    finalized_at: Optional[datetime] = None
+    finalized_by: Optional[str] = None
     model_config = ConfigDict(from_attributes=True, use_enum_values=True)
 
 

fides/api/service/connectors/base_connector.py

@@ -82,6 +82,7 @@ class BaseConnector(Generic[DB_CONNECTOR_TYPE], ABC):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Return the number of rows that have been updated
 
@@ -143,3 +144,17 @@ class BaseConnector(Generic[DB_CONNECTOR_TYPE], ABC):
         # Defaulting to true for now so we can keep the default behavior and
         # incrementally determine the need for primary keys across all connectors
         return True
+
+    def get_qualified_table_name(self, node: ExecutionNode) -> str:
+        """
+        Get the fully qualified table name for the given execution node.
+        """
+        raise NotImplementedError(
+            "get_qualified_table_name is not implemented by this connector"
+        )
+
+    def table_exists(self, qualified_table_name: str) -> bool:
+        """
+        Check if a table exists in the datastore.
+        """
+        raise NotImplementedError("table_exists is not implemented by this connector")

fides/api/service/connectors/bigquery_connector.py

@@ -1,13 +1,8 @@
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
 from loguru import logger
 from sqlalchemy import text
-from sqlalchemy.engine import (  # type: ignore
-    Connection,
-    Engine,
-    LegacyCursorResult,
-    create_engine,
-)
+from sqlalchemy.engine import Connection, Engine, create_engine  # type: ignore
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import Executable  # type: ignore
 from sqlalchemy.sql.elements import TextClause
@@ -17,6 +12,7 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.schemas.application_config import SqlDryRunMode
 from fides.api.schemas.connection_configuration.connection_secrets_bigquery import (
     BigQuerySchema,
 )
@@ -74,6 +70,11 @@ class BigQueryConnector(SQLConnector):
             node, SQLConnector.get_namespace_meta(db, node.address.dataset)
         )
 
+    def get_qualified_table_name(self, node: ExecutionNode) -> str:
+        """Get fully qualified BigQuery table name using existing query config logic"""
+        query_config = self.query_config(node)
+        return query_config.generate_table_name()
+
     def partitioned_retrieval(
         self,
         query_config: SQLQueryConfig,
@@ -99,6 +100,16 @@ class BigQueryConnector(SQLConnector):
         logger.info(
             f"Executing {len(partition_clauses)} partition queries for node '{query_config.node.address}' in DSR execution"
         )
+
+        if self.should_dry_run(SqlDryRunMode.access):
+            for partition_clause in partition_clauses:
+                existing_bind_params = stmt.compile().params
+                partitioned_stmt = text(
+                    f"{stmt} AND ({text(partition_clause)})"
+                ).params(existing_bind_params)
+                logger.warning(f"SQL DRY RUN - Would execute SQL: {partitioned_stmt}")
+            return []
+
         rows = []
         for partition_clause in partition_clauses:
             logger.debug(
@@ -135,6 +146,39 @@ class BigQueryConnector(SQLConnector):
             logger.exception(f"Error testing connection to remote BigQuery {str(e)}")
             raise ConnectionException(f"Connection error: {e}")
 
+    def _execute_statements_with_sql_dry_run(
+        self,
+        statements: List[Executable],
+        sql_dry_run_enabled: bool,
+        client: Engine,
+    ) -> int:
+        """
+        Execute SQL statements with sql_dry_run support.
+
+        Args:
+            statements: List of SQL statements to execute
+            sql_dry_run_enabled: Whether sql_dry_run mode is enabled
+            client: Database engine
+
+        Returns:
+            int: Number of affected rows (0 in sql_dry_run mode)
+        """
+        if not statements:
+            return 0
+
+        if sql_dry_run_enabled:
+            for stmt in statements:
+                logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
+            return 0
+
+        row_count = 0
+        with client.connect() as connection:
+            for stmt in statements:
+                results = connection.execute(stmt)
+                logger.debug(f"Affected {results.rowcount} rows")
+                row_count += results.rowcount
+        return row_count
+
     def mask_data(
         self,
         node: ExecutionNode,
@@ -142,22 +186,31 @@ class BigQueryConnector(SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Returns the number of records updated or deleted"""
+
         query_config = self.query_config(node)
         update_or_delete_ct = 0
         client = self.client()
-        for row in rows:
-            update_or_delete_stmts: List[Executable] = (
-                query_config.generate_masking_stmt(
-                    node, row, policy, privacy_request, client
-                )
+
+        if query_config.uses_delete_masking_strategy():
+            delete_stmts = query_config.generate_delete(client, input_data or {})
+            logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
+            update_or_delete_ct += self._execute_statements_with_sql_dry_run(
+                delete_stmts, self.should_dry_run(SqlDryRunMode.erasure), client
             )
-            if update_or_delete_stmts:
-                with client.connect() as connection:
-                    for update_or_delete_stmt in update_or_delete_stmts:
-                        results: LegacyCursorResult = connection.execute(
-                            update_or_delete_stmt
-                        )
-                        update_or_delete_ct = update_or_delete_ct + results.rowcount
+        else:
+            for row in rows:
+                update_or_delete_stmts: List[Executable] = query_config.generate_update(
+                    row, policy, privacy_request, client
+                )
+                logger.debug(
+                    f"Generated {len(update_or_delete_stmts)} UPDATE statements"
+                )
+                update_or_delete_ct += self._execute_statements_with_sql_dry_run(
+                    update_or_delete_stmts,
+                    self.should_dry_run(SqlDryRunMode.erasure),
+                    client,
+                )
         return update_or_delete_ct

fides/api/service/connectors/dynamodb_connector.py

@@ -140,8 +140,9 @@ class DynamoDBConnector(BaseConnector[Any]):  # type: ignore
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
-        """Execute a masking requestfor DynamoDB"""
+        """Execute a masking request for DynamoDB"""
 
         query_config = self.query_config(node)
         collection_name = node.address.collection

fides/api/service/connectors/fides_connector.py

@@ -142,6 +142,7 @@ class FidesConnector(BaseConnector[FidesClient]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute an erasure request on remote fides"""
         identity_data = {

fides/api/service/connectors/http_connector.py

@@ -90,6 +90,7 @@ class HTTPSConnector(BaseConnector[None]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Currently not supported as webhooks are not called at the collection level"""
         raise NotImplementedError(

fides/api/service/connectors/manual_task_connector.py

@@ -80,6 +80,7 @@ class ManualTaskConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """
         Manual tasks don't support erasure operations.

fides/api/service/connectors/manual_webhook_connector.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
@@ -53,6 +53,7 @@ class ManualWebhookConnector(BaseConnector[None]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[List[List[Row]]] = None,
     ) -> None:
         """
         Not applicable for a manual webhook.  Manual webhooks are not called as part of the traversal.

fides/api/service/connectors/mongodb_connector.py

@@ -126,6 +126,7 @@ class MongoDBConnector(BaseConnector[MongoClient]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request"""
         query_config = self.query_config(node)

fides/api/service/connectors/okta_connector.py

@@ -82,6 +82,7 @@ class OktaConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not supported for Okta connector"""
         return 0