ethyca-fides 2.66.1b0__py2.py3-none-any.whl → 2.66.1b1__py2.py3-none-any.whl

fides/api/api/v1/endpoints/user_endpoints.py

@@ -6,6 +6,7 @@ from typing import List, Optional
 
 import jose.exceptions
 from fastapi import Depends, HTTPException, Security
+from fastapi.security import SecurityScopes
 from fastapi_pagination import Page, Params
 from fastapi_pagination.bases import AbstractPage
 from fastapi_pagination.ext.sqlalchemy import paginate
@@ -39,7 +40,9 @@ from fides.api.oauth.roles import APPROVER, VIEWER
 from fides.api.oauth.utils import (
     create_temporary_user_for_login_flow,
     extract_payload,
+    extract_token_and_load_client,
     get_current_user,
+    has_permissions,
     oauth2_scheme,
     verify_oauth_client,
 )
@@ -65,6 +68,7 @@ from fides.common.api.scope_registry import (
     USER_DELETE,
     USER_PASSWORD_RESET,
     USER_READ,
+    USER_READ_OWN,
     USER_UPDATE,
 )
 from fides.common.api.v1 import urn_registry as urls
@@ -107,6 +111,37 @@ def _validate_current_user(user_id: str, user_from_token: FidesUser) -> None:
         )
 
 
+def verify_user_read_scopes(
+    authorization: str = Security(oauth2_scheme),
+    db: Session = Depends(get_db),
+) -> ClientDetail:
+    """
+    Custom dependency that verifies the user has either USER_READ or USER_READ_OWN scope.
+    Returns the client if authorized.
+    """
+    token_data, client = extract_token_and_load_client(authorization, db)
+
+    # Try USER_READ first
+    if has_permissions(
+        token_data=token_data,
+        client=client,
+        endpoint_scopes=SecurityScopes([USER_READ]),
+    ):
+        return client
+
+    if has_permissions(
+        token_data=token_data,
+        client=client,
+        endpoint_scopes=SecurityScopes([USER_READ_OWN]),
+    ):
+        return client
+
+    raise HTTPException(
+        status_code=HTTP_403_FORBIDDEN,
+        detail="Not authorized.",
+    )
+
+
 @router.put(
     urls.USER_DETAIL,
     dependencies=[Security(verify_oauth_client)],
@@ -498,14 +533,38 @@ def delete_user(
 
 @router.get(
     urls.USER_DETAIL,
-    dependencies=[Security(verify_oauth_client, scopes=[USER_READ])],
+    dependencies=[Security(verify_user_read_scopes)],
     response_model=UserResponse,
 )
-def get_user(*, db: Session = Depends(get_db), user_id: str) -> FidesUser:
-    """Returns a User based on an Id"""
+def get_user(
+    *,
+    db: Session = Depends(get_db),
+    user_id: str,
+    client: ClientDetail = Security(verify_user_read_scopes),
+    authorization: str = Security(oauth2_scheme),
+) -> FidesUser:
+    """Returns a User based on an Id. Users with USER_READ_OWN scope can only access their own data."""
     user: Optional[FidesUser] = FidesUser.get_by_key_or_id(db, data={"id": user_id})
     if user is None:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="User not found")
+    token_data, _ = extract_token_and_load_client(authorization, db)
+    # Check if user has USER_READ_OWN scope and is trying to access someone else's data
+    # The verify_user_read_scopes dependency already verified the user has either USER_READ or USER_READ_OWN
+    # We need to check if they have USER_READ_OWN and are accessing their own data
+    if has_permissions(
+        token_data=token_data,
+        client=client,
+        endpoint_scopes=SecurityScopes([USER_READ]),
+    ):
+        logger.debug("Returning user with id: '{}'.", user_id)
+        return user
+
+    # User has USER_READ_OWN scope, check if they're accessing their own data
+    if user.id != client.user_id:
+        raise HTTPException(
+            status_code=HTTP_403_FORBIDDEN,
+            detail="You can only access your own user data with USER_READ_OWN scope.",
+        )
 
     logger.debug("Returning user with id: '{}'.", user_id)
     return user
@@ -513,7 +572,7 @@ def get_user(*, db: Session = Depends(get_db), user_id: str) -> FidesUser:
 
 @router.get(
     urls.USERS,
-    dependencies=[Security(verify_oauth_client, scopes=[USER_READ])],
+    dependencies=[Security(verify_user_read_scopes)],
     response_model=Page[UserResponse],
 )
 def get_users(
@@ -521,11 +580,28 @@ def get_users(
     db: Session = Depends(get_db),
     params: Params = Depends(),
     username: Optional[str] = None,
+    client: ClientDetail = Security(verify_user_read_scopes),
+    authorization: str = Security(oauth2_scheme),
 ) -> AbstractPage[FidesUser]:
-    """Returns a paginated list of all users"""
+    """Returns a paginated list of users. Users with USER_READ_OWN scope only see their own data."""
     query = FidesUser.query(db)
-    if username:
-        query = query.filter(FidesUser.username.ilike(f"%{escape_like(username)}%"))
+
+    # Check if user has USER_READ_OWN scope and filter accordingly
+    # The verify_user_read_scopes dependency already verified the user has either USER_READ or USER_READ_OWN
+    token_data, _ = extract_token_and_load_client(authorization, db)
+    if has_permissions(
+        token_data=token_data,
+        client=client,
+        endpoint_scopes=SecurityScopes([USER_READ]),
+    ):
+        # User has USER_READ scope, can see all users
+        if username:
+            query = query.filter(FidesUser.username.ilike(f"%{escape_like(username)}%"))
+    else:
+        # User has USER_READ_OWN scope, only show their own data
+        query = query.filter(FidesUser.id == client.user_id)
+        if username:
+            query = query.filter(FidesUser.username.ilike(f"%{escape_like(username)}%"))
 
     logger.debug("Returning a paginated list of users.")
 

fides/api/graph/execution.py

@@ -1,6 +1,7 @@
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple
 
 from fideslang.validation import FidesKey
+from loguru import logger
 
 from fides.api.graph.config import (
     Collection,
@@ -117,6 +118,8 @@ class ExecutionNode(Contextualizable):  # pylint: disable=too-many-instance-attr
         The values are cast based on field types, if those types are specified.
         """
         out = {}
+        failed_conversions: Dict[str, Dict[str, Any]] = {}
+
         for key, values in input_data.items():
             path: FieldPath = FieldPath.parse(key)
             field: Optional[Field] = self.collection.field(path)
@@ -126,4 +129,31 @@ class ExecutionNode(Contextualizable):  # pylint: disable=too-many-instance-attr
                 filtered = list(filter(lambda x: x is not None, cast_values))
                 if filtered:
                     out[key] = filtered
+                elif values:  # Had input values but all failed conversion
+                    # Track conversion failures for logging
+                    failed_conversions[key] = {"field": field, "input_values": values}
+
+        # Log conversion failures if any occurred
+        if failed_conversions:
+            field_details = []
+            for field_name, failure_info in failed_conversions.items():
+                field = failure_info["field"]
+                values = failure_info["input_values"]
+
+                expected_type = field.data_type() if field else "unknown"
+                actual_types = {type(v).__name__ for v in values if v is not None}
+                actual_type_str = (
+                    ", ".join(sorted(actual_types)) if actual_types else "NoneType"
+                )
+
+                field_details.append(
+                    f"{field_name} (expected: {expected_type}, got: {actual_type_str})"
+                )
+
+            logger.warning(
+                "Type conversion failures for {}: {}",
+                self.address,
+                "; ".join(field_details),
+            )
+
         return out

fides/api/oauth/roles.py

@@ -47,6 +47,7 @@ from fides.common.api.scope_registry import (
     SYSTEM_READ,
     USER_PERMISSION_ASSIGN_OWNERS,
     USER_READ,
+    USER_READ_OWN,
     WEBHOOK_READ,
 )
 
@@ -126,6 +127,7 @@ viewer_scopes = [  # Intentionally omitted USER_PERMISSION_READ and PRIVACY_REQU
 
 respondent_scopes = [
     PRIVACY_REQUEST_MANUAL_STEPS_RESPOND,  # allows respondents to respond to assigned manual steps
+    USER_READ_OWN,
 ]
 
 external_respondent_scopes = [

fides/api/schemas/application_config.py

@@ -11,6 +11,14 @@ from fides.api.schemas.messaging.messaging import MessagingServiceType
 from fides.config.admin_ui_settings import ErrorNotificationMode
 
 
+class SqlDryRunMode(str, Enum):
+    """SQL dry run mode for controlling execution of SQL statements in privacy requests"""
+
+    none = "none"
+    access = "access"
+    erasure = "erasure"
+
+
 class StorageTypeApiAccepted(Enum):
     """Enum for storage destination types accepted in API updates"""
 
@@ -62,7 +70,9 @@ class ExecutionApplicationConfig(FidesSchema):
     subject_identity_verification_required: Optional[bool] = None
     disable_consent_identity_verification: Optional[bool] = None
     require_manual_request_approval: Optional[bool] = None
-    model_config = ConfigDict(extra="forbid")
+    sql_dry_run: Optional[SqlDryRunMode] = None
+
+    model_config = ConfigDict(use_enum_values=True, extra="forbid")
 
 
 class AdminUIConfig(FidesSchema):

fides/api/service/connectors/base_connector.py

@@ -82,6 +82,7 @@ class BaseConnector(Generic[DB_CONNECTOR_TYPE], ABC):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Return the number of rows that have been updated
 

fides/api/service/connectors/bigquery_connector.py

@@ -1,13 +1,8 @@
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
 from loguru import logger
 from sqlalchemy import text
-from sqlalchemy.engine import (  # type: ignore
-    Connection,
-    Engine,
-    LegacyCursorResult,
-    create_engine,
-)
+from sqlalchemy.engine import Connection, Engine, create_engine  # type: ignore
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import Executable  # type: ignore
 from sqlalchemy.sql.elements import TextClause
@@ -17,6 +12,7 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionTestStatus
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.schemas.application_config import SqlDryRunMode
 from fides.api.schemas.connection_configuration.connection_secrets_bigquery import (
     BigQuerySchema,
 )
@@ -99,6 +95,16 @@ class BigQueryConnector(SQLConnector):
         logger.info(
             f"Executing {len(partition_clauses)} partition queries for node '{query_config.node.address}' in DSR execution"
         )
+
+        if self.should_dry_run(SqlDryRunMode.access):
+            for partition_clause in partition_clauses:
+                existing_bind_params = stmt.compile().params
+                partitioned_stmt = text(
+                    f"{stmt} AND ({text(partition_clause)})"
+                ).params(existing_bind_params)
+                logger.warning(f"SQL DRY RUN - Would execute SQL: {partitioned_stmt}")
+            return []
+
         rows = []
         for partition_clause in partition_clauses:
             logger.debug(
@@ -135,6 +141,39 @@ class BigQueryConnector(SQLConnector):
             logger.exception(f"Error testing connection to remote BigQuery {str(e)}")
             raise ConnectionException(f"Connection error: {e}")
 
+    def _execute_statements_with_sql_dry_run(
+        self,
+        statements: List[Executable],
+        sql_dry_run_enabled: bool,
+        client: Engine,
+    ) -> int:
+        """
+        Execute SQL statements with sql_dry_run support.
+
+        Args:
+            statements: List of SQL statements to execute
+            sql_dry_run_enabled: Whether sql_dry_run mode is enabled
+            client: Database engine
+
+        Returns:
+            int: Number of affected rows (0 in sql_dry_run mode)
+        """
+        if not statements:
+            return 0
+
+        if sql_dry_run_enabled:
+            for stmt in statements:
+                logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
+            return 0
+
+        row_count = 0
+        with client.connect() as connection:
+            for stmt in statements:
+                results = connection.execute(stmt)
+                logger.debug(f"Affected {results.rowcount} rows")
+                row_count += results.rowcount
+        return row_count
+
     def mask_data(
         self,
         node: ExecutionNode,
@@ -142,22 +181,31 @@ class BigQueryConnector(SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Returns the number of records updated or deleted"""
+
         query_config = self.query_config(node)
         update_or_delete_ct = 0
         client = self.client()
-        for row in rows:
-            update_or_delete_stmts: List[Executable] = (
-                query_config.generate_masking_stmt(
-                    node, row, policy, privacy_request, client
-                )
+
+        if query_config.uses_delete_masking_strategy():
+            delete_stmts = query_config.generate_delete(client, input_data or {})
+            logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
+            update_or_delete_ct += self._execute_statements_with_sql_dry_run(
+                delete_stmts, self.should_dry_run(SqlDryRunMode.erasure), client
             )
-            if update_or_delete_stmts:
-                with client.connect() as connection:
-                    for update_or_delete_stmt in update_or_delete_stmts:
-                        results: LegacyCursorResult = connection.execute(
-                            update_or_delete_stmt
-                        )
-                        update_or_delete_ct = update_or_delete_ct + results.rowcount
+        else:
+            for row in rows:
+                update_or_delete_stmts: List[Executable] = query_config.generate_update(
+                    row, policy, privacy_request, client
+                )
+                logger.debug(
+                    f"Generated {len(update_or_delete_stmts)} UPDATE statements"
+                )
+                update_or_delete_ct += self._execute_statements_with_sql_dry_run(
+                    update_or_delete_stmts,
+                    self.should_dry_run(SqlDryRunMode.erasure),
+                    client,
+                )
         return update_or_delete_ct

fides/api/service/connectors/dynamodb_connector.py

@@ -140,8 +140,9 @@ class DynamoDBConnector(BaseConnector[Any]):  # type: ignore
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
-        """Execute a masking requestfor DynamoDB"""
+        """Execute a masking request for DynamoDB"""
 
         query_config = self.query_config(node)
         collection_name = node.address.collection

fides/api/service/connectors/fides_connector.py

@@ -142,6 +142,7 @@ class FidesConnector(BaseConnector[FidesClient]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute an erasure request on remote fides"""
         identity_data = {

fides/api/service/connectors/http_connector.py

@@ -90,6 +90,7 @@ class HTTPSConnector(BaseConnector[None]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Currently not supported as webhooks are not called at the collection level"""
         raise NotImplementedError(

fides/api/service/connectors/manual_task_connector.py

@@ -80,6 +80,7 @@ class ManualTaskConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """
         Manual tasks don't support erasure operations.

fides/api/service/connectors/manual_webhook_connector.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
@@ -53,6 +53,7 @@ class ManualWebhookConnector(BaseConnector[None]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[List[List[Row]]] = None,
     ) -> None:
         """
         Not applicable for a manual webhook.  Manual webhooks are not called as part of the traversal.

fides/api/service/connectors/mongodb_connector.py

@@ -126,6 +126,7 @@ class MongoDBConnector(BaseConnector[MongoClient]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request"""
         query_config = self.query_config(node)

fides/api/service/connectors/okta_connector.py

@@ -82,6 +82,7 @@ class OktaConnector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not supported for Okta connector"""
         return 0

fides/api/service/connectors/query_configs/bigquery_query_config.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Union, cast
 import pydash
 from fideslang.models import MaskingStrategies
 from loguru import logger
-from sqlalchemy import MetaData, Table, text
+from sqlalchemy import MetaData, Table, or_, text
 from sqlalchemy.engine import Engine
 from sqlalchemy.sql import Delete, Update
 from sqlalchemy.sql.elements import ColumnElement, TextClause
@@ -125,6 +125,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
         policy: Policy,
         request: PrivacyRequest,
         client: Engine,
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> Union[List[Update], List[Delete]]:
         """
         Generate a masking statement for BigQuery.
@@ -137,7 +138,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             logger.info(
                 f"Masking override detected for collection {node.address.value}: {masking_override.strategy.value}"
             )
-            return self.generate_delete(row, client)
+            return self.generate_delete(client, input_data or {})
         return self.generate_update(row, policy, request, client)
 
     def generate_update(
@@ -218,27 +219,30 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
 
         return [table.update().where(*where_clauses).values(**final_update_map)]
 
-    def generate_delete(self, row: Row, client: Engine) -> List[Delete]:
-        """Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
+    def generate_delete(
+        self,
+        client: Engine,
+        input_data: Optional[Dict[str, List[Any]]] = None,
+    ) -> List[Delete]:
+        """
+        Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
 
         Used when a collection-level masking override is present and the masking strategy is DELETE.
 
         A List of multiple DELETE statements are returned for partitioned tables; for a non-partitioned table,
         a single DELETE statement is returned in a List for consistent typing.
-
-        TODO: DRY up this method and `generate_update` a bit
         """
 
-        non_empty_reference_field_keys: Dict[str, Field] = filter_nonempty_values(
-            {
-                fpath.string_path: fld.cast(row[fpath.string_path])
-                for fpath, fld in self.reference_field_paths.items()
-                if fpath.string_path in row
-            }
-        )
+        if not input_data:
+            logger.warning(
+                "No input data provided for node '{}', skipping DELETE statement generation",
+                self.node.address,
+            )
+            return []
 
-        valid = len(non_empty_reference_field_keys) > 0
-        if not valid:
+        filtered_data = self.node.typed_filtered_values(input_data)
+
+        if not filtered_data:
             logger.warning(
                 "There is not enough data to generate a valid DELETE statement for {}",
                 self.node.address,
@@ -246,9 +250,17 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
             return []
 
         table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
-        where_clauses: List[ColumnElement] = [
-            getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items()
-        ]
+
+        # Build individual reference clauses
+        where_clauses: List[ColumnElement] = []
+        for column_name, values in filtered_data.items():
+            if len(values) == 1:
+                where_clauses.append(getattr(table.c, column_name) == values[0])
+            else:
+                where_clauses.append(getattr(table.c, column_name).in_(values))
+
+        # Combine reference clauses with OR instead of AND
+        combined_reference_clause = or_(*where_clauses)
 
         if self.partitioning:
             partition_clauses = self.get_partition_clauses()
@@ -259,12 +271,25 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
 
             for partition_clause in partition_clauses:
                 partitioned_queries.append(
-                    table.delete().where(*(where_clauses + [text(partition_clause)]))
+                    table.delete()
+                    .where(combined_reference_clause)
+                    .where(text(partition_clause))
                 )
 
             return partitioned_queries
 
-        return [table.delete().where(*where_clauses)]
+        return [table.delete().where(combined_reference_clause)]
+
+    def uses_delete_masking_strategy(self) -> bool:
+        """Check if this collection uses DELETE masking strategy.
+
+        Returns True if masking override is present and strategy is DELETE.
+        """
+        masking_override = self.node.collection.masking_strategy_override
+        return (
+            masking_override is not None
+            and masking_override.strategy == MaskingStrategies.DELETE
+        )
 
     def format_fields_for_query(
         self,

fides/api/service/connectors/rds_mysql_connector.py

@@ -158,6 +158,7 @@ class RDSMySQLConnector(RDSConnectorMixin, SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for RDS MySQL"""
         return 0

fides/api/service/connectors/rds_postgres_connector.py

@@ -147,6 +147,7 @@ class RDSPostgresConnector(RDSConnectorMixin, SQLConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for RDS Postgres"""
         return 0

fides/api/service/connectors/s3_connector.py

@@ -66,6 +66,7 @@ class S3Connector(BaseConnector):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """DSR execution not yet supported for S3"""
         return 0

fides/api/service/connectors/saas_connector.py

@@ -517,6 +517,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request. Return the number of rows that have been updated."""
         self.set_privacy_request_state(privacy_request, node, request_task)

fides/api/service/connectors/scylla_connector.py

@@ -150,6 +150,7 @@ class ScyllaConnector(BaseConnector[Cluster]):
         privacy_request: PrivacyRequest,
         request_task: RequestTask,
         rows: List[Row],
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> int:
         """Execute a masking request"""
         query_config = self.query_config(node)