ethyca-fides 2.70.0rc5__py2.py3-none-any.whl → 2.70.1b0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-12T11:27:49-0300",
+ "date": "2025-09-17T11:54:23-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "18d4179893f3166051eda599dc5c257d7dc34839",
- "version": "2.70.0rc5"
+ "full-revisionid": "f42db9940a5b753019726d317e93e58db908e929",
+ "version": "2.70.1b0"
 }
 '''  # END VERSION_JSON
 

fides/api/api/v1/endpoints/system.py

@@ -1,4 +1,4 @@
-from typing import Annotated, Dict, List, Optional, Union
+from typing import Annotated, Dict, List, Literal, Optional, Union
 
 from fastapi import Depends, HTTPException, Query, Response, Security
 from fastapi_pagination import Page, Params
@@ -509,6 +509,8 @@ async def ls(  # pylint: disable=invalid-name
     data_categories: Optional[List[FidesKey]] = Query(None),
     data_subjects: Optional[List[FidesKey]] = Query(None),
     show_deleted: bool = Query(False),
+    sort_by: Optional[List[Literal["name"]]] = Query(None),
+    sort_asc: Optional[bool] = Query(True),
 ) -> Union[List[System], Page[System]]:
     """Get a list of all of the Systems.
     If any parameters or filters are provided the response will be paginated and/or filtered.
@@ -524,6 +526,8 @@ async def ls(  # pylint: disable=invalid-name
         size=size,
         page=page,
         show_deleted=show_deleted,
+        sort_by=sort_by,
+        sort_asc=sort_asc,
     )
 
 

fides/api/db/crud.py

@@ -1,6 +1,6 @@
 """
-Contains all of the generic CRUD endpoints that can be
-generated programmatically for each resource.
+DEPRECATED: This module uses manual transaction handling which can
+lead to unexpected behavior. Use fides.api.db.safe_crud instead.
 """
 
 from collections import defaultdict
@@ -18,6 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 from sqlalchemy.sql import Select
 from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
+from typing_extensions import deprecated
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined]
     CustomField,
@@ -32,6 +33,9 @@ T = TypeVar("T", bound="FidesBase")
 
 
 # CRUD Functions
+@deprecated(
+    "This function uses a manual session.begin() which can lead to unexpected transaction handling. Use create_resource from safe_crud instead."
+)
 async def create_resource(
     sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
 ) -> T:
@@ -66,6 +70,9 @@ async def create_resource(
         return await get_resource(sql_model, resource_dict["fides_key"], async_session)
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use get_custom_fields_filtered from safe_crud instead."
+)
 async def get_custom_fields_filtered(
     async_session: AsyncSession,
     resource_types_to_ids: Dict[ResourceTypes, List[str]] = defaultdict(list),
@@ -111,6 +118,9 @@ async def get_custom_fields_filtered(
                 raise sa_error
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use get_resource from safe_crud instead."
+)
 async def get_resource(
     sql_model: Type[T],
     fides_key: str,
@@ -142,6 +152,9 @@ async def get_resource(
             return sql_resource
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use get_resource_with_custom_fields from safe_crud instead."
+)
 async def get_resource_with_custom_fields(
     sql_model: Type[T], fides_key: str, async_session: AsyncSession
 ) -> Dict[str, Any]:
@@ -193,6 +206,9 @@ async def get_resource_with_custom_fields(
     return resource_dict
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use list_resource from safe_crud instead."
+)
 async def list_resource(sql_model: Type[T], async_session: AsyncSession) -> List[T]:
     """
     Get a list of all of the resources of this type from the database.
@@ -203,6 +219,9 @@ async def list_resource(sql_model: Type[T], async_session: AsyncSession) -> List
     return await list_resource_query(async_session, query, sql_model)
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use list_resource_query from safe_crud instead."
+)
 async def list_resource_query(
     async_session: AsyncSession, query: Select, sql_model: Type[T]
 ) -> List[T]:
@@ -225,6 +244,9 @@ async def list_resource_query(
             return sql_resources
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use update_resource from safe_crud instead."
+)
 async def update_resource(
     sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
 ) -> Dict:
@@ -251,6 +273,9 @@ async def update_resource(
         return await get_resource(sql_model, resource_dict["fides_key"], async_session)
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use upsert_resources from safe_crud instead."
+)
 async def upsert_resources(
     sql_model: Type[T], resource_dicts: List[Dict], async_session: AsyncSession
 ) -> Tuple[int, int]:
@@ -300,6 +325,9 @@ async def upsert_resources(
                 raise sa_error
 
 
+@deprecated(
+    "This function uses manual session.begin() which can lead to unexpected transaction handling. Use delete_resource from safe_crud instead."
+)
 async def delete_resource(
     sql_model: Type[T], fides_key: str, async_session: AsyncSession
 ) -> T:

fides/api/db/safe_crud.py

@@ -0,0 +1,377 @@
+"""
+This module contains "safe" versions of CRUD operations that do NOT
+manually manage database transactions. Transaction management is left
+to the calling code (endpoints, services) to handle at the appropriate
+boundary.
+
+Use functions from this module instead of fides.api.db.crud
+"""
+
+from collections import defaultdict
+from typing import Any, Dict, List, Tuple, Type, TypeVar
+
+from fastapi import HTTPException
+from loguru import logger as log
+from sqlalchemy import and_, column
+from sqlalchemy import delete as _delete
+from sqlalchemy import or_
+from sqlalchemy import update as _update
+from sqlalchemy.dialects.postgresql import Insert as _insert
+from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.future import select
+from sqlalchemy.sql import Select
+from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
+
+from fides.api.models.sql_models import (  # type: ignore[attr-defined]
+    CustomField,
+    CustomFieldDefinition,
+    FidesBase,
+    ResourceTypes,
+)
+from fides.api.util import errors
+
+# Helps return type be linked to the type of the parameter
+T = TypeVar("T", bound="FidesBase")
+
+
+# CRUD Functions
+async def create_resource(
+    sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
+) -> T:
+    """
+    Create a resource in the database.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    with log.contextualize(
+        sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"]
+    ):
+
+        existing_resource = await get_resource(
+            sql_model, resource_dict["fides_key"], async_session, raise_not_found=False
+        )
+
+        if existing_resource is not None:
+            already_exists_error = errors.AlreadyExistsError(
+                sql_model.__name__, resource_dict["fides_key"]
+            )
+            log.bind(error=already_exists_error.detail["error"]).info(  # type: ignore[index]
+                "Failed to insert resource"
+            )
+            raise already_exists_error
+
+        try:
+            log.debug("Creating resource")
+            query = _insert(sql_model.__table__).values(resource_dict)
+            await async_session.execute(query)
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to create resource with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return await get_resource(sql_model, resource_dict["fides_key"], async_session)
+
+
+async def get_custom_fields_filtered(
+    async_session: AsyncSession,
+    resource_types_to_ids: Dict[ResourceTypes, List[str]] = defaultdict(list),
+) -> FidesBase:
+    """
+    Utility function to construct a filtered query for custom field values based on provided mapping of
+    resource types to resource IDs.
+
+    Only custom fields with an "active" CustomFieldDefinition are returned.
+
+    This is for use in bulk querying of custom fields, to avoid multiple round trips to the db.
+    """
+    with log.contextualize(model=CustomField):
+        try:
+            log.debug("Fetching resource")
+            query = select(
+                CustomField.resource_id,
+                CustomField.value,
+                CustomFieldDefinition.resource_type,
+                CustomFieldDefinition.name,
+                CustomFieldDefinition.field_type,
+            ).join(
+                CustomFieldDefinition,
+                CustomFieldDefinition.id == CustomField.custom_field_definition_id,
+            )
+
+            criteria = [
+                and_(
+                    CustomFieldDefinition.resource_type == resource_type.value,
+                    CustomField.resource_id.in_(resource_ids),
+                    # pylint: disable=singleton-comparison
+                    CustomFieldDefinition.active == True,
+                )
+                for resource_type, resource_ids in resource_types_to_ids.items()
+            ]
+            query = query.where(or_(False, *criteria))
+            result = await async_session.execute(query)
+            return result.mappings().all()
+        except SQLAlchemyError as e:
+            sa_error = errors.QueryError()
+            log.exception(f"Failed to fetch custom fields with error: '{e}'")
+            raise sa_error
+
+
+async def get_resource(
+    sql_model: Type[T],
+    fides_key: str,
+    async_session: AsyncSession,
+    raise_not_found: bool = True,
+) -> T:
+    """
+    Get a resource from the database by its FidesKey.
+
+    Returns a SQLAlchemy model of that resource.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
+        try:
+            log.debug("Fetching resource")
+            query = select(sql_model).where(sql_model.fides_key == fides_key)
+            result = await async_session.execute(query)
+        except SQLAlchemyError as e:
+            sa_error = errors.QueryError()
+            log.exception(f"Failed to fetch resource with error: '{e}'")
+            raise sa_error
+
+        sql_resource = result.scalars().first()
+        if sql_resource is None and raise_not_found:
+            not_found_error = errors.NotFoundError(sql_model.__name__, fides_key)
+            log.bind(error=not_found_error.detail["error"]).info("Resource not found")  # type: ignore[index]
+            raise not_found_error
+
+        return sql_resource
+
+
+async def get_resource_with_custom_fields(
+    sql_model: Type[T], fides_key: str, async_session: AsyncSession
+) -> Dict[str, Any]:
+    """Get a resource from the database by its FidesKey including it's custom fields.
+
+    Returns a dictionary of that resource.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    resource: T = await get_resource(sql_model, fides_key, async_session)
+    resource_dict = resource.__dict__
+    resource_dict.pop("_sa_instance_state", None)
+
+    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
+        try:
+            log.debug("Fetching custom fields for resource")
+            query = (
+                select(CustomFieldDefinition.name, CustomField.value)
+                .join(
+                    CustomField,
+                    CustomField.custom_field_definition_id == CustomFieldDefinition.id,
+                )
+                .where(
+                    (CustomField.resource_id == resource.fides_key)
+                    & (  # pylint: disable=singleton-comparison
+                        CustomFieldDefinition.active == True
+                    )
+                )
+            )
+            result = await async_session.execute(query)
+        except SQLAlchemyError as e:
+            sa_error = errors.QueryError()
+            log.exception(f"Failed to fetch custom fields with error: '{e}'")
+            raise sa_error
+
+        custom_fields = result.mappings().all()
+
+    if not custom_fields:
+        return resource_dict
+
+    for field in custom_fields:
+        if field["name"] in resource_dict:
+            resource_dict[field["name"]] = (
+                f"{resource_dict[field['name']]}, {', '.join(field['value'])}"
+            )
+        else:
+            resource_dict[field["name"]] = ", ".join(field["value"])
+
+    return resource_dict
+
+
+async def list_resource(sql_model: Type[T], async_session: AsyncSession) -> List[T]:
+    """
+    Get a list of all of the resources of this type from the database.
+
+    Returns a list of SQLAlchemy models of that resource type.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+    query = select(sql_model)
+    return await list_resource_query(async_session, query, sql_model)
+
+
+async def list_resource_query(
+    async_session: AsyncSession, query: Select, sql_model: Type[T]
+) -> List[T]:
+    """
+    Utility function to wrap a select query in generic "list_resource" execution handling.
+    Wrapping includes execution against the DB session, logging and error handling.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(sql_model=sql_model.__name__):
+        try:
+            log.debug("Fetching resources")
+            result = await async_session.execute(query)
+            sql_resources = result.scalars().all()
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to fetch resources with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return sql_resources
+
+
+async def update_resource(
+    sql_model: Type[T], resource_dict: Dict, async_session: AsyncSession
+) -> Dict:
+    """
+    Update a resource in the database by its fides_key.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(
+        sql_model=sql_model.__name__, fides_key=resource_dict["fides_key"]
+    ):
+        await get_resource(sql_model, resource_dict["fides_key"], async_session)
+
+        try:
+            log.debug("Updating resource")
+            await async_session.execute(
+                _update(sql_model.__table__)
+                .where(sql_model.fides_key == resource_dict["fides_key"])
+                .values(resource_dict)
+            )
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to update resource with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return await get_resource(sql_model, resource_dict["fides_key"], async_session)
+
+
+async def upsert_resources(
+    sql_model: Type[T], resource_dicts: List[Dict], async_session: AsyncSession
+) -> Tuple[int, int]:
+    """
+    Insert new resources into the database. If a resource already exists,
+    update it by it's fides_key.
+
+    Returns a Tuple containing the counts of inserted and updated rows.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(
+        sql_model=sql_model.__name__,
+        fides_keys=[resource["fides_key"] for resource in resource_dicts],
+    ):
+        try:
+            log.debug("Upserting resources")
+            insert_stmt = (
+                _insert(sql_model.__table__)
+                .values(resource_dicts)
+                .returning(
+                    (column("xmax") == 0),  # Row was inserted
+                    (column("xmax") != 0),  # Row was updated
+                )
+            )
+
+            excluded = dict(insert_stmt.excluded.items())  # type: ignore[attr-defined]
+            excluded.pop("id", None)  # If updating, don't update the "id"
+
+            result = await async_session.execute(
+                insert_stmt.on_conflict_do_update(
+                    index_elements=["fides_key"],
+                    set_=excluded,
+                )
+            )
+
+            inserts, updates = 0, 0
+            for xmax in result:
+                inserts += 1 if xmax[0] else 0
+                updates += 1 if xmax[1] else 0
+
+            return (inserts, updates)
+
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to upsert resources with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+
+async def delete_resource(
+    sql_model: Type[T], fides_key: str, async_session: AsyncSession
+) -> T:
+    """
+    Delete a resource by its fides_key.
+
+    If the resource has child keys referring to it, also delete those.
+
+    This version does NOT manually manage transactions - transaction management
+    is left to the calling code.
+    """
+
+    with log.contextualize(sql_model=sql_model.__name__, fides_key=fides_key):
+        sql_resource = await get_resource(sql_model, fides_key, async_session)
+
+        try:
+            # Automatically delete related resources if they are CTL objects
+            if hasattr(sql_model, "parent_key"):
+                log.debug("Deleting resource and its children")
+                query = (
+                    _delete(sql_model.__table__)
+                    .where(
+                        or_(
+                            sql_model.fides_key == fides_key,
+                            sql_model.fides_key.like(f"{fides_key}.%"),
+                        )
+                    )
+                    .execution_options(synchronize_session=False)
+                )
+            else:
+                log.debug("Deleting resource")
+                query = _delete(sql_model.__table__).where(
+                    sql_model.fides_key == fides_key
+                )
+            await async_session.execute(query)
+        except IntegrityError as err:
+            raw_error_text: str = err.orig.args[0]
+
+            if "violates foreign key constraint" in raw_error_text:
+                error_message = "Failed to delete resource! Foreign key constraint found, try deleting related resources first."
+            else:
+                error_message = "Failed to delete resource!"
+
+            log.bind(error="SQL Query integrity error!").error(raw_error_text)
+            raise HTTPException(
+                status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=error_message,
+            )
+        except SQLAlchemyError as e:
+            log.exception(f"Failed to delete resource with error: '{e}'")
+            sa_error = errors.QueryError()
+            raise sa_error
+
+        return sql_resource

fides/api/models/policy.py

@@ -177,6 +177,30 @@ class Policy(Base):
             for category in node.get_data_categories()
         )
 
+    def to_safe_dict(self) -> Dict[str, Any]:
+        """
+        Returns a dictionary representation of the Policy, excluding sensitive information.
+        """
+        return {
+            "id": self.id,
+            "name": self.name,
+            "key": self.key,
+            "drp_action": self.drp_action.value if self.drp_action else None,  # type: ignore[attr-defined]
+            "execution_timeframe": self.execution_timeframe,
+            "client_id": self.client_id,
+            "rules": [
+                {
+                    "id": rule.id,
+                    "name": rule.name,
+                    "key": rule.key,
+                    "action_type": rule.action_type.value,
+                    "masking_strategy": rule.masking_strategy,
+                    "storage_destination_id": rule.storage_destination_id,
+                }
+                for rule in self.rules  # type: ignore[attr-defined]
+            ],
+        }
+
 
 def _get_ref_from_taxonomy(
     fides_key: FidesKey,

fides/api/models/privacy_notice.py

@@ -41,7 +41,7 @@ class UserConsentPreference(Enum):
     tcf = "tcf"  # Overall preference set for TCF where there are numerous preferences under the single notice
 
 
-class ConsentMechanism(Enum):
+class ConsentMechanism(str, Enum):
     """
     Enum is not formalized in the DB because it may be subject to frequent change
     """

fides/api/models/privacy_request/privacy_request.py

@@ -355,6 +355,64 @@ class PrivacyRequest(
 
         return super().create(db=db, data=data, check_name=check_name)
 
+    def to_safe_dict(self) -> Dict[str, Any]:
+        """
+        Return a dict representation of the PrivacyRequest, excluding any fields
+        that may contain sensitive information.
+        """
+        return {
+            "id": self.id,
+            "external_id": self.external_id,
+            "status": self.status.value,
+            "requested_at": (
+                self.requested_at.isoformat() if self.requested_at else None
+            ),
+            "started_processing_at": (
+                self.started_processing_at.isoformat()
+                if self.started_processing_at
+                else None
+            ),
+            "finished_processing_at": (
+                self.finished_processing_at.isoformat()
+                if self.finished_processing_at
+                else None
+            ),
+            "reviewed_at": self.reviewed_at.isoformat() if self.reviewed_at else None,
+            "reviewed_by": self.reviewed_by,
+            "finalized_at": (
+                self.finalized_at.isoformat() if self.finalized_at else None
+            ),
+            "finalized_by": self.finalized_by,
+            "submitted_by": self.submitted_by,
+            "custom_privacy_request_fields_approved_by": (
+                self.custom_privacy_request_fields_approved_by
+            ),
+            "identity_verified_at": (
+                self.identity_verified_at.isoformat()
+                if self.identity_verified_at
+                else None
+            ),
+            "custom_privacy_request_fields_approved_at": (
+                self.custom_privacy_request_fields_approved_at.isoformat()
+                if self.custom_privacy_request_fields_approved_at
+                else None
+            ),
+            "client_id": self.client_id,
+            "origin": self.origin,
+            "policy_id": self.policy_id,
+            "policy": self.policy.to_safe_dict() if self.policy else None,
+            "property_id": self.property_id,
+            "cancel_reason": self.cancel_reason,
+            "canceled_at": self.canceled_at.isoformat() if self.canceled_at else None,
+            "consent_preferences": self.consent_preferences,
+            "source": self.source.value if self.source else None,
+            "paused_at": self.paused_at.isoformat() if self.paused_at else None,
+            "due_date": self.due_date.isoformat() if self.due_date else None,
+            "days_left": self.days_left,
+            "custom_fields": self.get_persisted_custom_privacy_request_fields() if self.custom_fields else None,  # type: ignore[attr-defined]
+            "location": self.location,
+        }
+
     def clear_cached_values(self) -> None:
         """
         Clears all cached values associated with this privacy request from Redis.
@@ -903,6 +961,8 @@ class PrivacyRequest(
             callback_type=CallbackType.pre_approval,
             identity=self.get_cached_identity_data(),
             policy_action=policy_action,
+            privacy_request=self.to_safe_dict(),
+            timestamp=datetime.utcnow(),
         )
         headers = {
             "reply-to-approve": f"/privacy-request/{self.id}/pre-approve/eligible",
@@ -943,6 +1003,8 @@ class PrivacyRequest(
             callback_type=webhook.prefix,
             identity=self.get_cached_identity_data(),
             policy_action=policy_action,
+            privacy_request=self.to_safe_dict(),
+            timestamp=datetime.utcnow(),
         )
 
         headers = {}

fides/api/models/privacy_request/webhook.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import json
 from datetime import datetime
 from enum import Enum as EnumType
-from typing import Optional
+from typing import Any, Dict, Optional
 
 from pydantic import BaseModel, ConfigDict
 
@@ -51,6 +51,8 @@ class SecondPartyRequestFormat(BaseModel):
     identity: Identity
     policy_action: Optional[ActionType] = None
     model_config = ConfigDict(use_enum_values=True)
+    privacy_request: Dict[str, Any]
+    timestamp: datetime
 
 
 def generate_request_callback_resume_jwe(webhook: PolicyPreWebhook) -> str:

fides/api/schemas/connection_configuration/connection_secrets_mongodb.py

@@ -1,4 +1,4 @@
-from typing import ClassVar, List
+from typing import ClassVar, List, Optional
 
 from pydantic import Field
 
@@ -30,10 +30,22 @@ class MongoDBSchema(ConnectionConfigSecretsSchema):
         json_schema_extra={"sensitive": True},
     )
     defaultauthdb: str = Field(
-        title="Default Auth DB",
+        title="Default auth DB",
         description="Used to specify the default authentication database.",
     )
 
+    use_srv: Optional[bool] = Field(
+        False,
+        title="Use SRV",
+        description="Enable SRV record lookup for service discovery (mongodb+srv://). Required for MongoDB Atlas. Enables SSL by default.",
+    )
+
+    ssl_enabled: Optional[bool] = Field(
+        None,
+        title="SSL enabled",
+        description="Enable SSL/TLS encryption. With SRV: defaults to enabled (can override). Without SRV: defaults to disabled.",
+    )
+
     _required_components: ClassVar[List[str]] = [
         "host",
         "username",