ethyca-fides 2.56.2rc3__py2.py3-none-any.whl → 2.56.3b0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-03-10T02:03:21-0400",
+ "date": "2025-03-10T08:30:39-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "9880864b483d2a61286e5e387cc99fa99f3098f1",
- "version": "2.56.2rc3"
+ "full-revisionid": "fb9dcf03471ea234b9b23a5925590aa995bdb8f5",
+ "version": "2.56.3b0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/3c58001ad310_add_messages_to_monitor_execution.py

@@ -0,0 +1,32 @@
+"""add errors to monitor execution
+
+Revision ID: 3c58001ad310
+Revises: bd875a8b5d96
+Create Date: 2025-02-27 20:45:38.462604
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3c58001ad310"
+down_revision = "bd875a8b5d96"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "monitorexecution",
+        sa.Column(
+            "messages",
+            sa.ARRAY(sa.String()),
+            nullable=False,
+            server_default="{}",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("monitorexecution", "messages")

fides/api/alembic/migrations/versions/bd875a8b5d96_add_attachments_and_.py

@@ -0,0 +1,96 @@
+"""Add Attachment and AttachmentReference tables
+
+Revision ID: bd875a8b5d96
+Revises: 82883b0df5e4
+Create Date: 2025-02-19 14:09:07.124680
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "bd875a8b5d96"
+down_revision = "82883b0df5e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # create attachment table
+    op.create_table(
+        "attachment",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column("user_id", sa.String(), nullable=True),
+        sa.Column("file_name", sa.String(), nullable=False),
+        sa.Column("attachment_type", sa.String(), nullable=False),
+        sa.Column("storage_key", sa.String(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["fidesuser.id"],
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_attachment_id"), "attachment", ["id"], unique=False)
+
+    # create attachment_reference table
+    op.create_table(
+        "attachment_reference",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("attachment_id", sa.String(), nullable=False),
+        sa.Column("reference_id", sa.String(), nullable=False),
+        sa.Column("reference_type", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["attachment_id"],
+            ["attachment.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_attachment_reference_id"),
+        "attachment_reference",
+        ["id"],
+        unique=False,
+    )
+    op.create_unique_constraint(
+        "_attachment_reference_uc",
+        "attachment_reference",
+        ["attachment_id", "reference_id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_attachment_reference_id"), table_name="attachment_reference")
+    op.drop_table("attachment_reference")
+    op.drop_index(op.f("ix_attachment_id"), table_name="attachment")
+    op.drop_table("attachment")
+    # ### end Alembic commands ###

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -174,7 +174,7 @@ from fides.service.privacy_request.privacy_request_service import (
     PrivacyRequestService,
     _process_privacy_request_restart,
     _requeue_privacy_request,
-    _trigger_pre_approval_webhooks,
+    handle_approval,
     queue_privacy_request,
 )
 
@@ -1310,19 +1310,7 @@ def verify_identification_code(
 
     logger.info("Identity verified for {}.", privacy_request.id)
 
-    if config_proxy.execution.require_manual_request_approval:
-        _trigger_pre_approval_webhooks(db, privacy_request)
-    else:
-        AuditLog.create(
-            db=db,
-            data={
-                "user_id": "system",
-                "privacy_request_id": privacy_request.id,
-                "action": AuditLogAction.approved,
-                "message": "",
-            },
-        )
-        queue_privacy_request(privacy_request.id)
+    handle_approval(db, config_proxy, privacy_request)
 
     return privacy_request  # type: ignore[return-value]
 

fides/api/graph/node_filters.py

@@ -1,19 +1,27 @@
-from typing import TYPE_CHECKING, Optional, Protocol
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
 
+from fides.api.common_exceptions import TraversalError, UnreachableNodesError
+from fides.api.graph.graph import DatasetGraph
 from fides.api.models.policy import Policy
+from fides.api.util.logger import suppress_logging
 
 if TYPE_CHECKING:
     from fides.api.graph.traversal import TraversalNode
 
 
-class NodeFilter(Protocol):
-    """Protocol for node filters that determine if nodes should be excluded from traversal"""
+class NodeFilter(ABC):
+    """Abstract base class for node filters that determine if nodes should be excluded from traversal"""
 
+    def __init__(self) -> None:
+        self.skipped_nodes: Dict[str, str] = {}
+
+    @abstractmethod
     def exclude_node(self, node: "TraversalNode") -> bool:
         """Returns True if the node should be excluded from traversal requirements"""
 
 
-class CustomRequestFieldFilter:
+class CustomRequestFieldFilter(NodeFilter):
     """
     Remove nodes that have custom request fields, since we don't care if these are reachable or not.
     They will be used independently by the Dynamic Email Erasure Connector.
@@ -26,16 +34,115 @@ class CustomRequestFieldFilter:
         return bool(node.node.collection.custom_request_fields())
 
 
-class PolicyDataCategoryFilter:
+class PolicyDataCategoryFilter(NodeFilter):
     """
     If a policy is provided, we will allow collections without relevant
     data categories to remain unreachable without raising any exceptions.
     """
 
     def __init__(self, policy: Optional[Policy]):
+        super().__init__()
         self.policy = policy
 
     def exclude_node(self, node: "TraversalNode") -> bool:
         if not self.policy:
             return False
         return not self.policy.applies_to(node)
+
+
+class OptionalIdentityFilter(NodeFilter):
+
+    def __init__(self, graph: DatasetGraph, identities: Dict[str, Any]):
+        """
+        Filter that excludes nodes that are reachable by an optional identity.
+        This prevents traversal errors for nodes that would be reachable if the required identities were provided.
+        """
+
+        super().__init__()
+        self.graph = graph
+        self.identities = identities
+        self.skipped_nodes: Dict[str, str] = {}
+
+        # Pre-compute which nodes are reachable with each identity
+        self.reachable_by_identity: Dict[str, Set[str]] = {}
+
+        # Get all collection addresses as strings
+        all_collection_addresses = {str(addr) for addr in graph.nodes.keys()}
+
+        # Test reachability with each possible identity seed
+        for identity_key in graph.identity_keys.values():
+            self._compute_reachable_nodes(identity_key, all_collection_addresses)
+
+    def exclude_node(self, node: "TraversalNode") -> bool:
+        """
+        Exclude nodes that are reachable by any identity, provided or optional.
+        Nodes (collections) not reachable by any identities should not be excluded and still be considered an error.
+        """
+        node_address = str(node.address)
+
+        # Check if node is reachable by any provided identity
+        for identity_key in self.identities:
+            if node_address in self.reachable_by_identity.get(identity_key, set()):
+                return True
+
+        # Find all optional identities that can reach this node
+        reaching_identities = []
+        for identity_key, reachable_nodes in self.reachable_by_identity.items():
+            if identity_key not in self.identities and node_address in reachable_nodes:
+                reaching_identities.append(identity_key)
+
+        # If any optional identities can reach this node, record them and exclude
+        if reaching_identities:
+            self.skipped_nodes[node_address] = self._format_skip_message(
+                node.address.value, reaching_identities
+            )
+            return True
+
+        # Node isn't reachable by any identity, don't exclude it (will be an error)
+        return False
+
+    def _compute_reachable_nodes(
+        self, identity_key: str, all_addresses: Set[str]
+    ) -> None:
+        """Helper method to compute which nodes are reachable with a given identity key"""
+
+        from fides.api.graph.traversal import BaseTraversal
+
+        try:
+            # Create a traversal object with just this identity
+            with suppress_logging():
+                # Suppress the logs since we don't want to flood the logs
+                # with traversal info for each identity we want to evaluate
+                BaseTraversal(self.graph, {identity_key: "dummy_value"})
+
+            # If successful, all nodes are reachable
+            self.reachable_by_identity[identity_key] = all_addresses
+        except UnreachableNodesError as exc:
+            # Get reachable nodes by removing unreachable ones from all nodes
+            unreachable = set(exc.errors)
+            self.reachable_by_identity[identity_key] = all_addresses - unreachable
+        except TraversalError:
+            # If traversal fails for other reasons, no nodes are reachable
+            self.reachable_by_identity[identity_key] = set()
+
+    def _format_skip_message(
+        self, node_address: str, reaching_identities: List[str]
+    ) -> str:
+        """
+        Format the skip message with proper grammar based on identity counts.
+        """
+
+        # Format lists of quoted identities
+        reachable_ids_str = ", ".join(f'"{id}"' for id in reaching_identities)
+        provided_ids = list(self.identities.keys())
+        provided_ids_str = ", ".join(f'"{id}"' for id in provided_ids)
+
+        # Determine singular/plural forms based on counts
+        reachable_term = "identity" if len(reaching_identities) == 1 else "identities"
+        provided_term = "identity" if len(provided_ids) == 1 else "identities"
+        was_were = "was" if len(provided_ids) == 1 else "were"
+
+        return (
+            f'Skipping the "{node_address}" collection, it is reachable by the {reachable_ids_str} '
+            f"{reachable_term} but only the {provided_ids_str} {provided_term} {was_were} provided"
+        )

fides/api/graph/traversal.py

@@ -28,6 +28,7 @@ from fides.api.graph.graph import DatasetGraph, Edge, Node
 from fides.api.graph.node_filters import (
     CustomRequestFieldFilter,
     NodeFilter,
+    OptionalIdentityFilter,
     PolicyDataCategoryFilter,
 )
 from fides.api.models.policy import Policy
@@ -64,7 +65,7 @@ def artificial_traversal_node(address: CollectionAddress) -> TraversalNode:
     return TraversalNode(node)
 
 
-class Traversal:
+class BaseTraversal:
     """Handling for a single reified traversal of a graph based on input (seed) data."""
 
     def extract_seed_field_addresses(
@@ -83,6 +84,7 @@ class Traversal:
         self,
         graph: DatasetGraph,
         data: Dict[str, Any],
+        *,
         policy: Optional[Policy] = None,
         node_filters: Optional[List[NodeFilter]] = None,
     ):
@@ -93,9 +95,6 @@ class Traversal:
         # Node filters are used to allow unreachable nodes to be ignored
         # if they don't apply to the given scenario
         self.node_filters = node_filters or []
-        self.node_filters.append(CustomRequestFieldFilter())
-        if policy:
-            self.node_filters.append(PolicyDataCategoryFilter(policy))
 
         self.traversal_node_dict = {k: TraversalNode(v) for k, v in graph.nodes.items()}
         self.edges: Set[Edge] = graph.edges.copy()
@@ -289,6 +288,37 @@ class Traversal:
             logger.debug("Found {} end nodes: {}", len(end_nodes), end_nodes)
         return end_nodes
 
+    @property
+    def skipped_nodes(self) -> Dict[str, str]:
+        """
+        Returns a dictionary of node addresses to skip reasons from all node filters.
+        """
+        result = {}
+        for node_filter in self.node_filters:
+            result.update(node_filter.skipped_nodes)
+        return result
+
+
+class Traversal(BaseTraversal):
+    """Handling for a single reified traversal of a graph based on input (seed) data."""
+
+    def __init__(
+        self,
+        graph: DatasetGraph,
+        data: Dict[str, Any],
+        *,
+        policy: Optional[Policy] = None,
+        node_filters: Optional[List[NodeFilter]] = None,
+    ):
+        filters = node_filters or []
+
+        filters.append(CustomRequestFieldFilter())
+        filters.append(OptionalIdentityFilter(graph, data))
+        if policy:
+            filters.append(PolicyDataCategoryFilter(policy))
+
+        super().__init__(graph, data, policy=policy, node_filters=filters)
+
 
 def log_traversal_error_and_update_privacy_request(
     privacy_request: PrivacyRequest, session: Session, err: TraversalError

fides/api/models/attachment.py

@@ -0,0 +1,153 @@
+from enum import Enum as EnumType
+from typing import Any, Optional
+
+from sqlalchemy import Column
+from sqlalchemy import Enum as EnumColumn
+from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+
+
+class AttachmentType(str, EnumType):
+    """
+    Enum for attachment types. Indicates attachment usage.
+    """
+
+    internal_use_only = "internal_use_only"
+    include_with_access_package = "include_with_access_package"
+
+
+class AttachmentReferenceType(str, EnumType):
+    """
+    Enum for attachment reference types. Indicates where attachment is referenced.
+    """
+
+    manual_step = "manual_step"
+    privacy_request = "privacy_request"
+    comment = "comment"
+
+
+class AttachmentReference(Base):
+    """
+    Stores information about an Attachment and any other element which may reference that attachment.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "attachment_reference"
+
+    attachment_id = Column(String, ForeignKey("attachment.id"), nullable=False)
+    reference_id = Column(String, nullable=False)
+    reference_type = Column(EnumColumn(AttachmentReferenceType), nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint(
+            "attachment_id", "reference_id", name="_attachment_reference_uc"
+        ),
+    )
+
+    attachment = relationship(
+        "Attachment",
+        back_populates="references",
+        uselist=False,
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> "AttachmentReference":
+        """Creates a new attachment reference record in the database."""
+        return super().create(db=db, data=data, check_name=check_name)
+
+
+class Attachment(Base):
+    """
+    Stores information about an Attachment.
+    """
+
+    user_id = Column(
+        String, ForeignKey("fidesuser.id", ondelete="SET NULL"), nullable=True
+    )
+    file_name = Column(String, nullable=False)
+    attachment_type = Column(EnumColumn(AttachmentType), nullable=False)
+    storage_key = Column(String, nullable=False)
+
+    user = relationship(
+        "FidesUser",
+        backref="attachments",
+        lazy="selectin",
+        uselist=False,
+    )
+
+    references = relationship(
+        "AttachmentReference",
+        back_populates="attachment",
+        cascade="all, delete",
+        uselist=True,
+    )
+
+    async def upload_attachment_to_s3(self, attachment: bytes) -> None:
+        """Upload an attachment to S3 to the storage_url."""
+        raise NotImplementedError("This method is not yet implemented")
+        # AuditLog.create(
+        #     db=db,
+        #     data={
+        #         "user_id": "system",
+        #         "privacy_request_id": privacy_request.id,
+        #         "action": AuditLogAction.attachment_uploaded,
+        #         "message": "",
+        #     },
+        # )
+
+    async def retrieve_attachment_from_s3(self) -> bytes:
+        """Retrieve an attachment from S3."""
+        raise NotImplementedError("This method is not yet implemented")
+        # AuditLog.create(
+        #     db=db,
+        #     data={
+        #         "user_id": "system",
+        #         "privacy_request_id": privacy_request.id,
+        #         "action": AuditLogAction.attachment_retrieved,
+        #         "message": "",
+        #     },
+        # )
+
+    async def delete_attachment_from_s3(self) -> None:
+        """Delete an attachment from S3."""
+        raise NotImplementedError("This method is not yet implemented")
+        # AuditLog.create(
+        #     db=db,
+        #     data={
+        #         "user_id": "system",
+        #         "privacy_request_id": privacy_request.id,
+        #         "action": AuditLogAction.attachment_deleted,
+        #         "message": "",
+        #     },
+        # )
+
+    @classmethod
+    def create(
+        cls,
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = False,
+        attachment: Optional[
+            bytes
+        ] = None,  # This will not be optional once the upload method is implemented.
+    ) -> "Attachment":
+        """Creates a new attachment record in the database and uploads the attachment to S3."""
+        # attachment_record.upload_attachment_to_s3(db, attachment)
+        # log.info(f"Uploaded attachment {attachment_record.id} to S3")
+        return super().create(db=db, data=data, check_name=check_name)
+
+    def delete(self, db: Session) -> None:
+        """Deletes an attachment record from the database and deletes the attachment from S3."""
+        # attachment_record = cls.get(db, id)
+        # attachment_record.delete_attachment_from_s3(db)
+        # log.info(f"Deleted attachment {attachment_record.id} from S3")
+        super().delete(db=db)

fides/api/models/audit_log.py

@@ -11,6 +11,9 @@ class AuditLogAction(str, EnumType):
     """Enum for audit log actions, reflecting what a user did."""
 
     approved = "approved"
+    attachment_uploaded = "attachment_uploaded"
+    attachment_deleted = "attachment_deleted"
+    attachment_retrieved = "attachment_retrieved"
     denied = "denied"
     email_sent = "email_sent"
     finished = "finished"

fides/api/models/detection_discovery.py

@@ -398,6 +398,13 @@ class MonitorExecution(Base):
         nullable=False,
         default=list,
     )
+    # stores additional information from monitor execution failures as an array of strings,
+    # e.g. error messages, stack traces, etc.
+    messages = Column(
+        ARRAY(String),
+        nullable=False,
+        default=list,
+    )
     created_at = Column(DateTime(timezone=True), server_default=func.now())
     updated_at = Column(
         DateTime(timezone=True),

fides/api/models/privacy_request.py

@@ -1278,6 +1278,28 @@ class PrivacyRequest(
             },
         )
 
+    def add_skipped_execution_log(
+        self,
+        db: Session,
+        connection_key: Optional[str],
+        dataset_name: Optional[str],
+        collection_name: Optional[str],
+        message: str,
+        action_type: ActionType,
+    ) -> ExecutionLog:
+        return ExecutionLog.create(
+            db=db,
+            data={
+                "privacy_request_id": self.id,
+                "connection_key": connection_key,
+                "dataset_name": dataset_name,
+                "collection_name": collection_name,
+                "status": ExecutionLogStatus.skipped,
+                "message": message,
+                "action_type": action_type,
+            },
+        )
+
     def add_error_execution_log(
         self,
         db: Session,
@@ -1861,6 +1883,7 @@ class TraversalDetails(FidesSchema):
     incoming_edges: List[Tuple[str, str]]
     outgoing_edges: List[Tuple[str, str]]
     input_keys: List[str]
+    skipped_nodes: Optional[List[Tuple[str, str]]] = None
 
     # TODO: remove this method once we support custom request fields in DSR graph.
     @classmethod
@@ -1868,15 +1891,15 @@ class TraversalDetails(FidesSchema):
         """
         Creates an "empty" TraversalDetails object that only has the dataset connection key set.
         This is a bit of a hacky workaround needed to implement the Dynamic Erasure Emails feature,
-        and should be needed only until we support custom request fields as entry points to the DSR graph.
-        This is needed because custom request field nodes aren't currently reachable, so they don't have
-        a real TraversalNode associated to them.
+        but we should no longer need it once the custom_request_fields are included in our graph
+        traversal
         """
         return cls(
             dataset_connection_key=connection_key,
             incoming_edges=[],
             outgoing_edges=[],
             input_keys=[],
+            skipped_nodes=[],
         )
 
 

fides/api/task/create_request_tasks.py

@@ -451,7 +451,7 @@ def run_access_request(
         )
     else:
         try:
-            traversal: Traversal = Traversal(graph, identity, policy)
+            traversal: Traversal = Traversal(graph, identity, policy=policy)
 
             # Traversal.traverse populates traversal_nodes in place, adding parents and children to each traversal_node.
             traversal_nodes: Dict[CollectionAddress, TraversalNode] = {}
@@ -485,14 +485,32 @@ def run_access_request(
                     connection_configs,
                 )
             )
-            privacy_request.add_success_execution_log(
-                session,
-                connection_key=None,
-                dataset_name="Dataset traversal",
-                collection_name=None,
-                message=f"Traversal successful for privacy request: {privacy_request.id}",
-                action_type=ActionType.access,
-            )
+
+            # Add execution logs for skipped nodes
+            if traversal.skipped_nodes:
+                logger.warning(
+                    "Some nodes were skipped, the identities provided were not sufficient to reach them"
+                )
+                for node_address, skip_message in traversal.skipped_nodes.items():
+                    logger.debug(skip_message)
+                    privacy_request.add_skipped_execution_log(
+                        session,
+                        connection_key=None,
+                        dataset_name="Dataset traversal",
+                        collection_name=node_address.replace(":", "."),
+                        message=skip_message,
+                        action_type=ActionType.access,
+                    )
+            # Or log success if all collections are reachable
+            else:
+                privacy_request.add_success_execution_log(
+                    session,
+                    connection_key=None,
+                    dataset_name="Dataset traversal",
+                    collection_name=None,
+                    message=f"Traversal successful for privacy request: {privacy_request.id}",
+                    action_type=ActionType.access,
+                )
         except TraversalError as err:
             log_traversal_error_and_update_privacy_request(
                 privacy_request, session, err