ethyca-fides 2.56.2rc3__py2.py3-none-any.whl → 2.56.3b1__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-03-10T02:03:21-0400",
+ "date": "2025-03-11T11:05:57-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "9880864b483d2a61286e5e387cc99fa99f3098f1",
- "version": "2.56.2rc3"
+ "full-revisionid": "26fed9fd3dc9101fa08d947da4153d60449f316c",
+ "version": "2.56.3b1"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/1152c1717849_fix_monitorexecution_started_default.py

@@ -0,0 +1,57 @@
+"""fix monitorexecution.started default
+
+Revision ID: 1152c1717849
+Revises: 3c58001ad310
+Create Date: 2025-03-10 16:04:57.104689
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "1152c1717849"
+down_revision = "3c58001ad310"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Set the new server default
+    # And set the datetime to be timezone aware
+    op.alter_column(
+        "monitorexecution",
+        "started",
+        existing_type=sa.DateTime(),
+        type_=sa.DateTime(timezone=True),
+        server_default=sa.text("now()"),
+    )
+
+    # Set the datetime to be timezone aware
+    op.alter_column(
+        "monitorexecution",
+        "completed",
+        existing_type=sa.DateTime(),
+        type_=sa.DateTime(timezone=True),
+    )
+
+
+def downgrade():
+    # Remove the server default
+    # And set the datetime to not be timezone aware
+    op.alter_column(
+        "monitorexecution",
+        "started",
+        server_default=None,
+        existing_type=sa.DateTime(timezone=True),
+        type_=sa.DateTime(),
+    )
+
+    # Set the datetime to not be timezone aware
+    op.alter_column(
+        "monitorexecution",
+        "completed",
+        existing_type=sa.DateTime(timezone=True),
+        type_=sa.DateTime(),
+    )

fides/api/alembic/migrations/versions/3c58001ad310_add_messages_to_monitor_execution.py

@@ -0,0 +1,32 @@
+"""add errors to monitor execution
+
+Revision ID: 3c58001ad310
+Revises: bd875a8b5d96
+Create Date: 2025-02-27 20:45:38.462604
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3c58001ad310"
+down_revision = "bd875a8b5d96"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "monitorexecution",
+        sa.Column(
+            "messages",
+            sa.ARRAY(sa.String()),
+            nullable=False,
+            server_default="{}",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("monitorexecution", "messages")

fides/api/alembic/migrations/versions/bd875a8b5d96_add_attachments_and_.py

@@ -0,0 +1,96 @@
+"""Add Attachment and AttachmentReference tables
+
+Revision ID: bd875a8b5d96
+Revises: 82883b0df5e4
+Create Date: 2025-02-19 14:09:07.124680
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "bd875a8b5d96"
+down_revision = "82883b0df5e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # create attachment table
+    op.create_table(
+        "attachment",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column("user_id", sa.String(), nullable=True),
+        sa.Column("file_name", sa.String(), nullable=False),
+        sa.Column("attachment_type", sa.String(), nullable=False),
+        sa.Column("storage_key", sa.String(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["fidesuser.id"],
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_attachment_id"), "attachment", ["id"], unique=False)
+
+    # create attachment_reference table
+    op.create_table(
+        "attachment_reference",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("attachment_id", sa.String(), nullable=False),
+        sa.Column("reference_id", sa.String(), nullable=False),
+        sa.Column("reference_type", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["attachment_id"],
+            ["attachment.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_attachment_reference_id"),
+        "attachment_reference",
+        ["id"],
+        unique=False,
+    )
+    op.create_unique_constraint(
+        "_attachment_reference_uc",
+        "attachment_reference",
+        ["attachment_id", "reference_id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_attachment_reference_id"), table_name="attachment_reference")
+    op.drop_table("attachment_reference")
+    op.drop_index(op.f("ix_attachment_id"), table_name="attachment")
+    op.drop_table("attachment")
+    # ### end Alembic commands ###

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -174,7 +174,7 @@ from fides.service.privacy_request.privacy_request_service import (
     PrivacyRequestService,
     _process_privacy_request_restart,
     _requeue_privacy_request,
-    _trigger_pre_approval_webhooks,
+    handle_approval,
     queue_privacy_request,
 )
 
@@ -1310,19 +1310,7 @@ def verify_identification_code(
 
     logger.info("Identity verified for {}.", privacy_request.id)
 
-    if config_proxy.execution.require_manual_request_approval:
-        _trigger_pre_approval_webhooks(db, privacy_request)
-    else:
-        AuditLog.create(
-            db=db,
-            data={
-                "user_id": "system",
-                "privacy_request_id": privacy_request.id,
-                "action": AuditLogAction.approved,
-                "message": "",
-            },
-        )
-        queue_privacy_request(privacy_request.id)
+    handle_approval(db, config_proxy, privacy_request)
 
     return privacy_request  # type: ignore[return-value]
 

fides/api/custom_types.py

@@ -116,6 +116,23 @@ def validate_path_of_url(value: AnyUrl) -> str:
 
     We perform the basic URL validation, but also prevent URLs with paths,
     as paths are not part of an origin.
+
+    As this is meant to be used as a validator _after_ the AnyUrl validator,
+    we implicitly disallow `*` to be specified as an origin value, since it is
+    rejected by the AnyUrl validator.
+
+    Note that `*` _is_ considered a valid origin value
+    (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin),
+    but we specifically disallow it to be set on the `cors_origins` security setting
+    in our application. This is because non-owner users (contributors)
+    are able to edit the `cors_origins` security setting (via API), and we do not
+    want to allow them to set a wildcard origin.
+
+    Instead, `.*` can be set via the `cors_origin_regex` security setting
+    to effectively allow all origins. This is roughly equivalent to setting
+    `"*"` as a `cors_origin`, but there is a slight behavioral difference:
+    the `Access-Control-Allow-Origin` in the response will be set to the
+    `Access-Control-Allow-Origin` specified in the request rather than `"*"`.
     """
     if value.path and value.path != "/":
         raise ValueError("URL origin values cannot contain a path.")
@@ -125,7 +142,13 @@ def validate_path_of_url(value: AnyUrl) -> str:
     return str(value).rstrip("/")
 
 
-URLOriginString = Annotated[AnyUrl, AfterValidator(validate_path_of_url)]
+# This custom type is used to validate HTTP origins, e.g. CORS origins
+# used in SecuritySettings.  See docstring for `validate_path_of_url`
+# for more details.
+URLOriginString = Annotated[
+    AnyUrl,
+    AfterValidator(validate_path_of_url),
+]
 
 
 def validate_css_str(value: str) -> str:

fides/api/db/base.py

@@ -4,6 +4,7 @@
 from fides.api.db.base_class import Base
 from fides.api.models.application_config import ApplicationConfig
 from fides.api.models.asset import Asset
+from fides.api.models.attachment import Attachment, AttachmentReference
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.authentication_request import AuthenticationRequest
 from fides.api.models.client import ClientDetail

fides/api/graph/node_filters.py

@@ -1,19 +1,27 @@
-from typing import TYPE_CHECKING, Optional, Protocol
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
 
+from fides.api.common_exceptions import TraversalError, UnreachableNodesError
+from fides.api.graph.graph import DatasetGraph
 from fides.api.models.policy import Policy
+from fides.api.util.logger import suppress_logging
 
 if TYPE_CHECKING:
     from fides.api.graph.traversal import TraversalNode
 
 
-class NodeFilter(Protocol):
-    """Protocol for node filters that determine if nodes should be excluded from traversal"""
+class NodeFilter(ABC):
+    """Abstract base class for node filters that determine if nodes should be excluded from traversal"""
 
+    def __init__(self) -> None:
+        self.skipped_nodes: Dict[str, str] = {}
+
+    @abstractmethod
     def exclude_node(self, node: "TraversalNode") -> bool:
         """Returns True if the node should be excluded from traversal requirements"""
 
 
-class CustomRequestFieldFilter:
+class CustomRequestFieldFilter(NodeFilter):
     """
     Remove nodes that have custom request fields, since we don't care if these are reachable or not.
     They will be used independently by the Dynamic Email Erasure Connector.
@@ -26,16 +34,115 @@ class CustomRequestFieldFilter:
         return bool(node.node.collection.custom_request_fields())
 
 
-class PolicyDataCategoryFilter:
+class PolicyDataCategoryFilter(NodeFilter):
     """
     If a policy is provided, we will allow collections without relevant
     data categories to remain unreachable without raising any exceptions.
     """
 
     def __init__(self, policy: Optional[Policy]):
+        super().__init__()
         self.policy = policy
 
     def exclude_node(self, node: "TraversalNode") -> bool:
         if not self.policy:
             return False
         return not self.policy.applies_to(node)
+
+
+class OptionalIdentityFilter(NodeFilter):
+
+    def __init__(self, graph: DatasetGraph, identities: Dict[str, Any]):
+        """
+        Filter that excludes nodes that are reachable by an optional identity.
+        This prevents traversal errors for nodes that would be reachable if the required identities were provided.
+        """
+
+        super().__init__()
+        self.graph = graph
+        self.identities = identities
+        self.skipped_nodes: Dict[str, str] = {}
+
+        # Pre-compute which nodes are reachable with each identity
+        self.reachable_by_identity: Dict[str, Set[str]] = {}
+
+        # Get all collection addresses as strings
+        all_collection_addresses = {str(addr) for addr in graph.nodes.keys()}
+
+        # Test reachability with each possible identity seed
+        for identity_key in graph.identity_keys.values():
+            self._compute_reachable_nodes(identity_key, all_collection_addresses)
+
+    def exclude_node(self, node: "TraversalNode") -> bool:
+        """
+        Exclude nodes that are reachable by any identity, provided or optional.
+        Nodes (collections) not reachable by any identities should not be excluded and still be considered an error.
+        """
+        node_address = str(node.address)
+
+        # Check if node is reachable by any provided identity
+        for identity_key in self.identities:
+            if node_address in self.reachable_by_identity.get(identity_key, set()):
+                return True
+
+        # Find all optional identities that can reach this node
+        reaching_identities = []
+        for identity_key, reachable_nodes in self.reachable_by_identity.items():
+            if identity_key not in self.identities and node_address in reachable_nodes:
+                reaching_identities.append(identity_key)
+
+        # If any optional identities can reach this node, record them and exclude
+        if reaching_identities:
+            self.skipped_nodes[node_address] = self._format_skip_message(
+                node.address.value, reaching_identities
+            )
+            return True
+
+        # Node isn't reachable by any identity, don't exclude it (will be an error)
+        return False
+
+    def _compute_reachable_nodes(
+        self, identity_key: str, all_addresses: Set[str]
+    ) -> None:
+        """Helper method to compute which nodes are reachable with a given identity key"""
+
+        from fides.api.graph.traversal import BaseTraversal
+
+        try:
+            # Create a traversal object with just this identity
+            with suppress_logging():
+                # Suppress the logs since we don't want to flood the logs
+                # with traversal info for each identity we want to evaluate
+                BaseTraversal(self.graph, {identity_key: "dummy_value"})
+
+            # If successful, all nodes are reachable
+            self.reachable_by_identity[identity_key] = all_addresses
+        except UnreachableNodesError as exc:
+            # Get reachable nodes by removing unreachable ones from all nodes
+            unreachable = set(exc.errors)
+            self.reachable_by_identity[identity_key] = all_addresses - unreachable
+        except TraversalError:
+            # If traversal fails for other reasons, no nodes are reachable
+            self.reachable_by_identity[identity_key] = set()
+
+    def _format_skip_message(
+        self, node_address: str, reaching_identities: List[str]
+    ) -> str:
+        """
+        Format the skip message with proper grammar based on identity counts.
+        """
+
+        # Format lists of quoted identities
+        reachable_ids_str = ", ".join(f'"{id}"' for id in reaching_identities)
+        provided_ids = list(self.identities.keys())
+        provided_ids_str = ", ".join(f'"{id}"' for id in provided_ids)
+
+        # Determine singular/plural forms based on counts
+        reachable_term = "identity" if len(reaching_identities) == 1 else "identities"
+        provided_term = "identity" if len(provided_ids) == 1 else "identities"
+        was_were = "was" if len(provided_ids) == 1 else "were"
+
+        return (
+            f'Skipping the "{node_address}" collection, it is reachable by the {reachable_ids_str} '
+            f"{reachable_term} but only the {provided_ids_str} {provided_term} {was_were} provided"
+        )

fides/api/graph/traversal.py

@@ -28,6 +28,7 @@ from fides.api.graph.graph import DatasetGraph, Edge, Node
 from fides.api.graph.node_filters import (
     CustomRequestFieldFilter,
     NodeFilter,
+    OptionalIdentityFilter,
     PolicyDataCategoryFilter,
 )
 from fides.api.models.policy import Policy
@@ -64,7 +65,7 @@ def artificial_traversal_node(address: CollectionAddress) -> TraversalNode:
     return TraversalNode(node)
 
 
-class Traversal:
+class BaseTraversal:
     """Handling for a single reified traversal of a graph based on input (seed) data."""
 
     def extract_seed_field_addresses(
@@ -83,6 +84,7 @@ class Traversal:
         self,
         graph: DatasetGraph,
         data: Dict[str, Any],
+        *,
         policy: Optional[Policy] = None,
         node_filters: Optional[List[NodeFilter]] = None,
     ):
@@ -93,9 +95,6 @@ class Traversal:
         # Node filters are used to allow unreachable nodes to be ignored
         # if they don't apply to the given scenario
         self.node_filters = node_filters or []
-        self.node_filters.append(CustomRequestFieldFilter())
-        if policy:
-            self.node_filters.append(PolicyDataCategoryFilter(policy))
 
         self.traversal_node_dict = {k: TraversalNode(v) for k, v in graph.nodes.items()}
         self.edges: Set[Edge] = graph.edges.copy()
@@ -289,6 +288,37 @@ class Traversal:
             logger.debug("Found {} end nodes: {}", len(end_nodes), end_nodes)
         return end_nodes
 
+    @property
+    def skipped_nodes(self) -> Dict[str, str]:
+        """
+        Returns a dictionary of node addresses to skip reasons from all node filters.
+        """
+        result = {}
+        for node_filter in self.node_filters:
+            result.update(node_filter.skipped_nodes)
+        return result
+
+
+class Traversal(BaseTraversal):
+    """Handling for a single reified traversal of a graph based on input (seed) data."""
+
+    def __init__(
+        self,
+        graph: DatasetGraph,
+        data: Dict[str, Any],
+        *,
+        policy: Optional[Policy] = None,
+        node_filters: Optional[List[NodeFilter]] = None,
+    ):
+        filters = node_filters or []
+
+        filters.append(CustomRequestFieldFilter())
+        filters.append(OptionalIdentityFilter(graph, data))
+        if policy:
+            filters.append(PolicyDataCategoryFilter(policy))
+
+        super().__init__(graph, data, policy=policy, node_filters=filters)
+
 
 def log_traversal_error_and_update_privacy_request(
     privacy_request: PrivacyRequest, session: Session, err: TraversalError

fides/api/models/attachment.py

@@ -0,0 +1,153 @@
+from enum import Enum as EnumType
+from typing import Any, Optional
+
+from sqlalchemy import Column
+from sqlalchemy import Enum as EnumColumn
+from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+
+
+class AttachmentType(str, EnumType):
+    """
+    Enum for attachment types. Indicates attachment usage.
+    """
+
+    internal_use_only = "internal_use_only"
+    include_with_access_package = "include_with_access_package"
+
+
+class AttachmentReferenceType(str, EnumType):
+    """
+    Enum for attachment reference types. Indicates where attachment is referenced.
+    """
+
+    manual_step = "manual_step"
+    privacy_request = "privacy_request"
+    comment = "comment"
+
+
+class AttachmentReference(Base):
+    """
+    Stores information about an Attachment and any other element which may reference that attachment.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "attachment_reference"
+
+    attachment_id = Column(String, ForeignKey("attachment.id"), nullable=False)
+    reference_id = Column(String, nullable=False)
+    reference_type = Column(EnumColumn(AttachmentReferenceType), nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint(
+            "attachment_id", "reference_id", name="_attachment_reference_uc"
+        ),
+    )
+
+    attachment = relationship(
+        "Attachment",
+        back_populates="references",
+        uselist=False,
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> "AttachmentReference":
+        """Creates a new attachment reference record in the database."""
+        return super().create(db=db, data=data, check_name=check_name)
+
+
+class Attachment(Base):
+    """
+    Stores information about an Attachment.
+    """
+
+    user_id = Column(
+        String, ForeignKey("fidesuser.id", ondelete="SET NULL"), nullable=True
+    )
+    file_name = Column(String, nullable=False)
+    attachment_type = Column(EnumColumn(AttachmentType), nullable=False)
+    storage_key = Column(String, nullable=False)
+
+    user = relationship(
+        "FidesUser",
+        backref="attachments",
+        lazy="selectin",
+        uselist=False,
+    )
+
+    references = relationship(
+        "AttachmentReference",
+        back_populates="attachment",
+        cascade="all, delete",
+        uselist=True,
+    )
+
+    async def upload_attachment_to_s3(self, attachment: bytes) -> None:
+        """Upload an attachment to S3 to the storage_url."""
+        raise NotImplementedError("This method is not yet implemented")
+        # AuditLog.create(
+        #     db=db,
+        #     data={
+        #         "user_id": "system",
+        #         "privacy_request_id": privacy_request.id,
+        #         "action": AuditLogAction.attachment_uploaded,
+        #         "message": "",
+        #     },
+        # )
+
+    async def retrieve_attachment_from_s3(self) -> bytes:
+        """Retrieve an attachment from S3."""
+        raise NotImplementedError("This method is not yet implemented")
+        # AuditLog.create(
+        #     db=db,
+        #     data={
+        #         "user_id": "system",
+        #         "privacy_request_id": privacy_request.id,
+        #         "action": AuditLogAction.attachment_retrieved,
+        #         "message": "",
+        #     },
+        # )
+
+    async def delete_attachment_from_s3(self) -> None:
+        """Delete an attachment from S3."""
+        raise NotImplementedError("This method is not yet implemented")
+        # AuditLog.create(
+        #     db=db,
+        #     data={
+        #         "user_id": "system",
+        #         "privacy_request_id": privacy_request.id,
+        #         "action": AuditLogAction.attachment_deleted,
+        #         "message": "",
+        #     },
+        # )
+
+    @classmethod
+    def create(
+        cls,
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = False,
+        attachment: Optional[
+            bytes
+        ] = None,  # This will not be optional once the upload method is implemented.
+    ) -> "Attachment":
+        """Creates a new attachment record in the database and uploads the attachment to S3."""
+        # attachment_record.upload_attachment_to_s3(db, attachment)
+        # log.info(f"Uploaded attachment {attachment_record.id} to S3")
+        return super().create(db=db, data=data, check_name=check_name)
+
+    def delete(self, db: Session) -> None:
+        """Deletes an attachment record from the database and deletes the attachment from S3."""
+        # attachment_record = cls.get(db, id)
+        # attachment_record.delete_attachment_from_s3(db)
+        # log.info(f"Deleted attachment {attachment_record.id} from S3")
+        super().delete(db=db)

fides/api/models/audit_log.py

@@ -11,6 +11,9 @@ class AuditLogAction(str, EnumType):
     """Enum for audit log actions, reflecting what a user did."""
 
     approved = "approved"
+    attachment_uploaded = "attachment_uploaded"
+    attachment_deleted = "attachment_deleted"
+    attachment_retrieved = "attachment_retrieved"
     denied = "denied"
     email_sent = "email_sent"
     finished = "finished"