ethyca-fides 2.62.1rc0__py2.py3-none-any.whl → 2.63.0__py2.py3-none-any.whl

fides/api/models/detection_discovery.py

@@ -3,15 +3,27 @@ from __future__ import annotations
 from datetime import datetime
 from enum import Enum
 from re import match
-from typing import Any, Dict, Iterable, List, Optional, Type
+from typing import Any, Dict, Iterable, List, Optional, Set, Type
 
 from loguru import logger
-from sqlalchemy import ARRAY, Boolean, Column, DateTime, ForeignKey, String, func
+from sqlalchemy import (
+    ARRAY,
+    Boolean,
+    Column,
+    DateTime,
+    ForeignKey,
+    Index,
+    String,
+    UniqueConstraint,
+    func,
+    text,
+)
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.future import select
-from sqlalchemy.orm import Session, relationship
+from sqlalchemy.orm import RelationshipProperty, Session, relationship
 from sqlalchemy.orm.query import Query
 
 from fides.api.db.base_class import Base, FidesBase
@@ -47,6 +59,29 @@ class MonitorFrequency(Enum):
 QUARTERLY_MONTH_PATTERN = r"^\d+,\d+,\d+,\d+$"
 
 
+class SharedMonitorConfig(Base, FidesBase):
+    """SQL model for shareable monitor configurations"""
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "shared_monitor_config"
+
+    # Basic info
+    name = Column(String, nullable=False)
+    key = Column(String, unique=True, nullable=False)
+    description = Column(String, nullable=True)
+
+    # Classification parameters (including regex patterns)
+    classify_params = Column(
+        MutableDict.as_mutable(JSONB),
+        index=False,
+        unique=False,
+        nullable=False,
+        server_default="{}",
+        default=dict,
+    )
+
+
 class MonitorConfig(Base):
     """
     Monitor configuration used for data detection and discovery.
@@ -87,7 +122,9 @@ class MonitorConfig(Base):
     )  # stores the cron-based kwargs for scheduling the monitor execution.
     # see https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html
 
-    classify_params = Column(
+    # We use _classify_params for the actual column to prevent direct access
+    _classify_params = Column(
+        "classify_params",
         MutableDict.as_mutable(JSONB),
         index=False,
         unique=False,
@@ -124,6 +161,45 @@ class MonitorConfig(Base):
         backref="monitor_config",
     )
 
+    shared_config_id = Column(
+        String,
+        ForeignKey(SharedMonitorConfig.id_field_path, ondelete="RESTRICT"),
+        nullable=True,
+        index=True,
+    )
+
+    shared_config = relationship(SharedMonitorConfig)
+
+    @property
+    def classify_params(self) -> dict:
+        """
+        Returns the merged classify parameters from both the monitor config and
+        the shared config (if it exists).
+
+        The shared config parameters take precedence over the monitor's own parameters,
+        but only for values that are not falsy (None, empty, etc.).
+        """
+        # Start with an empty dict
+        merged_params = {}
+
+        # Add this monitor's params if available
+        if self._classify_params:
+            merged_params.update(self._classify_params)
+
+        # Add/override with shared config params if available
+        if self.shared_config and self.shared_config.classify_params:
+            # Only update with non-falsy values from shared config
+            for key, value in self.shared_config.classify_params.items():
+                if value:  # Only override if the value is not falsy
+                    merged_params[key] = value
+
+        return merged_params
+
+    @classify_params.setter
+    def classify_params(self, value: Dict[str, Any]) -> None:
+        """Setter for the classify_params to maintain compatibility with existing code"""
+        self._classify_params = value
+
     @property
     def connection_config_key(self) -> str:
         """Derives the `connection_config_key`"""
@@ -255,6 +331,86 @@ class MonitorConfig(Base):
             data["monitor_execution_trigger"] = cron_trigger_dict
 
 
+class StagedResourceAncestor(Base):
+    """
+    A simple junction table that is used to store the many-to-many relationship
+    between staged resources and their ancestors.
+
+    This table is used to easily lookup all ancestors of a given staged resource,
+    as its indexed by both ancestor and descendant URN columns.
+
+    Its entries should be deleted when the staged resource is deleted, via cascade.
+    """
+
+    ancestor_urn = Column(
+        String,
+        ForeignKey("stagedresource.urn", ondelete="CASCADE"),
+        primary_key=True,
+        nullable=False,
+    )
+    descendant_urn = Column(
+        String,
+        ForeignKey("stagedresource.urn", ondelete="CASCADE"),
+        primary_key=True,
+        nullable=False,
+    )
+
+    ancestor_staged_resource = relationship(
+        "StagedResource",
+        back_populates="ancestor_links",
+        lazy="selectin",
+        foreign_keys=[ancestor_urn],
+    )
+    descendant_staged_resource = relationship(
+        "StagedResource",
+        back_populates="descendant_links",
+        lazy="selectin",
+        foreign_keys=[descendant_urn],
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "ancestor_urn", "descendant_urn", name="uq_staged_resource_ancestor"
+        ),
+        Index("ix_staged_resource_ancestor_ancestor", "ancestor_urn"),
+        Index("ix_staged_resource_ancestor_descendant", "descendant_urn"),
+    )
+
+    @classmethod
+    def create_staged_resource_ancestor_links(
+        cls,
+        db: Session,
+        resource_urn: str,
+        ancestor_urns: Set[str],
+    ) -> None:
+        """
+        Bulk inserts entries in the StagedResourceAncestor table
+        based on the provided resource URN and the set of its ancestor URNs.
+
+        We execute the bulk INSERT with the provided (synchronous) db session,
+        but the transaction is _not_ committed, so the caller must commit the transaction
+        to persist the changes.
+        """
+        links_to_insert = []
+
+        for ancestor_urn in ancestor_urns:
+            links_to_insert.append(
+                {"ancestor_urn": ancestor_urn, "descendant_urn": resource_urn}
+            )
+
+        if links_to_insert:
+            # Using raw SQL for ON CONFLICT with parameters for safety
+            stmt_text = text(
+                """
+                INSERT INTO stagedresourceancestor (id, ancestor_urn, descendant_urn)
+                VALUES ('srl_' || gen_random_uuid(), :ancestor_urn, :descendant_urn)
+                ON CONFLICT (ancestor_urn, descendant_urn) DO NOTHING;
+                """
+            )
+
+            db.execute(stmt_text, links_to_insert)
+
+
 class StagedResource(Base):
     """
     Base DB model that represents a staged resource, fields common to all types of staged resources
@@ -331,14 +487,36 @@ class StagedResource(Base):
     parent = Column(String, nullable=True)
 
     # diff-related fields
-    diff_status = Column(String, nullable=True)
-    child_diff_statuses = Column(
-        MutableDict.as_mutable(JSONB),
-        nullable=False,
-        server_default="{}",
-        default=dict,
+    diff_status = Column(String, nullable=True, index=True)
+
+    ancestor_links: RelationshipProperty[List[StagedResourceAncestor]] = relationship(
+        "StagedResourceAncestor",
+        back_populates="descendant_staged_resource",
+        lazy="dynamic",
+        cascade="all, delete",
+        foreign_keys=[StagedResourceAncestor.descendant_urn],
+    )
+
+    descendant_links: RelationshipProperty[List[StagedResourceAncestor]] = relationship(
+        "StagedResourceAncestor",
+        back_populates="ancestor_staged_resource",
+        lazy="dynamic",
+        cascade="all, delete",
+        foreign_keys=[StagedResourceAncestor.ancestor_urn],
     )
 
+    def ancestors(self) -> List[StagedResource]:
+        """
+        Returns the ancestors of the staged resource
+        """
+        return [link.ancestor_staged_resource for link in self.ancestor_links]
+
+    def descendants(self) -> List[StagedResource]:
+        """
+        Returns the descendants of the staged resource
+        """
+        return [link.descendant_staged_resource for link in self.descendant_links]
+
     # placeholder for additional attributes
     meta = Column(
         MutableDict.as_mutable(JSONB),
@@ -391,28 +569,15 @@ class StagedResource(Base):
         )
         return results.scalars().all()
 
-    def add_child_diff_status(self, diff_status: DiffStatus) -> None:
-        """Increments the specified child diff status"""
-        self.child_diff_statuses[diff_status.value] = (
-            self.child_diff_statuses.get(diff_status.value, 0) + 1
-        )
-
     def mark_as_addition(
         self,
         db: Session,
         parent_resource_urns: Iterable[str] = [],
     ) -> None:
         """
-        Marks the resource as an addition and the child diff status of
-        the given parent resource URNs accordingly
+        Marks the resource as an addition
         """
         self.diff_status = DiffStatus.ADDITION.value
-        for parent_resource_urn in parent_resource_urns:
-            parent_resource: Optional[StagedResource] = StagedResource.get_urn(
-                db, parent_resource_urn
-            )
-            if parent_resource:
-                parent_resource.add_child_diff_status(DiffStatus.ADDITION)
 
 
 class MonitorExecution(Base):

fides/api/models/fides_user.py

@@ -23,12 +23,16 @@ from fides.api.db.base_class import Base
 from fides.api.models.audit_log import AuditLog
 
 # Intentionally importing SystemManager here to build the FidesUser.systems relationship
-from fides.api.models.system_manager import SystemManager  # type: ignore[unused-import]
 from fides.api.schemas.user import DisabledReason
 from fides.config import CONFIG
 
 if TYPE_CHECKING:
+    from fides.api.models.fides_user_permissions import FidesUserPermissions
+    from fides.api.models.fides_user_respondent_email_verification import (
+        FidesUserRespondentEmailVerification,
+    )
     from fides.api.models.sql_models import System  # type: ignore[attr-defined]
+    from fides.api.models.system_manager import SystemManager
 
 
 class FidesUser(Base):
@@ -71,6 +75,20 @@ class FidesUser(Base):
     )
 
     systems = relationship("System", secondary="systemmanager", back_populates="data_stewards")  # type: ignore
+    # permissions relationship is defined via backref in FidesUserPermissions
+    email_verifications = relationship(
+        "FidesUserRespondentEmailVerification",
+        back_populates="user",
+        cascade="all,delete",
+        lazy="dynamic",
+        foreign_keys="[FidesUserRespondentEmailVerification.user_id]",
+    )
+    permissions = relationship(
+        "FidesUserPermissions",
+        back_populates="user",
+        cascade="all,delete",
+        uselist=False,
+    )
 
     @property
     def system_ids(self) -> List[str]:
@@ -92,12 +110,11 @@ class FidesUser(Base):
     ) -> FidesUser:
         """Create a FidesUser by hashing the password with a generated salt
         and storing the hashed password and the salt"""
+        hashed_password = None
+        salt = None
 
         if password := data.get("password"):
             hashed_password, salt = FidesUser.hash_password(password)
-        else:
-            hashed_password = None
-            salt = None
 
         user = super().create(
             db,
@@ -117,6 +134,18 @@ class FidesUser(Base):
 
         return user  # type: ignore
 
+    @classmethod
+    def create_respondent(cls, db: Session, data: dict[str, Any]) -> FidesUser:
+        """Create a respondent user. This user will not be able to login with a password and
+        requires an email address to be provided.
+        """
+        if not data.get("email_address"):
+            raise ValueError("Email address is required for external respondents")
+        if data.get("password"):
+            raise ValueError("Password login is not allowed for external respondents")
+        data["password_login_enabled"] = False
+        return cls.create(db, data)
+
     def credentials_valid(self, password: str, encoding: str = "UTF-8") -> bool:
         """Verifies that the provided password is correct."""
         if self.salt is None:
@@ -134,6 +163,9 @@ class FidesUser(Base):
 
         No validations are performed on the old/existing password within this function.
         """
+        if self.permissions is not None:
+            if self.permissions.is_respondent():
+                raise ValueError("Password changes are not allowed for respondents")
 
         hashed_password, salt = FidesUser.hash_password(new_password)
         self.hashed_password = hashed_password  # type: ignore
@@ -141,6 +173,17 @@ class FidesUser(Base):
         self.password_reset_at = datetime.utcnow()  # type: ignore
         self.save(db)
 
+    def update_email_address(self, db: Session, new_email_address: str) -> None:
+        """Updates the user's email address to the specified value."""
+        if self.permissions is not None:
+            if self.permissions.is_respondent():
+                raise ValueError(
+                    "Email address changes are not allowed for respondents"
+                )
+
+        self.email_address = new_email_address  # type: ignore
+        self.save(db)
+
     def set_as_system_manager(self, db: Session, system: System) -> None:
         """Add a user as one of the system managers for the given system
         If applicable, also update the systems on the user's client
@@ -157,6 +200,10 @@ class FidesUser(Base):
                 f"User '{self.username}' is already a system manager of '{system.name}'."
             )
 
+        if self.permissions is not None:
+            if self.permissions.is_respondent():
+                raise SystemManagerException("Respondents cannot be system managers.")
+
         self.systems.append(system)
         self.save(db=db)
 

fides/api/models/fides_user_permissions.py

@@ -1,11 +1,15 @@
-from typing import List
+from typing import List, cast
 
 from sqlalchemy import ARRAY, Column, ForeignKey, String
-from sqlalchemy.orm import backref, relationship
+from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import Base
 from fides.api.models.fides_user import FidesUser
-from fides.api.oauth.roles import ROLES_TO_SCOPES_MAPPING
+from fides.api.oauth.roles import (
+    EXTERNAL_RESPONDENT,
+    RESPONDENT,
+    ROLES_TO_SCOPES_MAPPING,
+)
 
 
 class FidesUserPermissions(Base):
@@ -13,10 +17,7 @@ class FidesUserPermissions(Base):
 
     user_id = Column(String, ForeignKey(FidesUser.id), nullable=False, unique=True)
     roles = Column(ARRAY(String), nullable=False, server_default="{}", default=dict)
-    user = relationship(
-        FidesUser,
-        backref=backref("permissions", cascade="all,delete", uselist=False),
-    )
+    user = relationship(FidesUser, back_populates="permissions", uselist=False)
 
     @property
     def total_scopes(self) -> List[str]:
@@ -26,3 +27,20 @@ class FidesUserPermissions(Base):
             all_scopes += ROLES_TO_SCOPES_MAPPING.get(role, [])
 
         return sorted(list(set(all_scopes)))
+
+    def is_respondent(self) -> bool:
+        """Check if the user is a respondent."""
+        return any(role in self.roles for role in [RESPONDENT, EXTERNAL_RESPONDENT])
+
+    def update_roles(self, db: Session, new_roles: List[str]) -> None:
+        """Update the user's roles if allowed.
+        Raises ValueError if role changes are not allowed."""
+        if self.is_respondent():
+            raise ValueError("Role changes are not allowed for respondents")
+
+        self.roles = new_roles
+        self.save(db)
+
+    def update(self, db: Session, *, data: dict) -> "FidesUserPermissions":
+        """Update the user permissions with the provided data."""
+        return cast(FidesUserPermissions, super().update(db, data=data))

fides/api/models/fides_user_respondent_email_verification.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Any, Optional
+
+from sqlalchemy import Column, DateTime, ForeignKey, String
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.cryptography.cryptographic_util import generate_secure_random_string
+from fides.api.db.base_class import Base
+from fides.api.util.identity_verification import IdentityVerificationMixin
+from fides.config import get_config
+
+CONFIG = get_config()
+
+
+if TYPE_CHECKING:
+    from fides.api.models.fides_user import FidesUser
+
+# Access links stay active for 45 days - the same as the DSR expiration. A new link is generated for each email.
+# The emails are created for new DSRs which are assigned to the respondent.
+ACCESS_LINK_TTL_DAYS = 45
+
+
+class FidesUserRespondentEmailVerification(Base, IdentityVerificationMixin):
+    """Model for handling email verification for external respondents.
+
+    This handles two types of verification:
+    1. Access links - long-lived (45 days) for initial access
+    2. Verification codes - short-lived (1 hour) for actual verification
+
+    When an email is sent to an external respondent, a new verification is created with a new access token is created.
+    When a respondent clicks the link in the email, the access token is verified and a verification code is generated.
+    The verification code is sent to the respondent's email address and the respondent is prompted to enter the code.
+    Verification is handled by the `IdentityVerificationMixin` class.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "fides_user_respondent_email_verification"
+
+    user_id = Column(
+        String,
+        ForeignKey("fidesuser.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    access_token = Column(
+        String, nullable=False, unique=True, index=True
+    )  # Token for the access link
+    access_token_expires_at = Column(DateTime(timezone=True), nullable=False)
+    identity_verified_at = Column(DateTime(timezone=True), nullable=True)
+
+    user = relationship(
+        "FidesUser",
+        back_populates="email_verifications",
+        foreign_keys=[user_id],
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> FidesUserRespondentEmailVerification:
+        """
+        Create a FidesUserEmailVerification record with a new access token.
+        The verification code will be generated when the access link is used.
+        """
+        # Generate a secure token for the access link
+        access_token = generate_secure_random_string(32)
+        expires_at = datetime.now(timezone.utc) + timedelta(days=ACCESS_LINK_TTL_DAYS)
+
+        verification = super().create(
+            db,
+            data={
+                "user_id": data["user_id"],
+                "access_token": access_token,
+                "access_token_expires_at": expires_at,
+            },
+        )
+
+        return verification
+
+    def is_access_token_expired(self) -> bool:
+        """Check if the access token has expired."""
+        if not self.access_token_expires_at:
+            return True
+
+        current_time_utc = datetime.now(timezone.utc)
+        return current_time_utc > self.access_token_expires_at
+
+    def verify_access_token(self, token: str) -> bool:
+        """Verify the access token and generate a verification code if valid."""
+        if self.is_access_token_expired():
+            return False
+
+        return self.access_token == token
+
+    def verify_identity(
+        self,
+        db: Session,
+        provided_code: Optional[str] = None,
+    ) -> None:
+        """A method to call the internal identity verification method provided by the
+        `IdentityVerificationMixin`."""
+        if self.is_access_token_expired():
+            raise ValueError("Access token has expired.")
+
+        self._verify_identity(provided_code=provided_code)
+        self.identity_verified_at = datetime.now(timezone.utc)
+        self.save(db)

fides/api/models/privacy_request/privacy_request.py

@@ -1089,6 +1089,46 @@ class PrivacyRequest(
         if attachment:
             attachment.delete(db)
 
+    def _get_manual_webhook_attachments(
+        self, db: Session, manual_webhook_id: str, reference_type: str
+    ) -> List[Attachment]:
+        """Helper method to get attachments that have references to both this privacy request and the specified manual webhook"""
+        query = """
+            SELECT DISTINCT a.*
+            FROM attachment a
+            INNER JOIN attachment_reference ar1 ON a.id = ar1.attachment_id
+            INNER JOIN attachment_reference ar2 ON a.id = ar2.attachment_id
+            WHERE ar1.reference_id = :privacy_request_id
+            AND ar1.reference_type = 'privacy_request'
+            AND ar2.reference_id = :manual_webhook_id
+            AND ar2.reference_type = :reference_type
+        """
+        result = db.execute(
+            query,
+            {
+                "privacy_request_id": self.id,
+                "manual_webhook_id": manual_webhook_id,
+                "reference_type": reference_type,
+            },
+        )
+        return [Attachment(**row) for row in result]
+
+    def get_access_manual_webhook_attachments(
+        self, db: Session, manual_webhook_id: str
+    ) -> List[Attachment]:
+        """Get all attachments that have references to both this privacy request and the specified access manual webhook"""
+        return self._get_manual_webhook_attachments(
+            db, manual_webhook_id, "access_manual_webhook"
+        )
+
+    def get_erasure_manual_webhook_attachments(
+        self, db: Session, manual_webhook_id: str
+    ) -> List[Attachment]:
+        """Get all attachments that have references to both this privacy request and the specified erasure manual webhook"""
+        return self._get_manual_webhook_attachments(
+            db, manual_webhook_id, "erasure_manual_webhook"
+        )
+
     def get_existing_request_task(
         self,
         db: Session,
@@ -1228,7 +1268,6 @@ class PrivacyRequest(
         """
         if not self.policy.get_rules_for_action(action_type=ActionType.access):
             return None
-
         self.filtered_final_upload = results
         self.save(db)
 

fides/api/oauth/roles.py

@@ -27,6 +27,8 @@ from fides.common.api.scope_registry import (
     PRIVACY_NOTICE_READ,
     PRIVACY_REQUEST_CALLBACK_RESUME,
     PRIVACY_REQUEST_DELETE,
+    PRIVACY_REQUEST_MANUAL_STEPS_RESPOND,
+    PRIVACY_REQUEST_MANUAL_STEPS_REVIEW,
     PRIVACY_REQUEST_NOTIFICATIONS_CREATE_OR_UPDATE,
     PRIVACY_REQUEST_NOTIFICATIONS_READ,
     PRIVACY_REQUEST_READ,
@@ -52,6 +54,8 @@ CONTRIBUTOR = "contributor"
 OWNER = "owner"
 VIEWER = "viewer"
 VIEWER_AND_APPROVER = "viewer_and_approver"
+RESPONDENT = "respondent"
+EXTERNAL_RESPONDENT = "external_respondent"
 
 
 class RoleRegistryEnum(Enum):
@@ -62,6 +66,8 @@ class RoleRegistryEnum(Enum):
     Approver - Limited viewer but can approve Privacy Requests
     Viewer + Approver = Full View and can approve Privacy Requests
     Contributor - Can't configure storage and messaging
+    Respondent - Internal user who can respond to manual steps
+    External Respondent - External user who can only respond to assigned manual steps
     """
 
     owner = OWNER
@@ -69,6 +75,8 @@ class RoleRegistryEnum(Enum):
     viewer = VIEWER
     approver = APPROVER
     contributor = CONTRIBUTOR
+    respondent = RESPONDENT
+    external_respondent = EXTERNAL_RESPONDENT
 
 
 approver_scopes = [
@@ -79,6 +87,7 @@ approver_scopes = [
     PRIVACY_REQUEST_VIEW_DATA,
     PRIVACY_REQUEST_DELETE,
     USER_READ,  # allows approver to view user management table and update their own password
+    PRIVACY_REQUEST_MANUAL_STEPS_REVIEW,  # allows approvers to see all manual steps
 ]
 
 
@@ -114,6 +123,14 @@ viewer_scopes = [  # Intentionally omitted USER_PERMISSION_READ and PRIVACY_REQU
     USER_READ,
 ]
 
+respondent_scopes = [
+    PRIVACY_REQUEST_MANUAL_STEPS_RESPOND,  # allows respondents to respond to assigned manual steps
+]
+
+external_respondent_scopes = [
+    PRIVACY_REQUEST_MANUAL_STEPS_RESPOND,  # allows external respondents to respond to assigned manual steps
+]
+
 not_contributor_scopes = [
     CONNECTOR_TEMPLATE_REGISTER,
     STORAGE_CREATE_OR_UPDATE,
@@ -130,6 +147,8 @@ ROLES_TO_SCOPES_MAPPING: Dict[str, List] = {
     VIEWER: sorted(viewer_scopes),
     APPROVER: sorted(approver_scopes),
     CONTRIBUTOR: sorted(list(set(SCOPE_REGISTRY) - set(not_contributor_scopes))),
+    RESPONDENT: sorted(respondent_scopes),
+    EXTERNAL_RESPONDENT: sorted(external_respondent_scopes),
 }