ethyca-fides 2.58.1rc0__py2.py3-none-any.whl → 2.58.2b0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-04-02T12:06:35-0600",
+ "date": "2025-04-02T16:21:34-0300",
  "dirty": false,
  "error": null,
- "full-revisionid": "8acf1908449d28a9d2b040b52b42b44afbc23628",
- "version": "2.58.1rc0"
+ "full-revisionid": "fe5691256243517ec2cac351e75000e269a4fa50",
+ "version": "2.58.2b0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/67d01c4e124e_add_reject_all_mechanism_to_privacy_.py

@@ -0,0 +1,56 @@
+"""add_reject_all_mechanism_to_privacy_experience_config
+
+Revision ID: 67d01c4e124e
+Revises: 1d2f04ac19f2
+Create Date: 2025-03-27 15:26:24.635947
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "67d01c4e124e"
+down_revision = "1d2f04ac19f2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    # Add the reject all mechanism enum and column
+    op.execute(
+        "CREATE TYPE rejectallmechanism AS ENUM ('REJECT_ALL', 'REJECT_CONSENT_ONLY')"
+    )
+    op.add_column(
+        "privacyexperienceconfig",
+        sa.Column(
+            "reject_all_mechanism",
+            sa.Enum("REJECT_ALL", "REJECT_CONSENT_ONLY", name="rejectallmechanism"),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "privacyexperienceconfighistory",
+        sa.Column(
+            "reject_all_mechanism",
+            sa.Enum("REJECT_ALL", "REJECT_CONSENT_ONLY", name="rejectallmechanism"),
+            nullable=True,
+        ),
+    )
+
+    # Data migration - set the reject_all_mechanism for all TCF experience configs to REJECT_ALL
+    op.execute(
+        "UPDATE privacyexperienceconfig SET reject_all_mechanism = 'REJECT_ALL' WHERE component = 'tcf_overlay'"
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("privacyexperienceconfig", "reject_all_mechanism")
+    op.drop_column("privacyexperienceconfighistory", "reject_all_mechanism")
+    op.execute("DROP TYPE IF EXISTS rejectallmechanism")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/6e565c16dae1_add_tcf_publisher_restrictions.py

@@ -0,0 +1,107 @@
+"""Add TCF Publisher Restrictions
+
+Revision ID: 6e565c16dae1
+Revises: 67d01c4e124e
+Create Date: 2025-04-02 12:35:34.105607
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "6e565c16dae1"
+down_revision = "67d01c4e124e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Create tcf_configuration table
+    op.create_table(
+        "tcf_configuration",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
+    )
+    # Create tcf_publisher_restriction table
+    op.create_table(
+        "tcf_publisher_restriction",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("tcf_configuration_id", sa.String(255), nullable=False),
+        sa.Column("purpose_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "restriction_type",
+            sa.Enum(
+                "purpose_restriction",
+                "require_consent",
+                "require_legitimate_interest",
+                name="tcfrestrictiontype",
+            ),
+            nullable=False,
+        ),
+        sa.Column(
+            "vendor_restriction",
+            sa.Enum(
+                "restrict_all_vendors",
+                "allow_specific_vendors",
+                "restrict_specific_vendors",
+                name="tcfvendorrestriction",
+            ),
+            nullable=False,
+        ),
+        sa.Column(
+            "range_entries",
+            postgresql.ARRAY(postgresql.JSONB(astext_type=sa.Text())),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["tcf_configuration_id"],
+            ["tcf_configuration.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_tcf_publisher_restriction_config_purpose"),
+        "tcf_publisher_restriction",
+        ["tcf_configuration_id", "purpose_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    # Drop tables
+    op.drop_table("tcf_publisher_restriction")
+    op.drop_table("tcf_configuration")
+
+    # Drop enums
+    op.execute("DROP TYPE tcfrestrictiontype")
+    op.execute("DROP TYPE tcfvendorrestriction")

fides/api/api/deps.py

@@ -29,8 +29,14 @@ def get_db() -> Generator:
 
 
 @contextmanager
-def get_db_contextmanager() -> Generator[Session, None, None]:
-    """Return our database session as a context manager"""
+def get_autoclose_db_session() -> Generator[Session, None, None]:
+    """
+    Return a database session as a context manager that automatically closes when the context exits.
+
+    Unlike get_api_session which is managed by FastAPI's dependency injection,
+    this context manager explicitly closes the session when exiting the context.
+    Use this when you need manual control over the session lifecycle outside of API endpoints.
+    """
     try:
         db = get_api_session()
         yield db

fides/api/cryptography/identity_salt.py

@@ -3,9 +3,8 @@ from functools import cache
 
 from loguru import logger
 
-from fides.api.db.session import get_db_session
+from fides.api.api.deps import get_autoclose_db_session
 from fides.api.models.identity_salt import IdentitySalt
-from fides.config import CONFIG
 
 
 @cache
@@ -15,14 +14,14 @@ def get_identity_salt() -> str:
     This function is cached to avoid repeated calls to the database for the same value.
     """
 
-    SessionLocal = get_db_session(CONFIG)
-    db = SessionLocal()
-    existing_salt = db.query(IdentitySalt).first()
-    if existing_salt is None:
-        new_salt = IdentitySalt.create(
-            db, data={"encrypted_value": {"value": secrets.token_hex(32)}}
-        )
-        logger.info("Created new identity salt.")
-        return new_salt.encrypted_value.get("value")
-    logger.info("Caching existing identity salt.")
-    return existing_salt.encrypted_value.get("value")
+    with get_autoclose_db_session() as db:
+        existing_salt = db.query(IdentitySalt).first()
+        if existing_salt is None:
+            new_salt = IdentitySalt.create(
+                db, data={"encrypted_value": {"value": secrets.token_hex(32)}}
+            )
+            logger.info("Created new identity salt.")
+            return new_salt.encrypted_value.get("value")
+
+        logger.info("Caching existing identity salt.")
+        return existing_salt.encrypted_value.get("value")

fides/api/custom_types.py

@@ -66,8 +66,13 @@ def validate_html_str(val: str) -> str:
         "strong",
         "u",
     }
+
+    ALLOWED_ATTRIBUTES = {
+        "a": {"href", "hreflang", "target"},
+    }
+
     if val:
-        return clean(val, tags=ALLOWED_HTML_TAGS)
+        return clean(val, tags=ALLOWED_HTML_TAGS, attributes=ALLOWED_ATTRIBUTES)
     return val
 
 

fides/api/db/base.py

@@ -8,6 +8,7 @@ from fides.api.models.attachment import Attachment, AttachmentReference
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.authentication_request import AuthenticationRequest
 from fides.api.models.client import ClientDetail
+from fides.api.models.comment import Comment, CommentReference
 from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.consent_automation import ConsentAutomation
 from fides.api.models.custom_asset import CustomAsset
@@ -58,4 +59,8 @@ from fides.api.models.storage import StorageConfig
 from fides.api.models.system_compass_sync import SystemCompassSync
 from fides.api.models.system_history import SystemHistory
 from fides.api.models.system_manager import SystemManager
+from fides.api.models.tcf_publisher_restrictions import (
+    TCFConfiguration,
+    TCFPublisherRestriction,
+)
 from fides.api.models.tcf_purpose_overrides import TCFPurposeOverride

fides/api/migrations/hash_migration_job.py

@@ -2,7 +2,7 @@ from loguru import logger
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
-from fides.api.api.deps import get_db_contextmanager
+from fides.api.api.deps import get_autoclose_db_session
 from fides.api.db.base_class import FidesBase
 from fides.api.migrations.hash_migration_mixin import HashMigrationMixin
 from fides.api.migrations.hash_migration_tracker import HashMigrationTracker
@@ -47,7 +47,7 @@ def bcrypt_migration_task() -> None:
     Job to migrate all the tables using bcrypt hashes for general data (excludes tables with credentials).
     """
 
-    with get_db_contextmanager() as db:
+    with get_autoclose_db_session() as db:
         # Do a single pass to check if any of the models have already been migrated.
         # This will allow us to optimize searching for these models by not calling
         # the previously used bcrypt hash.

fides/api/models/attachment.py

@@ -1,11 +1,13 @@
 import os
 from enum import Enum as EnumType
-from typing import IO, Any, Optional
+from io import BytesIO
+from typing import IO, TYPE_CHECKING, Any, Optional, Tuple, Union
 
+from fideslang.validation import AnyHttpUrlString
 from loguru import logger as log
 from sqlalchemy import Column
 from sqlalchemy import Enum as EnumColumn
-from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy import ForeignKey, String, UniqueConstraint, orm
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
@@ -23,6 +25,10 @@ from fides.api.service.storage.util import (
     get_local_filename,
 )
 
+if TYPE_CHECKING:
+    from fides.api.models.comment import Comment
+    from fides.api.models.privacy_request import PrivacyRequest
+
 
 class AttachmentType(str, EnumType):
     """
@@ -53,7 +59,9 @@ class AttachmentReference(Base):
         """Overriding base class method to set the table name."""
         return "attachment_reference"
 
-    attachment_id = Column(String, ForeignKey("attachment.id"), nullable=False)
+    attachment_id = Column(
+        String, ForeignKey("attachment.id", ondelete="CASCADE"), nullable=False
+    )
     reference_id = Column(String, nullable=False)
     reference_type = Column(EnumColumn(AttachmentReferenceType), nullable=False)
 
@@ -63,11 +71,8 @@ class AttachmentReference(Base):
         ),
     )
 
-    attachment = relationship(
-        "Attachment",
-        back_populates="references",
-        uselist=False,
-    )
+    # Relationships
+    attachment = relationship("Attachment", back_populates="references")
 
     @classmethod
     def create(
@@ -100,8 +105,11 @@ class Attachment(Base):
     references = relationship(
         "AttachmentReference",
         back_populates="attachment",
-        cascade="all, delete",
+        cascade="all, delete-orphan",
         uselist=True,
+        foreign_keys=[AttachmentReference.attachment_id],
+        primaryjoin=lambda: Attachment.id
+        == orm.foreign(AttachmentReference.attachment_id),
     )
 
     config = relationship(
@@ -127,13 +135,34 @@ class Attachment(Base):
 
         if self.config.type == StorageType.local:
             filename = get_local_filename(self.id)
+
+            # Validate that attachment is a file-like object
+            if not hasattr(attachment, "read"):
+                raise TypeError(f"Expected a file-like object, got {type(attachment)}")
+
+            # Reset the file pointer to the beginning
+            try:
+                attachment.seek(0)
+            except Exception as e:
+                raise ValueError(f"Failed to reset file pointer for attachment: {e}")
+
+            # Write the file in chunks to avoid loading the entire content into memory
             with open(filename, "wb") as file:
-                file.write(attachment.read())
+                for chunk in iter(
+                    lambda: attachment.read(1024 * 1024), b""
+                ):  # 1 MB chunks
+                    if not isinstance(chunk, bytes):
+                        raise TypeError(f"Expected bytes, got {type(chunk)}")
+                    file.write(chunk)
+
+            log.info(f"Uploaded {self.file_name} to local storage at {filename}")
             return
 
         raise ValueError(f"Unsupported storage type: {self.config.type}")
 
-    def retrieve_attachment(self) -> Optional[bytes]:
+    def retrieve_attachment(
+        self,
+    ) -> Optional[Tuple[BytesIO, Union[AnyHttpUrlString, str]]]:
         """Returns the attachment from S3 in bytes form."""
         if self.config.type == StorageType.s3:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
@@ -208,4 +237,44 @@ class Attachment(Base):
     def delete(self, db: Session) -> None:
         """Deletes an attachment record from the database and deletes the attachment from S3."""
         self.delete_attachment_from_storage()
+        for attachment_reference in self.references:
+            attachment_reference.delete(db)
         super().delete(db=db)
+
+    @staticmethod
+    def delete_attachments_for_reference_and_type(
+        db: Session, reference_id: str, reference_type: AttachmentReferenceType
+    ) -> None:
+        """
+        Deletes attachments associated with a given reference_id and reference_type.
+        Deletes all references to the attachments.
+
+        Args:
+            db: Database session
+            reference_id: ID of the reference
+            reference_type: Type of the reference
+
+        Examples:
+
+        - Delete all attachments associated with a comment.
+           ``Attachment.delete_attachments_for_reference_and_type(
+               db, comment.id, AttachmentReferenceType.comment
+           )``
+        - Delete all attachments associated with a privacy request.
+           ``Attachment.delete_attachments_for_reference_and_type(
+               db, privacy_request.id, AttachmentReferenceType.privacy_request
+            )``
+        """
+        # Query attachments explicitly to avoid lazy loading
+        attachments = (
+            db.query(Attachment)
+            .join(AttachmentReference)
+            .filter(
+                AttachmentReference.reference_id == reference_id,
+                AttachmentReference.reference_type == reference_type,
+            )
+            .all()
+        )
+
+        for attachment in attachments:
+            attachment.delete(db)

fides/api/models/comment.py

@@ -3,15 +3,17 @@ from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import Column
 from sqlalchemy import Enum as EnumColumn
-from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy import ForeignKey, String, UniqueConstraint, orm
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import Base
+from fides.api.models.attachment import Attachment, AttachmentReferenceType
 
 if TYPE_CHECKING:
-    from fides.api.models.attachment import Attachment
+    from fides.api.models.attachment import AttachmentReference
     from fides.api.models.fides_user import FidesUser
+    from fides.api.models.privacy_request import PrivacyRequest
 
 
 class CommentType(str, EnumType):
@@ -87,23 +89,51 @@ class Comment(Base):
     references = relationship(
         "CommentReference",
         back_populates="comment",
-        cascade="all, delete",
-        uselist=True,
-    )
-
-    attachments = relationship(
-        "Attachment",
-        secondary="attachment_reference",
-        primaryjoin="Comment.id == AttachmentReference.reference_id",
-        secondaryjoin="Attachment.id == AttachmentReference.attachment_id",
-        order_by="Attachment.created_at",
+        cascade="all, delete-orphan",
         uselist=True,
+        foreign_keys=[CommentReference.comment_id],
+        primaryjoin=lambda: Comment.id == orm.foreign(CommentReference.comment_id),
     )
 
     def delete(self, db: Session) -> None:
         """Delete the comment and all associated references."""
         # Delete the comment
-        for attachment in self.attachments:
-            if len(attachment.references) == 1:
-                attachment.delete(db)
+        Attachment.delete_attachments_for_reference_and_type(
+            db, self.id, AttachmentReferenceType.comment
+        )
+        for reference in self.references:
+            reference.delete(db)
         db.delete(self)
+
+    @staticmethod
+    def delete_comments_for_reference_and_type(
+        db: Session, reference_id: str, reference_type: CommentReferenceType
+    ) -> None:
+        """
+        Deletes comments associated with a given reference_id and reference_type.
+        Delete all references to the comments.
+
+        Args:
+            db: Database session.
+            reference_id: The reference id to delete.
+            reference_type: The reference type to delete
+
+        Examples:
+          - Delete all comments associated with a privacy request.
+            ``Comment.delete_comments_for_reference_and_type(
+                db, privacy_request.id, CommentReferenceType.privacy_request
+            )``
+        """
+        # Query comments explicitly to avoid lazy loading
+        comments = (
+            db.query(Comment)
+            .join(CommentReference)
+            .filter(
+                CommentReference.reference_id == reference_id,
+                CommentReference.reference_type == reference_type,
+            )
+            .all()
+        )
+
+        for comment in comments:
+            comment.delete(db)

fides/api/models/privacy_experience.py

@@ -46,6 +46,21 @@ class Layer1ButtonOption(Enum):
 
     ACKNOWLEDGE = "acknowledge"
     OPT_IN_OPT_OUT = "opt_in_opt_out"
+    OPT_IN_ONLY = "opt_in_only"
+
+
+class RejectAllMechanism(Enum):
+    """
+    Reject all mechanism options - not formalized in the db.
+    Used to configure the behavior of the reject all button in TCF experiences
+    """
+
+    # Reject both consent and legitimate interest preferences (all purposes, special features, vendors)
+    # This is the default behavior
+    REJECT_ALL = "reject_all"
+    # Reject only consent preferences (all purposes, special features, vendors).
+    # Do not reject any legitimate interest preferences.
+    REJECT_CONSENT_ONLY = "reject_consent_only"
 
 
 # Fides JS UX Types - there should only be one of these defined per region
@@ -179,6 +194,10 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
     The Privacy Experience Configuration model that stores shared configuration for Privacy Experiences.
 
     - Translations, Notices, and Regions (via Privacy Experiences) are linked to this resource.
+
+    If you're adding a new PrivacyExperienceConfig, make sure to use the `create` method since it has
+    custom logic that ensures other resources are created/updated as needed, as well as setting the
+    expected values for some fields in the experience config itself.
     """
 
     allow_language_selection = Column(
@@ -203,6 +222,13 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
         String, ForeignKey(ExperienceConfigTemplate.id_field_path)
     )  # The template from which this config was created if applicable
 
+    # Mechanism to use when the reject all button is clicked in a TCF experience
+    # Nullable because this is not applicable for other experience types
+    reject_all_mechanism = Column(
+        EnumColumn(RejectAllMechanism),
+        nullable=True,
+    )
+
     # Relationships
     experiences = relationship(
         "PrivacyExperience",
@@ -304,6 +330,15 @@ class PrivacyExperienceConfig(PrivacyExperienceConfigBase, Base):
         # Link Properties to this Privacy Experience config via the PrivacyExperienceConfigProperty table
         link_properties_to_experience_config(db, properties, experience_config)
 
+        # If the reject all mechanism is not set and the experience config is a TCF experience,
+        # set the reject all mechanism to REJECT_ALL
+        if (
+            experience_config.component == ComponentType.tcf_overlay
+            and experience_config.reject_all_mechanism is None
+        ):
+            experience_config.reject_all_mechanism = RejectAllMechanism.REJECT_ALL  # type: ignore
+            experience_config.save(db)
+
         return experience_config
 
     def update(self, db: Session, *, data: dict[str, Any]) -> PrivacyExperienceConfig:
@@ -507,6 +542,13 @@ class PrivacyExperienceConfigHistory(
         index=True,
     )  # If a translation is deleted, this is set to null, but the overall record remains in the database for reporting purposes
 
+    # Mechanism to use when the reject all button is clicked in a TCF experience
+    # Nullable because this is not applicable for other experience types
+    reject_all_mechanism = Column(
+        EnumColumn(RejectAllMechanism),
+        nullable=True,
+    )
+
     version = Column(Float, nullable=False, default=1.0)
 
 

fides/api/models/privacy_request/privacy_request.py

@@ -39,10 +39,14 @@ from fides.api.graph.config import (
     CollectionAddress,
 )
 from fides.api.migrations.hash_migration_mixin import HashMigrationMixin
-from fides.api.models.attachment import Attachment, AttachmentReference
+from fides.api.models.attachment import (
+    Attachment,
+    AttachmentReference,
+    AttachmentReferenceType,
+)
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.client import ClientDetail
-from fides.api.models.comment import Comment, CommentReference
+from fides.api.models.comment import Comment, CommentReference, CommentReferenceType
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.policy import (
@@ -169,18 +173,25 @@ class PrivacyRequest(
         backref="privacy_requests",
     )
     attachments = relationship(
-        Attachment,
+        "Attachment",
         secondary="attachment_reference",
-        primaryjoin="PrivacyRequest.id == AttachmentReference.reference_id",
+        primaryjoin="and_(PrivacyRequest.id == AttachmentReference.reference_id, "
+        "AttachmentReference.reference_type == 'privacy_request')",
         secondaryjoin="Attachment.id == AttachmentReference.attachment_id",
         order_by="Attachment.created_at",
+        viewonly=True,
+        uselist=True,
     )
+
     comments = relationship(
-        Comment,
+        "Comment",
         secondary="comment_reference",
-        primaryjoin="PrivacyRequest.id == CommentReference.reference_id",
+        primaryjoin="and_(PrivacyRequest.id == CommentReference.reference_id, "
+        "CommentReference.reference_type == 'privacy_request')",
         secondaryjoin="Comment.id == CommentReference.comment_id",
         order_by="Comment.created_at",
+        viewonly=True,
+        uselist=True,
     )
     property_id = Column(String, nullable=True)
 
@@ -323,6 +334,12 @@ class PrivacyRequest(
         deleting this object from the database
         """
         self.clear_cached_values()
+        Attachment.delete_attachments_for_reference_and_type(
+            db, self.id, AttachmentReferenceType.privacy_request
+        )
+        Comment.delete_comments_for_reference_and_type(
+            db, self.id, CommentReferenceType.privacy_request
+        )
 
         for provided_identity in self.provided_identities:  # type: ignore[attr-defined]
             provided_identity.delete(db=db)