ethyca-fides 2.59.1rc0__py2.py3-none-any.whl → 2.59.2b1__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-04-16T11:42:20+0200",
+ "date": "2025-04-16T22:35:59-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "943a31433f896fe25cd9884d836cc029465c1e3b",
- "version": "2.59.1rc0"
+ "full-revisionid": "f64e628f1502e4a7c7045e517a74550c9d5cd661",
+ "version": "2.59.2b1"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/c9c72b3d550b_data_migration_cookie_asset.py

@@ -0,0 +1,218 @@
+"""data migration converting existing 'Cookies' resources to 'Asset' resources with type 'Cookie'
+
+Revision ID: c9c72b3d550b
+Revises: 9288f729cac4
+Create Date: 2025-02-18 18:33:56.039924
+
+"""
+
+import uuid
+from typing import Dict, Tuple
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+from fides.api.models.asset import Asset
+
+# revision identifiers, used by Alembic.
+revision = "c9c72b3d550b"
+down_revision = "9288f729cac4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # migrate existing cookies to assets
+    connection = op.get_bind()
+    result = connection.execute(
+        sa.text(
+            "SELECT c.id, c.created_at, c.updated_at, c.name, c.domain, c.system_id, c.path, c.privacy_declaration_id, pd.system_id AS pud_system_id, pd.data_use "
+            "FROM cookies c "
+            "LEFT JOIN privacydeclaration pd ON c.privacy_declaration_id = pd.id"
+        )
+    )
+
+    logger.debug("Converting existing cookies to assets")
+    assets_to_create: Dict[str, Asset] = {}
+    for row in result:
+        if row.privacy_declaration_id is None:
+            identifier = f"{row.name}_{row.system_id}"
+            if identifier not in assets_to_create.keys():
+                assets_to_create[identifier] = Asset(
+                    id=str(uuid.uuid4()),
+                    created_at=row.created_at,
+                    updated_at=row.updated_at,
+                    name=row.name,
+                    domain=row.domain,
+                    system_id=row.system_id,
+                    data_uses=[],
+                    asset_type="Cookie",
+                )
+            continue
+
+        if row.pud_system_id:
+            system_id = row.pud_system_id
+            data_uses = [row.data_use]
+        else:
+            system_id = row.system_id
+            data_uses = []
+
+        # Asset unique identifier is a composite of name, asset_type, domain, base_url, system_id. All assets here are cookies.
+        identifier = f"{row.name}_{'Cookie'}_{row.domain}_{row.path}_{system_id}"
+        if identifier in assets_to_create.keys():
+            # If the asset already exists, append the data_use to the existing asset
+            assets_to_create[identifier].data_uses.extend(data_uses)
+        else:
+            # If the asset does not exist, create a new asset with the data_use
+            assets_to_create[identifier] = Asset(
+                id=str(uuid.uuid4()),
+                created_at=row.created_at,
+                updated_at=row.updated_at,
+                name=row.name,
+                domain=row.domain,
+                system_id=system_id,
+                data_uses=data_uses,
+                asset_type="Cookie",
+            )
+
+    # Insert the assets into the asset table
+    assets_list = [
+        {
+            "id": asset.id if asset.id else str(uuid.uuid4()),
+            "created_at": asset.created_at,
+            "updated_at": asset.updated_at,
+            "name": asset.name,
+            "domain": asset.domain,
+            "system_id": asset.system_id,
+            "data_uses": asset.data_uses,
+            "asset_type": "Cookie",
+        }
+        for asset in assets_to_create.values()
+    ]
+
+    if assets_list:
+        logger.debug("Inserting assets into asset table ")
+        connection.execute(
+            sa.text(
+                "INSERT INTO asset (id, created_at, updated_at, name, domain, system_id, data_uses, asset_type, with_consent) "
+                "VALUES (:id, :created_at, :updated_at, :name, :domain, :system_id, :data_uses, :asset_type, false) "
+                "ON CONFLICT DO NOTHING"
+            ),
+            assets_list,
+        )
+    else:
+        logger.debug("No assets to insert into asset table. Skipping.")
+
+    # Delete the cookies table
+    logger.debug("Deleting cookies table")
+    connection.execute(sa.text("DROP TABLE cookies"))
+
+
+def downgrade():
+    # Recreate the cookies table
+    logger.debug("Recreating cookies table")
+    op.create_table(
+        "cookies",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("domain", sa.String(), nullable=True),
+        sa.Column("path", sa.String(), nullable=True),
+        sa.Column("system_id", sa.String(), nullable=True),
+        sa.Column("privacy_declaration_id", sa.String(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["privacy_declaration_id"], ["privacydeclaration.id"], ondelete="SET NULL"
+        ),
+        sa.ForeignKeyConstraint(["system_id"], ["ctl_systems.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "name", "privacy_declaration_id", name="_cookie_name_privacy_declaration_uc"
+        ),
+    )
+    op.create_index(op.f("ix_cookies_id"), "cookies", ["id"], unique=False)
+    op.create_index(op.f("ix_cookies_name"), "cookies", ["name"], unique=False)
+    op.create_index(
+        op.f("ix_cookies_privacy_declaration_id"),
+        "cookies",
+        ["privacy_declaration_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_cookies_system_id"), "cookies", ["system_id"], unique=False
+    )
+    # ### end Alembic commands ###
+
+    connection = op.get_bind()
+    result = connection.execute(
+        sa.text(
+            "SELECT id, name, domain, system_id, data_uses FROM asset WHERE asset_type = 'Cookie'"
+        )
+    )
+
+    # fetch all privacy declarations
+    privacy_declarations = connection.execute(
+        sa.text("SELECT id, system_id, data_use FROM privacydeclaration")
+    ).fetchall()
+    # create a mapping of system_id and data_use to privacy_declaration_id
+    privacy_declaration_mapping: Dict[Tuple[str, str], str] = {}
+    for pud in privacy_declarations:
+        privacy_declaration_mapping[(pud.system_id, pud.data_use)] = pud.id
+
+    logger.debug("Migrating existing assets to cookies")
+    for row in result:
+        # Find the privacy declaration ID matching the system ID and data use
+        privacy_declaration_ids = []
+        for data_use in row.data_uses:
+            pud_id = privacy_declaration_mapping.get((row.system_id, data_use), None)
+            if pud_id:
+                privacy_declaration_ids.append(pud_id)
+
+        cookies_to_insert = []
+
+        if len(privacy_declaration_ids) == 0:
+            # If no privacy declaration match we attach to system_id
+            cookies_to_insert.append(
+                {
+                    "id": row.id,
+                    "name": row.name,
+                    "domain": row.domain,
+                    "system_id": row.system_id,
+                    "privacy_declaration_id": None,
+                }
+            )
+        else:
+            for privacy_declaration_id in privacy_declaration_ids:
+                # Create a cookie for each privacy declaration ID
+                # generate a new ID
+                cookie_id = str(uuid.uuid4())
+                cookies_to_insert.append(
+                    {
+                        "id": cookie_id,
+                        "name": row.name,
+                        "domain": row.domain,
+                        "system_id": None,
+                        "privacy_declaration_id": privacy_declaration_id,
+                    }
+                )
+
+        if cookies_to_insert:
+            connection.execute(
+                sa.text(
+                    "INSERT INTO cookies (id, created_at, updated_at, name, domain, system_id, privacy_declaration_id) "
+                    "VALUES (:id, NOW(), NOW(), :name, :domain, :system_id, :privacy_declaration_id) "
+                    "ON CONFLICT DO NOTHING"
+                ),
+                cookies_to_insert,
+            )

fides/api/api/v1/endpoints/storage_endpoints.py

@@ -40,6 +40,7 @@ from fides.api.schemas.storage.storage import (
     FULLY_CONFIGURED_STORAGE_TYPES,
     AWSAuthMethod,
     BulkPutStorageConfigResponse,
+    GCSAuthMethod,
     StorageConfigStatus,
     StorageConfigStatusMessage,
     StorageDestination,
@@ -418,10 +419,10 @@ def get_storage_status(
 
 
 def _storage_config_requires_secrets(storage_config: StorageConfig) -> bool:
-    return (
-        storage_config.details.get(StorageDetails.AUTH_METHOD.value, None)
-        == AWSAuthMethod.SECRET_KEYS.value
-    )
+    return storage_config.details.get(StorageDetails.AUTH_METHOD.value, None) in [
+        AWSAuthMethod.SECRET_KEYS.value,
+        GCSAuthMethod.SERVICE_ACCOUNT_KEYS.value,
+    ]
 
 
 @router.put(

fides/api/db/system.py

@@ -8,19 +8,15 @@ from typing import Any, Dict, List, Optional, Tuple, Type, Union
 
 from deepdiff import DeepDiff
 from fastapi import HTTPException
-from fideslang.models import Cookies as CookieSchema
 from fideslang.models import System as SystemSchema
 from fideslang.validation import FidesKey
 from loguru import logger as log
-from sqlalchemy import and_, delete, insert, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session
-from sqlalchemy.sql.elements import BinaryExpression
 from starlette.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND
 
 from fides.api.db.crud import create_resource, get_resource, update_resource
 from fides.api.models.sql_models import (  # type: ignore[attr-defined]
-    Cookies,
     DataCategory,
     DataSubject,
     DataUse,
@@ -155,7 +151,6 @@ async def upsert_privacy_declarations(
         for privacy_declaration in resource.privacy_declarations:
             # prepare our 'payload' for either create or update
             data = privacy_declaration.model_dump(mode="json")
-            privacy_declaration_cookies: List[Dict] = data.pop("cookies", None)
             data["system_id"] = system.id  # include FK back to system
 
             # if we find matching declaration, remove it from our map
@@ -168,102 +163,11 @@ async def upsert_privacy_declarations(
                 # otherwise, create a new declaration record
                 declaration = PrivacyDeclaration.create(db, data=data)
 
-            # Upsert cookies for the given privacy declaration
-            await upsert_cookies(
-                db, privacy_declaration_cookies, declaration, system=None
-            )
-
         # delete any existing privacy declarations that have not been "matched" in the request
         for existing_declarations in existing_declarations.values():
             await db.delete(existing_declarations)
 
 
-async def upsert_cookies(
-    async_session: AsyncSession,
-    cookies: Optional[List[Dict]],
-    privacy_declaration: Optional[PrivacyDeclaration],
-    system: Optional[System],
-) -> None:
-    """
-    Upsert cookies for the given system or privacy_declaration.  Cookies can be attached at the system-level
-    or the privacy declaration-level.
-
-    Remove any existing cookies that aren't specified here.
-    """
-    if not (privacy_declaration or system):
-        # Dev-level error
-        raise ValueError(
-            "Either system or privacy declaration must be supplied to upsert cookies"
-        )
-
-    if privacy_declaration and system:
-        # Dev-level error
-        raise ValueError(
-            "Supply either system or privacy declaration, not both, to upsert cookies"
-        )
-
-    parsed_cookies = (
-        [CookieSchema.model_validate(cookie) for cookie in cookies] if cookies else []
-    )
-
-    resource_filter: BinaryExpression = (
-        Cookies.system_id == system.id
-        if system
-        else Cookies.privacy_declaration_id == privacy_declaration.id  # type: ignore[union-attr]
-    )
-
-    for cookie_data in parsed_cookies:
-        # Check if cookie exists on the resource
-        result = await async_session.execute(
-            select(Cookies).where(
-                and_(Cookies.name == cookie_data.name, resource_filter)
-            )
-        )
-        row: Optional[Cookies] = result.scalars().first()
-        if row:
-            # Update existing cookie
-            await async_session.execute(
-                update(Cookies)
-                .where(Cookies.id == row.id)
-                .values(cookie_data.model_dump(mode="json"))
-            )
-
-        else:
-            # Insert new cookie
-            await async_session.execute(
-                insert(Cookies).values(
-                    {
-                        "name": cookie_data.name,
-                        "path": cookie_data.path,
-                        "domain": cookie_data.domain,
-                        "privacy_declaration_id": (
-                            privacy_declaration.id if privacy_declaration else None
-                        ),
-                        "system_id": system.id if system else None,
-                    }
-                )
-            )
-
-    # Select cookies which are currently on the application resource, but not included in the request
-    delete_result = await async_session.execute(
-        select(Cookies).where(
-            and_(
-                Cookies.name.notin_([cookie.name for cookie in parsed_cookies]),
-                resource_filter,
-            )
-        )
-    )
-
-    # Remove those cookies altogether
-    await async_session.execute(
-        delete(Cookies).where(
-            Cookies.id.in_(
-                [cookie.id for cookie in delete_result.scalars().unique().all()]
-            )
-        )
-    )
-
-
 async def update_system(
     resource: SystemSchema, db: AsyncSession, current_user_id: Optional[str] = None
 ) -> Tuple[Dict, bool]:
@@ -288,18 +192,10 @@ async def update_system(
         resource, "privacy_declarations"
     )  # remove the attribute on the system since we've already updated declarations
 
-    # Remove system-level cookies from request before updating the system.
-    # Cookies are upserted separately
-    proposed_system_cookies: Optional[List[Cookies]] = resource.cookies
-    delattr(resource, "cookies")
-
     # perform any updates on the system resource itself
     updated_system = await update_resource(System, resource.model_dump(), db)
 
     async with db.begin():
-        await upsert_cookies(
-            db, proposed_system_cookies, privacy_declaration=None, system=updated_system
-        )  # Upsert the associated cookies at the System-level
 
         await db.refresh(updated_system)
 
@@ -400,10 +296,6 @@ async def create_system(
     # remove the attribute on the system update since the declarations will be created separately
     delattr(resource, "privacy_declarations")
 
-    # Remove system-level cookies from request; these will be processed after the system is added
-    proposed_system_cookies: Optional[List[Cookies]] = resource.cookies
-    delattr(resource, "cookies")
-
     # create the system resource using generic creation
     # the system must be created before the privacy declarations so that it can be referenced
     resource_dict = resource.model_dump()
@@ -418,27 +310,15 @@ async def create_system(
     privacy_declaration_exception = None
     try:
         async with db.begin():
-            await upsert_cookies(
-                db,
-                proposed_system_cookies,
-                privacy_declaration=None,
-                system=created_system,
-            )  # Create the associated cookies at the System-level
 
             # create the specified declarations as records in their own table
             for privacy_declaration in privacy_declarations:
                 data = privacy_declaration.model_dump(mode="json")
                 data["system_id"] = created_system.id  # add FK back to system
-                privacy_declaration_cookies: List[Dict] = data.pop("cookies", [])
                 privacy_declaration = PrivacyDeclaration.create(
                     db, data=data
                 )  # create the associated PrivacyDeclaration
-                await upsert_cookies(
-                    db,
-                    privacy_declaration_cookies,
-                    privacy_declaration=privacy_declaration,
-                    system=None,
-                )  # Create the associated cookies on the Privacy Declaration
+
     except Exception as e:
         log.error(
             f"Error adding privacy declarations, reverting system creation: {str(privacy_declaration_exception)}"

fides/api/models/asset.py

@@ -25,8 +25,7 @@ from fides.api.models.sql_models import System  # type: ignore[attr-defined]
 
 class Asset(Base):
     """
-    Web assets associated with a system. This model will supersede `Cookies` once we have established
-    a migration path and backward compatibility with all `Cookies` related APIs.
+    Web assets associated with a system
     """
 
     # Common attributes

fides/api/models/fides_user.py

@@ -23,6 +23,7 @@ from fides.api.db.base_class import Base
 from fides.api.models.audit_log import AuditLog
 
 # Intentionally importing SystemManager here to build the FidesUser.systems relationship
+from fides.api.models.system_manager import SystemManager  # type: ignore[unused-import]
 from fides.api.schemas.user import DisabledReason
 from fides.config import CONFIG
 

fides/api/models/privacy_notice.py

@@ -20,10 +20,9 @@ from fides.api.models import (
     dry_update_data,
     update_if_modified,
 )
+from fides.api.models.asset import Asset
 from fides.api.models.location_regulation_selections import DeprecatedNoticeRegion
 from fides.api.models.sql_models import (  # type: ignore[attr-defined]
-    Cookies,
-    PrivacyDeclaration,
     System,
     get_system_data_uses,
 )
@@ -191,24 +190,20 @@ class PrivacyNotice(PrivacyNoticeBase, Base):
         raise Exception("Invalid notice consent mechanism.")
 
     @property
-    def cookies(self) -> List[Cookies]:
-        """Return relevant cookie names (via the data use)"""
+    def cookies(self) -> List[Asset]:
+        """Return relevant assets of type 'cookie' (via the data use)"""
         db = Session.object_session(self)
-        return (
-            db.query(Cookies)
-            .join(
-                PrivacyDeclaration,
-                PrivacyDeclaration.id == Cookies.privacy_declaration_id,
-            )
-            .filter(
-                or_(
-                    *[
-                        PrivacyDeclaration.data_use.like(f"{notice_use}%")
-                        for notice_use in self.data_uses
-                    ]
-                )
-            )
-        ).all()
+        or_queries = [
+            f"array_to_string(data_uses, ',') ILIKE '{data_use}%'"
+            for data_use in self.data_uses
+        ]
+
+        query = db.query(Asset).filter(
+            Asset.asset_type == "Cookie",
+            or_(*[text(query) for query in or_queries]),
+        )
+
+        return query.all()
 
     @property
     def calculated_systems_applicable(self) -> bool:

fides/api/models/sql_models.py

@@ -457,10 +457,6 @@ class System(Base, FidesBase):
         lazy="selectin",
     )
 
-    cookies = relationship(
-        "Cookies", back_populates="system", lazy="selectin", uselist=True, viewonly=True
-    )
-
     assets = relationship(
         "Asset", back_populates="system", lazy="selectin", uselist=True, viewonly=True
     )
@@ -560,9 +556,6 @@ class PrivacyDeclaration(Base):
     system = relationship(
         System, back_populates="privacy_declarations", lazy="selectin"
     )
-    cookies = relationship(
-        "Cookies", back_populates="privacy_declaration", lazy="joined", uselist=True
-    )
 
     @classmethod
     def create(
@@ -866,50 +859,6 @@ class AuditLogResource(Base):
     extra_data = Column(JSON, nullable=True)
 
 
-class Cookies(Base):
-    """
-    Stores cookies.  Cookies have a FK to system and privacy declaration. If a privacy declaration is deleted,
-    the cookie can still remain linked to the system but unassociated with a data use.
-    """
-
-    name = Column(String, index=True, nullable=False)
-    path = Column(String)
-    domain = Column(String)
-
-    system_id = Column(
-        String, ForeignKey(System.id_field_path, ondelete="CASCADE"), index=True
-    )  # If system is deleted, remove the associated cookies.
-
-    privacy_declaration_id = Column(
-        String,
-        ForeignKey(PrivacyDeclaration.id_field_path, ondelete="CASCADE"),
-        index=True,
-    )  # If privacy declaration is deleted, remove the associated cookies.
-
-    system = relationship(
-        "System",
-        back_populates="cookies",
-        cascade="all,delete",
-        uselist=False,
-        lazy="selectin",
-    )
-
-    privacy_declaration = relationship(
-        "PrivacyDeclaration",
-        back_populates="cookies",
-        cascade="all,delete",
-        uselist=False,
-        lazy="joined",  # Joined is intentional, instead of selectin
-    )
-
-    __table_args__ = (
-        UniqueConstraint(
-            "name", "privacy_declaration_id", name="_cookie_name_privacy_declaration_uc"
-        ),
-        UniqueConstraint("name", "system_id", name="_cookie_name_system_uc"),
-    )
-
-
 def get_system_data_uses(db: Session, include_parents: bool) -> Set:
     """Get data uses across all systems and those data uses' parents if include_parents is True
 

fides/api/schemas/application_config.py

@@ -14,6 +14,7 @@ class StorageTypeApiAccepted(Enum):
     """Enum for storage destination types accepted in API updates"""
 
     s3 = "s3"
+    gcs = "gcs"
     local = "local"  # local should be used for testing only, not for processing real-world privacy requests
 
 

fides/api/schemas/connection_configuration/connection_secrets_datahub.py

@@ -20,11 +20,11 @@ class PeriodicIntegrationFrequency(Enum):
 
 class DatahubSchema(ConnectionConfigSecretsSchema):
     datahub_server_url: AnyHttpUrlStringRemovesSlash = Field(
-        title="DataHub Server URL",
+        title="DataHub server URL",
         description="The URL of your DataHub server.",
     )
     datahub_token: str = Field(
-        title="DataHub Token",
+        title="DataHub token",
         description="The token used to authenticate with your DataHub server.",
         json_schema_extra={"sensitive": True},
     )
@@ -33,7 +33,7 @@ class DatahubSchema(ConnectionConfigSecretsSchema):
         description="The frequency at which the integration should run. Available options are daily, weekly, and monthly.",
     )
     glossary_node: str = Field(
-        title="Glossary Node",
+        title="Glossary node",
         description="The glossary node name to use on Datahub for Fides Data Categories. (e.g. FidesDataCategories)",
     )
 

fides/api/schemas/connection_configuration/connection_secrets_postgres.py

@@ -31,7 +31,13 @@ class PostgreSQLSchema(ConnectionConfigSecretsSchema):
         description="The password used to authenticate and access the database.",
         json_schema_extra={"sensitive": True},
     )
-    dbname: str = Field(
+    ssh_required: bool = Field(
+        False,
+        title="SSH required",
+        description="Indicates whether an SSH tunnel is required for the connection. Enable this option if your PostgreSQL server is behind a firewall and requires SSH tunneling for remote connections.",
+    )
+    dbname: Optional[str] = Field(
+        default=None,
         title="Database",
         description="The name of the specific database within the database server that you want to connect to.",
     )
@@ -40,15 +46,9 @@ class PostgreSQLSchema(ConnectionConfigSecretsSchema):
         title="Schema",
         description="The default schema to be used for the database connection (defaults to public).",
     )
-    ssh_required: bool = Field(
-        False,
-        title="SSH required",
-        description="Indicates whether an SSH tunnel is required for the connection. Enable this option if your PostgreSQL server is behind a firewall and requires SSH tunneling for remote connections.",
-    )
 
     _required_components: ClassVar[List[str]] = [
         "host",
-        "dbname",
     ]
 
 

fides/api/schemas/messaging/messaging.py

@@ -286,12 +286,19 @@ class MessagingServiceDetailsTwilioEmail(BaseModel):
 class MessagingServiceDetailsAWS_SES(BaseModel):
     """The details required to represent an AWS SES email configuration."""
 
-    email_from: str
-    domain: str
+    email_from: Optional[str] = None
+    domain: Optional[str] = None
     aws_region: str
 
     model_config = ConfigDict(extra="forbid")
 
+    @model_validator(mode="before")
+    @classmethod
+    def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if not values.get("domain") and not values.get("email_from"):
+            raise ValueError("Either 'email_from' or 'domain' must be provided.")
+        return values
+
 
 class MessagingServiceSecrets(Enum):
     """Enum for message service secrets"""