ethyca-fides 2.57.1rc0__py2.py3-none-any.whl → 2.58.0rc0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-03-25T12:32:49-0300",
+ "date": "2025-03-26T15:45:28-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "e52bec9197d4b3ac45f875238ec25f7af6cf5db7",
- "version": "2.57.1rc0"
+ "full-revisionid": "66a04f627fb146468d57489cd6a92628dfa3703e",
+ "version": "2.58.0rc0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/1d2f04ac19f2_add_page_column_to_asset_table.py

@@ -0,0 +1,49 @@
+"""
+- add page column to asset table and
+- change `Browser Request` values to `Browser request` StagedResource.resource_type column
+- change `Browser Request` values to `Browser request` Asset.asset_type column
+
+Revision ID: 1d2f04ac19f2
+Revises: 7c3fbee90c78
+Create Date: 2025-03-17 19:36:43.016383
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "1d2f04ac19f2"
+down_revision = "7c3fbee90c78"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # adds page column to asset table
+    op.add_column(
+        "asset",
+        sa.Column("page", sa.ARRAY(sa.String()), server_default="{}", nullable=False),
+    )
+
+    # changes `Browser Request` values to `Browser request` in the two tables where it shows up`
+    op.execute(
+        "UPDATE stagedresource SET resource_type = 'Browser request' WHERE resource_type = 'Browser Request'"
+    )
+    op.execute(
+        "UPDATE asset SET asset_type = 'Browser request' WHERE asset_type = 'Browser Request'"
+    )
+
+
+def downgrade():
+    # drops page column from asset table
+    op.drop_column("asset", "page")
+
+    # changes `Browser request` values back to `Browser Request` in the two tables where it shows up
+    op.execute(
+        "UPDATE stagedresource SET resource_type = 'Browser Request' WHERE resource_type = 'Browser request'"
+    )
+    op.execute(
+        "UPDATE asset SET asset_type = 'Browser Request' WHERE asset_type = 'Browser request'"
+    )

fides/api/alembic/migrations/versions/7c3fbee90c78_add_user_assigned_data_uses_and_system_.py

@@ -0,0 +1,54 @@
+"""add user assigned data_uses and system_id columns to stagedresource
+
+Revision ID: 7c3fbee90c78
+Revises: 9a1eacd2666b
+Create Date: 2025-03-04 15:00:23.874270
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "7c3fbee90c78"
+down_revision = "9a1eacd2666b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "stagedresource",
+        sa.Column(
+            "user_assigned_system_id", sa.VARCHAR(), autoincrement=False, nullable=True
+        ),
+    )
+    op.add_column(
+        "stagedresource",
+        sa.Column(
+            "user_assigned_data_uses",
+            postgresql.ARRAY(sa.VARCHAR()),
+            server_default=sa.text("'{}'::character varying[]"),
+            autoincrement=False,
+            nullable=False,
+        ),
+    )
+    # ### end Alembic commands ###
+    op.create_index(
+        "ix_stagedresource_user_assigned_system_id",
+        "stagedresource",
+        ["user_assigned_system_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("stagedresource", "user_assigned_data_uses")
+    op.drop_column("stagedresource", "user_assigned_system_id")
+    # ### end Alembic commands ###
+    op.drop_index(
+        "ix_stagedresource_user_assigned_system_id", table_name="stagedresource"
+    )

fides/api/alembic/migrations/versions/9a1eacd2666b_add_description_column_to_asset.py

@@ -0,0 +1,25 @@
+"""add description column to asset
+
+Revision ID: 9a1eacd2666b
+Revises: 69ad6d844e21
+Create Date: 2025-02-28 13:59:26.022822
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "9a1eacd2666b"
+down_revision = "69ad6d844e21"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("asset", sa.Column("description", sa.String(), nullable=True))
+
+
+def downgrade():
+    op.drop_column("asset", "description")

fides/api/api/v1/endpoints/config_endpoints.py

@@ -58,8 +58,10 @@ def patch_settings(
     Only keys provided will be updated, others will be unaffected,
     i.e. true PATCH behavior.
     """
-
-    pruned_data = data.model_dump(exclude_none=True)
+    # We use exclude_unset=True to ensure that only the provided keys are updated.
+    # This is particularly useful for allowing setting a specific key to None, while
+    # keeping the existing values for other keys that aren't provided in the payload data.
+    pruned_data = data.model_dump(exclude_unset=True)
     logger.info("PATCHing application settings")
     update_config: ApplicationConfig = ApplicationConfig.update_api_set(db, pruned_data)
 

fides/api/api/v1/endpoints/generic_overrides.py

@@ -9,7 +9,7 @@ from fideslang.models import Dataset as FideslangDataset
 from pydantic import ValidationError as PydanticValidationError
 from sqlalchemy import not_, select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, load_only
 from starlette import status
 from starlette.status import (
     HTTP_200_OK,
@@ -22,9 +22,10 @@ from fides.api.common_exceptions import KeyOrNameAlreadyExists, ValidationError
 from fides.api.db.base_class import get_key_from_data
 from fides.api.db.crud import list_resource_query
 from fides.api.db.ctl_session import get_async_db
-from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
 from fides.api.models.datasetconfig import DatasetConfig
 from fides.api.oauth.utils import verify_oauth_client
+from fides.api.schemas.dataset import DatasetResponse
 from fides.api.schemas.filter_params import FilterParams
 from fides.api.schemas.taxonomy_extensions import (
     DataCategory,
@@ -98,6 +99,11 @@ async def create_dataset(
                 else exc.message
             ),
         )
+    except KeyOrNameAlreadyExists as exc:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(exc),
+        )
 
 
 @dataset_router.put(
@@ -133,7 +139,7 @@ async def update_dataset(
 @dataset_router.get(
     "/dataset",
     dependencies=[Security(verify_oauth_client, scopes=[CTL_DATASET_READ])],
-    response_model=Union[Page[FideslangDataset], List[FideslangDataset]],
+    response_model=Union[Page[DatasetResponse], List[DatasetResponse]],
     name="List datasets (optionally paginated)",
 )
 async def list_dataset_paginated(
@@ -144,7 +150,9 @@ async def list_dataset_paginated(
     data_categories: Optional[List[str]] = Query(None),
     exclude_saas_datasets: Optional[bool] = Query(False),
     only_unlinked_datasets: Optional[bool] = Query(False),
-) -> Union[Page[FideslangDataset], List[FideslangDataset]]:
+    connection_type: Optional[ConnectionType] = Query(None),
+    minimal: Optional[bool] = Query(False),
+) -> Union[Page[DatasetResponse], List[DatasetResponse]]:
     """
     Get a list of all of the Datasets.
     If any pagination parameters (size or page) are provided, then the response will be paginated.
@@ -156,6 +164,25 @@ async def list_dataset_paginated(
 
     query = select(CtlDataset)
 
+    if minimal:
+        # .options() allows us to modify how the query loads data by configuring the query's loading behavior
+        # load_only() optimizes the query by only loading the specified columns from the database
+        # This reduces memory usage and query time by not loading unnecessary columns
+        # The columns specified below are the minimal set needed for the DatasetResponse model
+        query = query.options(  # type: ignore[attr-defined]
+            load_only(
+                CtlDataset.id,
+                CtlDataset.fides_key,
+                CtlDataset.organization_fides_key,
+                CtlDataset.name,
+                CtlDataset.created_at,
+                CtlDataset.updated_at,
+                CtlDataset.meta,
+                CtlDataset.fides_meta,
+                CtlDataset.description,
+            )
+        )
+
     # Add filters for search and data categories
     filter_params = FilterParams(search=search, data_categories=data_categories)
     filtered_query = apply_filters_to_query(
@@ -165,6 +192,13 @@ async def list_dataset_paginated(
         filter_params=filter_params,
     )
 
+    # If applicable, filter by connection type
+    if connection_type:
+        filtered_query = filtered_query.where(
+            CtlDataset.fides_meta["namespace"]["connection_type"].as_string()
+            == connection_type.value
+        )
+
     # If applicable, keep only unlinked datasets
     if only_unlinked_datasets:
         linked_datasets = select([DatasetConfig.ctl_dataset_id])
@@ -182,10 +216,18 @@ async def list_dataset_paginated(
         )
 
     if not page and not size:
-        return await list_resource_query(db, filtered_query, CtlDataset)
+        results = await list_resource_query(db, filtered_query, CtlDataset)
+        response = [
+            DatasetResponse.model_validate(result.__dict__) for result in results
+        ]
+        return response
 
     pagination_params = Params(page=page or 1, size=size or 50)
-    return await async_paginate(db, filtered_query, pagination_params)
+    results = await async_paginate(db, filtered_query, pagination_params)
+    results.items = [  # type: ignore[attr-defined]
+        DatasetResponse.model_validate(result.__dict__) for result in results.items  # type: ignore[attr-defined]
+    ]
+    return results
 
 
 @dataset_router.get(

fides/api/graph/traversal.py

@@ -270,6 +270,14 @@ class BaseTraversal:
                 [key.value for key in remaining_node_keys],
             )
 
+        # filter out remaining_edges if the nodes they link are allowed to remain unreachable
+        remaining_edges = {
+            edge
+            for edge in remaining_edges
+            if edge.f1.collection_address() in remaining_node_keys
+            and edge.f2.collection_address() in remaining_node_keys
+        }
+
         # error if there are edges that have not been visited
         if remaining_edges:
             logger.error(

fides/api/models/asset.py

@@ -38,6 +38,8 @@ class Asset(Base):
     locations = Column(ARRAY(String), server_default="{}", nullable=False)
     with_consent = Column(BOOLEAN, default=False, nullable=False)
     data_uses = Column(ARRAY(String), server_default="{}", nullable=False)
+    description = Column(String, nullable=True)
+    page = Column(ARRAY(String), server_default="{}", nullable=False)
 
     # generic object to store additional attributes, specific to asset type
     meta = Column(
@@ -93,9 +95,11 @@ class Asset(Base):
         - asset_type
         - domain
         - base_url (if applicable)
-        - system_id.
+        - system_id
+
+        If you provide the ID of an existing asset it will be updated with any provided data
         """
-        if (
+        if "id" not in data and (
             "name" not in data
             or "asset_type" not in data
             or "domain" not in data
@@ -105,25 +109,40 @@ class Asset(Base):
                 "name, asset_type, domain, and system_id are required fields on assets"
             )
 
-        result = await async_session.execute(
-            select(cls).where(  # type: ignore[arg-type, call-arg]
-                cls.name == data["name"],
-                cls.asset_type == data["asset_type"],
-                cls.domain == data["domain"],
-                cls.system_id == data["system_id"],
-                cls.base_url == data.get("base_url"),
-            )
-        )
-        existing_record = result.scalars().first()
         record_id: str
-        if existing_record:
-            await async_session.execute(
-                update(cls).where(cls.id == existing_record.id).values(data)  # type: ignore[arg-type]
+
+        if "id" in data:
+            result = await async_session.execute(
+                select(cls).where(cls.id == data["id"])  # type: ignore[arg-type]
             )
-            record_id = existing_record.id
+            existing_record = result.scalars().first()
+            if existing_record:
+                await async_session.execute(
+                    update(cls).where(cls.id == existing_record.id).values(data)  # type: ignore[arg-type]
+                )
+                record_id = existing_record.id
+            else:
+                raise ValueError(f"Asset with id {data['id']} does not exist")
         else:
-            result = await async_session.execute(insert(cls).values(data))  # type: ignore[arg-type]
-            record_id = result.inserted_primary_key.id
+            result = await async_session.execute(
+                select(cls).where(  # type: ignore[arg-type, call-arg]
+                    cls.name == data["name"],
+                    cls.asset_type == data["asset_type"],
+                    cls.domain == data["domain"],
+                    cls.base_url == data.get("base_url"),
+                    cls.system_id == data["system_id"],
+                )
+            )
+
+            existing_record = result.scalars().first()
+            if existing_record:
+                await async_session.execute(
+                    update(cls).where(cls.id == existing_record.id).values(data)  # type: ignore[arg-type]
+                )
+                record_id = existing_record.id
+            else:
+                result = await async_session.execute(insert(cls).values(data))  # type: ignore[arg-type]
+                record_id = result.inserted_primary_key.id
 
         result = await async_session.execute(select(cls).where(cls.id == record_id))  # type: ignore[arg-type]
         return result.scalars().first()

fides/api/models/attachment.py

@@ -1,6 +1,6 @@
 import os
 from enum import Enum as EnumType
-from typing import Any, Optional
+from typing import IO, Any, Optional
 
 from loguru import logger as log
 from sqlalchemy import Column
@@ -13,12 +13,14 @@ from fides.api.db.base_class import Base
 from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
 from fides.api.models.storage import StorageConfig  # pylint: disable=unused-import
 from fides.api.schemas.storage.storage import StorageDetails, StorageType
-from fides.api.tasks.storage import (
-    LOCAL_FIDES_UPLOAD_DIRECTORY,
+from fides.api.service.storage.s3 import (
     generic_delete_from_s3,
     generic_retrieve_from_s3,
+    generic_upload_to_s3,
+)
+from fides.api.service.storage.util import (
+    LOCAL_FIDES_UPLOAD_DIRECTORY,
     get_local_filename,
-    upload_to_s3,
 )
 
 
@@ -91,7 +93,6 @@ class Attachment(Base):
 
     user = relationship(
         "FidesUser",
-        backref="attachments",
         lazy="selectin",
         uselist=False,
     )
@@ -109,18 +110,15 @@ class Attachment(Base):
         uselist=False,
     )
 
-    def upload(self, attachment: bytes) -> None:
+    def upload(self, attachment: IO[bytes]) -> None:
         """Uploads an attachment to S3 or local storage."""
         if self.config.type == StorageType.s3:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
-            upload_to_s3(
+            generic_upload_to_s3(
                 storage_secrets=self.config.secrets,
-                data={},
                 bucket_name=bucket_name,
                 file_key=self.id,
-                resp_format=self.config.format,
-                privacy_request=None,
                 document=attachment,
                 auth_method=auth_method,
             )
@@ -130,7 +128,7 @@ class Attachment(Base):
         if self.config.type == StorageType.local:
             filename = get_local_filename(self.id)
             with open(filename, "wb") as file:
-                file.write(attachment)
+                file.write(attachment.read())
             return
 
         raise ValueError(f"Unsupported storage type: {self.config.type}")
@@ -150,7 +148,7 @@ class Attachment(Base):
         if self.config.type == StorageType.local:
             filename = f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{self.id}"
             with open(filename, "rb") as file:
-                return file.read()
+                return file.read(), filename
 
         raise ValueError(f"Unsupported storage type: {self.config.type}")
 
@@ -180,7 +178,7 @@ class Attachment(Base):
         db: Session,
         *,
         data: dict[str, Any],
-        attachment_file: bytes,
+        attachment_file: IO[bytes],
         check_name: bool = False,
     ) -> "Attachment":
         """Creates a new attachment record in the database and uploads the attachment to S3."""

fides/api/models/comment.py

@@ -1,5 +1,5 @@
 from enum import Enum as EnumType
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import Column
 from sqlalchemy import Enum as EnumColumn
@@ -8,8 +8,10 @@ from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import Base
-from fides.api.models.attachment import Attachment, AttachmentReference
-from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+
+if TYPE_CHECKING:
+    from fides.api.models.attachment import Attachment
+    from fides.api.models.fides_user import FidesUser
 
 
 class CommentType(str, EnumType):
@@ -78,7 +80,6 @@ class Comment(Base):
 
     user = relationship(
         "FidesUser",
-        backref="comments",
         lazy="selectin",
         uselist=False,
     )
@@ -90,20 +91,19 @@ class Comment(Base):
         uselist=True,
     )
 
-    def get_attachments(self, db: Session) -> list[Attachment]:
-        """Retrieve all attachments associated with this comment."""
-        stmt = (
-            db.query(Attachment)
-            .join(
-                AttachmentReference, Attachment.id == AttachmentReference.attachment_id
-            )
-            .where(AttachmentReference.reference_id == self.id)
-        )
-        return db.execute(stmt).scalars().all()
+    attachments = relationship(
+        "Attachment",
+        secondary="attachment_reference",
+        primaryjoin="Comment.id == AttachmentReference.reference_id",
+        secondaryjoin="Attachment.id == AttachmentReference.attachment_id",
+        order_by="Attachment.created_at",
+        uselist=True,
+    )
 
     def delete(self, db: Session) -> None:
         """Delete the comment and all associated references."""
-        attachments = self.get_attachments(db)
-        for attachment in attachments:
-            attachment.delete(db)
+        # Delete the comment
+        for attachment in self.attachments:
+            if len(attachment.references) == 1:
+                attachment.delete(db)
         db.delete(self)

fides/api/models/connectionconfig.py

@@ -186,7 +186,7 @@ class ConnectionConfig(Base):
             )
         ),
         nullable=True,
-    )  # Type bytea in the db
+    )  # Type bytes in the db
     last_test_timestamp = Column(DateTime(timezone=True))
     last_test_succeeded = Column(Boolean)
     disabled = Column(Boolean, server_default="f", default=False)

fides/api/models/detection_discovery.py

@@ -277,6 +277,18 @@ class StagedResource(Base):
         server_default="{}",
         default=dict,
     )
+    user_assigned_data_uses = Column(
+        ARRAY(String),
+        nullable=False,
+        server_default="{}",
+        default=dict,
+    )
+    user_assigned_system_id = Column(
+        String,
+        ForeignKey(System.id_field_path),
+        nullable=True,
+        index=True,
+    )
 
     # pointers to child and parent URNs
     children = Column(

fides/api/models/location_regulation_selections.py

@@ -317,6 +317,7 @@ class Location(LocationRegulationBase):
 
     belongs_to: List[str] = []
     regulation: List[str] = []
+    is_country: bool = False
 
 
 class LocationGroup(Location):

fides/api/models/privacy_request/__init__.py

@@ -0,0 +1,44 @@
+from .consent import Consent, ConsentRequest
+from .execution_log import (
+    COMPLETED_EXECUTION_LOG_STATUSES,
+    EXITED_EXECUTION_LOG_STATUSES,
+    ExecutionLog,
+    can_run_checkpoint,
+)
+from .privacy_request import (
+    CustomPrivacyRequestField,
+    PrivacyRequest,
+    PrivacyRequestError,
+    PrivacyRequestNotifications,
+)
+from .provided_identity import ProvidedIdentity, ProvidedIdentityType
+from .request_task import RequestTask, TraversalDetails
+from .webhook import (
+    CallbackType,
+    SecondPartyRequestFormat,
+    generate_request_callback_pre_approval_jwe,
+    generate_request_callback_resume_jwe,
+    generate_request_task_callback_jwe,
+)
+
+__all__ = [
+    "CallbackType",
+    "Consent",
+    "ConsentRequest",
+    "COMPLETED_EXECUTION_LOG_STATUSES",
+    "EXITED_EXECUTION_LOG_STATUSES",
+    "CustomPrivacyRequestField",
+    "ExecutionLog",
+    "PrivacyRequest",
+    "PrivacyRequestError",
+    "PrivacyRequestNotifications",
+    "ProvidedIdentity",
+    "ProvidedIdentityType",
+    "RequestTask",
+    "SecondPartyRequestFormat",
+    "TraversalDetails",
+    "can_run_checkpoint",
+    "generate_request_callback_pre_approval_jwe",
+    "generate_request_callback_resume_jwe",
+    "generate_request_task_callback_jwe",
+]