ethyca-fides 2.57.1b6__py2.py3-none-any.whl → 2.57.1b8__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-03-20T13:38:41-0600",
+ "date": "2025-03-24T16:32:51-0600",
  "dirty": false,
  "error": null,
- "full-revisionid": "22f49c9755a002d9026f5ad9766c3d3aa7162e71",
- "version": "2.57.1b6"
+ "full-revisionid": "c371f453f33d1af4a35dfe6ae1d78c16b72576d8",
+ "version": "2.57.1b8"
 }
 '''  # END VERSION_JSON
 

fides/api/api/v1/endpoints/config_endpoints.py

@@ -58,8 +58,10 @@ def patch_settings(
     Only keys provided will be updated, others will be unaffected,
     i.e. true PATCH behavior.
     """
-
-    pruned_data = data.model_dump(exclude_none=True)
+    # We use exclude_unset=True to ensure that only the provided keys are updated.
+    # This is particularly useful for allowing setting a specific key to None, while
+    # keeping the existing values for other keys that aren't provided in the payload data.
+    pruned_data = data.model_dump(exclude_unset=True)
     logger.info("PATCHing application settings")
     update_config: ApplicationConfig = ApplicationConfig.update_api_set(db, pruned_data)
 

fides/api/api/v1/endpoints/generic_overrides.py

@@ -9,7 +9,7 @@ from fideslang.models import Dataset as FideslangDataset
 from pydantic import ValidationError as PydanticValidationError
 from sqlalchemy import not_, select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, load_only
 from starlette import status
 from starlette.status import (
     HTTP_200_OK,
@@ -22,9 +22,10 @@ from fides.api.common_exceptions import KeyOrNameAlreadyExists, ValidationError
 from fides.api.db.base_class import get_key_from_data
 from fides.api.db.crud import list_resource_query
 from fides.api.db.ctl_session import get_async_db
-from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
 from fides.api.models.datasetconfig import DatasetConfig
 from fides.api.oauth.utils import verify_oauth_client
+from fides.api.schemas.dataset import DatasetResponse
 from fides.api.schemas.filter_params import FilterParams
 from fides.api.schemas.taxonomy_extensions import (
     DataCategory,
@@ -138,7 +139,7 @@ async def update_dataset(
 @dataset_router.get(
     "/dataset",
     dependencies=[Security(verify_oauth_client, scopes=[CTL_DATASET_READ])],
-    response_model=Union[Page[FideslangDataset], List[FideslangDataset]],
+    response_model=Union[Page[DatasetResponse], List[DatasetResponse]],
     name="List datasets (optionally paginated)",
 )
 async def list_dataset_paginated(
@@ -149,7 +150,9 @@ async def list_dataset_paginated(
     data_categories: Optional[List[str]] = Query(None),
     exclude_saas_datasets: Optional[bool] = Query(False),
     only_unlinked_datasets: Optional[bool] = Query(False),
-) -> Union[Page[FideslangDataset], List[FideslangDataset]]:
+    connection_type: Optional[ConnectionType] = Query(None),
+    minimal: Optional[bool] = Query(False),
+) -> Union[Page[DatasetResponse], List[DatasetResponse]]:
     """
     Get a list of all of the Datasets.
     If any pagination parameters (size or page) are provided, then the response will be paginated.
@@ -161,6 +164,25 @@ async def list_dataset_paginated(
 
     query = select(CtlDataset)
 
+    if minimal:
+        # .options() allows us to modify how the query loads data by configuring the query's loading behavior
+        # load_only() optimizes the query by only loading the specified columns from the database
+        # This reduces memory usage and query time by not loading unnecessary columns
+        # The columns specified below are the minimal set needed for the DatasetResponse model
+        query = query.options(  # type: ignore[attr-defined]
+            load_only(
+                CtlDataset.id,
+                CtlDataset.fides_key,
+                CtlDataset.organization_fides_key,
+                CtlDataset.name,
+                CtlDataset.created_at,
+                CtlDataset.updated_at,
+                CtlDataset.meta,
+                CtlDataset.fides_meta,
+                CtlDataset.description,
+            )
+        )
+
     # Add filters for search and data categories
     filter_params = FilterParams(search=search, data_categories=data_categories)
     filtered_query = apply_filters_to_query(
@@ -170,6 +192,13 @@ async def list_dataset_paginated(
         filter_params=filter_params,
     )
 
+    # If applicable, filter by connection type
+    if connection_type:
+        filtered_query = filtered_query.where(
+            CtlDataset.fides_meta["namespace"]["connection_type"].as_string()
+            == connection_type.value
+        )
+
     # If applicable, keep only unlinked datasets
     if only_unlinked_datasets:
         linked_datasets = select([DatasetConfig.ctl_dataset_id])
@@ -187,10 +216,18 @@ async def list_dataset_paginated(
         )
 
     if not page and not size:
-        return await list_resource_query(db, filtered_query, CtlDataset)
+        results = await list_resource_query(db, filtered_query, CtlDataset)
+        response = [
+            DatasetResponse.model_validate(result.__dict__) for result in results
+        ]
+        return response
 
     pagination_params = Params(page=page or 1, size=size or 50)
-    return await async_paginate(db, filtered_query, pagination_params)
+    results = await async_paginate(db, filtered_query, pagination_params)
+    results.items = [  # type: ignore[attr-defined]
+        DatasetResponse.model_validate(result.__dict__) for result in results.items  # type: ignore[attr-defined]
+    ]
+    return results
 
 
 @dataset_router.get(

fides/api/main.py

@@ -232,10 +232,13 @@ async def log_request(request: Request, call_next: Callable) -> Response:
         response = Response(status_code=500)
 
     handler_time = datetime.now() - start
+
+    # Take the total time in seconds and convert it to milliseconds, rounding to 3 decimal places
+    total_time = round(handler_time.total_seconds() * 1000, 3)
     logger.bind(
         method=request.method,
         status_code=response.status_code,
-        handler_time=f"{round(handler_time.microseconds * 0.001,3)}ms",
+        handler_time=f"{total_time}ms",
         path=request.url.path,
     ).info("Request received")
     return response

fides/api/models/attachment.py

@@ -13,12 +13,14 @@ from fides.api.db.base_class import Base
 from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
 from fides.api.models.storage import StorageConfig  # pylint: disable=unused-import
 from fides.api.schemas.storage.storage import StorageDetails, StorageType
-from fides.api.tasks.storage import (
-    LOCAL_FIDES_UPLOAD_DIRECTORY,
+from fides.api.service.storage.s3 import (
     generic_delete_from_s3,
     generic_retrieve_from_s3,
+    generic_upload_to_s3,
+)
+from fides.api.service.storage.util import (
+    LOCAL_FIDES_UPLOAD_DIRECTORY,
     get_local_filename,
-    upload_to_s3,
 )
 
 
@@ -91,7 +93,6 @@ class Attachment(Base):
 
     user = relationship(
         "FidesUser",
-        backref="attachments",
         lazy="selectin",
         uselist=False,
     )
@@ -114,13 +115,10 @@ class Attachment(Base):
         if self.config.type == StorageType.s3:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
-            upload_to_s3(
+            generic_upload_to_s3(
                 storage_secrets=self.config.secrets,
-                data={},
                 bucket_name=bucket_name,
                 file_key=self.id,
-                resp_format=self.config.format,
-                privacy_request=None,
                 document=attachment,
                 auth_method=auth_method,
             )

fides/api/models/comment.py

@@ -1,5 +1,5 @@
 from enum import Enum as EnumType
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import Column
 from sqlalchemy import Enum as EnumColumn
@@ -8,8 +8,10 @@ from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import Base
-from fides.api.models.attachment import Attachment, AttachmentReference
-from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+
+if TYPE_CHECKING:
+    from fides.api.models.attachment import Attachment
+    from fides.api.models.fides_user import FidesUser
 
 
 class CommentType(str, EnumType):
@@ -78,7 +80,6 @@ class Comment(Base):
 
     user = relationship(
         "FidesUser",
-        backref="comments",
         lazy="selectin",
         uselist=False,
     )
@@ -90,20 +91,19 @@ class Comment(Base):
         uselist=True,
     )
 
-    def get_attachments(self, db: Session) -> list[Attachment]:
-        """Retrieve all attachments associated with this comment."""
-        stmt = (
-            db.query(Attachment)
-            .join(
-                AttachmentReference, Attachment.id == AttachmentReference.attachment_id
-            )
-            .where(AttachmentReference.reference_id == self.id)
-        )
-        return db.execute(stmt).scalars().all()
+    attachments = relationship(
+        "Attachment",
+        secondary="attachment_reference",
+        primaryjoin="Comment.id == AttachmentReference.reference_id",
+        secondaryjoin="Attachment.id == AttachmentReference.attachment_id",
+        order_by="Attachment.created_at",
+        uselist=True,
+    )
 
     def delete(self, db: Session) -> None:
         """Delete the comment and all associated references."""
-        attachments = self.get_attachments(db)
-        for attachment in attachments:
-            attachment.delete(db)
+        # Delete the comment
+        for attachment in self.attachments:
+            if len(attachment.references) == 1:
+                attachment.delete(db)
         db.delete(self)

fides/api/models/connectionconfig.py

@@ -186,7 +186,7 @@ class ConnectionConfig(Base):
             )
         ),
         nullable=True,
-    )  # Type bytea in the db
+    )  # Type bytes in the db
     last_test_timestamp = Column(DateTime(timezone=True))
     last_test_succeeded = Column(Boolean)
     disabled = Column(Boolean, server_default="f", default=False)

fides/api/models/privacy_request/privacy_request.py

@@ -39,8 +39,10 @@ from fides.api.graph.config import (
     CollectionAddress,
 )
 from fides.api.migrations.hash_migration_mixin import HashMigrationMixin
+from fides.api.models.attachment import Attachment, AttachmentReference
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.client import ClientDetail
+from fides.api.models.comment import Comment, CommentReference
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.policy import (
@@ -166,6 +168,20 @@ class PrivacyRequest(
         Policy,
         backref="privacy_requests",
     )
+    attachments = relationship(
+        Attachment,
+        secondary="attachment_reference",
+        primaryjoin="PrivacyRequest.id == AttachmentReference.reference_id",
+        secondaryjoin="Attachment.id == AttachmentReference.attachment_id",
+        order_by="Attachment.created_at",
+    )
+    comments = relationship(
+        Comment,
+        secondary="comment_reference",
+        primaryjoin="PrivacyRequest.id == CommentReference.reference_id",
+        secondaryjoin="Comment.id == CommentReference.comment_id",
+        order_by="Comment.created_at",
+    )
     property_id = Column(String, nullable=True)
 
     cancel_reason = Column(String(200))
@@ -1011,6 +1027,51 @@ class PrivacyRequest(
         """Return existing Consent Request Tasks for the current privacy request"""
         return self.request_tasks.filter(RequestTask.action_type == ActionType.consent)
 
+    def get_comment_by_id(self, db: Session, comment_id: str) -> Optional[Comment]:
+        """Get the comment associated with the privacy request"""
+        comment = (
+            db.query(Comment)
+            .join(CommentReference, Comment.id == CommentReference.comment_id)
+            .filter(
+                CommentReference.reference_id
+                == self.id,  # Ensure the comment is linked to this privacy request
+                Comment.id == comment_id,  # Match the specific comment ID
+            )
+            .first()
+        )
+        if not comment:
+            logger.info(
+                f"Comment with id {comment_id} not found on privacy request {self.id}"
+            )
+        return comment
+
+    def get_attachment_by_id(
+        self, db: Session, attachment_id: str
+    ) -> Optional[Attachment]:
+        """Get the attachment associated with the privacy request"""
+        attachment = (
+            db.query(Attachment)
+            .join(
+                AttachmentReference, Attachment.id == AttachmentReference.attachment_id
+            )
+            .filter(
+                AttachmentReference.reference_id == self.id,
+                Attachment.id == attachment_id,
+            )
+            .first()
+        )
+        if not attachment:
+            logger.info(
+                f"Attachment with id {attachment_id} not found on privacy request {self.id}"
+            )
+        return attachment
+
+    def delete_attachment_by_id(self, db: Session, attachment_id: str) -> None:
+        """Delete the attachment associated with the privacy request"""
+        attachment = self.get_attachment_by_id(db, attachment_id)
+        if attachment:
+            attachment.delete(db)
+
     def get_existing_request_task(
         self,
         db: Session,

fides/api/oauth/utils.py

@@ -339,7 +339,16 @@ def has_permissions(
     has_role: bool = _has_scope_via_role(
         token_data=token_data, client=client, endpoint_scopes=endpoint_scopes
     )
-    return has_direct_scope or has_role
+
+    has_required_permissions = has_direct_scope or has_role
+    if not has_required_permissions:
+        scopes_required = ",".join(endpoint_scopes.scopes)
+        logger.debug(
+            "Authorization failed. Missing required scopes: {}. Neither direct scopes nor role-derived scopes were sufficient.",
+            scopes_required,
+        )
+
+    return has_required_permissions
 
 
 def _has_scope_via_role(
@@ -385,16 +394,7 @@ def _has_direct_scopes(
 
 def has_scope_subset(user_scopes: List[str], endpoint_scopes: SecurityScopes) -> bool:
     """Are the required scopes a subset of the scopes belonging to the user?"""
-    if not set(endpoint_scopes.scopes).issubset(user_scopes):
-        scopes_required = ",".join(endpoint_scopes.scopes)
-        scopes_provided = ",".join(user_scopes)
-        logger.debug(
-            "Auth token missing required scopes: {}. Scopes provided: {}.",
-            scopes_required,
-            scopes_provided,
-        )
-        return False
-    return True
+    return set(endpoint_scopes.scopes).issubset(user_scopes)
 
 
 def create_temporary_user_for_login_flow(config: FidesConfig) -> FidesUser:

fides/api/schemas/dataset.py

@@ -1,8 +1,8 @@
 from typing import Any, Dict, List, Optional, Union
 
-from fideslang.models import Dataset
+from fideslang.models import Dataset, DatasetCollection
 from fideslang.validation import FidesKey
-from pydantic import ConfigDict
+from pydantic import ConfigDict, Field
 
 from fides.api.schemas.api import BulkResponse, BulkUpdateFailed
 from fides.api.schemas.base_class import FidesSchema
@@ -72,3 +72,23 @@ class DatasetReachability(FidesSchema):
 
     reachable: bool
     details: Optional[Union[str, List[Dict[str, Any]]]]
+
+
+class DatasetResponse(Dataset):
+    """
+    Dataset response model for API endpoints.
+
+    Note: This class extends the Dataset model from fideslang rather than having a proper
+    dedicated API response model. We had to make the collections field
+    Optional and allow null values to support the minimal=true parameter in API responses,
+    even though collections is required in the base fideslang Dataset model.
+    """
+
+    collections: Optional[List[DatasetCollection]] = Field(  # type: ignore
+        description="An array of objects that describe the Dataset's collections.",
+        default=None,
+    )
+
+    model_config = ConfigDict(
+        extra="ignore", from_attributes=False, coerce_numbers_to_str=True
+    )

fides/api/service/privacy_request/request_runner_service.py

@@ -67,7 +67,6 @@ from fides.api.tasks import DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
 from fides.api.util.collection_util import Row
 from fides.api.util.logger import Pii, _log_exception, _log_warning
-from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
 from fides.common.api.v1.urn_registry import (
     PRIVACY_REQUEST_TRANSFER_TO_PARENT,
     V1_URL_PREFIX,
@@ -270,7 +269,8 @@ def upload_access_results(  # pylint: disable=R0912
 
 
 @celery_app.task(base=DatabaseTask, bind=True)
-@log_context(capture_args={"privacy_request_id": LoggerContextKeys.privacy_request_id})
+# TODO: Add log_context back in, this is just for some temporary testing
+# @log_context(capture_args={"privacy_request_id": LoggerContextKeys.privacy_request_id})
 def run_privacy_request(
     self: DatabaseTask,
     privacy_request_id: str,
@@ -290,7 +290,10 @@ def run_privacy_request(
     """
     resume_step: Optional[CurrentStep] = CurrentStep(from_step) if from_step else None  # type: ignore
     if from_step:
-        logger.info("Resuming privacy request from checkpoint: '{}'", from_step)
+        with logger.contextualize(
+            privacy_request_id=privacy_request_id,
+        ):
+            logger.info("Resuming privacy request from checkpoint: '{}'", from_step)
 
     with self.get_new_session() as session:
         privacy_request = PrivacyRequest.get(db=session, object_id=privacy_request_id)
@@ -302,7 +305,8 @@ def run_privacy_request(
         with logger.contextualize(
             privacy_request_source=(
                 privacy_request.source.value if privacy_request.source else None
-            )
+            ),
+            privacy_request_id=privacy_request.id,
         ):
             if privacy_request.status == PrivacyRequestStatus.canceled:
                 logger.info("Terminating privacy request: request canceled.")

fides/api/service/storage/s3.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+from io import BytesIO
+from typing import Any, Dict, Optional
+
+from boto3.s3.transfer import TransferConfig
+from botocore.exceptions import ClientError, ParamValidationError
+from fideslang.validation import AnyHttpUrlString
+from loguru import logger
+
+from fides.api.schemas.storage.storage import StorageSecrets
+from fides.api.util.aws_util import get_s3_client
+from fides.config import CONFIG
+
+
+def create_presigned_url_for_s3(
+    s3_client: Any, bucket_name: str, file_key: str
+) -> AnyHttpUrlString:
+    """ "Generate a presigned URL to share an S3 object
+
+    :param s3_client: s3 base client
+    :param bucket_name: string
+    :param file_key: string
+    :return: Presigned URL as string.
+    """
+    response = s3_client.generate_presigned_url(
+        "get_object",
+        Params={"Bucket": bucket_name, "Key": file_key},
+        ExpiresIn=CONFIG.security.subject_request_download_link_ttl_seconds,
+    )
+
+    # The response contains the presigned URL
+    return response
+
+
+def generic_upload_to_s3(  # pylint: disable=R0913
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+    document: BytesIO,
+) -> Optional[AnyHttpUrlString]:
+    """
+    Uploads arbitrary data to S3 returned from an access request.
+    Handles both small and large uploads.
+    """
+    logger.info("Starting S3 Upload of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+
+        # Define a transfer configuration for multipart uploads
+        transfer_config = TransferConfig(
+            multipart_threshold=5 * 1024 * 1024,  # 5 MB threshold for multipart uploads
+            multipart_chunksize=5 * 1024 * 1024,  # 5 MB chunk size
+        )
+
+        try:
+            # Check if the document is a file-like object or raw bytes
+            if isinstance(document, bytes):
+                document = BytesIO(document)  # Wrap raw bytes in a file-like object
+
+            # Use upload_fileobj for efficient uploads (handles both small and large files)
+            s3_client.upload_fileobj(
+                Fileobj=document,
+                Bucket=bucket_name,
+                Key=file_key,
+                Config=transfer_config,
+            )
+        except Exception as e:
+            logger.error("Encountered error while uploading S3 object: {}", e)
+            raise e
+
+        # Generate a presigned URL for the uploaded file
+        presigned_url: AnyHttpUrlString = create_presigned_url_for_s3(
+            s3_client, bucket_name, file_key
+        )
+
+        return presigned_url
+    except ClientError as e:
+        logger.error(
+            "Encountered error while uploading and generating link for S3 object: {}", e
+        )
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
+def generic_retrieve_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> Optional[BytesIO]:
+    """Retrieves arbitrary data from s3"""
+    logger.info("Starting S3 Retrieve of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
+            return response["Body"].read()
+        except Exception as e:
+            logger.error("Encountered error while retrieving s3 object: {}", e)
+            raise e
+    except ClientError as e:
+        logger.error("Encountered error while retrieving s3 object: {}", e)
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
+def generic_delete_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> None:
+    """Deletes arbitrary data from s3"""
+    logger.info("Starting S3 Delete of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            s3_client.delete_object(Bucket=bucket_name, Key=file_key)
+        except Exception as e:
+            logger.error("Encountered error while deleting s3 object: {}", e)
+            raise e
+    except ClientError as e:
+        logger.error("Encountered error while deleting s3 object: {}", e)
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")

fides/api/service/storage/util.py

@@ -0,0 +1,10 @@
+import os
+
+LOCAL_FIDES_UPLOAD_DIRECTORY = "fides_uploads"
+
+
+def get_local_filename(file_key: str) -> str:
+    """Verifies that the local storage directory exists"""
+    if not os.path.exists(LOCAL_FIDES_UPLOAD_DIRECTORY):
+        os.makedirs(LOCAL_FIDES_UPLOAD_DIRECTORY)
+    return f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{file_key}"

fides/api/tasks/__init__.py

@@ -79,7 +79,9 @@ class DatabaseTask(Task):  # pylint: disable=W0223
         # but a new session is instantiated each time the method is invoked
         # to prevent session overlap when requests are executing concurrently
         # when in task_always_eager mode (i.e. without proper workers)
-        return self._sessionmaker()
+        new_session = self._sessionmaker()
+        logger.debug(f"DatabaseTaskSession ID: {id(new_session)}. Self ID: {id(self)}")
+        return new_session
 
 
 def _create_celery(config: FidesConfig = CONFIG) -> Celery: