ethyca-fides 2.63.1rc0__py2.py3-none-any.whl → 2.63.3b0__py2.py3-none-any.whl

fides/api/schemas/connection_configuration/connection_config.py

@@ -111,10 +111,10 @@ class ConnectionConfigSecretsMixin(BaseModel):
         return self
 
 
-class ConnectionConfigurationResponse(ConnectionConfigSecretsMixin):
+class ConnectionConfigurationResponseBase(ConnectionConfigSecretsMixin):
     """
-    Describes the returned schema for a ConnectionConfiguration.
-
+    Base schema for ConnectionConfiguration responses,
+    excluding fields that might be conditionally omitted.
     The mixin base class ensures that `secrets` sensitive values are masked.
     """
 
@@ -129,10 +129,25 @@ class ConnectionConfigurationResponse(ConnectionConfigSecretsMixin):
     last_test_succeeded: Optional[bool] = None
     authorized: Optional[bool] = False
     enabled_actions: Optional[List[ActionType]] = None
-
     model_config = ConfigDict(from_attributes=True)
 
 
+class ConnectionConfigurationResponseWithSystemKey(ConnectionConfigurationResponseBase):
+    """
+    Describes the full returned schema for a ConnectionConfiguration.
+    """
+
+    # Using this response with models returned from an async DB session will error
+    # because the system_key is lazy loaded. Just a quirk of SQLAlchemy 1.4.
+    system_key: Optional[str] = None
+
+
+class ConnectionConfigurationResponse(ConnectionConfigurationResponseBase):
+    """
+    Describes the returned schema for a ConnectionConfiguration.
+    """
+
+
 class BulkPutConnectionConfiguration(BulkResponse):
     """Schema with mixed success/failure responses for Bulk Create/Update of ConnectionConfiguration responses."""
 

fides/api/schemas/manual_tasks/manual_task_schemas.py

@@ -0,0 +1,79 @@
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Any, Optional
+
+from pydantic import ConfigDict, Field
+
+from fides.api.schemas.base_class import FidesSchema
+
+
+class ManualTaskType(str, Enum):
+    """Enum for manual task types."""
+
+    privacy_request = "privacy_request"
+    # Add more task types as needed
+
+
+class ManualTaskParentEntityType(str, Enum):
+    """Enum for manual task parent entity types."""
+
+    connection_config = (
+        "connection_config"  # used for access and erasure privacy requests
+    )
+    # Add more parent entity types as needed
+
+
+class ManualTaskReferenceType(str, Enum):
+    """Enum for manual task reference types."""
+
+    privacy_request = "privacy_request"
+    connection_config = "connection_config"
+    manual_task_config = "manual_task_config"
+    assigned_user = "assigned_user"  # Reference to the user assigned to the task
+    # Add more reference types as needed
+
+
+class ManualTaskLogStatus(str, Enum):
+    """Enum for manual task log status."""
+
+    created = "created"
+    updated = "updated"
+    in_processing = "in_processing"
+    complete = "complete"
+    error = "error"
+    retrying = "retrying"
+    paused = "paused"
+    awaiting_input = "awaiting_input"
+
+
+class ManualTaskLogCreate(FidesSchema):
+    """Schema for creating a manual task log entry."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    task_id: Annotated[str, Field(..., description="ID of the task")]
+    status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
+    message: Annotated[Optional[str], Field(None, description="Log message")]
+    details: Annotated[
+        Optional[dict[str, Any]], Field(None, description="Additional details")
+    ]
+    config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
+    instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
+
+
+class ManualTaskLogResponse(FidesSchema):
+    """Schema for manual task log response."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: Annotated[str, Field(..., description="Log ID")]
+    task_id: Annotated[str, Field(..., description="Task ID")]
+    status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
+    message: Annotated[Optional[str], Field(None, description="Log message")]
+    details: Annotated[
+        Optional[dict[str, Any]], Field(None, description="Additional details")
+    ]
+    config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
+    instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
+    created_at: Annotated[datetime, Field(..., description="Creation timestamp")]
+    updated_at: Annotated[datetime, Field(..., description="Last update timestamp")]

fides/api/schemas/manual_tasks/manual_task_status.py

@@ -0,0 +1,151 @@
+from datetime import datetime, timezone
+from enum import Enum as EnumType
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+
+class StatusTransitionNotAllowed(Exception):
+    """Exception raised when a status transition is not allowed."""
+
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+
+class StatusType(str, EnumType):
+    """Enum for manual task status."""
+
+    pending = "pending"
+    in_progress = "in_progress"
+    completed = "completed"
+    failed = "failed"
+
+    @classmethod
+    def get_valid_transitions(cls, current_status: "StatusType") -> list["StatusType"]:
+        """Get valid transitions from the current status.
+
+        Args:
+            current_status: The current status
+
+        Returns:
+            list[StatusType]: List of valid transitions
+        """
+        if current_status == cls.pending:
+            return [cls.in_progress, cls.failed]
+        if current_status == cls.in_progress:
+            return [cls.completed, cls.failed]
+        if current_status == cls.completed:
+            return []
+        if current_status == cls.failed:
+            return [cls.pending, cls.in_progress]
+        return []
+
+
+class StatusTransitionMixin:
+    """Mixin for handling status transitions.
+
+    This mixin provides methods for managing status transitions and completion tracking.
+    It can be used by any model that needs status management.
+    """
+
+    # These should be overridden by the implementing class
+    status: StatusType
+    completed_at: Optional[datetime]
+    completed_by_id: Optional[str]
+
+    def _get_valid_transitions(self) -> list[StatusType]:
+        """Get valid transitions from the current status.
+
+        Returns:
+            list[StatusType]: List of valid transitions
+        """
+        return StatusType.get_valid_transitions(self.status)
+
+    def _validate_status_transition(self, new_status: StatusType) -> None:
+        """Validate that a status transition is allowed.
+
+        Args:
+            new_status: The new status to transition to
+
+        Raises:
+            StatusTransitionNotAllowed: If the transition is not allowed
+        """
+        # Don't allow transitions to the same status
+        if new_status == self.status:
+            raise StatusTransitionNotAllowed(
+                f"Invalid status transition: already in status {new_status}"
+            )
+
+        # Get valid transitions for current status
+        valid_transitions = self._get_valid_transitions()
+        if new_status not in valid_transitions:
+            raise StatusTransitionNotAllowed(
+                f"Invalid status transition from {self.status} to {new_status}. "
+                f"Valid transitions are: {valid_transitions}"
+            )
+
+    def update_status(
+        self, db: Session, new_status: StatusType, user_id: Optional[str] = None
+    ) -> None:
+        """Update the status with validation and completion handling.
+
+        Args:
+            db: Database session
+            new_status: New status to set
+            user_id: Optional user ID who is making the change
+        """
+        self._validate_status_transition(new_status)
+
+        if new_status == StatusType.completed:
+            self.completed_at = datetime.now(timezone.utc)
+            self.completed_by_id = user_id
+        elif new_status == StatusType.pending:
+            # Reset completion fields if going back to pending
+            self.completed_at = None
+            self.completed_by_id = None
+
+        self.status = new_status
+        db.add(self)
+        db.commit()
+
+    def mark_completed(self, db: Session, user_id: str) -> None:
+        """Mark as completed.
+
+        Args:
+            db: Database session
+            user_id: user ID who completed the task
+        """
+        self.update_status(db, StatusType.completed, user_id)
+
+    def mark_failed(self, db: Session) -> None:
+        """Mark as failed."""
+        self.update_status(db, StatusType.failed)
+
+    def start_progress(self, db: Session) -> None:
+        """Mark as in progress."""
+        self.update_status(db, StatusType.in_progress)
+
+    def reset_to_pending(self, db: Session) -> None:
+        """Reset to pending status."""
+        self.update_status(db, StatusType.pending)
+
+    @property
+    def is_completed(self) -> bool:
+        """Check if completed."""
+        return self.status == StatusType.completed
+
+    @property
+    def is_failed(self) -> bool:
+        """Check if failed."""
+        return self.status == StatusType.failed
+
+    @property
+    def is_in_progress(self) -> bool:
+        """Check if in progress."""
+        return self.status == StatusType.in_progress
+
+    @property
+    def is_pending(self) -> bool:
+        """Check if pending."""
+        return self.status == StatusType.pending

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -43,6 +43,7 @@ from fides.api.util.saas_util import (
     ALL_OBJECT_FIELDS,
     CUSTOM_PRIVACY_REQUEST_FIELDS,
     FIDESOPS_GROUPED_INPUTS,
+    FIELD_LIST,
     ISO_8601_DATETIME,
     MASKED_OBJECT_FIELDS,
     PRIVACY_REQUEST_ID,
@@ -353,6 +354,13 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
 
         param_values[UUID] = str(uuid4())
         param_values[ISO_8601_DATETIME] = datetime.now().date().isoformat()
+        param_values[FIELD_LIST] = ",".join(
+            [
+                field.name
+                for field in self.top_level_field_map().values()
+                if field.data_type() != "None"
+            ]
+        )
         if self.request_task and self.request_task.id:
             param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
                 self.request_task
@@ -458,6 +466,13 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         param_values[CUSTOM_PRIVACY_REQUEST_FIELDS] = custom_privacy_request_fields
         param_values[UUID] = str(uuid4())
         param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
+        param_values[FIELD_LIST] = ",".join(
+            [
+                field.name
+                for field in self.top_level_field_map().values()
+                if field.data_type() != "None"
+            ]
+        )
         if self.request_task and self.request_task.id:
             param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
                 self.request_task

fides/api/service/privacy_request/attachment_handling.py

@@ -0,0 +1,132 @@
+import time as time_module
+from dataclasses import dataclass
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+from loguru import logger
+
+from fides.api.models.attachment import Attachment, AttachmentType
+from fides.api.schemas.storage.storage import StorageDetails
+
+
+@dataclass
+class AttachmentData:
+    """Data structure for attachment metadata and content.
+    Using a dataclass rather than a Pydantic model here for the following reasons:
+    - The data structure is simple and doesn't need complex validation.
+    - The fields being used have already been validated and are properly typed.
+    - The class is used internally for data transfer, not for API serialization.
+    - Performance is important since this is used in a data processing pipeline.
+    """
+
+    file_name: str
+    file_size: Optional[int]
+    download_url: Optional[str]
+    content_type: str
+    bucket_name: str
+    file_key: str
+    storage_key: str
+
+    def to_upload_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for upload, including presigned URL."""
+        return {
+            "file_name": self.file_name,
+            "file_size": self.file_size,
+            "download_url": self.download_url,
+            "content_type": self.content_type,
+        }
+
+    def to_storage_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for storage, including the elements needed to recreated the presigned URL."""
+        return {
+            "file_name": self.file_name,
+            "file_size": self.file_size,
+            "content_type": self.content_type,
+            "bucket_name": self.bucket_name,
+            "file_key": self.file_key,
+            "storage_key": self.storage_key,
+        }
+
+
+def get_attachments_content(
+    loaded_attachments: List[Attachment],
+) -> Iterator[AttachmentData]:
+    """
+    Retrieves all attachments associated with a privacy request that are marked to be included with the access package.
+    Yields AttachmentData objects containing attachment metadata and download urls.
+    Uses generators to minimize memory usage.
+
+    Args:
+        loaded_attachments: List of Attachment objects to process
+
+    Yields:
+        AttachmentData object containing attachment metadata and url
+    """
+    start_time = time_module.time()
+    processed_count = 0
+    skipped_count = 0
+    error_count = 0
+    total_size = 0
+
+    for attachment in loaded_attachments:
+        if attachment.attachment_type != AttachmentType.include_with_access_package:
+            skipped_count += 1
+            continue
+
+        try:
+            # Get size and download URL using retrieve_attachment
+            size, url = attachment.retrieve_attachment()
+            total_size += size if size else 0
+            if url is None:
+                logger.warning(
+                    "No download URL retrieved for attachment {}", attachment.file_name
+                )
+                skipped_count += 1
+                continue
+
+            processed_count += 1
+            yield AttachmentData(
+                file_name=attachment.file_name,
+                file_size=size,
+                download_url=str(url) if url else None,
+                content_type=attachment.content_type,
+                bucket_name=attachment.config.details[StorageDetails.BUCKET.value],
+                file_key=attachment.file_key,
+                storage_key=attachment.storage_key,
+            )
+
+        except Exception as e:
+            error_count += 1
+            logger.error(
+                "Error processing attachment {}: {}", attachment.file_name, str(e)
+            )
+            continue
+
+    # Log final metrics
+    time_taken = time_module.time() - start_time
+    logger.bind(
+        time_to_process=time_taken,
+        total_attachments=len(loaded_attachments),
+        processed_attachments=processed_count,
+        skipped_attachments=skipped_count,
+        error_attachments=error_count,
+        total_size_bytes=total_size,
+    ).info("Attachment processing complete")
+
+
+def process_attachments_for_upload(
+    attachments: Iterator[AttachmentData],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    """
+    Process attachments into separate upload and storage formats.
+    Returns both formats:
+    - upload_attachments: Used for uploading to access packages
+    - storage_attachments: Used for saving filtered access results
+    """
+    upload_attachments = []
+    storage_attachments = []
+
+    for attachment in attachments:
+        storage_attachments.append(attachment.to_storage_dict())
+        upload_attachments.append(attachment.to_upload_dict())
+
+    return upload_attachments, storage_attachments