ethyca-fides 2.63.3rc0__py2.py3-none-any.whl → 2.64.1b0__py2.py3-none-any.whl

fides/api/schemas/manual_tasks/manual_task_config.py

@@ -0,0 +1,311 @@
+from datetime import datetime
+from enum import Enum
+from typing import TYPE_CHECKING, Annotated, Literal, Optional, Union
+
+from pydantic import ConfigDict, Field, ValidationInfo, field_validator, model_validator
+
+from fides.api.schemas.base_class import FidesSchema
+
+if TYPE_CHECKING:  # pragma: no cover
+    from fides.api.models.manual_tasks.manual_task import ManualTask  # pragma: no cover
+
+
+class ManualTaskConfigurationType(str, Enum):
+    """Enum for manual task configuration types."""
+
+    access_privacy_request = "access_privacy_request"
+    erasure_privacy_request = "erasure_privacy_request"
+    # Add more configuration types as needed
+
+
+class ManualTaskFieldType(str, Enum):
+    """Enum for manual task field types."""
+
+    text = "text"  # Key-value pairs
+    checkbox = "checkbox"  # Boolean value
+    attachment = "attachment"  # File upload
+    # Add more field types as needed
+
+
+class ManualTaskFieldMetadata(FidesSchema):
+    """Base schema for manual task field metadata."""
+
+    model_config = ConfigDict(extra="allow")
+
+    label: Annotated[str, Field(description="Display label for the field")]
+    required: Annotated[
+        bool, Field(default=False, description="Whether the field is required")
+    ]
+    help_text: Annotated[
+        Optional[str],
+        Field(default=None, description="Help text to display with the field"),
+    ]
+    data_uses: Annotated[
+        Optional[list[str]],
+        Field(
+            default=None,
+            description="List of data uses associated with this field",
+        ),
+    ]
+
+
+class ManualTaskTextFieldMetadata(ManualTaskFieldMetadata):
+    """Schema text field metadata."""
+
+    placeholder: Annotated[
+        Optional[str], Field(default=None, description="Placeholder text for the field")
+    ]
+    default_value: Annotated[
+        Optional[str], Field(default=None, description="Default value for the field")
+    ]
+    max_length: Annotated[
+        Optional[int], Field(default=None, description="Maximum length for text fields")
+    ]
+    min_length: Annotated[
+        Optional[int], Field(default=None, description="Minimum length for text fields")
+    ]
+    pattern: Annotated[
+        Optional[str],
+        Field(
+            default=None,
+            description=(
+                "Regex pattern for validation, for example if the field is an email address, "
+                r"you could use: ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"
+            ),
+        ),
+    ]
+
+    @field_validator("min_length", "max_length")
+    @classmethod
+    def validate_lengths(cls, v: Optional[int], info: ValidationInfo) -> Optional[int]:
+        """Validate that min_length is not greater than max_length."""
+        if v is None:
+            return v
+
+        if info.field_name == "min_length":
+            max_length = info.data.get("max_length")
+            if max_length is not None and v > max_length:
+                raise ValueError("min_length cannot be greater than max_length")
+        elif info.field_name == "max_length":
+            min_length = info.data.get("min_length")
+            if min_length is not None and v < min_length:
+                raise ValueError("min_length cannot be greater than max_length")
+
+        return v
+
+
+class ManualTaskCheckboxFieldMetadata(ManualTaskFieldMetadata):
+    """Schema for checkbox field metadata."""
+
+    default_value: Annotated[
+        Optional[bool],
+        Field(default=False, description="Default value for the checkbox"),
+    ]
+    label_true: Annotated[
+        Optional[str], Field(default=None, description="Label for checked state")
+    ]
+    label_false: Annotated[
+        Optional[str], Field(default=None, description="Label for unchecked state")
+    ]
+
+
+class ManualTaskAttachmentFieldMetadata(ManualTaskFieldMetadata):
+    """Schema for attachment field metadata."""
+
+    file_types: Annotated[
+        list[str], Field(description="Allowed file types for upload fields")
+    ]
+    max_file_size: Annotated[
+        int, Field(description="Maximum file size in bytes for upload fields")
+    ]
+    multiple: Annotated[
+        bool, Field(default=False, description="Whether multiple files can be uploaded")
+    ]
+    max_files: Annotated[
+        Optional[int],
+        Field(
+            default=None,
+            description="Maximum number of files allowed when multiple is true",
+        ),
+    ]
+    require_attachment_id: Annotated[
+        bool,
+        Field(
+            default=True,
+            description="Whether an attachment ID is required for this field",
+        ),
+    ]
+    attachment_ids: Annotated[
+        Optional[list[str]],
+        Field(
+            default=None,
+            description="List of attachment IDs associated with this field",
+        ),
+    ]
+
+    @model_validator(mode="after")
+    def validate_multiple_max_files(self) -> "ManualTaskAttachmentFieldMetadata":
+        """Validate that max_files is set when multiple is True."""
+        if self.multiple and self.max_files is None:
+            raise ValueError("max_files must be set when multiple is True")
+        return self
+
+
+class ManualTaskFieldBase(FidesSchema):
+    """Base schema for all field definitions."""
+
+    model_config = ConfigDict(extra="allow")
+
+    field_key: Annotated[
+        str, Field(description="Unique key for the field", min_length=1)
+    ]
+    field_type: Annotated[ManualTaskFieldType, Field(description="Type of the field")]
+    field_metadata: Annotated[
+        ManualTaskFieldMetadata, Field(description="Field metadata and configuration")
+    ]
+
+    @classmethod
+    def get_field_model_for_type(
+        cls, field_type: Union[str, ManualTaskFieldType]
+    ) -> type["ManualTaskFieldBase"]:
+        """Get the appropriate field model class for a given field type.
+
+        Args:
+            field_type: The field type as a string or ManualTaskFieldType enum value
+
+        Returns:
+            The appropriate field model class
+
+        Raises:
+            ValueError: If the field type is not recognized
+        """
+        # Handle non-string/non-enum types
+        if not isinstance(field_type, (str, ManualTaskFieldType)):
+            raise ValueError(
+                f"Invalid field type: expected string or ManualTaskFieldType, got {type(field_type).__name__}"
+            )
+
+        # Convert string to enum if needed
+        if isinstance(field_type, str):
+            try:
+                field_type = ManualTaskFieldType(field_type)
+            except ValueError:
+                raise ValueError(
+                    f"Invalid field type: '{field_type}' is not a valid ManualTaskFieldType"
+                )
+
+        # Map field types to their model classes
+        field_type_to_model = {
+            ManualTaskFieldType.text: ManualTaskTextField,
+            ManualTaskFieldType.checkbox: ManualTaskCheckboxField,
+            ManualTaskFieldType.attachment: ManualTaskAttachmentField,
+        }
+
+        model_class = field_type_to_model.get(field_type)
+        if not model_class:
+            raise ValueError(f"No model class found for field type: {field_type}")
+
+        return model_class
+
+
+class ManualTaskTextField(ManualTaskFieldBase):
+    """Schema for text field definition."""
+
+    field_type: Annotated[
+        Literal[ManualTaskFieldType.text], Field(description="Type of the field")
+    ]
+    field_metadata: Annotated[
+        ManualTaskTextFieldMetadata,
+        Field(description="Field metadata and configuration"),
+    ]
+
+
+class ManualTaskCheckboxField(ManualTaskFieldBase):
+    """Schema for checkbox field definition."""
+
+    field_type: Annotated[
+        Literal[ManualTaskFieldType.checkbox], Field(description="Type of the field")
+    ]
+    field_metadata: Annotated[
+        ManualTaskCheckboxFieldMetadata,
+        Field(description="Field metadata and configuration"),
+    ]
+
+
+class ManualTaskAttachmentField(ManualTaskFieldBase):
+    """Schema for attachment field definition."""
+
+    field_type: Annotated[
+        Literal[ManualTaskFieldType.attachment], Field(description="Type of the field")
+    ]
+    field_metadata: Annotated[
+        ManualTaskAttachmentFieldMetadata,
+        Field(description="Field metadata and configuration"),
+    ]
+
+
+# Type alias for all field types
+ManualTaskField = ManualTaskFieldBase
+
+
+class ManualTaskConfigCreate(FidesSchema):
+    """Schema for creating a manual task configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    task_id: Annotated[
+        str, Field(description="ID of the task this configuration belongs to")
+    ]
+    config_type: Annotated[
+        ManualTaskConfigurationType, Field(description="Type of configuration")
+    ]
+    fields: Annotated[
+        list[ManualTaskFieldBase], Field(description="List of field definitions")
+    ]
+
+
+class ManualTaskConfigResponse(FidesSchema):
+    """Schema for manual task configuration response."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: Annotated[str, Field(description="Configuration ID")]
+    task_id: Annotated[str, Field(description="Task ID")]
+    config_type: Annotated[
+        ManualTaskConfigurationType, Field(description="Type of configuration")
+    ]
+    version: Annotated[int, Field(description="Version of the configuration")]
+    is_current: Annotated[
+        bool, Field(description="Whether this is the current version")
+    ]
+    fields: Annotated[
+        list[ManualTaskFieldBase], Field(description="List of field definitions")
+    ]
+    created_at: Annotated[datetime, Field(description="Creation timestamp")]
+    updated_at: Annotated[datetime, Field(description="Last update timestamp")]
+
+
+class ManualTaskConfigUpdate(FidesSchema):
+    """Schema for updating a manual task configuration."""
+
+    model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
+
+    task_id: Annotated[str, Field(description="ID of the task")]
+    config_type: Annotated[
+        ManualTaskConfigurationType, Field(description="Type of configuration")
+    ]
+    fields: Annotated[
+        list[ManualTaskFieldBase], Field(description="List of field definitions")
+    ]
+
+
+class ManualTaskConfigFieldResponse(FidesSchema):
+    """Schema for manual task configuration field response."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    field_key: Annotated[str, Field(description="Unique key for the field")]
+    field_type: Annotated[ManualTaskFieldType, Field(description="Type of the field")]
+    field_metadata: Annotated[
+        ManualTaskFieldMetadata, Field(description="Field metadata and configuration")
+    ]

fides/api/schemas/manual_tasks/manual_task_schemas.py

@@ -38,7 +38,7 @@ class ManualTaskLogStatus(str, Enum):
 
     created = "created"
     updated = "updated"
-    in_processing = "in_processing"
+    in_progress = "in_progress"
     complete = "complete"
     error = "error"
     retrying = "retrying"

fides/api/schemas/manual_tasks/manual_task_status.py

@@ -32,7 +32,7 @@ class StatusType(str, EnumType):
             list[StatusType]: List of valid transitions
         """
         if current_status == cls.pending:
-            return [cls.in_progress, cls.failed]
+            return [cls.in_progress, cls.failed, cls.completed]
         if current_status == cls.in_progress:
             return [cls.completed, cls.failed]
         if current_status == cls.completed:

fides/api/schemas/privacy_request.py

@@ -77,6 +77,7 @@ class PrivacyRequestSource(str, EnumType):
     - Consent Webhook: Request created as a side-effect of a consent webhook request (bidirectional consent)
     - Fides.js: Request created as a side-effect of a privacy preference update from Fides.js
     - Dataset Test: Standalone dataset test
+    - Janus SDK: Request created from the Mobile SDK
     """
 
     privacy_center = "Privacy Center"
@@ -84,6 +85,7 @@ class PrivacyRequestSource(str, EnumType):
     consent_webhook = "Consent Webhook"
     fides_js = "Fides.js"
     dataset_test = "Dataset Test"
+    janus_sdk = "Janus SDK"
 
 
 class PrivacyRequestCreate(FidesSchema):

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -43,6 +43,7 @@ from fides.api.util.saas_util import (
     ALL_OBJECT_FIELDS,
     CUSTOM_PRIVACY_REQUEST_FIELDS,
     FIDESOPS_GROUPED_INPUTS,
+    FIELD_LIST,
     ISO_8601_DATETIME,
     MASKED_OBJECT_FIELDS,
     PRIVACY_REQUEST_ID,
@@ -353,6 +354,13 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
 
         param_values[UUID] = str(uuid4())
         param_values[ISO_8601_DATETIME] = datetime.now().date().isoformat()
+        param_values[FIELD_LIST] = ",".join(
+            [
+                field.name
+                for field in self.top_level_field_map().values()
+                if field.data_type() != "None"
+            ]
+        )
         if self.request_task and self.request_task.id:
             param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
                 self.request_task
@@ -458,6 +466,13 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         param_values[CUSTOM_PRIVACY_REQUEST_FIELDS] = custom_privacy_request_fields
         param_values[UUID] = str(uuid4())
         param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
+        param_values[FIELD_LIST] = ",".join(
+            [
+                field.name
+                for field in self.top_level_field_map().values()
+                if field.data_type() != "None"
+            ]
+        )
         if self.request_task and self.request_task.id:
             param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
                 self.request_task

fides/api/service/privacy_request/attachment_handling.py

@@ -0,0 +1,132 @@
+import time as time_module
+from dataclasses import dataclass
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+from loguru import logger
+
+from fides.api.models.attachment import Attachment, AttachmentType
+from fides.api.schemas.storage.storage import StorageDetails
+
+
+@dataclass
+class AttachmentData:
+    """Data structure for attachment metadata and content.
+    Using a dataclass rather than a Pydantic model here for the following reasons:
+    - The data structure is simple and doesn't need complex validation.
+    - The fields being used have already been validated and are properly typed.
+    - The class is used internally for data transfer, not for API serialization.
+    - Performance is important since this is used in a data processing pipeline.
+    """
+
+    file_name: str
+    file_size: Optional[int]
+    download_url: Optional[str]
+    content_type: str
+    bucket_name: str
+    file_key: str
+    storage_key: str
+
+    def to_upload_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for upload, including presigned URL."""
+        return {
+            "file_name": self.file_name,
+            "file_size": self.file_size,
+            "download_url": self.download_url,
+            "content_type": self.content_type,
+        }
+
+    def to_storage_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for storage, including the elements needed to recreated the presigned URL."""
+        return {
+            "file_name": self.file_name,
+            "file_size": self.file_size,
+            "content_type": self.content_type,
+            "bucket_name": self.bucket_name,
+            "file_key": self.file_key,
+            "storage_key": self.storage_key,
+        }
+
+
+def get_attachments_content(
+    loaded_attachments: List[Attachment],
+) -> Iterator[AttachmentData]:
+    """
+    Retrieves all attachments associated with a privacy request that are marked to be included with the access package.
+    Yields AttachmentData objects containing attachment metadata and download urls.
+    Uses generators to minimize memory usage.
+
+    Args:
+        loaded_attachments: List of Attachment objects to process
+
+    Yields:
+        AttachmentData object containing attachment metadata and url
+    """
+    start_time = time_module.time()
+    processed_count = 0
+    skipped_count = 0
+    error_count = 0
+    total_size = 0
+
+    for attachment in loaded_attachments:
+        if attachment.attachment_type != AttachmentType.include_with_access_package:
+            skipped_count += 1
+            continue
+
+        try:
+            # Get size and download URL using retrieve_attachment
+            size, url = attachment.retrieve_attachment()
+            total_size += size if size else 0
+            if url is None:
+                logger.warning(
+                    "No download URL retrieved for attachment {}", attachment.file_name
+                )
+                skipped_count += 1
+                continue
+
+            processed_count += 1
+            yield AttachmentData(
+                file_name=attachment.file_name,
+                file_size=size,
+                download_url=str(url) if url else None,
+                content_type=attachment.content_type,
+                bucket_name=attachment.config.details[StorageDetails.BUCKET.value],
+                file_key=attachment.file_key,
+                storage_key=attachment.storage_key,
+            )
+
+        except Exception as e:
+            error_count += 1
+            logger.error(
+                "Error processing attachment {}: {}", attachment.file_name, str(e)
+            )
+            continue
+
+    # Log final metrics
+    time_taken = time_module.time() - start_time
+    logger.bind(
+        time_to_process=time_taken,
+        total_attachments=len(loaded_attachments),
+        processed_attachments=processed_count,
+        skipped_attachments=skipped_count,
+        error_attachments=error_count,
+        total_size_bytes=total_size,
+    ).info("Attachment processing complete")
+
+
+def process_attachments_for_upload(
+    attachments: Iterator[AttachmentData],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    """
+    Process attachments into separate upload and storage formats.
+    Returns both formats:
+    - upload_attachments: Used for uploading to access packages
+    - storage_attachments: Used for saving filtered access results
+    """
+    upload_attachments = []
+    storage_attachments = []
+
+    for attachment in attachments:
+        storage_attachments.append(attachment.to_storage_dict())
+        upload_attachments.append(attachment.to_upload_dict())
+
+    return upload_attachments, storage_attachments