ethyca-fides 2.63.1b4__py2.py3-none-any.whl → 2.63.1rc0__py2.py3-none-any.whl

fides/api/schemas/manual_tasks/manual_task_status.py

@@ -1,151 +0,0 @@
-from datetime import datetime, timezone
-from enum import Enum as EnumType
-from typing import Optional
-
-from sqlalchemy.orm import Session
-
-
-class StatusTransitionNotAllowed(Exception):
-    """Exception raised when a status transition is not allowed."""
-
-    def __init__(self, message: str):
-        self.message = message
-        super().__init__(self.message)
-
-
-class StatusType(str, EnumType):
-    """Enum for manual task status."""
-
-    pending = "pending"
-    in_progress = "in_progress"
-    completed = "completed"
-    failed = "failed"
-
-    @classmethod
-    def get_valid_transitions(cls, current_status: "StatusType") -> list["StatusType"]:
-        """Get valid transitions from the current status.
-
-        Args:
-            current_status: The current status
-
-        Returns:
-            list[StatusType]: List of valid transitions
-        """
-        if current_status == cls.pending:
-            return [cls.in_progress, cls.failed]
-        if current_status == cls.in_progress:
-            return [cls.completed, cls.failed]
-        if current_status == cls.completed:
-            return []
-        if current_status == cls.failed:
-            return [cls.pending, cls.in_progress]
-        return []
-
-
-class StatusTransitionMixin:
-    """Mixin for handling status transitions.
-
-    This mixin provides methods for managing status transitions and completion tracking.
-    It can be used by any model that needs status management.
-    """
-
-    # These should be overridden by the implementing class
-    status: StatusType
-    completed_at: Optional[datetime]
-    completed_by_id: Optional[str]
-
-    def _get_valid_transitions(self) -> list[StatusType]:
-        """Get valid transitions from the current status.
-
-        Returns:
-            list[StatusType]: List of valid transitions
-        """
-        return StatusType.get_valid_transitions(self.status)
-
-    def _validate_status_transition(self, new_status: StatusType) -> None:
-        """Validate that a status transition is allowed.
-
-        Args:
-            new_status: The new status to transition to
-
-        Raises:
-            StatusTransitionNotAllowed: If the transition is not allowed
-        """
-        # Don't allow transitions to the same status
-        if new_status == self.status:
-            raise StatusTransitionNotAllowed(
-                f"Invalid status transition: already in status {new_status}"
-            )
-
-        # Get valid transitions for current status
-        valid_transitions = self._get_valid_transitions()
-        if new_status not in valid_transitions:
-            raise StatusTransitionNotAllowed(
-                f"Invalid status transition from {self.status} to {new_status}. "
-                f"Valid transitions are: {valid_transitions}"
-            )
-
-    def update_status(
-        self, db: Session, new_status: StatusType, user_id: Optional[str] = None
-    ) -> None:
-        """Update the status with validation and completion handling.
-
-        Args:
-            db: Database session
-            new_status: New status to set
-            user_id: Optional user ID who is making the change
-        """
-        self._validate_status_transition(new_status)
-
-        if new_status == StatusType.completed:
-            self.completed_at = datetime.now(timezone.utc)
-            self.completed_by_id = user_id
-        elif new_status == StatusType.pending:
-            # Reset completion fields if going back to pending
-            self.completed_at = None
-            self.completed_by_id = None
-
-        self.status = new_status
-        db.add(self)
-        db.commit()
-
-    def mark_completed(self, db: Session, user_id: str) -> None:
-        """Mark as completed.
-
-        Args:
-            db: Database session
-            user_id: user ID who completed the task
-        """
-        self.update_status(db, StatusType.completed, user_id)
-
-    def mark_failed(self, db: Session) -> None:
-        """Mark as failed."""
-        self.update_status(db, StatusType.failed)
-
-    def start_progress(self, db: Session) -> None:
-        """Mark as in progress."""
-        self.update_status(db, StatusType.in_progress)
-
-    def reset_to_pending(self, db: Session) -> None:
-        """Reset to pending status."""
-        self.update_status(db, StatusType.pending)
-
-    @property
-    def is_completed(self) -> bool:
-        """Check if completed."""
-        return self.status == StatusType.completed
-
-    @property
-    def is_failed(self) -> bool:
-        """Check if failed."""
-        return self.status == StatusType.failed
-
-    @property
-    def is_in_progress(self) -> bool:
-        """Check if in progress."""
-        return self.status == StatusType.in_progress
-
-    @property
-    def is_pending(self) -> bool:
-        """Check if pending."""
-        return self.status == StatusType.pending

fides/api/service/privacy_request/attachment_handling.py

@@ -1,132 +0,0 @@
-import time as time_module
-from dataclasses import dataclass
-from typing import Any, Dict, Iterator, List, Optional, Tuple
-
-from loguru import logger
-
-from fides.api.models.attachment import Attachment, AttachmentType
-from fides.api.schemas.storage.storage import StorageDetails
-
-
-@dataclass
-class AttachmentData:
-    """Data structure for attachment metadata and content.
-    Using a dataclass rather than a Pydantic model here for the following reasons:
-    - The data structure is simple and doesn't need complex validation.
-    - The fields being used have already been validated and are properly typed.
-    - The class is used internally for data transfer, not for API serialization.
-    - Performance is important since this is used in a data processing pipeline.
-    """
-
-    file_name: str
-    file_size: Optional[int]
-    download_url: Optional[str]
-    content_type: str
-    bucket_name: str
-    file_key: str
-    storage_key: str
-
-    def to_upload_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary for upload, including presigned URL."""
-        return {
-            "file_name": self.file_name,
-            "file_size": self.file_size,
-            "download_url": self.download_url,
-            "content_type": self.content_type,
-        }
-
-    def to_storage_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary for storage, including the elements needed to recreated the presigned URL."""
-        return {
-            "file_name": self.file_name,
-            "file_size": self.file_size,
-            "content_type": self.content_type,
-            "bucket_name": self.bucket_name,
-            "file_key": self.file_key,
-            "storage_key": self.storage_key,
-        }
-
-
-def get_attachments_content(
-    loaded_attachments: List[Attachment],
-) -> Iterator[AttachmentData]:
-    """
-    Retrieves all attachments associated with a privacy request that are marked to be included with the access package.
-    Yields AttachmentData objects containing attachment metadata and download urls.
-    Uses generators to minimize memory usage.
-
-    Args:
-        loaded_attachments: List of Attachment objects to process
-
-    Yields:
-        AttachmentData object containing attachment metadata and url
-    """
-    start_time = time_module.time()
-    processed_count = 0
-    skipped_count = 0
-    error_count = 0
-    total_size = 0
-
-    for attachment in loaded_attachments:
-        if attachment.attachment_type != AttachmentType.include_with_access_package:
-            skipped_count += 1
-            continue
-
-        try:
-            # Get size and download URL using retrieve_attachment
-            size, url = attachment.retrieve_attachment()
-            total_size += size if size else 0
-            if url is None:
-                logger.warning(
-                    "No download URL retrieved for attachment {}", attachment.file_name
-                )
-                skipped_count += 1
-                continue
-
-            processed_count += 1
-            yield AttachmentData(
-                file_name=attachment.file_name,
-                file_size=size,
-                download_url=str(url) if url else None,
-                content_type=attachment.content_type,
-                bucket_name=attachment.config.details[StorageDetails.BUCKET.value],
-                file_key=attachment.file_key,
-                storage_key=attachment.storage_key,
-            )
-
-        except Exception as e:
-            error_count += 1
-            logger.error(
-                "Error processing attachment {}: {}", attachment.file_name, str(e)
-            )
-            continue
-
-    # Log final metrics
-    time_taken = time_module.time() - start_time
-    logger.bind(
-        time_to_process=time_taken,
-        total_attachments=len(loaded_attachments),
-        processed_attachments=processed_count,
-        skipped_attachments=skipped_count,
-        error_attachments=error_count,
-        total_size_bytes=total_size,
-    ).info("Attachment processing complete")
-
-
-def process_attachments_for_upload(
-    attachments: Iterator[AttachmentData],
-) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
-    """
-    Process attachments into separate upload and storage formats.
-    Returns both formats:
-    - upload_attachments: Used for uploading to access packages
-    - storage_attachments: Used for saving filtered access results
-    """
-    upload_attachments = []
-    storage_attachments = []
-
-    for attachment in attachments:
-        storage_attachments.append(attachment.to_storage_dict())
-        upload_attachments.append(attachment.to_upload_dict())
-
-    return upload_attachments, storage_attachments

fides/api/service/privacy_request/dsr_package/templates/attachments_index.html

@@ -1,33 +0,0 @@
-<html>
-   <head>
-      <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600">
-      <link rel="stylesheet" href="../data/main.css">
-   </head>
-   <body>
-      <div class="container">
-         <div class="header"></div>
-         <div class="content">
-            <div class="button-container">
-               <a href="../welcome.html">
-                  <div class="button"><img src="../data/back.svg"></div>
-                  <span>Back to main page</span>
-               </a>
-            </div>
-            <h1>Attachments</h1>
-            <p class="expiration-notice">Note: All download links will expire in 7 days.</p>
-            <div class="table table-hover">
-               <div class="table-row">
-                  <div class="table-cell" style="text-align: left;">File Name</div>
-                  <div class="table-cell" style="text-align: left;">Size</div>
-               </div>
-               {% for name, info in data.items() %}
-               <a href="{{ info.url }}" class="table-row" target="_blank">
-                  <div class="table-cell" style="text-align: left;">{{ name }}</div>
-                  <div class="table-cell" style="text-align: left;">{{ info.size }}</div>
-               </a>
-               {% endfor %}
-            </div>
-         </div>
-      </div>
-   </body>
-</html>

fides/api/tasks/csv_utils.py

@@ -1,170 +0,0 @@
-import zipfile
-from io import BytesIO
-from typing import Any, Optional
-
-import pandas as pd
-
-from fides.api.tasks.encryption_utils import encrypt_access_request_results
-from fides.config import CONFIG
-
-
-def create_csv_from_dataframe(df: pd.DataFrame) -> BytesIO:
-    """Create a CSV file from a pandas DataFrame.
-
-    Args:
-        df: The DataFrame to convert to CSV
-
-    Returns:
-        BytesIO: A file-like object containing the CSV data
-    """
-    buffer = BytesIO()
-    df.to_csv(buffer, index=False, encoding=CONFIG.security.encoding)
-    buffer.seek(0)
-    return buffer
-
-
-def create_attachment_csv(attachments: list[dict[str, Any]]) -> Optional[BytesIO]:
-    """Create a CSV file containing attachment metadata.
-
-    Args:
-        attachments: List of attachment dictionaries
-        privacy_request_id: The ID of the privacy request for encryption
-
-    Returns:
-        Optional[BytesIO]: A file-like object containing the CSV data, or None if no attachments
-    """
-    if not attachments:
-        return None
-
-    # Filter out invalid attachments and create a list of valid ones
-    valid_attachments = []
-    for a in attachments:
-        if not isinstance(a, dict):
-            continue
-
-        # Check if the attachment has at least one of the required fields
-        if not any(
-            key in a
-            for key in ["file_name", "file_size", "content_type", "download_url"]
-        ):
-            continue
-
-        valid_attachments.append(
-            {
-                "file_name": a.get("file_name", ""),
-                "file_size": a.get("file_size", 0),
-                "content_type": a.get("content_type", "application/octet-stream"),
-                "download_url": a.get("download_url", ""),
-            }
-        )
-
-    # Return None if there are no valid attachments
-    if not valid_attachments:
-        return None
-
-    df = pd.DataFrame(valid_attachments)
-
-    if df.empty:
-        return None
-
-    return create_csv_from_dataframe(df)
-
-
-def _write_attachment_csv(
-    zip_file: zipfile.ZipFile,
-    key: str,
-    idx: int,
-    attachments: list[dict[str, Any]],
-    privacy_request_id: str,
-) -> None:
-    """Write attachment data to a CSV file in the zip archive.
-
-    Args:
-        zip_file: The zip file to write to
-        key: The key for the data
-        idx: The index of the item in the list
-        attachments: List of attachment dictionaries
-        privacy_request_id: The ID of the privacy request for encryption
-    """
-    buffer = create_attachment_csv(attachments)
-    if buffer:
-        zip_file.writestr(
-            f"{key}/{idx + 1}/attachments.csv",
-            encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
-        )
-
-
-def _write_item_csv(
-    zip_file: zipfile.ZipFile,
-    key: str,
-    items: list[dict[str, Any]],
-    privacy_request_id: str,
-) -> None:
-    """Write item data to a CSV file in the zip archive.
-
-    Args:
-        zip_file: The zip file to write to
-        key: The key for the data
-        items: List of items to write
-        privacy_request_id: The ID of the privacy request for encryption
-    """
-    if items:
-        df = pd.DataFrame(items)
-        buffer = create_csv_from_dataframe(df)
-        zip_file.writestr(
-            f"{key}.csv",
-            encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
-        )
-
-
-def _write_simple_csv(
-    zip_file: zipfile.ZipFile,
-    key: str,
-    value: Any,
-    privacy_request_id: str,
-) -> None:
-    """Write simple key-value data to a CSV file in the zip archive.
-
-    Args:
-        zip_file: The zip file to write to
-        key: The key for the data
-        value: The value to write
-        privacy_request_id: The ID of the privacy request for encryption
-    """
-    df = pd.json_normalize({key: value})
-    buffer = create_csv_from_dataframe(df)
-    zip_file.writestr(
-        f"{key}.csv",
-        encrypt_access_request_results(buffer.getvalue(), privacy_request_id),
-    )
-
-
-def write_csv_to_zip(
-    zip_file: zipfile.ZipFile, data: dict[str, Any], privacy_request_id: str
-) -> None:
-    """Write data to a zip file in CSV format.
-
-    Args:
-        zip_file: The zip file to write to
-        data: The data to convert to CSV
-        privacy_request_id: The ID of the privacy request for encryption
-    """
-    for key, value in data.items():
-        if (
-            isinstance(value, list)
-            and value
-            and all(isinstance(item, dict) for item in value)
-        ):
-            # Handle lists of dictionaries
-            items: list[dict[str, Any]] = []
-            for item in value:
-                # Extract attachments if they exist
-                attachments = item.pop("attachments", [])
-                if attachments:
-                    _write_attachment_csv(
-                        zip_file, key, len(items), attachments, privacy_request_id
-                    )
-                items.append(item)
-            _write_item_csv(zip_file, key, items, privacy_request_id)
-        else:
-            _write_simple_csv(zip_file, key, value, privacy_request_id)

fides/api/tasks/encryption_utils.py

@@ -1,42 +0,0 @@
-import secrets
-from typing import Optional, Union
-
-from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
-from fides.api.util.cache import get_cache, get_encryption_cache_key
-from fides.api.util.encryption.aes_gcm_encryption_scheme import (
-    encrypt_to_bytes_verify_secrets_length,
-)
-from fides.config import CONFIG
-
-
-def encrypt_access_request_results(data: Union[str, bytes], request_id: str) -> str:
-    """Encrypt data with encryption key if provided, otherwise return unencrypted data.
-
-    Args:
-        data: The data to encrypt
-        request_id: The ID of the privacy request for encryption key lookup
-
-    Returns:
-        str: The encrypted data as a string
-    """
-    cache = get_cache()
-    encryption_cache_key = get_encryption_cache_key(
-        privacy_request_id=request_id,
-        encryption_attr="key",
-    )
-    if isinstance(data, bytes):
-        data = data.decode(CONFIG.security.encoding)
-
-    encryption_key: Optional[str] = cache.get(encryption_cache_key)
-    if not encryption_key:
-        return data
-
-    bytes_encryption_key: bytes = encryption_key.encode(
-        encoding=CONFIG.security.encoding
-    )
-    nonce: bytes = secrets.token_bytes(CONFIG.security.aes_gcm_nonce_length)
-    # b64encode the entire nonce and the encrypted message together
-    return bytes_to_b64_str(
-        nonce
-        + encrypt_to_bytes_verify_secrets_length(data, bytes_encryption_key, nonce)
-    )

fides/service/manual_tasks/manual_task_service.py

@@ -1,150 +0,0 @@
-from typing import Optional
-
-from loguru import logger
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from fides.api.models.fides_user import FidesUser
-from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
-from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
-from fides.api.schemas.manual_tasks.manual_task_schemas import (
-    ManualTaskLogStatus,
-    ManualTaskParentEntityType,
-    ManualTaskReferenceType,
-    ManualTaskType,
-)
-
-
-class ManualTaskService:
-    def __init__(self, db: Session):
-        self.db = db
-
-    def get_task(
-        self,
-        task_id: Optional[str] = None,
-        parent_entity_id: Optional[str] = None,
-        parent_entity_type: Optional[ManualTaskParentEntityType] = None,
-        task_type: Optional[ManualTaskType] = None,
-    ) -> Optional[ManualTask]:
-        """Get the manual task using provided filters.
-
-        Args:
-            task_id: The task ID
-            parent_entity_id: The parent entity ID
-            parent_entity_type: The parent entity type
-            task_type: The task type
-
-        Returns:
-            Optional[ManualTask]: The manual task for the connection, if it exists
-        """
-        if not any([task_id, parent_entity_id, parent_entity_type, task_type]):
-            logger.warning("No filters provided to get_task. Returning None.")
-            return None
-
-        stmt = select(ManualTask)  # type: ignore[arg-type]
-        if task_id:
-            stmt = stmt.where(ManualTask.id == task_id)
-        if parent_entity_id:
-            stmt = stmt.where(ManualTask.parent_entity_id == parent_entity_id)
-        if parent_entity_type:
-            stmt = stmt.where(ManualTask.parent_entity_type == parent_entity_type)
-        if task_type:
-            stmt = stmt.where(ManualTask.task_type == task_type)
-        return self.db.execute(stmt).scalar_one_or_none()
-
-    # User Management
-    def assign_users_to_task(
-        self, db: Session, task: ManualTask, user_ids: list[str]
-    ) -> None:
-        """Assigns users to this task. We can assign one or more users to a task.
-
-        Args:
-            db: Database session
-            task: The task to assign users to
-            user_ids: List of user IDs to assign
-        """
-        user_ids = list(set(user_ids))
-        if not user_ids:
-            raise ValueError("User ID is required for assignment")
-
-        # Create new user assignment
-        for user_id in user_ids:
-            # if user is already assigned, skip
-            if user_id in task.assigned_users:
-                continue
-            # verify user exists
-            user = db.query(FidesUser).filter_by(id=user_id).first()
-            if not user:
-                ManualTaskLog.create_error_log(
-                    db=db,
-                    task_id=task.id,
-                    message=f"Failed to add user {user_id} to task {task.id}: user does not exist",
-                    details={"user_id": user_id},
-                )
-                continue
-
-            ManualTaskReference.create(
-                db=db,
-                data={
-                    "task_id": task.id,
-                    "reference_id": user_id,
-                    "reference_type": ManualTaskReferenceType.assigned_user,
-                },
-            )
-
-            # Log the user assignment
-            ManualTaskLog.create_log(
-                db=db,
-                task_id=task.id,
-                status=ManualTaskLogStatus.updated,
-                message=f"User {user_id} assigned to task",
-                details={"assigned_user_id": user_id},
-            )
-
-    def unassign_users_from_task(
-        self, db: Session, task: ManualTask, user_ids: list[str]
-    ) -> None:
-        """Remove the user assignment from this task.
-
-        Args:
-            db: Database session
-            task: The task to unassign users from
-            user_ids: List of user IDs to unassign
-        """
-        user_ids = list(set(user_ids))
-        if not user_ids:
-            raise ValueError("User ID is required for unassignment")
-
-        # Get references to unassign
-        references_to_unassign = (
-            db.query(ManualTaskReference)
-            .filter(
-                ManualTaskReference.task_id == task.id,
-                ManualTaskReference.reference_type
-                == ManualTaskReferenceType.assigned_user,
-                ManualTaskReference.reference_id.in_(user_ids),
-            )
-            .all()
-        )
-
-        # Delete references and log unassignments
-        for ref in references_to_unassign:
-            ref.delete(db)
-            ManualTaskLog.create_log(
-                db=db,
-                task_id=task.id,
-                status=ManualTaskLogStatus.updated,
-                message=f"User {ref.reference_id} unassigned from task",
-                details={"unassigned_user_id": ref.reference_id},
-            )
-
-        # Check if any users weren't unassigned
-        unassigned_user_ids = [ref.reference_id for ref in references_to_unassign]
-        left_over_user_ids = [
-            user_id for user_id in user_ids if user_id not in unassigned_user_ids
-        ]
-        if left_over_user_ids:
-            logger.warning(
-                f"Failed to unassign users {left_over_user_ids} from task {task.id}: "
-                "users were not assigned to the task"
-            )