ethyca-fides 2.63.1b0__py2.py3-none-any.whl → 2.63.1b3__py2.py3-none-any.whl

fides/api/alembic/migrations/versions/bf713b5a021d_staged_resource_ancestor_link_data_.py

@@ -74,6 +74,11 @@ def upgrade():
             children = result.children
             if children:
                 resource_children[urn] = children
+            else:
+                # even if no children, we still add the resource to the map
+                # so that we maintain a record of all resources and can
+                # check for orphaned descendant records later
+                resource_children[urn] = []
 
     # Build list of ancestor-descendant pairs
     ancestor_links = []
@@ -86,17 +91,21 @@ def upgrade():
         for child_urn in children:
             if child_urn not in visited:
                 visited.add(child_urn)
-                # Add direct ancestor link
-                ancestor_links.append(
-                    {
-                        "id": f"srl_{uuid.uuid4()}",
-                        "ancestor_urn": ancestor_urn,
-                        "descendant_urn": child_urn,
-                    }
-                )
-
-                # Recursively process this child's children
-                if child_urn in resource_children:
+                if child_urn not in resource_children:
+                    logger.warning(
+                        f"Found orphaned descendant URN: {child_urn}, not adding ancestor link"
+                    )
+                else:
+                    # Add direct ancestor link
+                    ancestor_links.append(
+                        {
+                            "id": f"srl_{uuid.uuid4()}",
+                            "ancestor_urn": ancestor_urn,
+                            "descendant_urn": child_urn,
+                        }
+                    )
+
+                    # Recursively process this child's children
                     process_children(
                         ancestor_urn, resource_children[child_urn], visited
                     )

fides/api/db/base.py

@@ -27,6 +27,8 @@ from fides.api.models.fides_user_respondent_email_verification import (
 )
 from fides.api.models.identity_salt import IdentitySalt
 from fides.api.models.location_regulation_selections import LocationRegulationSelections
+from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.messaging import MessagingConfig
 from fides.api.models.messaging_template import MessagingTemplate

fides/api/migrations/post_upgrade_index_creation.py

@@ -182,7 +182,7 @@ def create_object(db: Session, object_statement: str, object_name: str) -> None:
 def check_and_create_objects(
     db: Session, table_object_map: Dict[str, List[Dict[str, str]]], lock: Lock
 ) -> Dict[str, str]:
-    """Returns a dictionary of any indices or constraints that are in the process of being created."""
+    """Returns a dictionary of any indices or constraints that were created."""
     object_info: Dict[str, str] = {}
     for _, objects in table_object_map.items():
         for object_data in objects:
@@ -203,7 +203,7 @@ def check_and_create_objects(
                             continue
 
                 create_object(db, object_statement, object_name)
-                object_info[object_name] = "in progress"
+                object_info[object_name] = "created"
             else:
                 logger.debug(
                     f"Object {object_name} already exists, skipping index/constraint creation"
@@ -248,7 +248,7 @@ def post_upgrade_index_creation_task() -> None:
                     f"Post upgrade index creation output: {json.dumps(object_info)}"
                 )
             else:
-                logger.debug("All indices and constraints created")
+                logger.debug("All indices and constraints already created")
 
 
 def initiate_post_upgrade_index_creation() -> None:

fides/api/models/attachment.py

@@ -1,5 +1,6 @@
 import os
 from enum import Enum as EnumType
+from io import BytesIO
 from typing import IO, TYPE_CHECKING, Any, Tuple
 
 from fideslang.validation import AnyHttpUrlString
@@ -19,7 +20,6 @@ from fides.api.service.storage.s3 import (
     generic_upload_to_s3,
 )
 from fides.api.service.storage.util import AllowedFileType, get_local_filename
-from fides.config import CONFIG
 
 if TYPE_CHECKING:
     from fides.api.models.comment import Comment
@@ -28,6 +28,11 @@ if TYPE_CHECKING:
     from fides.api.models.storage import StorageConfig
 
 
+# This is 7 days in seconds and is currently the max allowed
+# configurable expiration time for presigned URLs for both s3 and gcs.
+MAX_TTL_SECONDS = 604800
+
+
 class AttachmentType(str, EnumType):
     """
     Enum for attachment types. Indicates attachment usage.
@@ -122,6 +127,11 @@ class Attachment(Base):
         """Returns the content type of the attachment."""
         return AllowedFileType[self.file_name.split(".")[-1]].value
 
+    @property
+    def file_key(self) -> str:
+        """Returns the file key of the attachment."""
+        return f"{self.id}/{self.file_name}"
+
     def upload(self, attachment: IO[bytes]) -> None:
         """Uploads an attachment to S3, GCS, or local storage."""
         if self.config.type == StorageType.s3:
@@ -130,7 +140,7 @@ class Attachment(Base):
             generic_upload_to_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 document=attachment,
                 auth_method=auth_method,
             )
@@ -142,7 +152,7 @@ class Attachment(Base):
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(f"{self.id}/{self.file_name}")
+            blob = bucket.blob(self.file_key)
 
             # Reset the file pointer to the beginning
             try:
@@ -155,7 +165,7 @@ class Attachment(Base):
             return
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(f"{self.id}/{self.file_name}")
+            filename = get_local_filename(self.file_key)
 
             # Validate that attachment is a file-like object
             if not hasattr(attachment, "read"):
@@ -205,9 +215,10 @@ class Attachment(Base):
             size, url = generic_retrieve_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 auth_method=auth_method,
                 get_content=False,
+                ttl_seconds=MAX_TTL_SECONDS,
             )
             return size, url
 
@@ -216,19 +227,20 @@ class Attachment(Base):
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(f"{self.id}/{self.file_name}")
+            blob = bucket.blob(self.file_key)
 
             # Ensure we have the blob metadata
             blob.reload()
+            # Expiration is set to 7 days
             url = blob.generate_signed_url(
                 version="v4",
-                expiration=CONFIG.security.subject_request_download_link_ttl_seconds,
+                expiration=MAX_TTL_SECONDS,
                 method="GET",
             )
             return blob.size, url
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(f"{self.id}/{self.file_name}")
+            filename = get_local_filename(self.file_key)
             size = os.path.getsize(filename)
             return size, filename
 
@@ -236,7 +248,7 @@ class Attachment(Base):
 
     def retrieve_attachment_content(
         self,
-    ) -> Tuple[int, bytes]:
+    ) -> Tuple[int, IO[bytes]]:
         """
         Retrieves the size of the attachment and its actual content.
         - For s3:
@@ -252,30 +264,33 @@ class Attachment(Base):
         if self.config.type == StorageType.s3:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
-            size, content = generic_retrieve_from_s3(
+            size, fileobj = generic_retrieve_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 auth_method=auth_method,
                 get_content=True,
             )
-            return size, content
+            return size, fileobj
 
         if self.config.type == StorageType.gcs:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(f"{self.id}/{self.file_name}")
+            blob = bucket.blob(self.file_key)
 
-            content = blob.download_as_bytes()
-            return len(content), content
+            fileobj = BytesIO()
+            blob.download_to_file(fileobj)
+            fileobj.seek(0)  # Reset pointer to beginning after download
+            return blob.size, fileobj
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(f"{self.id}/{self.file_name}")
-            with open(filename, "rb") as file:
-                content = file.read()
-                size = len(content)
+            filename = get_local_filename(self.file_key)
+            size = os.path.getsize(filename)
+            with open(filename, "rb") as fileobj:
+                content = BytesIO(fileobj.read())
+                content.seek(0)  # Reset pointer to beginning
                 return size, content
 
         raise ValueError(f"Unsupported storage type: {self.config.type}")
@@ -288,7 +303,7 @@ class Attachment(Base):
             generic_delete_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 auth_method=auth_method,
             )
             return
@@ -307,9 +322,7 @@ class Attachment(Base):
             return
 
         if self.config.type == StorageType.local:
-            folder_path = os.path.dirname(
-                get_local_filename(f"{self.id}/{self.file_name}")
-            )
+            folder_path = os.path.dirname(get_local_filename(self.file_key))
             if os.path.exists(folder_path):
                 import shutil
 

fides/api/models/manual_tasks/__init__.py

@@ -0,0 +1,8 @@
+from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
+
+__all__ = [
+    "ManualTask",
+    "ManualTaskReference",
+    "ManualTaskLog",
+]

fides/api/models/manual_tasks/manual_task.py

@@ -0,0 +1,110 @@
+from typing import Any
+
+from sqlalchemy import Column, DateTime, ForeignKey, String
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.db.util import EnumColumn
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
+from fides.api.schemas.manual_tasks.manual_task_schemas import (
+    ManualTaskLogStatus,
+    ManualTaskParentEntityType,
+    ManualTaskReferenceType,
+    ManualTaskType,
+)
+
+
+class ManualTask(Base):
+    """Model for storing manual tasks.
+
+    This model can be used for both privacy request tasks and general tasks.
+    For privacy requests, it replaces the functionality of manual webhooks.
+    For other use cases, it provides a flexible task management system.
+
+    There can only be one ManualTask per parent entity.
+    You can create multiple Configs for the same ManualTask.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task"
+
+    # Database columns
+    task_type = Column(
+        EnumColumn(ManualTaskType),
+        nullable=False,
+        default=ManualTaskType.privacy_request,
+    )
+    parent_entity_id = Column(String, nullable=False)
+    parent_entity_type = Column(
+        EnumColumn(ManualTaskParentEntityType),
+        nullable=False,
+        default=ManualTaskParentEntityType.connection_config,
+    )
+    due_date = Column(DateTime, nullable=True)
+
+    # Relationships
+    references = relationship(
+        "ManualTaskReference",
+        back_populates="task",
+        uselist=True,
+        cascade="all, delete-orphan",
+    )
+    logs = relationship(
+        "ManualTaskLog",
+        back_populates="task",
+        primaryjoin="and_(ManualTask.id == ManualTaskLog.task_id)",
+        viewonly=True,
+        order_by="ManualTaskLog.created_at",
+    )
+
+    # Properties
+    @property
+    def assigned_users(self) -> list[str]:
+        """Get all users assigned to this task."""
+        if not self.references:
+            return []
+        return [
+            ref.reference_id
+            for ref in self.references
+            if ref.reference_type == ManualTaskReferenceType.assigned_user
+        ]
+
+    # CRUD Operations
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = True
+    ) -> "ManualTask":
+        """Create a new manual task."""
+        task = super().create(db=db, data=data, check_name=check_name)
+        ManualTaskLog.create_log(
+            db=db,
+            task_id=task.id,
+            status=ManualTaskLogStatus.created,
+            message=f"Created manual task for {data['task_type']}",
+        )
+        return task
+
+
+class ManualTaskReference(Base):
+    """Join table to associate manual tasks with multiple references.
+
+    A single task may have many references including privacy requests, configurations, and assigned users.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_reference"
+
+    # Database columns
+    task_id = Column(
+        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+    )
+    reference_id = Column(String, nullable=False)
+    reference_type = Column(EnumColumn(ManualTaskReferenceType), nullable=False)
+
+    # Relationships
+    task = relationship("ManualTask", back_populates="references")

fides/api/models/manual_tasks/manual_task_log.py

@@ -0,0 +1,100 @@
+from typing import TYPE_CHECKING, Any, Optional
+
+from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.schemas.manual_tasks.manual_task_schemas import ManualTaskLogStatus
+
+if TYPE_CHECKING:
+    from fides.api.models.manual_tasks.manual_task import ManualTask
+
+
+class ManualTaskLog(Base):
+    """Model for storing manual task execution logs."""
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_log"
+
+    task_id = Column(
+        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+    )
+    # TODO: Add foreign key constraints when config and instance are implemented
+    config_id = Column(String, nullable=True)
+    instance_id = Column(String, nullable=True)
+    status = Column(String, nullable=False)
+    message = Column(String, nullable=True)
+    details = Column(JSONB, nullable=True)
+
+    # Relationships - using string references to avoid circular imports
+    task = relationship("ManualTask", back_populates="logs", foreign_keys=[task_id])
+    # TODO: Add config and instance relationships when they are implemented
+    # config = relationship("ManualTaskConfig", back_populates="logs")
+    # instance = relationship("ManualTaskInstance", back_populates="logs")
+
+    @classmethod
+    def create_log(
+        cls,
+        db: Session,
+        status: ManualTaskLogStatus,
+        task_id: str,
+        config_id: Optional[str] = None,
+        instance_id: Optional[str] = None,
+        message: Optional[str] = None,
+        details: Optional[dict[str, Any]] = None,
+    ) -> "ManualTaskLog":
+        """Create a new task log entry.
+
+        Args:
+            db: Database session
+            task_id: ID of the task
+            status: Status of the log entry
+            message: Optional message describing the event
+            details: Optional additional details about the event
+        """
+        data = {
+            "task_id": task_id,
+            "config_id": config_id,
+            "instance_id": instance_id,
+            "status": status,
+            "message": message,
+            "details": details,
+        }
+        return cls.create(db=db, data=data)
+
+    @classmethod
+    def create_error_log(
+        cls,
+        db: Session,
+        task_id: str,
+        message: str,
+        config_id: Optional[str] = None,
+        instance_id: Optional[str] = None,
+        details: Optional[dict[str, Any]] = None,
+    ) -> "ManualTaskLog":
+        """Create a new error log entry.
+
+        Args:
+            db: Database session
+            task_id: ID of the task
+            message: Error message describing what went wrong
+            config_id: Optional ID of the configuration
+            instance_id: Optional ID of the instance
+            details: Optional additional details about the error
+
+        Returns:
+            The created error log entry
+        """
+        return cls.create_log(
+            db=db,
+            status=ManualTaskLogStatus.error,
+            task_id=task_id,
+            config_id=config_id,
+            instance_id=instance_id,
+            message=message,
+            details=details,
+        )

fides/api/schemas/manual_tasks/manual_task_schemas.py

@@ -0,0 +1,79 @@
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Any, Optional
+
+from pydantic import ConfigDict, Field
+
+from fides.api.schemas.base_class import FidesSchema
+
+
+class ManualTaskType(str, Enum):
+    """Enum for manual task types."""
+
+    privacy_request = "privacy_request"
+    # Add more task types as needed
+
+
+class ManualTaskParentEntityType(str, Enum):
+    """Enum for manual task parent entity types."""
+
+    connection_config = (
+        "connection_config"  # used for access and erasure privacy requests
+    )
+    # Add more parent entity types as needed
+
+
+class ManualTaskReferenceType(str, Enum):
+    """Enum for manual task reference types."""
+
+    privacy_request = "privacy_request"
+    connection_config = "connection_config"
+    manual_task_config = "manual_task_config"
+    assigned_user = "assigned_user"  # Reference to the user assigned to the task
+    # Add more reference types as needed
+
+
+class ManualTaskLogStatus(str, Enum):
+    """Enum for manual task log status."""
+
+    created = "created"
+    updated = "updated"
+    in_processing = "in_processing"
+    complete = "complete"
+    error = "error"
+    retrying = "retrying"
+    paused = "paused"
+    awaiting_input = "awaiting_input"
+
+
+class ManualTaskLogCreate(FidesSchema):
+    """Schema for creating a manual task log entry."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    task_id: Annotated[str, Field(..., description="ID of the task")]
+    status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
+    message: Annotated[Optional[str], Field(None, description="Log message")]
+    details: Annotated[
+        Optional[dict[str, Any]], Field(None, description="Additional details")
+    ]
+    config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
+    instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
+
+
+class ManualTaskLogResponse(FidesSchema):
+    """Schema for manual task log response."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: Annotated[str, Field(..., description="Log ID")]
+    task_id: Annotated[str, Field(..., description="Task ID")]
+    status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
+    message: Annotated[Optional[str], Field(None, description="Log message")]
+    details: Annotated[
+        Optional[dict[str, Any]], Field(None, description="Additional details")
+    ]
+    config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
+    instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
+    created_at: Annotated[datetime, Field(..., description="Creation timestamp")]
+    updated_at: Annotated[datetime, Field(..., description="Last update timestamp")]