ethyca-fides 2.63.1rc0__py2.py3-none-any.whl → 2.63.3b0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-06-12T09:56:31-0700",
+ "date": "2025-06-12T18:57:31-0300",
  "dirty": false,
  "error": null,
- "full-revisionid": "273c62880da650c9930f107ddc22491c7caff338",
- "version": "2.63.1rc0"
+ "full-revisionid": "14baaf4b4e688a68ff32098124e78e6bab89dcb5",
+ "version": "2.63.3b0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/5efcdf18438e_add_manual_task_tables.py

@@ -0,0 +1,160 @@
+"""add manual task tables
+
+Revision ID: 5efcdf18438e
+Revises: c586a56c25e7
+Create Date: 2025-06-04 17:24:00.300170
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.sql import func
+
+# revision identifiers, used by Alembic.
+revision = "5efcdf18438e"
+down_revision = "c586a56c25e7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "manual_task",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=func.now(),
+        ),
+        sa.Column(
+            "task_type", sa.String(), nullable=False, server_default="privacy_request"
+        ),
+        sa.Column("parent_entity_id", sa.String(), nullable=False),
+        sa.Column("parent_entity_type", sa.String(), nullable=False),
+        sa.Column("due_date", sa.DateTime(timezone=True), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "parent_entity_id",
+            "parent_entity_type",
+            name="uq_manual_task_parent_entity",
+        ),
+    )
+
+    op.create_table(
+        "manual_task_reference",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=func.now(),
+        ),
+        sa.Column("task_id", sa.String(), nullable=False),
+        sa.Column("reference_id", sa.String(), nullable=False),
+        sa.Column("reference_type", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["task_id"],
+            ["manual_task.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+    op.create_table(
+        "manual_task_log",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=func.now(),
+        ),
+        sa.Column("task_id", sa.String(), nullable=False),
+        sa.Column("config_id", sa.String(), nullable=True),
+        sa.Column("instance_id", sa.String(), nullable=True),
+        sa.Column("status", sa.String(), nullable=False),
+        sa.Column("message", sa.String(), nullable=True),
+        sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["task_id"],
+            ["manual_task.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+    # Create indexes for manual_task
+    op.create_index("ix_manual_task_task_type", "manual_task", ["task_type"])
+    op.create_index(
+        "ix_manual_task_parent_entity",
+        "manual_task",
+        ["parent_entity_type", "parent_entity_id"],
+    )
+    op.create_index("ix_manual_task_due_date", "manual_task", ["due_date"])
+
+    # Create indexes for manual_task_reference
+    op.create_index(
+        "ix_manual_task_reference_task_id", "manual_task_reference", ["task_id"]
+    )
+    op.create_index(
+        "ix_manual_task_reference_reference",
+        "manual_task_reference",
+        ["reference_id", "reference_type"],
+    )
+
+    # Create indexes for manual_task_log
+    op.create_index("ix_manual_task_log_task_id", "manual_task_log", ["task_id"])
+    op.create_index("ix_manual_task_log_config_id", "manual_task_log", ["config_id"])
+    op.create_index(
+        "ix_manual_task_log_instance_id", "manual_task_log", ["instance_id"]
+    )
+    op.create_index("ix_manual_task_log_status", "manual_task_log", ["status"])
+    op.create_index("ix_manual_task_log_created_at", "manual_task_log", ["created_at"])
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Drop indexes first
+    op.drop_index("ix_manual_task_log_created_at", table_name="manual_task_log")
+    op.drop_index("ix_manual_task_log_status", table_name="manual_task_log")
+    op.drop_index("ix_manual_task_log_instance_id", table_name="manual_task_log")
+    op.drop_index("ix_manual_task_log_config_id", table_name="manual_task_log")
+    op.drop_index("ix_manual_task_log_task_id", table_name="manual_task_log")
+    op.drop_index(
+        "ix_manual_task_reference_reference", table_name="manual_task_reference"
+    )
+    op.drop_index(
+        "ix_manual_task_reference_task_id", table_name="manual_task_reference"
+    )
+    op.drop_index("ix_manual_task_due_date", table_name="manual_task")
+    op.drop_index("ix_manual_task_parent_entity", table_name="manual_task")
+    op.drop_index("ix_manual_task_task_type", table_name="manual_task")
+
+    # Then drop tables
+    op.drop_table("manual_task_log")
+    op.drop_table("manual_task_reference")
+    op.drop_table("manual_task")
+    # ### end Alembic commands ###

fides/api/api/v1/endpoints/connection_endpoints.py

@@ -22,6 +22,7 @@ from fides.api.schemas.connection_configuration import connection_secrets_schema
 from fides.api.schemas.connection_configuration.connection_config import (
     BulkPutConnectionConfiguration,
     ConnectionConfigurationResponse,
+    ConnectionConfigurationResponseWithSystemKey,
     CreateConnectionConfigurationWithSecrets,
 )
 from fides.api.schemas.connection_configuration.connection_secrets import (
@@ -154,13 +155,25 @@ def get_connections(
 @router.get(
     CONNECTION_BY_KEY,
     dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_READ])],
-    response_model=ConnectionConfigurationResponse,
+    response_model=ConnectionConfigurationResponseWithSystemKey,
 )
 def get_connection_detail(
     connection_key: FidesKey, db: Session = Depends(deps.get_db)
-) -> ConnectionConfig:
+) -> ConnectionConfigurationResponseWithSystemKey:
     """Returns connection configuration with matching key."""
-    return get_connection_config_or_error(db, connection_key)
+    connection_config = get_connection_config_or_error(db, connection_key)
+
+    # Convert to Pydantic model with all fields
+    response = ConnectionConfigurationResponseWithSystemKey.model_validate(
+        connection_config
+    )
+
+    # Override just the system_key field to use only the system's fides_key without fallback
+    response.system_key = (
+        connection_config.system.fides_key if connection_config.system else None
+    )
+
+    return response
 
 
 @router.patch(

fides/api/api/v1/endpoints/oauth_endpoints.py

@@ -274,9 +274,15 @@ def oauth_callback(code: str, state: str, db: Session = Depends(get_db)) -> Resp
 
             parsed_url = urlparse(authentication_request.referer)
             base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
-            system_key = connection_config.system.fides_key
+
+            if connection_config.system is not None:
+                system_key = connection_config.system.fides_key
+                redirect_path = f"/systems/configure/{system_key}"
+            else:
+                redirect_path = f"/integrations/{connection_config.key}"
+
             return RedirectResponse(
-                url=f"{base_url}/systems/configure/{system_key}?status={test_status_value}"
+                url=f"{base_url}{redirect_path}?status={test_status_value}"
             )
         return PlainTextResponse(
             content=f"Connection test status: {test_status_value}. No referer URL available. Please navigate back to the Fides Admin UI.",

fides/api/db/base.py

@@ -31,6 +31,8 @@ from fides.api.models.fides_user_respondent_email_verification import (
 )
 from fides.api.models.identity_salt import IdentitySalt
 from fides.api.models.location_regulation_selections import LocationRegulationSelections
+from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.messaging import MessagingConfig
 from fides.api.models.messaging_template import MessagingTemplate

fides/api/main.py

@@ -244,6 +244,7 @@ async def log_request(request: Request, call_next: Callable) -> Response:
         status_code=response.status_code,
         handler_time=f"{total_time}ms",
         path=request.url.path,
+        fides_client=request.headers.get("Fides-Client", "unknown"),
     ).info("Request received")
     return response
 

fides/api/models/attachment.py

@@ -1,5 +1,6 @@
 import os
 from enum import Enum as EnumType
+from io import BytesIO
 from typing import IO, TYPE_CHECKING, Any, Tuple
 
 from fideslang.validation import AnyHttpUrlString
@@ -19,7 +20,6 @@ from fides.api.service.storage.s3 import (
     generic_upload_to_s3,
 )
 from fides.api.service.storage.util import AllowedFileType, get_local_filename
-from fides.config import CONFIG
 
 if TYPE_CHECKING:
     from fides.api.models.comment import Comment
@@ -28,6 +28,11 @@ if TYPE_CHECKING:
     from fides.api.models.storage import StorageConfig
 
 
+# This is 7 days in seconds and is currently the max allowed
+# configurable expiration time for presigned URLs for both s3 and gcs.
+MAX_TTL_SECONDS = 604800
+
+
 class AttachmentType(str, EnumType):
     """
     Enum for attachment types. Indicates attachment usage.
@@ -122,6 +127,11 @@ class Attachment(Base):
         """Returns the content type of the attachment."""
         return AllowedFileType[self.file_name.split(".")[-1]].value
 
+    @property
+    def file_key(self) -> str:
+        """Returns the file key of the attachment."""
+        return f"{self.id}/{self.file_name}"
+
     def upload(self, attachment: IO[bytes]) -> None:
         """Uploads an attachment to S3, GCS, or local storage."""
         if self.config.type == StorageType.s3:
@@ -130,7 +140,7 @@ class Attachment(Base):
             generic_upload_to_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 document=attachment,
                 auth_method=auth_method,
             )
@@ -142,7 +152,7 @@ class Attachment(Base):
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(f"{self.id}/{self.file_name}")
+            blob = bucket.blob(self.file_key)
 
             # Reset the file pointer to the beginning
             try:
@@ -155,7 +165,7 @@ class Attachment(Base):
             return
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(f"{self.id}/{self.file_name}")
+            filename = get_local_filename(self.file_key)
 
             # Validate that attachment is a file-like object
             if not hasattr(attachment, "read"):
@@ -205,9 +215,10 @@ class Attachment(Base):
             size, url = generic_retrieve_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 auth_method=auth_method,
                 get_content=False,
+                ttl_seconds=MAX_TTL_SECONDS,
             )
             return size, url
 
@@ -216,19 +227,20 @@ class Attachment(Base):
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(f"{self.id}/{self.file_name}")
+            blob = bucket.blob(self.file_key)
 
             # Ensure we have the blob metadata
             blob.reload()
+            # Expiration is set to 7 days
             url = blob.generate_signed_url(
                 version="v4",
-                expiration=CONFIG.security.subject_request_download_link_ttl_seconds,
+                expiration=MAX_TTL_SECONDS,
                 method="GET",
             )
             return blob.size, url
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(f"{self.id}/{self.file_name}")
+            filename = get_local_filename(self.file_key)
             size = os.path.getsize(filename)
             return size, filename
 
@@ -236,7 +248,7 @@ class Attachment(Base):
 
     def retrieve_attachment_content(
         self,
-    ) -> Tuple[int, bytes]:
+    ) -> Tuple[int, IO[bytes]]:
         """
         Retrieves the size of the attachment and its actual content.
         - For s3:
@@ -252,30 +264,33 @@ class Attachment(Base):
         if self.config.type == StorageType.s3:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
-            size, content = generic_retrieve_from_s3(
+            size, fileobj = generic_retrieve_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 auth_method=auth_method,
                 get_content=True,
             )
-            return size, content
+            return size, fileobj
 
         if self.config.type == StorageType.gcs:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(f"{self.id}/{self.file_name}")
+            blob = bucket.blob(self.file_key)
 
-            content = blob.download_as_bytes()
-            return len(content), content
+            fileobj = BytesIO()
+            blob.download_to_file(fileobj)
+            fileobj.seek(0)  # Reset pointer to beginning after download
+            return blob.size, fileobj
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(f"{self.id}/{self.file_name}")
-            with open(filename, "rb") as file:
-                content = file.read()
-                size = len(content)
+            filename = get_local_filename(self.file_key)
+            size = os.path.getsize(filename)
+            with open(filename, "rb") as fileobj:
+                content = BytesIO(fileobj.read())
+                content.seek(0)  # Reset pointer to beginning
                 return size, content
 
         raise ValueError(f"Unsupported storage type: {self.config.type}")
@@ -288,7 +303,7 @@ class Attachment(Base):
             generic_delete_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 auth_method=auth_method,
             )
             return
@@ -307,9 +322,7 @@ class Attachment(Base):
             return
 
         if self.config.type == StorageType.local:
-            folder_path = os.path.dirname(
-                get_local_filename(f"{self.id}/{self.file_name}")
-            )
+            folder_path = os.path.dirname(get_local_filename(self.file_key))
             if os.path.exists(folder_path):
                 import shutil
 

fides/api/models/detection_discovery/monitor_task.py

@@ -26,6 +26,7 @@ class MonitorTaskType(Enum):
     DETECTION = "detection"
     CLASSIFICATION = "classification"
     PROMOTION = "promotion"
+    REMOVAL_PROMOTION = "removal_promotion"
 
 
 class MonitorTask(WorkerTask, Base):

fides/api/models/manual_tasks/__init__.py

@@ -0,0 +1,8 @@
+from fides.api.models.manual_tasks.manual_task import ManualTask, ManualTaskReference
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
+
+__all__ = [
+    "ManualTask",
+    "ManualTaskReference",
+    "ManualTaskLog",
+]

fides/api/models/manual_tasks/manual_task.py

@@ -0,0 +1,110 @@
+from typing import Any
+
+from sqlalchemy import Column, DateTime, ForeignKey, String
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.db.util import EnumColumn
+from fides.api.models.manual_tasks.manual_task_log import ManualTaskLog
+from fides.api.schemas.manual_tasks.manual_task_schemas import (
+    ManualTaskLogStatus,
+    ManualTaskParentEntityType,
+    ManualTaskReferenceType,
+    ManualTaskType,
+)
+
+
+class ManualTask(Base):
+    """Model for storing manual tasks.
+
+    This model can be used for both privacy request tasks and general tasks.
+    For privacy requests, it replaces the functionality of manual webhooks.
+    For other use cases, it provides a flexible task management system.
+
+    There can only be one ManualTask per parent entity.
+    You can create multiple Configs for the same ManualTask.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task"
+
+    # Database columns
+    task_type = Column(
+        EnumColumn(ManualTaskType),
+        nullable=False,
+        default=ManualTaskType.privacy_request,
+    )
+    parent_entity_id = Column(String, nullable=False)
+    parent_entity_type = Column(
+        EnumColumn(ManualTaskParentEntityType),
+        nullable=False,
+        default=ManualTaskParentEntityType.connection_config,
+    )
+    due_date = Column(DateTime, nullable=True)
+
+    # Relationships
+    references = relationship(
+        "ManualTaskReference",
+        back_populates="task",
+        uselist=True,
+        cascade="all, delete-orphan",
+    )
+    logs = relationship(
+        "ManualTaskLog",
+        back_populates="task",
+        primaryjoin="and_(ManualTask.id == ManualTaskLog.task_id)",
+        viewonly=True,
+        order_by="ManualTaskLog.created_at",
+    )
+
+    # Properties
+    @property
+    def assigned_users(self) -> list[str]:
+        """Get all users assigned to this task."""
+        if not self.references:
+            return []
+        return [
+            ref.reference_id
+            for ref in self.references
+            if ref.reference_type == ManualTaskReferenceType.assigned_user
+        ]
+
+    # CRUD Operations
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = True
+    ) -> "ManualTask":
+        """Create a new manual task."""
+        task = super().create(db=db, data=data, check_name=check_name)
+        ManualTaskLog.create_log(
+            db=db,
+            task_id=task.id,
+            status=ManualTaskLogStatus.created,
+            message=f"Created manual task for {data['task_type']}",
+        )
+        return task
+
+
+class ManualTaskReference(Base):
+    """Join table to associate manual tasks with multiple references.
+
+    A single task may have many references including privacy requests, configurations, and assigned users.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_reference"
+
+    # Database columns
+    task_id = Column(
+        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+    )
+    reference_id = Column(String, nullable=False)
+    reference_type = Column(EnumColumn(ManualTaskReferenceType), nullable=False)
+
+    # Relationships
+    task = relationship("ManualTask", back_populates="references")

fides/api/models/manual_tasks/manual_task_log.py

@@ -0,0 +1,100 @@
+from typing import TYPE_CHECKING, Any, Optional
+
+from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.schemas.manual_tasks.manual_task_schemas import ManualTaskLogStatus
+
+if TYPE_CHECKING:
+    from fides.api.models.manual_tasks.manual_task import ManualTask
+
+
+class ManualTaskLog(Base):
+    """Model for storing manual task execution logs."""
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_log"
+
+    task_id = Column(
+        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+    )
+    # TODO: Add foreign key constraints when config and instance are implemented
+    config_id = Column(String, nullable=True)
+    instance_id = Column(String, nullable=True)
+    status = Column(String, nullable=False)
+    message = Column(String, nullable=True)
+    details = Column(JSONB, nullable=True)
+
+    # Relationships - using string references to avoid circular imports
+    task = relationship("ManualTask", back_populates="logs", foreign_keys=[task_id])
+    # TODO: Add config and instance relationships when they are implemented
+    # config = relationship("ManualTaskConfig", back_populates="logs")
+    # instance = relationship("ManualTaskInstance", back_populates="logs")
+
+    @classmethod
+    def create_log(
+        cls,
+        db: Session,
+        status: ManualTaskLogStatus,
+        task_id: str,
+        config_id: Optional[str] = None,
+        instance_id: Optional[str] = None,
+        message: Optional[str] = None,
+        details: Optional[dict[str, Any]] = None,
+    ) -> "ManualTaskLog":
+        """Create a new task log entry.
+
+        Args:
+            db: Database session
+            task_id: ID of the task
+            status: Status of the log entry
+            message: Optional message describing the event
+            details: Optional additional details about the event
+        """
+        data = {
+            "task_id": task_id,
+            "config_id": config_id,
+            "instance_id": instance_id,
+            "status": status,
+            "message": message,
+            "details": details,
+        }
+        return cls.create(db=db, data=data)
+
+    @classmethod
+    def create_error_log(
+        cls,
+        db: Session,
+        task_id: str,
+        message: str,
+        config_id: Optional[str] = None,
+        instance_id: Optional[str] = None,
+        details: Optional[dict[str, Any]] = None,
+    ) -> "ManualTaskLog":
+        """Create a new error log entry.
+
+        Args:
+            db: Database session
+            task_id: ID of the task
+            message: Error message describing what went wrong
+            config_id: Optional ID of the configuration
+            instance_id: Optional ID of the instance
+            details: Optional additional details about the error
+
+        Returns:
+            The created error log entry
+        """
+        return cls.create_log(
+            db=db,
+            status=ManualTaskLogStatus.error,
+            task_id=task_id,
+            config_id=config_id,
+            instance_id=instance_id,
+            message=message,
+            details=details,
+        )