ethyca-fides 2.63.1b1__py2.py3-none-any.whl → 2.63.1b3__py2.py3-none-any.whl

fides/api/models/manual_tasks/manual_task_log.py

@@ -0,0 +1,100 @@
+from typing import TYPE_CHECKING, Any, Optional
+
+from sqlalchemy import Column, ForeignKey, String
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.schemas.manual_tasks.manual_task_schemas import ManualTaskLogStatus
+
+if TYPE_CHECKING:
+    from fides.api.models.manual_tasks.manual_task import ManualTask
+
+
+class ManualTaskLog(Base):
+    """Model for storing manual task execution logs."""
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "manual_task_log"
+
+    task_id = Column(
+        String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
+    )
+    # TODO: Add foreign key constraints when config and instance are implemented
+    config_id = Column(String, nullable=True)
+    instance_id = Column(String, nullable=True)
+    status = Column(String, nullable=False)
+    message = Column(String, nullable=True)
+    details = Column(JSONB, nullable=True)
+
+    # Relationships - using string references to avoid circular imports
+    task = relationship("ManualTask", back_populates="logs", foreign_keys=[task_id])
+    # TODO: Add config and instance relationships when they are implemented
+    # config = relationship("ManualTaskConfig", back_populates="logs")
+    # instance = relationship("ManualTaskInstance", back_populates="logs")
+
+    @classmethod
+    def create_log(
+        cls,
+        db: Session,
+        status: ManualTaskLogStatus,
+        task_id: str,
+        config_id: Optional[str] = None,
+        instance_id: Optional[str] = None,
+        message: Optional[str] = None,
+        details: Optional[dict[str, Any]] = None,
+    ) -> "ManualTaskLog":
+        """Create a new task log entry.
+
+        Args:
+            db: Database session
+            task_id: ID of the task
+            status: Status of the log entry
+            message: Optional message describing the event
+            details: Optional additional details about the event
+        """
+        data = {
+            "task_id": task_id,
+            "config_id": config_id,
+            "instance_id": instance_id,
+            "status": status,
+            "message": message,
+            "details": details,
+        }
+        return cls.create(db=db, data=data)
+
+    @classmethod
+    def create_error_log(
+        cls,
+        db: Session,
+        task_id: str,
+        message: str,
+        config_id: Optional[str] = None,
+        instance_id: Optional[str] = None,
+        details: Optional[dict[str, Any]] = None,
+    ) -> "ManualTaskLog":
+        """Create a new error log entry.
+
+        Args:
+            db: Database session
+            task_id: ID of the task
+            message: Error message describing what went wrong
+            config_id: Optional ID of the configuration
+            instance_id: Optional ID of the instance
+            details: Optional additional details about the error
+
+        Returns:
+            The created error log entry
+        """
+        return cls.create_log(
+            db=db,
+            status=ManualTaskLogStatus.error,
+            task_id=task_id,
+            config_id=config_id,
+            instance_id=instance_id,
+            message=message,
+            details=details,
+        )

fides/api/schemas/manual_tasks/manual_task_schemas.py

@@ -0,0 +1,79 @@
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Any, Optional
+
+from pydantic import ConfigDict, Field
+
+from fides.api.schemas.base_class import FidesSchema
+
+
+class ManualTaskType(str, Enum):
+    """Enum for manual task types."""
+
+    privacy_request = "privacy_request"
+    # Add more task types as needed
+
+
+class ManualTaskParentEntityType(str, Enum):
+    """Enum for manual task parent entity types."""
+
+    connection_config = (
+        "connection_config"  # used for access and erasure privacy requests
+    )
+    # Add more parent entity types as needed
+
+
+class ManualTaskReferenceType(str, Enum):
+    """Enum for manual task reference types."""
+
+    privacy_request = "privacy_request"
+    connection_config = "connection_config"
+    manual_task_config = "manual_task_config"
+    assigned_user = "assigned_user"  # Reference to the user assigned to the task
+    # Add more reference types as needed
+
+
+class ManualTaskLogStatus(str, Enum):
+    """Enum for manual task log status."""
+
+    created = "created"
+    updated = "updated"
+    in_processing = "in_processing"
+    complete = "complete"
+    error = "error"
+    retrying = "retrying"
+    paused = "paused"
+    awaiting_input = "awaiting_input"
+
+
+class ManualTaskLogCreate(FidesSchema):
+    """Schema for creating a manual task log entry."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    task_id: Annotated[str, Field(..., description="ID of the task")]
+    status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
+    message: Annotated[Optional[str], Field(None, description="Log message")]
+    details: Annotated[
+        Optional[dict[str, Any]], Field(None, description="Additional details")
+    ]
+    config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
+    instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
+
+
+class ManualTaskLogResponse(FidesSchema):
+    """Schema for manual task log response."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: Annotated[str, Field(..., description="Log ID")]
+    task_id: Annotated[str, Field(..., description="Task ID")]
+    status: Annotated[ManualTaskLogStatus, Field(..., description="Log status")]
+    message: Annotated[Optional[str], Field(None, description="Log message")]
+    details: Annotated[
+        Optional[dict[str, Any]], Field(None, description="Additional details")
+    ]
+    config_id: Annotated[Optional[str], Field(None, description="Configuration ID")]
+    instance_id: Annotated[Optional[str], Field(None, description="Instance ID")]
+    created_at: Annotated[datetime, Field(..., description="Creation timestamp")]
+    updated_at: Annotated[datetime, Field(..., description="Last update timestamp")]

fides/api/schemas/manual_tasks/manual_task_status.py

@@ -0,0 +1,151 @@
+from datetime import datetime, timezone
+from enum import Enum as EnumType
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+
+class StatusTransitionNotAllowed(Exception):
+    """Exception raised when a status transition is not allowed."""
+
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+
+class StatusType(str, EnumType):
+    """Enum for manual task status."""
+
+    pending = "pending"
+    in_progress = "in_progress"
+    completed = "completed"
+    failed = "failed"
+
+    @classmethod
+    def get_valid_transitions(cls, current_status: "StatusType") -> list["StatusType"]:
+        """Get valid transitions from the current status.
+
+        Args:
+            current_status: The current status
+
+        Returns:
+            list[StatusType]: List of valid transitions
+        """
+        if current_status == cls.pending:
+            return [cls.in_progress, cls.failed]
+        if current_status == cls.in_progress:
+            return [cls.completed, cls.failed]
+        if current_status == cls.completed:
+            return []
+        if current_status == cls.failed:
+            return [cls.pending, cls.in_progress]
+        return []
+
+
+class StatusTransitionMixin:
+    """Mixin for handling status transitions.
+
+    This mixin provides methods for managing status transitions and completion tracking.
+    It can be used by any model that needs status management.
+    """
+
+    # These should be overridden by the implementing class
+    status: StatusType
+    completed_at: Optional[datetime]
+    completed_by_id: Optional[str]
+
+    def _get_valid_transitions(self) -> list[StatusType]:
+        """Get valid transitions from the current status.
+
+        Returns:
+            list[StatusType]: List of valid transitions
+        """
+        return StatusType.get_valid_transitions(self.status)
+
+    def _validate_status_transition(self, new_status: StatusType) -> None:
+        """Validate that a status transition is allowed.
+
+        Args:
+            new_status: The new status to transition to
+
+        Raises:
+            StatusTransitionNotAllowed: If the transition is not allowed
+        """
+        # Don't allow transitions to the same status
+        if new_status == self.status:
+            raise StatusTransitionNotAllowed(
+                f"Invalid status transition: already in status {new_status}"
+            )
+
+        # Get valid transitions for current status
+        valid_transitions = self._get_valid_transitions()
+        if new_status not in valid_transitions:
+            raise StatusTransitionNotAllowed(
+                f"Invalid status transition from {self.status} to {new_status}. "
+                f"Valid transitions are: {valid_transitions}"
+            )
+
+    def update_status(
+        self, db: Session, new_status: StatusType, user_id: Optional[str] = None
+    ) -> None:
+        """Update the status with validation and completion handling.
+
+        Args:
+            db: Database session
+            new_status: New status to set
+            user_id: Optional user ID who is making the change
+        """
+        self._validate_status_transition(new_status)
+
+        if new_status == StatusType.completed:
+            self.completed_at = datetime.now(timezone.utc)
+            self.completed_by_id = user_id
+        elif new_status == StatusType.pending:
+            # Reset completion fields if going back to pending
+            self.completed_at = None
+            self.completed_by_id = None
+
+        self.status = new_status
+        db.add(self)
+        db.commit()
+
+    def mark_completed(self, db: Session, user_id: str) -> None:
+        """Mark as completed.
+
+        Args:
+            db: Database session
+            user_id: user ID who completed the task
+        """
+        self.update_status(db, StatusType.completed, user_id)
+
+    def mark_failed(self, db: Session) -> None:
+        """Mark as failed."""
+        self.update_status(db, StatusType.failed)
+
+    def start_progress(self, db: Session) -> None:
+        """Mark as in progress."""
+        self.update_status(db, StatusType.in_progress)
+
+    def reset_to_pending(self, db: Session) -> None:
+        """Reset to pending status."""
+        self.update_status(db, StatusType.pending)
+
+    @property
+    def is_completed(self) -> bool:
+        """Check if completed."""
+        return self.status == StatusType.completed
+
+    @property
+    def is_failed(self) -> bool:
+        """Check if failed."""
+        return self.status == StatusType.failed
+
+    @property
+    def is_in_progress(self) -> bool:
+        """Check if in progress."""
+        return self.status == StatusType.in_progress
+
+    @property
+    def is_pending(self) -> bool:
+        """Check if pending."""
+        return self.status == StatusType.pending

fides/api/util/cache.py

@@ -1,4 +1,5 @@
 import json
+import os
 from typing import Any, Dict, List, Optional, Union
 from urllib.parse import unquote_to_bytes
 
@@ -27,6 +28,7 @@ from fides.config import CONFIG
 RedisValue = Union[bytes, float, int, str]
 
 _connection = None
+_read_only_connection = None
 
 
 class FidesopsRedis(Redis):
@@ -157,6 +159,36 @@ class FidesopsRedis(Redis):
         return list_length
 
 
+# FIXME: Ideally we don't want our code to be aware of the way tests are run,
+# e.g that we run them in parallel with pytest-xdist. We need to find a way
+# to change the pytest_configure_node hook to set the correct environment variable
+# like we do for the readonly database. It wasn't working so we're using this workaround for now.
+def _determine_redis_db_index(
+    read_only: Optional[bool] = False,
+) -> int:  # pragma: no cover
+    """Return the Redis DB index that should be used for the current process.
+
+    Behavior:
+    1. Test mode:
+       - If running under xdist, map `gwN` → DB `N + 1` (reserve DB 0).
+       - If *not* running under xdist, always use DB 1.
+
+    2. Non-test mode: return the value already present in `CONFIG.redis.db_index`
+    """
+
+    # 1. Test mode logic
+    if CONFIG.test_mode:
+        worker_id = os.getenv("PYTEST_XDIST_WORKER")
+        if worker_id and worker_id.startswith("gw"):
+            suffix = worker_id[2:]
+            if suffix.isdigit():
+                return int(suffix) + 1  # gw0 -> 1, gw1 -> 2, etc.
+        return CONFIG.redis.test_db_index
+
+    # 2. Non-test mode
+    return CONFIG.redis.read_only_db_index if read_only else CONFIG.redis.db_index
+
+
 def get_cache(should_log: Optional[bool] = False) -> FidesopsRedis:
     """Return a singleton connection to our Redis cache"""
 
@@ -173,7 +205,7 @@ def get_cache(should_log: Optional[bool] = False) -> FidesopsRedis:
             decode_responses=CONFIG.redis.decode_responses,
             host=CONFIG.redis.host,
             port=CONFIG.redis.port,
-            db=CONFIG.redis.db_index,
+            db=_determine_redis_db_index(),
             username=CONFIG.redis.user,
             password=CONFIG.redis.password,
             ssl=CONFIG.redis.ssl,
@@ -202,6 +234,50 @@ def get_cache(should_log: Optional[bool] = False) -> FidesopsRedis:
     return _connection
 
 
+def get_read_only_cache() -> FidesopsRedis:
+    """
+    Return a singleton connection to the read-only Redis cache.
+    If read-only is not enabled, return the regular cache.
+    """
+    # If read-only is not enabled, return the regular cache
+    if not CONFIG.redis.read_only_enabled:
+        logger.debug(
+            "Read-only Redis is not enabled. Returning writeable cache connection instead."
+        )
+        return get_cache()
+
+    global _read_only_connection  # pylint: disable=W0603
+    if _read_only_connection is None:
+        logger.debug("Creating new read-only Redis connection...")
+        _read_only_connection = FidesopsRedis(  # type: ignore[call-overload]
+            charset=CONFIG.redis.charset,
+            decode_responses=CONFIG.redis.decode_responses,
+            host=CONFIG.redis.read_only_host,
+            port=CONFIG.redis.read_only_port,
+            db=_determine_redis_db_index(read_only=True),
+            username=CONFIG.redis.read_only_user,
+            password=CONFIG.redis.read_only_password,
+            ssl=CONFIG.redis.read_only_ssl,
+            ssl_ca_certs=CONFIG.redis.read_only_ssl_ca_certs,
+            ssl_cert_reqs=CONFIG.redis.read_only_ssl_cert_reqs,
+        )
+        logger.debug("New read-only Redis connection created.")
+
+    try:
+        connected = _read_only_connection.ping()
+        logger.debug("Read-only Redis connection succeeded.")
+    except ConnectionErrorFromRedis:
+        connected = False
+
+    if not connected:
+        logger.error(
+            "Unable to establish read-only Redis connection. Returning writeable cache connection instead."
+        )
+        return get_cache()
+
+    return _read_only_connection
+
+
 def get_identity_cache_key(privacy_request_id: str, identity_attribute: str) -> str:
     """Return the key at which to save this PrivacyRequest's identity for the passed in attribute"""
     # TODO: Remove this prefix

fides/config/redis_settings.py

@@ -20,6 +20,10 @@ class RedisSettings(FidesSettings):
         default=0,
         description="The application will use this index in the Redis cache to cache data.",
     )
+    test_db_index: int = Field(
+        default=1,
+        description="The application will use this index in the Redis cache to cache data for testing.",
+    )
     decode_responses: bool = Field(
         default=True,
         description="Whether or not to automatically decode the values fetched from Redis. Decodes using the `charset` configuration value.",
@@ -64,14 +68,57 @@ class RedisSettings(FidesSettings):
         default="", description="The user with which to login to the Redis cache."
     )
 
+    # Read-only Redis settings
+    read_only_enabled: bool = Field(
+        default=False,
+        description="Whether a read-only Redis cache is enabled.",
+    )
+    read_only_host: str = Field(
+        default="",
+        description="The network address for the read-only Redis cache.",
+    )
+    read_only_port: int = Field(
+        default=6379,
+        description="The port at which the read-only Redis cache will be accessible.",
+    )
+    read_only_user: str = Field(
+        default="",
+        description="The user with which to login to the read-only Redis cache.",
+    )
+    read_only_password: str = Field(
+        default="",
+        description="The password with which to login to the read-only Redis cache.",
+    )
+    read_only_db_index: int = Field(
+        default=0,
+        description="The application will use this index in the read-only Redis cache to cache data.",
+    )
+    read_only_ssl: bool = Field(
+        default=False,
+        description="Whether the application's connections to the read-only cache should be encrypted using TLS.",
+    )
+    read_only_ssl_cert_reqs: Optional[str] = Field(
+        default="required",
+        description="If using TLS encryption, set this to 'required' if you wish to enforce the read-only Redis cache to provide a certificate. Note that not all cache providers support this without setting ssl_ca_certs (e.g. AWS Elasticache).",
+    )
+    read_only_ssl_ca_certs: str = Field(
+        default="",
+        description="If using TLS encryption rooted with a custom Certificate Authority, set this to the path of the CA certificate.",
+    )
+
     # This relies on other values to get built so must be last
     connection_url: Optional[str] = Field(
         default=None,
         description="A full connection URL to the Redis cache. If not specified, this URL is automatically assembled from the host, port, password and db_index specified above.",
         exclude=True,
     )
+    read_only_connection_url: Optional[str] = Field(
+        default=None,
+        description="A full connection URL to the read-only Redis cache. If not specified, this URL is automatically assembled from the read_only_host, read_only_port, read_only_password and read_only_db_index specified above.",
+        exclude=True,
+    )
 
-    @field_validator("connection_url", mode="before")
+    @field_validator("connection_url", "read_only_connection_url", mode="before")
     @classmethod
     def assemble_connection_url(
         cls,
@@ -83,22 +130,50 @@ class RedisSettings(FidesSettings):
             # If the whole URL is provided via the config, preference that
             return v
 
+        is_read_only = info.field_name == "read_only_connection_url"
+
         connection_protocol = "redis"
         params_str = ""
-        use_tls = info.data.get("ssl", False)
+        use_tls = (
+            info.data.get("read_only_ssl")
+            if is_read_only
+            else info.data.get("ssl", False)
+        )
 
         # These vars are intentionally fetched with `or ""` as the default to account
         # for the edge case where `None` is explicitly set in `values` by Pydantic because
         # it is not overridden by the config file or an env var
-        user = info.data.get("user") or ""
-        password = info.data.get("password") or ""
-        db_index = info.data.get("db_index") or ""
+        user = (
+            info.data.get("read_only_user", "")
+            if is_read_only
+            else info.data.get("user", "")
+        )
+        password = (
+            info.data.get("read_only_password", "")
+            if is_read_only
+            else info.data.get("password", "")
+        )
+        db_index = (
+            info.data.get("read_only_db_index", "")
+            if is_read_only
+            else info.data.get("db_index", "")
+        )
         if use_tls:
             # If using TLS update the connection URL format
             connection_protocol = "rediss"
-            cert_reqs = info.data.get("ssl_cert_reqs", "none")
+            cert_reqs = (
+                info.data.get("read_only_ssl_cert_reqs", "none")
+                if is_read_only
+                else info.data.get("ssl_cert_reqs", "none")
+            )
             params = {"ssl_cert_reqs": quote_plus(cert_reqs)}
-            if ssl_ca_certs := info.data.get("ssl_ca_certs", ""):
+
+            ssl_ca_certs = (
+                info.data.get("read_only_ssl_ca_certs", "")
+                if is_read_only
+                else info.data.get("ssl_ca_certs", "")
+            )
+            if ssl_ca_certs:
                 params["ssl_ca_certs"] = quote(ssl_ca_certs, safe="/")
             params_str = "?" + urlencode(params, quote_via=quote, safe="/")
 
@@ -108,7 +183,23 @@ class RedisSettings(FidesSettings):
         if password or user:
             auth_prefix = f"{quote_plus(user)}:{quote_plus(password)}@"
 
-        connection_url = f"{connection_protocol}://{auth_prefix}{info.data.get('host', '')}:{info.data.get('port', '')}/{db_index}{params_str}"
+        host = (
+            info.data.get("read_only_host", "")
+            if is_read_only
+            else info.data.get("host", "")
+        )
+        port = (
+            info.data.get("read_only_port", "")
+            if is_read_only
+            else info.data.get("port", "")
+        )
+
+        # Only include database index in URL if it's not the default (0)
+        db_path = f"{db_index}" if db_index != 0 else ""
+
+        connection_url = (
+            f"{connection_protocol}://{auth_prefix}{host}:{port}/{db_path}{params_str}"
+        )
         return connection_url
 
     model_config = SettingsConfigDict(env_prefix=ENV_PREFIX)