ethyca-fides 2.63.3rc0__py2.py3-none-any.whl → 2.64.1b0__py2.py3-none-any.whl

fides/config/redis_settings.py

@@ -77,35 +77,99 @@ class RedisSettings(FidesSettings):
         default="",
         description="The network address for the read-only Redis cache.",
     )
-    read_only_port: int = Field(
-        default=6379,
-        description="The port at which the read-only Redis cache will be accessible.",
+    # Read-only Redis settings with automatic fallback behavior
+    # Field validators will populate these fields with the corresponding writer settings if not provided
+    # The order here is important, these must be defined after the writer settings
+    read_only_port: int = Field(  # type: ignore[assignment]
+        default=None,
+        description="The port at which the read-only Redis cache will be accessible. If not provided, the `port` setting will be used.",
     )
-    read_only_user: str = Field(
-        default="",
-        description="The user with which to login to the read-only Redis cache.",
+    read_only_user: str = Field(  # type: ignore[assignment]
+        default=None,
+        description="The user with which to login to the read-only Redis cache. If not provided, the `user` setting will be used.",
     )
-    read_only_password: str = Field(
-        default="",
-        description="The password with which to login to the read-only Redis cache.",
+    read_only_password: str = Field(  # type: ignore[assignment]
+        default=None,
+        description="The password with which to login to the read-only Redis cache. If not provided, the `password` setting will be used.",
     )
-    read_only_db_index: int = Field(
-        default=0,
-        description="The application will use this index in the read-only Redis cache to cache data.",
+    read_only_db_index: int = Field(  # type: ignore[assignment]
+        default=None,
+        description="The application will use this index in the read-only Redis cache to cache data. If not provided, the `db_index` setting will be used.",
     )
-    read_only_ssl: bool = Field(
-        default=False,
-        description="Whether the application's connections to the read-only cache should be encrypted using TLS.",
+    read_only_ssl: bool = Field(  # type: ignore[assignment]
+        default=None,
+        description="Whether the application's connections to the read-only cache should be encrypted using TLS. If not provided, the `ssl` setting will be used.",
     )
     read_only_ssl_cert_reqs: Optional[str] = Field(
-        default="required",
-        description="If using TLS encryption, set this to 'required' if you wish to enforce the read-only Redis cache to provide a certificate. Note that not all cache providers support this without setting ssl_ca_certs (e.g. AWS Elasticache).",
+        default=None,
+        description="If using TLS encryption, set this to 'required' if you wish to enforce the read-only Redis cache to provide a certificate. Note that not all cache providers support this without setting ssl_ca_certs (e.g. AWS Elasticache). If not provided, the `ssl_cert_reqs` setting will be used.",
     )
-    read_only_ssl_ca_certs: str = Field(
-        default="",
-        description="If using TLS encryption rooted with a custom Certificate Authority, set this to the path of the CA certificate.",
+    read_only_ssl_ca_certs: str = Field(  # type: ignore[assignment]
+        default=None,
+        description="If using TLS encryption rooted with a custom Certificate Authority, set this to the path of the CA certificate. If not provided, the `ssl_ca_certs` setting will be used.",
     )
 
+    # Field validators for automatic fallback behavior
+    @field_validator("read_only_port", mode="before")
+    @classmethod
+    def resolve_read_only_port(cls, v: Optional[int], info: ValidationInfo) -> int:
+        """Resolves the read-only port setting by falling back to the `port` value if no `read_only_port` is specified."""
+        if v is None:
+            return info.data["port"]
+        return v
+
+    @field_validator("read_only_user", mode="before")
+    @classmethod
+    def resolve_read_only_user(cls, v: Optional[str], info: ValidationInfo) -> str:
+        """Resolves the read-only user setting by falling back to the `user` value if no `read_only_user` is specified."""
+        if v is None:
+            return info.data["user"]
+        return v
+
+    @field_validator("read_only_password", mode="before")
+    @classmethod
+    def resolve_read_only_password(cls, v: Optional[str], info: ValidationInfo) -> str:
+        """Resolves the read-only password setting by falling back to the `password` value if no `read_only_password` is specified."""
+        if v is None:
+            return info.data["password"]
+        return v
+
+    @field_validator("read_only_db_index", mode="before")
+    @classmethod
+    def resolve_read_only_db_index(cls, v: Optional[int], info: ValidationInfo) -> int:
+        """Resolves the read-only db_index setting by falling back to the `db_index` value if no `read_only_db_index` is specified."""
+        if v is None:
+            return info.data["db_index"]
+        return v
+
+    @field_validator("read_only_ssl", mode="before")
+    @classmethod
+    def resolve_read_only_ssl(cls, v: Optional[bool], info: ValidationInfo) -> bool:
+        """Resolves the read-only ssl setting by falling back to the `ssl` value if no `read_only_ssl` is specified."""
+        if v is None:
+            return info.data["ssl"]
+        return v
+
+    @field_validator("read_only_ssl_cert_reqs", mode="before")
+    @classmethod
+    def resolve_read_only_ssl_cert_reqs(
+        cls, v: Optional[str], info: ValidationInfo
+    ) -> Optional[str]:
+        """Resolves the read-only ssl_cert_reqs setting by falling back to the `ssl_cert_reqs` value if no `read_only_ssl_cert_reqs` is specified."""
+        if v is None:
+            return info.data["ssl_cert_reqs"]
+        return v
+
+    @field_validator("read_only_ssl_ca_certs", mode="before")
+    @classmethod
+    def resolve_read_only_ssl_ca_certs(
+        cls, v: Optional[str], info: ValidationInfo
+    ) -> str:
+        """Resolves the read-only ssl_ca_certs setting by falling back to the `ssl_ca_certs` value if no `read_only_ssl_ca_certs` is specified."""
+        if v is None:
+            return info.data["ssl_ca_certs"]
+        return v
+
     # This relies on other values to get built so must be last
     connection_url: Optional[str] = Field(
         default=None,
@@ -130,49 +194,48 @@ class RedisSettings(FidesSettings):
             # If the whole URL is provided via the config, preference that
             return v
 
+        # Determine which set of settings to use based on field name
         is_read_only = info.field_name == "read_only_connection_url"
 
-        connection_protocol = "redis"
-        params_str = ""
-        use_tls = (
-            info.data.get("read_only_ssl")
-            if is_read_only
-            else info.data.get("ssl", False)
-        )
-
-        # These vars are intentionally fetched with `or ""` as the default to account
-        # for the edge case where `None` is explicitly set in `values` by Pydantic because
-        # it is not overridden by the config file or an env var
+        # Extract settings - fallbacks already resolved by field validators for read-only fields
         user = (
-            info.data.get("read_only_user", "")
-            if is_read_only
-            else info.data.get("user", "")
+            info.data["read_only_user"] if is_read_only else info.data.get("user", "")
         )
         password = (
-            info.data.get("read_only_password", "")
+            info.data["read_only_password"]
             if is_read_only
             else info.data.get("password", "")
         )
+
         db_index = (
-            info.data.get("read_only_db_index", "")
+            info.data["read_only_db_index"]
             if is_read_only
             else info.data.get("db_index", "")
         )
+        use_tls = (
+            info.data["read_only_ssl"] if is_read_only else info.data.get("ssl", False)
+        )
+
+        connection_protocol = "redis"
+        params_str = ""
+
         if use_tls:
             # If using TLS update the connection URL format
             connection_protocol = "rediss"
-            cert_reqs = (
+
+            ssl_cert_reqs = (
                 info.data.get("read_only_ssl_cert_reqs", "none")
                 if is_read_only
                 else info.data.get("ssl_cert_reqs", "none")
             )
-            params = {"ssl_cert_reqs": quote_plus(cert_reqs)}
-
             ssl_ca_certs = (
-                info.data.get("read_only_ssl_ca_certs", "")
+                info.data["read_only_ssl_ca_certs"]
                 if is_read_only
                 else info.data.get("ssl_ca_certs", "")
             )
+
+            # Build SSL parameters
+            params = {"ssl_cert_reqs": quote_plus(ssl_cert_reqs or "none")}
             if ssl_ca_certs:
                 params["ssl_ca_certs"] = quote(ssl_ca_certs, safe="/")
             params_str = "?" + urlencode(params, quote_via=quote, safe="/")
@@ -183,17 +246,15 @@ class RedisSettings(FidesSettings):
         if password or user:
             auth_prefix = f"{quote_plus(user)}:{quote_plus(password)}@"
 
+        # For host, we don't have a fallback - read replica should be a different host
         host = (
             info.data.get("read_only_host", "")
             if is_read_only
             else info.data.get("host", "")
         )
         port = (
-            info.data.get("read_only_port", "")
-            if is_read_only
-            else info.data.get("port", "")
+            info.data["read_only_port"] if is_read_only else info.data.get("port", "")
         )
-
         # Only include database index in URL if it's not the default (0)
         db_path = f"{db_index}" if db_index != 0 else ""
 

fides/service/manual_tasks/manual_task_config_service.py

@@ -0,0 +1,370 @@
+from typing import Any, Optional
+
+from loguru import logger
+from sqlalchemy.orm import Query, Session, selectinload
+
+from fides.api.models.manual_tasks.manual_task import ManualTask
+from fides.api.models.manual_tasks.manual_task_config import (
+    ManualTaskConfig,
+    ManualTaskConfigField,
+)
+from fides.api.schemas.manual_tasks.manual_task_config import (
+    ManualTaskConfigResponse,
+    ManualTaskConfigurationType,
+)
+from fides.service.manual_tasks.utils import validate_fields, with_task_logging
+
+
+class ManualTaskConfigService:
+    def __init__(self, db: Session):
+        self.db = db
+
+    def _create_log_data(
+        self, task_id: str, config_id: Optional[str], details: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Create standard log data structure."""
+        return {"task_id": task_id, "config_id": config_id, "details": details}
+
+    def _get_base_config_query(self) -> Query:
+        """Get base config query with field definitions loaded."""
+        return self.db.query(ManualTaskConfig).options(
+            selectinload(ManualTaskConfig.field_definitions)
+        )
+
+    def _create_config_version(
+        self,
+        task: ManualTask,
+        config_type: str,
+        version: int,
+        is_current: bool = True,
+    ) -> ManualTaskConfig:
+        """Create a new config version. This is a helper method that creates a new config
+        version and sets it as the current version.
+
+        Args:
+            task: The task to create the config for
+            config_type: The type of config to create
+            version: The version number of the config
+            is_current: Whether the config is the current version
+        Returns:
+            The new config
+        """
+        try:
+            ManualTaskConfigurationType(config_type)
+        except ValueError:
+            raise ValueError(f"Invalid config type: {config_type}")
+
+        # Set all existing versions to non-current
+        if is_current:
+            self.db.query(ManualTaskConfig).filter(
+                ManualTaskConfig.task_id == task.id,
+                ManualTaskConfig.config_type == config_type,
+            ).update({"is_current": False})
+            self.db.flush()
+
+        return ManualTaskConfig.create(
+            db=self.db,
+            data={
+                "task_id": task.id,
+                "config_type": config_type,
+                "version": version,
+                "is_current": is_current,
+            },
+        )
+
+    def _handle_field_updates(
+        self,
+        config: ManualTaskConfig,
+        field_updates: Optional[list[dict[str, Any]]] = None,
+        fields_to_remove: Optional[list[str]] = None,
+        previous_config: Optional[ManualTaskConfig] = None,
+    ) -> set[str]:
+        """Handle field updates, removals, and recreation from previous version. This is a helper
+        function that:
+        - Creates new fields from field_updates
+        - Recreates unmodified fields from previous version so they will remain in the config
+        - Removes fields that are no longer in the config
+
+        Args:
+            config: The config to update
+            field_updates: The fields to update
+            fields_to_remove: The fields to remove
+            previous_config: The previous config
+        Returns:
+            The modified keys
+        """
+        modified_keys = set(fields_to_remove or [])
+
+        if field_updates:
+            validate_fields(field_updates, is_submission=False)
+            fields_to_create = [
+                {
+                    "task_id": config.task_id,
+                    "config_id": config.id,
+                    "field_key": f["field_key"],
+                    "field_type": f.get("field_type"),
+                    "field_metadata": f.get("field_metadata", {}),
+                }
+                for f in field_updates
+            ]
+            if fields_to_create:
+                self.db.bulk_insert_mappings(ManualTaskConfigField, fields_to_create)
+                modified_keys.update(f["field_key"] for f in field_updates)
+
+        if previous_config:
+            # Recreate unmodified fields from previous version
+            unmodified = [
+                {
+                    "task_id": config.task_id,
+                    "config_id": config.id,
+                    "field_key": f.field_key,
+                    "field_type": f.field_type,
+                    "field_metadata": f.field_metadata,
+                }
+                for f in previous_config.field_definitions
+                if f.field_key not in modified_keys
+            ]
+            if unmodified:
+                self.db.bulk_insert_mappings(ManualTaskConfigField, unmodified)
+
+        self.db.flush()
+        return modified_keys
+
+    def to_response(self, config: ManualTaskConfig) -> ManualTaskConfigResponse:
+        """Convert config model to response object."""
+        return ManualTaskConfigResponse.model_validate(
+            {
+                "id": config.id,
+                "task_id": config.task_id,
+                "config_type": config.config_type,
+                "version": config.version,
+                "is_current": config.is_current,
+                "fields": [
+                    {
+                        "field_key": f.field_key,
+                        "field_type": f.field_type,
+                        "field_metadata": f.field_metadata,
+                    }
+                    for f in config.field_definitions
+                ],
+                "created_at": config.created_at,
+                "updated_at": config.updated_at,
+            }
+        )
+
+    def get_current_config(
+        self, task: ManualTask, config_type: str
+    ) -> ManualTaskConfig:
+        """Get current config for task by type."""
+        config = (
+            self._get_base_config_query()
+            .filter(
+                ManualTaskConfig.task_id == task.id,
+                ManualTaskConfig.config_type == config_type,
+                ManualTaskConfig.is_current.is_(True),
+            )
+            .first()
+        )
+
+        if not config:
+            raise ValueError(
+                f"No current config found for task {task.id} and type {config_type}"
+            )
+        return config
+
+    def list_config_type_versions(
+        self, task: ManualTask, config_type: str
+    ) -> list[ManualTaskConfig]:
+        """List all versions of a config type for a task."""
+        return (
+            self._get_base_config_query()
+            .filter(
+                ManualTaskConfig.task_id == task.id,
+                ManualTaskConfig.config_type == config_type,
+            )
+            .order_by(ManualTaskConfig.version.desc())
+            .all()
+        )
+
+    def get_config(
+        self,
+        task: ManualTask,
+        config_type: str,
+        field_id: str,
+        config_id: str,
+        field_key: str,
+        version: int,
+    ) -> Optional[ManualTaskConfig]:
+        """Get config by various filters. This is a flexible lookup method that can find configs based on various filters.
+
+        Args:
+            task: The task to get the config for
+            config_type: The type of config to get
+            field_id: The ID of a field in the config
+            config_id: The ID of the config
+            field_key: The key of a field in the config
+            version: The version number of the config
+        Returns:
+            The matching config if found, None otherwise
+        """
+        if not any([task, config_id, field_id, config_type]):
+            logger.debug("No filters provided to get_config")
+            return None
+
+        query = self._get_base_config_query()
+        if field_id or field_key:
+            query = query.join(ManualTaskConfig.field_definitions)
+
+        filters = [
+            f
+            for f in [
+                task and ManualTaskConfig.task_id == task.id,
+                config_id and ManualTaskConfig.id == config_id,
+                field_id and ManualTaskConfigField.id == field_id,
+                field_key and ManualTaskConfigField.field_key == field_key,
+                version and ManualTaskConfig.version == version,
+                config_type and ManualTaskConfig.config_type == config_type,
+            ]
+            if f
+        ]
+
+        return query.filter(*filters).first()
+
+    @with_task_logging("Creating new configuration version")
+    def create_new_version(
+        self,
+        task: ManualTask,
+        config_type: str,
+        field_updates: Optional[list[dict[str, Any]]] = None,
+        fields_to_remove: Optional[list[str]] = None,
+        previous_config: Optional[ManualTaskConfig] = None,
+    ) -> tuple[ManualTaskConfig, dict[str, Any]]:
+        """Create new version of configuration.
+
+        Args:
+            task: The task to create the config for
+            config_type: The type of config to create
+            field_updates: The fields to update
+            fields_to_remove: The fields to remove
+            previous_config: The previous config
+
+        Returns:
+            Tuple containing the new config and log data, the log data is
+            captured by the with_task_logging decorator. and the new config is
+            returned to the caller.
+        """
+        new_config = self._create_config_version(
+            task, config_type, (previous_config.version + 1 if previous_config else 1)
+        )
+
+        self._handle_field_updates(
+            new_config, field_updates, fields_to_remove, previous_config
+        )
+
+        return new_config, self._create_log_data(
+            task.id,
+            new_config.id,
+            {
+                "config_type": config_type,
+                "version": new_config.version,
+                "added_field_keys": [f.get("field_key") for f in (field_updates or [])],
+                "removed_field_keys": fields_to_remove or [],
+            },
+        )
+
+    @with_task_logging("Adding fields to configuration")
+    def add_fields(
+        self, task: ManualTask, config_type: str, fields: list[dict[str, Any]]
+    ) -> tuple[ManualTaskConfig, dict[str, Any]]:
+        """Add fields to configuration.
+
+        Args:
+            task: The task to add the fields to
+            config_type: The type of config to add the fields to
+            fields: The fields to add
+
+        Returns:
+            Tuple containing the new config and log data, the log data is
+            captured by the with_task_logging decorator. and the new config is
+            returned to the caller.
+        """
+        current = self.get_current_config(task, config_type)
+        self.create_new_version(task, config_type, fields, previous_config=current)
+        new_config = self.get_current_config(task, config_type)
+
+        return new_config, self._create_log_data(
+            task.id,
+            current.id,
+            {
+                "config_type": config_type,
+                "previous_version": current.version,
+                "added_field_keys": [f.get("field_key") for f in fields],
+                "new_config_id": new_config.id,
+                "new_config_version": new_config.version,
+            },
+        )
+
+    @with_task_logging("Removing fields from configuration")
+    def remove_fields(
+        self, task: ManualTask, config_type: str, field_keys: list[str]
+    ) -> tuple[ManualTaskConfig, dict[str, Any]]:
+        """Remove fields from configuration.
+
+        Args:
+            task: The task to remove the fields from
+            config_type: The type of config to remove the fields from
+            field_keys: The keys of the fields to remove
+
+        Returns:
+            Tuple containing the new config and log data, the log data is
+            captured by the with_task_logging decorator. and the new config is
+            returned to the caller.
+        """
+        current = self.get_current_config(task, config_type)
+        self.create_new_version(
+            task, config_type, fields_to_remove=field_keys, previous_config=current
+        )
+        new_config = self.get_current_config(task, config_type)
+
+        return new_config, self._create_log_data(
+            task.id,
+            current.id,
+            {
+                "config_type": config_type,
+                "version": current.version,
+                "deleted_field_keys": field_keys,
+                "new_config_id": new_config.id,
+                "new_config_version": new_config.version,
+            },
+        )
+
+    @with_task_logging("Deleting Manual Task configuration")
+    def delete_config(
+        self, task: ManualTask, config_id: str
+    ) -> tuple[ManualTaskConfig, dict[str, Any]]:
+        """Delete config for task.
+
+        Args:
+            task: The task to delete the config for
+            config_id: The ID of the config to delete
+
+        Returns:
+            Tuple containing the deleted config and log data, the log data is
+            captured by the with_task_logging decorator. and the deleted config is
+            returned to the caller.
+        """
+        config = self.db.query(ManualTaskConfig).filter_by(id=config_id).first()
+        if not config:
+            raise ValueError(f"Config with ID {config_id} not found")
+
+        log_data = self._create_log_data(
+            task.id,
+            None,
+            {
+                "config_type": config.config_type,
+                "version": config.version,
+                "deleted_config_id": config_id,
+            },
+        )
+        config.delete(self.db)
+        return config, log_data