ethyca-fides 2.58.2rc0__py2.py3-none-any.whl → 2.58.3b1__py2.py3-none-any.whl

fides/api/util/collection_util.py

@@ -122,6 +122,7 @@ def extract_key_for_address(
     return f"{dataset}:{collection}"
 
 
+# pylint: disable=too-many-branches
 def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str, Any]:
     """
     Converts a dictionary of paths/values into a nested dictionary
@@ -149,17 +150,29 @@ def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str,
         for i, current_key in enumerate(keys[:-1]):
             next_key = keys[i + 1]
             if next_key.isdigit():
-                target = target.setdefault(current_key, [])
+                if isinstance(target, dict):  # Only call setdefault on dictionaries
+                    target = target.setdefault(current_key, [])
+                elif isinstance(
+                    target, list
+                ):  # If target is a list, handle differently
+                    idx = int(current_key)
+                    while len(target) <= idx:
+                        target.append([])  # Add a list since next_key is a digit
+                    target = target[idx]
             else:
                 if isinstance(target, dict):
                     target = target.setdefault(current_key, {})
                 elif isinstance(target, list):
-                    while len(target) <= int(current_key):
+                    idx = int(current_key)
+                    while len(target) <= idx:
                         target.append({})
-                    target = target[int(current_key)]
+                    target = target[idx]
         try:
             if isinstance(target, list):
-                target.append(value)
+                idx = int(keys[-1]) if keys[-1].isdigit() else len(target)
+                while len(target) <= idx:
+                    target.append(None)
+                target[idx] = value
             else:
                 # If the value is a dictionary, add its components to the queue for processing
                 if isinstance(value, dict):
@@ -176,10 +189,12 @@ def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str,
     return output
 
 
+# pylint: disable=too-many-branches
 def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str, Any]:
     """
     Recursively flatten a dictionary or list into a flat dictionary with dot-notation keys.
     Handles nested dictionaries and arrays with proper indices.
+    Preserves empty lists and dictionaries.
 
     example:
 
@@ -191,7 +206,9 @@ def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str,
         "D": [
             {"E": "3"},
             {"E": "4"}
-        ]
+        ],
+        "E": [],
+        "F": {}
     }
 
     becomes
@@ -200,7 +217,9 @@ def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str,
         "A.B": "1",
         "A.C": "2",
         "D.0.E": "3",
-        "D.1.E": "4"
+        "D.1.E": "4",
+        "E": [],
+        "F": {}
     }
 
     Args:
@@ -211,20 +230,40 @@ def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str,
     Returns:
         A flattened dictionary with dot-notation keys
     """
-    items = {}
+    items: Dict[str, Any] = {}
 
     if isinstance(data, dict):
+        # Handle top-level empty dictionary case
+        if not data and not prefix:
+            return {}
+
+        # If the dictionary is empty but has a prefix, store it as is
+        if not data:
+            items[prefix] = {}
+            return items
+
         for k, v in data.items():
             new_key = f"{prefix}{separator}{k}" if prefix else k
             if isinstance(v, (dict, list)):
-                items.update(flatten_dict(v, new_key, separator))
+                if not v:  # Handle empty dict or list
+                    items[new_key] = v
+                else:
+                    items.update(flatten_dict(v, new_key, separator))
             else:
                 items[new_key] = v
     elif isinstance(data, list):
+        # If the list is empty, store it as is
+        if not data:
+            items[prefix] = []
+            return items
+
         for i, v in enumerate(data):
             new_key = f"{prefix}{separator}{i}"
             if isinstance(v, (dict, list)):
-                items.update(flatten_dict(v, new_key, separator))
+                if not v:  # Handle empty dict or list
+                    items[new_key] = v
+                else:
+                    items.update(flatten_dict(v, new_key, separator))
             else:
                 items[new_key] = v
     else:

fides/cli/commands/pull.py

@@ -5,9 +5,11 @@ from click_default_group import DefaultGroup
 
 from fides.cli.options import fides_key_argument, manifests_dir_argument
 from fides.cli.utils import with_analytics, with_server_health_check
-from fides.common.utils import echo_red
+from fides.common.utils import echo_green, echo_red
 from fides.core import parse as _parse
 from fides.core import pull as _pull
+from fides.core.api_helpers import list_server_resources
+from fides.core.pull import remove_nulls, write_manifest_file
 from fides.core.utils import git_is_dirty
 
 
@@ -60,26 +62,88 @@ def pull_all(
 
 @pull.command(name="dataset")  # type: ignore
 @click.pass_context
-@fides_key_argument
+@click.argument("fides_key", required=False)
 @manifests_dir_argument
-def dataset(
+@click.option(
+    "--all-resources",
+    "-a",
+    is_flag=True,
+    default=False,
+    help="Pull all datasets from the server.",
+)
+@click.option(
+    "--separate-files",
+    is_flag=True,
+    default=False,
+    help="Write each dataset to a separate file named after its fides_key.",
+)
+@with_analytics
+@with_server_health_check
+def pull_dataset(
     ctx: click.Context,
-    fides_key: str,
+    fides_key: Optional[str],
     manifests_dir: str,
+    all_resources: bool,
+    separate_files: bool,
 ) -> None:
     """
-    Retrieve a specific dataset from the server and update the local manifest files.
+    Retrieve datasets from the server and update the local manifest files.
+
+    If FIDES_KEY is provided, only that dataset will be pulled.
+    If --all-resources is specified, all datasets will be pulled.
+    If --separate-files is specified, each dataset will be written to a separate file.
     """
+    if not fides_key and not all_resources:
+        echo_red("Error: Either FIDES_KEY or --all-resources must be specified.")
+        raise SystemExit(1)
 
     config = ctx.obj["CONFIG"]
-    _pull.pull(
-        url=config.cli.server_url,
-        manifests_dir=manifests_dir,
-        headers=config.user.auth_header,
-        fides_key=fides_key,
-        resource_type="dataset",
-        all_resources_file=None,
-    )
+
+    # Check for unstaged git changes before proceeding
+    if git_is_dirty(manifests_dir):
+        echo_red(
+            f"There are unstaged changes in your manifest directory: '{manifests_dir}' \nAborting pull!"
+        )
+        raise SystemExit(1)
+
+    if fides_key:
+        _pull.pull(
+            url=config.cli.server_url,
+            manifests_dir=manifests_dir,
+            headers=config.user.auth_header,
+            fides_key=fides_key,
+            resource_type="dataset",
+            all_resources_file=None,
+        )
+    elif all_resources:
+        # Get all available datasets from server
+        datasets = list_server_resources(
+            url=config.cli.server_url,
+            headers=config.user.auth_header,
+            resource_type="dataset",
+            exclude_keys=[],
+        )
+
+        if not datasets:
+            echo_red("No datasets found on the server.")
+            return
+
+        # Remove null values
+        datasets = [remove_nulls(dataset) for dataset in datasets]
+
+        if separate_files:
+            # Write each dataset to a separate file
+            for dataset in datasets:
+                if "fides_key" in dataset:
+                    fides_key = dataset["fides_key"]
+                    manifest_path = f"{manifests_dir.rstrip('/')}/{fides_key}.yml"
+                    write_manifest_file(manifest_path, {"dataset": [dataset]})
+        else:
+            # Write all datasets to a single file
+            all_datasets_file = f"{manifests_dir.rstrip('/')}/datasets.yml"
+            write_manifest_file(all_datasets_file, {"dataset": datasets})
+
+        echo_green("Pull complete.")
 
 
 @pull.command(name="system")  # type: ignore

fides/config/database_settings.py

@@ -65,6 +65,17 @@ class DatabaseSettings(FidesSettings):
         default="default-db",
         description="The hostname of the application database server.",
     )
+    readonly_server: Optional[str] = Field(
+        default=None,
+        description="The hostname of the application read database server.",
+    )
+
+    # TODO (LJ-663): add optional readonly_user
+    # TODO (LJ-663): add optional readonly_password
+    # TODO (LJ-663): add optional readonly_port
+    # TODO (LJ-663): add optional readonly_params
+    # TODO (LJ-663): add optional readonly_db
+
     task_engine_pool_size: int = Field(
         default=50,
         description="Number of concurrent database connections Fides will use for executing privacy request tasks, either locally or on each worker. Note that the pool begins with no connections, but as they are requested the connections are maintained and reused up to this limit.",
@@ -105,6 +116,11 @@ class DatabaseSettings(FidesSettings):
         description="Programmatically created connection string for the application database.",
         exclude=True,
     )
+    sqlalchemy_readonly_database_uri: Optional[str] = Field(
+        default=None,
+        description="Programmatically created connection string for the read-only application database.",
+        exclude=True,
+    )
     sqlalchemy_test_database_uri: str = Field(
         default="",
         description="Programmatically created connection string for the test database.",
@@ -218,6 +234,36 @@ class DatabaseSettings(FidesSettings):
             )
         )
 
+    @field_validator("sqlalchemy_readonly_database_uri", mode="before")
+    @classmethod
+    def assemble_readonly_db_connection(
+        cls, v: Optional[str], info: ValidationInfo
+    ) -> Optional[str]:
+        """Join DB connection credentials into a synchronous connection string."""
+        if isinstance(v, str) and v:
+            return v
+        if not info.data.get("readonly_server"):
+            return None
+        port: int = port_integer_converter(info)
+        return str(
+            # TODO: support optional readonly params for user, password, etc.
+            PostgresDsn.build(  # pylint: disable=no-member
+                scheme="postgresql",
+                username=info.data.get("user"),
+                password=info.data.get("password"),
+                host=info.data.get("readonly_server"),
+                port=port,
+                path=f"{info.data.get('db') or ''}",
+                query=(
+                    urlencode(
+                        cast(Dict, info.data.get("params")), quote_via=quote, safe="/"
+                    )
+                    if info.data.get("params")
+                    else None
+                ),
+            )
+        )
+
     @field_validator("sqlalchemy_test_database_uri", mode="before")
     @classmethod
     def assemble_test_db_connection(cls, v: Optional[str], info: ValidationInfo) -> str:

fides/core/api.py

@@ -4,7 +4,8 @@ from typing import Dict, List, Optional, Union
 
 import requests
 
-from fides.api.util.endpoint_utils import API_PREFIX
+# Not using the constant value from fides.api.util.endpoint_utils to reduce the startup time for the CLI
+API_PREFIX = "/api/v1"
 
 
 def generate_resource_url(

fides/core/pull.py

@@ -1,6 +1,6 @@
 """This module handles the logic for syncing remote resource versions into their local file."""
 
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 import yaml
 from fideslang import model_list
@@ -13,6 +13,30 @@ from fides.core.utils import get_manifest_list
 MODEL_LIST = model_list
 
 
+def remove_nulls(obj: Any) -> Any:
+    """
+    Recursively remove all null values from dictionaries and lists.
+
+    Args:
+        obj: Any Python object (dict, list, etc.)
+
+    Returns:
+        The transformed data structure with null values removed
+    """
+    if isinstance(obj, list):
+        # For lists: process each item and filter out None values
+        return [remove_nulls(item) for item in obj if item is not None]
+
+    if isinstance(obj, dict):
+        # For dictionaries: process each key-value pair and filter out pairs with None values
+        return {
+            key: remove_nulls(value) for key, value in obj.items() if value is not None
+        }
+
+    # Return all other data types unchanged
+    return obj
+
+
 def write_manifest_file(manifest_path: str, manifest: Dict) -> None:
     """
     Write a manifest file out.
@@ -54,6 +78,8 @@ def pull_existing_resources(
                 )
 
                 if server_resource:
+                    # Remove null values from the server resource
+                    server_resource = remove_nulls(server_resource)
                     updated_resource_list.append(server_resource)
                     print(
                         f" - {resource_type.capitalize()} with fides_key: {fides_key} is being updated from the server..."
@@ -77,16 +103,17 @@ def pull_resource_by_key(
 ) -> None:
     """
     Pull a resource from the server by its fides_key and update the local manifest file if it exists,
-    otherwise a new manifest file at {manifests_dir}/{resource_type}.yaml
+    otherwise a new manifest file at {manifests_dir}/{resource_type}.yml
     """
     if manifests_dir[-1] == "/":
         manifests_dir = manifests_dir[:-1]
-    manifest_path = f"{manifests_dir}/{resource_type}.yaml"
-    print(f"Pulling {resource_type} with fides_key: {fides_key}...", end=" ")
+    manifest_path = f"{manifests_dir}/{fides_key}.yml"
+    print(f"Pulling {resource_type} with fides_key: {fides_key}...", end="\n")
     server_resource = get_server_resource(url, resource_type, fides_key, headers)
-    print("done.")
 
     if server_resource:
+        # Remove null values from the server resource
+        server_resource = remove_nulls(server_resource)
         try:
             manifest = load_yaml_into_dict(manifest_path)
         except FileNotFoundError:
@@ -94,10 +121,8 @@ def pull_resource_by_key(
                 f"Manifest file {manifest_path} does not already exist and will be created"
             )
             manifest = {}
-        print("Writing out resource to file...", end=" ")
         manifest[resource_type] = [server_resource]
         write_manifest_file(manifest_path, manifest)
-        print("done.")
 
     else:
         echo_red(
@@ -127,6 +152,12 @@ def pull_missing_resources(
         for resource in MODEL_LIST
     }
 
+    # Remove null values from all resources
+    for resource_type, resources in resource_manifest.items():
+        resource_manifest[resource_type] = [
+            remove_nulls(resource) for resource in (resources or [])
+        ]
+
     resource_manifest = {
         key: value for key, value in resource_manifest.items() if value
     }

fides/service/error_handling/error_handler.py

@@ -0,0 +1,202 @@
+from functools import wraps
+from typing import Any, Callable, Optional, TypeVar
+
+from fastapi import HTTPException
+from loguru import logger
+from starlette.status import HTTP_400_BAD_REQUEST, HTTP_422_UNPROCESSABLE_ENTITY
+
+T = TypeVar("T")
+
+
+class ErrorHandler:
+    """Utility class for handling errors consistently throughout the application.
+
+    Usage Examples:
+    -----------------------------------------------------------------------------
+
+    1. Basic Validation:
+    ```python
+    from fastapi import FastAPI
+    from fides.service.error_handling.error_handler import ErrorHandler
+
+    app = FastAPI()
+
+    @app.post("/users")
+    def create_user(age: int):
+        # Simple validation
+        ErrorHandler.validate(age >= 18, "User must be 18 or older")
+        return {"message": "User created"}
+
+    @app.get("/items/{item_id}")
+    def get_item(item_id: str):
+        # Direct error raising
+        if not item_id:
+            ErrorHandler.raise_error("Item ID is required", status_code=400)
+        return {"item_id": item_id}
+    ```
+
+    2. Exception Handling Decorator:
+    ```python
+    @app.post("/orders")
+    @ErrorHandler.handle_exceptions("Failed to create order")
+    def create_order(order_data: dict):
+        if not order_data.get("items"):
+            raise ValueError("Order must contain items")
+        # Process order...
+        return {"message": "Order created"}
+
+    @app.get("/products/{product_id}")
+    @ErrorHandler.handle_exceptions("Failed to fetch product", status_code=404)
+    def get_product(product_id: str):
+        product = database.get_product(product_id)
+        if not product:
+            raise ValueError("Product not found")
+        return product
+    ```
+
+    3. Complex Validation:
+    ```python
+    @app.post("/payments")
+    @ErrorHandler.handle_exceptions("Payment processing failed")
+    def process_payment(payment: dict):
+        # Validate amount
+        ErrorHandler.validate(
+            payment.get("amount", 0) > 0,
+            "Payment amount must be positive",
+            HTTP_400_BAD_REQUEST
+        )
+
+        # Validate currency
+        ErrorHandler.validate(
+            payment.get("currency") in ["USD", "EUR"],
+            "Invalid currency",
+            HTTP_400_BAD_REQUEST,
+            "Unsupported currency provided"  # Optional log message
+        )
+
+        # Custom error for insufficient funds
+        if payment.get("amount", 0) > get_balance():
+            ErrorHandler.raise_error(
+                "Insufficient funds",
+                status_code=HTTP_400_BAD_REQUEST,
+                log_message="User attempted payment exceeding balance"
+            )
+
+        return {"status": "payment processed"}
+    ```
+
+    4. Error Logging:
+    ```python
+    @app.post("/imports")
+    @ErrorHandler.handle_exceptions("Import failed")
+    def import_data(data: dict):
+        try:
+            process_import(data)
+        except Exception as e:
+            # Log error with custom message before raising
+            ErrorHandler.raise_error(
+                "Import failed: invalid format",
+                status_code=400,
+                log_message=f"Import failed with error: {str(e)}"
+            )
+        return {"status": "import complete"}
+    ```
+
+    Key Features:
+    - Simple validation with custom error messages
+    - Consistent error handling across endpoints
+    - Built-in error logging
+    - HTTP status code customization
+    - Exception handling decorator for common patterns
+    """
+
+    @staticmethod
+    def raise_error(
+        detail: str,
+        status_code: int = HTTP_422_UNPROCESSABLE_ENTITY,
+        log_message: Optional[str] = None,
+    ) -> None:
+        """Raise an HTTPException with consistent logging.
+
+        Args:
+            detail: Error message to include in the HTTPException
+            status_code: HTTP status code to use (default: 422)
+            log_message: Optional message to log before raising the exception
+
+        Raises:
+            HTTPException: Always raised with the provided details
+        """
+        if log_message:
+            logger.error(log_message)
+        raise HTTPException(status_code=status_code, detail=detail)
+
+    @staticmethod
+    def validate(
+        condition: bool,
+        detail: str,
+        status_code: int = HTTP_400_BAD_REQUEST,
+        log_message: Optional[str] = None,
+    ) -> None:
+        """Validate a condition and raise an error if it's False.
+
+        Args:
+            condition: The condition to check
+            detail: Error message if condition is False
+            status_code: HTTP status code to use if condition is False
+            log_message: Optional message to log before raising the exception
+
+        Raises:
+            HTTPException: If the condition is False
+        """
+        if not condition:
+            ErrorHandler.raise_error(detail, status_code, log_message)
+
+    @classmethod
+    def handle_exceptions(
+        cls, error_message: str, status_code: int = HTTP_422_UNPROCESSABLE_ENTITY
+    ) -> Callable[[Callable[..., T]], Callable[..., T]]:
+        """Decorator to handle exceptions consistently.
+
+        Args:
+            error_message: Base error message to use
+            status_code: HTTP status code to use for unexpected errors
+
+        Returns:
+            Callable: The decorated function that handles exceptions
+
+        Note:
+            This decorator will catch specific exceptions and convert them to HTTPExceptions.
+            HTTPExceptions are re-raised as is, while other exceptions are wrapped with
+            additional context.
+        """
+
+        def decorator(func: Callable[..., T]) -> Callable[..., T]:
+            @wraps(func)
+            def wrapper(*args: Any, **kwargs: Any) -> Optional[T]:
+                try:
+                    return func(*args, **kwargs)
+                except HTTPException:
+                    # Re-raise HTTP exceptions without modification
+                    raise
+                except (ValueError, TypeError, AttributeError, KeyError) as e:
+                    # Handle common validation and data access errors
+                    cls.raise_error(
+                        f"{error_message}: {str(e)}",
+                        status_code,
+                        f"{error_message}: {e}",
+                    )
+                except Exception as e:  # pylint: disable=broad-except
+                    # Log unexpected errors with full context but present a sanitized message
+                    logger.error(
+                        f"Unexpected error in {func.__name__}: {str(e)}", exc_info=True
+                    )
+                    cls.raise_error(
+                        f"{error_message}: An unexpected error occurred",
+                        status_code,
+                        f"{error_message}: Unexpected {type(e).__name__}",
+                    )
+                return None  # This line is never reached but satisfies the return type checker
+
+            return wrapper  # type: ignore[return-value]
+
+        return decorator