ethyca-fides 2.58.2rc0__py2.py3-none-any.whl → 2.58.3b0__py2.py3-none-any.whl

fides/api/util/collection_util.py

@@ -122,6 +122,7 @@ def extract_key_for_address(
     return f"{dataset}:{collection}"
 
 
+# pylint: disable=too-many-branches
 def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str, Any]:
     """
     Converts a dictionary of paths/values into a nested dictionary
@@ -149,17 +150,29 @@ def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str,
         for i, current_key in enumerate(keys[:-1]):
             next_key = keys[i + 1]
             if next_key.isdigit():
-                target = target.setdefault(current_key, [])
+                if isinstance(target, dict):  # Only call setdefault on dictionaries
+                    target = target.setdefault(current_key, [])
+                elif isinstance(
+                    target, list
+                ):  # If target is a list, handle differently
+                    idx = int(current_key)
+                    while len(target) <= idx:
+                        target.append([])  # Add a list since next_key is a digit
+                    target = target[idx]
             else:
                 if isinstance(target, dict):
                     target = target.setdefault(current_key, {})
                 elif isinstance(target, list):
-                    while len(target) <= int(current_key):
+                    idx = int(current_key)
+                    while len(target) <= idx:
                         target.append({})
-                    target = target[int(current_key)]
+                    target = target[idx]
         try:
             if isinstance(target, list):
-                target.append(value)
+                idx = int(keys[-1]) if keys[-1].isdigit() else len(target)
+                while len(target) <= idx:
+                    target.append(None)
+                target[idx] = value
             else:
                 # If the value is a dictionary, add its components to the queue for processing
                 if isinstance(value, dict):
@@ -176,10 +189,12 @@ def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str,
     return output
 
 
+# pylint: disable=too-many-branches
 def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str, Any]:
     """
     Recursively flatten a dictionary or list into a flat dictionary with dot-notation keys.
     Handles nested dictionaries and arrays with proper indices.
+    Preserves empty lists and dictionaries.
 
     example:
 
@@ -191,7 +206,9 @@ def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str,
         "D": [
             {"E": "3"},
             {"E": "4"}
-        ]
+        ],
+        "E": [],
+        "F": {}
     }
 
     becomes
@@ -200,7 +217,9 @@ def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str,
         "A.B": "1",
         "A.C": "2",
         "D.0.E": "3",
-        "D.1.E": "4"
+        "D.1.E": "4",
+        "E": [],
+        "F": {}
     }
 
     Args:
@@ -211,20 +230,40 @@ def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str,
     Returns:
         A flattened dictionary with dot-notation keys
     """
-    items = {}
+    items: Dict[str, Any] = {}
 
     if isinstance(data, dict):
+        # Handle top-level empty dictionary case
+        if not data and not prefix:
+            return {}
+
+        # If the dictionary is empty but has a prefix, store it as is
+        if not data:
+            items[prefix] = {}
+            return items
+
         for k, v in data.items():
             new_key = f"{prefix}{separator}{k}" if prefix else k
             if isinstance(v, (dict, list)):
-                items.update(flatten_dict(v, new_key, separator))
+                if not v:  # Handle empty dict or list
+                    items[new_key] = v
+                else:
+                    items.update(flatten_dict(v, new_key, separator))
             else:
                 items[new_key] = v
     elif isinstance(data, list):
+        # If the list is empty, store it as is
+        if not data:
+            items[prefix] = []
+            return items
+
         for i, v in enumerate(data):
             new_key = f"{prefix}{separator}{i}"
             if isinstance(v, (dict, list)):
-                items.update(flatten_dict(v, new_key, separator))
+                if not v:  # Handle empty dict or list
+                    items[new_key] = v
+                else:
+                    items.update(flatten_dict(v, new_key, separator))
             else:
                 items[new_key] = v
     else:

fides/cli/commands/pull.py

@@ -5,9 +5,11 @@ from click_default_group import DefaultGroup
 
 from fides.cli.options import fides_key_argument, manifests_dir_argument
 from fides.cli.utils import with_analytics, with_server_health_check
-from fides.common.utils import echo_red
+from fides.common.utils import echo_green, echo_red
 from fides.core import parse as _parse
 from fides.core import pull as _pull
+from fides.core.api_helpers import list_server_resources
+from fides.core.pull import remove_nulls, write_manifest_file
 from fides.core.utils import git_is_dirty
 
 
@@ -60,26 +62,88 @@ def pull_all(
 
 @pull.command(name="dataset")  # type: ignore
 @click.pass_context
-@fides_key_argument
+@click.argument("fides_key", required=False)
 @manifests_dir_argument
-def dataset(
+@click.option(
+    "--all-resources",
+    "-a",
+    is_flag=True,
+    default=False,
+    help="Pull all datasets from the server.",
+)
+@click.option(
+    "--separate-files",
+    is_flag=True,
+    default=False,
+    help="Write each dataset to a separate file named after its fides_key.",
+)
+@with_analytics
+@with_server_health_check
+def pull_dataset(
     ctx: click.Context,
-    fides_key: str,
+    fides_key: Optional[str],
     manifests_dir: str,
+    all_resources: bool,
+    separate_files: bool,
 ) -> None:
     """
-    Retrieve a specific dataset from the server and update the local manifest files.
+    Retrieve datasets from the server and update the local manifest files.
+
+    If FIDES_KEY is provided, only that dataset will be pulled.
+    If --all-resources is specified, all datasets will be pulled.
+    If --separate-files is specified, each dataset will be written to a separate file.
     """
+    if not fides_key and not all_resources:
+        echo_red("Error: Either FIDES_KEY or --all-resources must be specified.")
+        raise SystemExit(1)
 
     config = ctx.obj["CONFIG"]
-    _pull.pull(
-        url=config.cli.server_url,
-        manifests_dir=manifests_dir,
-        headers=config.user.auth_header,
-        fides_key=fides_key,
-        resource_type="dataset",
-        all_resources_file=None,
-    )
+
+    # Check for unstaged git changes before proceeding
+    if git_is_dirty(manifests_dir):
+        echo_red(
+            f"There are unstaged changes in your manifest directory: '{manifests_dir}' \nAborting pull!"
+        )
+        raise SystemExit(1)
+
+    if fides_key:
+        _pull.pull(
+            url=config.cli.server_url,
+            manifests_dir=manifests_dir,
+            headers=config.user.auth_header,
+            fides_key=fides_key,
+            resource_type="dataset",
+            all_resources_file=None,
+        )
+    elif all_resources:
+        # Get all available datasets from server
+        datasets = list_server_resources(
+            url=config.cli.server_url,
+            headers=config.user.auth_header,
+            resource_type="dataset",
+            exclude_keys=[],
+        )
+
+        if not datasets:
+            echo_red("No datasets found on the server.")
+            return
+
+        # Remove null values
+        datasets = [remove_nulls(dataset) for dataset in datasets]
+
+        if separate_files:
+            # Write each dataset to a separate file
+            for dataset in datasets:
+                if "fides_key" in dataset:
+                    fides_key = dataset["fides_key"]
+                    manifest_path = f"{manifests_dir.rstrip('/')}/{fides_key}.yml"
+                    write_manifest_file(manifest_path, {"dataset": [dataset]})
+        else:
+            # Write all datasets to a single file
+            all_datasets_file = f"{manifests_dir.rstrip('/')}/datasets.yml"
+            write_manifest_file(all_datasets_file, {"dataset": datasets})
+
+        echo_green("Pull complete.")
 
 
 @pull.command(name="system")  # type: ignore

fides/core/api.py

@@ -4,7 +4,8 @@ from typing import Dict, List, Optional, Union
 
 import requests
 
-from fides.api.util.endpoint_utils import API_PREFIX
+# Not using the constant value from fides.api.util.endpoint_utils to reduce the startup time for the CLI
+API_PREFIX = "/api/v1"
 
 
 def generate_resource_url(

fides/core/pull.py

@@ -1,6 +1,6 @@
 """This module handles the logic for syncing remote resource versions into their local file."""
 
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 import yaml
 from fideslang import model_list
@@ -13,6 +13,30 @@ from fides.core.utils import get_manifest_list
 MODEL_LIST = model_list
 
 
+def remove_nulls(obj: Any) -> Any:
+    """
+    Recursively remove all null values from dictionaries and lists.
+
+    Args:
+        obj: Any Python object (dict, list, etc.)
+
+    Returns:
+        The transformed data structure with null values removed
+    """
+    if isinstance(obj, list):
+        # For lists: process each item and filter out None values
+        return [remove_nulls(item) for item in obj if item is not None]
+
+    if isinstance(obj, dict):
+        # For dictionaries: process each key-value pair and filter out pairs with None values
+        return {
+            key: remove_nulls(value) for key, value in obj.items() if value is not None
+        }
+
+    # Return all other data types unchanged
+    return obj
+
+
 def write_manifest_file(manifest_path: str, manifest: Dict) -> None:
     """
     Write a manifest file out.
@@ -54,6 +78,8 @@ def pull_existing_resources(
                 )
 
                 if server_resource:
+                    # Remove null values from the server resource
+                    server_resource = remove_nulls(server_resource)
                     updated_resource_list.append(server_resource)
                     print(
                         f" - {resource_type.capitalize()} with fides_key: {fides_key} is being updated from the server..."
@@ -77,16 +103,17 @@ def pull_resource_by_key(
 ) -> None:
     """
     Pull a resource from the server by its fides_key and update the local manifest file if it exists,
-    otherwise a new manifest file at {manifests_dir}/{resource_type}.yaml
+    otherwise a new manifest file at {manifests_dir}/{resource_type}.yml
     """
     if manifests_dir[-1] == "/":
         manifests_dir = manifests_dir[:-1]
-    manifest_path = f"{manifests_dir}/{resource_type}.yaml"
-    print(f"Pulling {resource_type} with fides_key: {fides_key}...", end=" ")
+    manifest_path = f"{manifests_dir}/{fides_key}.yml"
+    print(f"Pulling {resource_type} with fides_key: {fides_key}...", end="\n")
     server_resource = get_server_resource(url, resource_type, fides_key, headers)
-    print("done.")
 
     if server_resource:
+        # Remove null values from the server resource
+        server_resource = remove_nulls(server_resource)
         try:
             manifest = load_yaml_into_dict(manifest_path)
         except FileNotFoundError:
@@ -94,10 +121,8 @@ def pull_resource_by_key(
                 f"Manifest file {manifest_path} does not already exist and will be created"
             )
             manifest = {}
-        print("Writing out resource to file...", end=" ")
         manifest[resource_type] = [server_resource]
         write_manifest_file(manifest_path, manifest)
-        print("done.")
 
     else:
         echo_red(
@@ -127,6 +152,12 @@ def pull_missing_resources(
         for resource in MODEL_LIST
     }
 
+    # Remove null values from all resources
+    for resource_type, resources in resource_manifest.items():
+        resource_manifest[resource_type] = [
+            remove_nulls(resource) for resource in (resources or [])
+        ]
+
     resource_manifest = {
         key: value for key, value in resource_manifest.items() if value
     }

fides/service/error_handling/error_handler.py

@@ -0,0 +1,202 @@
+from functools import wraps
+from typing import Any, Callable, Optional, TypeVar
+
+from fastapi import HTTPException
+from loguru import logger
+from starlette.status import HTTP_400_BAD_REQUEST, HTTP_422_UNPROCESSABLE_ENTITY
+
+T = TypeVar("T")
+
+
+class ErrorHandler:
+    """Utility class for handling errors consistently throughout the application.
+
+    Usage Examples:
+    -----------------------------------------------------------------------------
+
+    1. Basic Validation:
+    ```python
+    from fastapi import FastAPI
+    from fides.service.error_handling.error_handler import ErrorHandler
+
+    app = FastAPI()
+
+    @app.post("/users")
+    def create_user(age: int):
+        # Simple validation
+        ErrorHandler.validate(age >= 18, "User must be 18 or older")
+        return {"message": "User created"}
+
+    @app.get("/items/{item_id}")
+    def get_item(item_id: str):
+        # Direct error raising
+        if not item_id:
+            ErrorHandler.raise_error("Item ID is required", status_code=400)
+        return {"item_id": item_id}
+    ```
+
+    2. Exception Handling Decorator:
+    ```python
+    @app.post("/orders")
+    @ErrorHandler.handle_exceptions("Failed to create order")
+    def create_order(order_data: dict):
+        if not order_data.get("items"):
+            raise ValueError("Order must contain items")
+        # Process order...
+        return {"message": "Order created"}
+
+    @app.get("/products/{product_id}")
+    @ErrorHandler.handle_exceptions("Failed to fetch product", status_code=404)
+    def get_product(product_id: str):
+        product = database.get_product(product_id)
+        if not product:
+            raise ValueError("Product not found")
+        return product
+    ```
+
+    3. Complex Validation:
+    ```python
+    @app.post("/payments")
+    @ErrorHandler.handle_exceptions("Payment processing failed")
+    def process_payment(payment: dict):
+        # Validate amount
+        ErrorHandler.validate(
+            payment.get("amount", 0) > 0,
+            "Payment amount must be positive",
+            HTTP_400_BAD_REQUEST
+        )
+
+        # Validate currency
+        ErrorHandler.validate(
+            payment.get("currency") in ["USD", "EUR"],
+            "Invalid currency",
+            HTTP_400_BAD_REQUEST,
+            "Unsupported currency provided"  # Optional log message
+        )
+
+        # Custom error for insufficient funds
+        if payment.get("amount", 0) > get_balance():
+            ErrorHandler.raise_error(
+                "Insufficient funds",
+                status_code=HTTP_400_BAD_REQUEST,
+                log_message="User attempted payment exceeding balance"
+            )
+
+        return {"status": "payment processed"}
+    ```
+
+    4. Error Logging:
+    ```python
+    @app.post("/imports")
+    @ErrorHandler.handle_exceptions("Import failed")
+    def import_data(data: dict):
+        try:
+            process_import(data)
+        except Exception as e:
+            # Log error with custom message before raising
+            ErrorHandler.raise_error(
+                "Import failed: invalid format",
+                status_code=400,
+                log_message=f"Import failed with error: {str(e)}"
+            )
+        return {"status": "import complete"}
+    ```
+
+    Key Features:
+    - Simple validation with custom error messages
+    - Consistent error handling across endpoints
+    - Built-in error logging
+    - HTTP status code customization
+    - Exception handling decorator for common patterns
+    """
+
+    @staticmethod
+    def raise_error(
+        detail: str,
+        status_code: int = HTTP_422_UNPROCESSABLE_ENTITY,
+        log_message: Optional[str] = None,
+    ) -> None:
+        """Raise an HTTPException with consistent logging.
+
+        Args:
+            detail: Error message to include in the HTTPException
+            status_code: HTTP status code to use (default: 422)
+            log_message: Optional message to log before raising the exception
+
+        Raises:
+            HTTPException: Always raised with the provided details
+        """
+        if log_message:
+            logger.error(log_message)
+        raise HTTPException(status_code=status_code, detail=detail)
+
+    @staticmethod
+    def validate(
+        condition: bool,
+        detail: str,
+        status_code: int = HTTP_400_BAD_REQUEST,
+        log_message: Optional[str] = None,
+    ) -> None:
+        """Validate a condition and raise an error if it's False.
+
+        Args:
+            condition: The condition to check
+            detail: Error message if condition is False
+            status_code: HTTP status code to use if condition is False
+            log_message: Optional message to log before raising the exception
+
+        Raises:
+            HTTPException: If the condition is False
+        """
+        if not condition:
+            ErrorHandler.raise_error(detail, status_code, log_message)
+
+    @classmethod
+    def handle_exceptions(
+        cls, error_message: str, status_code: int = HTTP_422_UNPROCESSABLE_ENTITY
+    ) -> Callable[[Callable[..., T]], Callable[..., T]]:
+        """Decorator to handle exceptions consistently.
+
+        Args:
+            error_message: Base error message to use
+            status_code: HTTP status code to use for unexpected errors
+
+        Returns:
+            Callable: The decorated function that handles exceptions
+
+        Note:
+            This decorator will catch specific exceptions and convert them to HTTPExceptions.
+            HTTPExceptions are re-raised as is, while other exceptions are wrapped with
+            additional context.
+        """
+
+        def decorator(func: Callable[..., T]) -> Callable[..., T]:
+            @wraps(func)
+            def wrapper(*args: Any, **kwargs: Any) -> Optional[T]:
+                try:
+                    return func(*args, **kwargs)
+                except HTTPException:
+                    # Re-raise HTTP exceptions without modification
+                    raise
+                except (ValueError, TypeError, AttributeError, KeyError) as e:
+                    # Handle common validation and data access errors
+                    cls.raise_error(
+                        f"{error_message}: {str(e)}",
+                        status_code,
+                        f"{error_message}: {e}",
+                    )
+                except Exception as e:  # pylint: disable=broad-except
+                    # Log unexpected errors with full context but present a sanitized message
+                    logger.error(
+                        f"Unexpected error in {func.__name__}: {str(e)}", exc_info=True
+                    )
+                    cls.raise_error(
+                        f"{error_message}: An unexpected error occurred",
+                        status_code,
+                        f"{error_message}: Unexpected {type(e).__name__}",
+                    )
+                return None  # This line is never reached but satisfies the return type checker
+
+            return wrapper  # type: ignore[return-value]
+
+        return decorator

fides/service/user/user_service.py

@@ -0,0 +1,140 @@
+import uuid
+from datetime import datetime
+from typing import Optional, Tuple
+
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from fides.api.api.v1.endpoints.messaging_endpoints import user_email_invite_status
+from fides.api.common_exceptions import AuthorizationError
+from fides.api.models.client import ClientDetail
+from fides.api.models.fides_user import FidesUser
+from fides.api.models.fides_user_invite import FidesUserInvite
+from fides.api.schemas.messaging.messaging import (
+    MessagingActionType,
+    UserInviteBodyParams,
+)
+from fides.api.schemas.redis_cache import Identity
+from fides.api.service.messaging.message_dispatch_service import dispatch_message
+from fides.config import FidesConfig
+from fides.config.config_proxy import ConfigProxy
+
+
+class UserService:
+    def __init__(self, db: Session, config: FidesConfig, config_proxy: ConfigProxy):
+        self.db = db
+        self.config = config
+        self.config_proxy = config_proxy
+
+    def invite_user(self, user: FidesUser) -> None:
+        """
+        Generates a user invite and sends the invite code to the user via email.
+
+        This is a no-op if email messaging isn't configured.
+        """
+
+        # invite user via email if email messaging is enabled and the Admin UI URL is defined
+        if user_email_invite_status(db=self.db, config_proxy=self.config_proxy).enabled:
+            invite_code = str(uuid.uuid4())
+            FidesUserInvite.create(
+                db=self.db, data={"username": user.username, "invite_code": invite_code}
+            )
+            user.update(self.db, data={"disabled": True})
+            # TODO: refactor to use MessagingService
+            dispatch_message(
+                self.db,
+                action_type=MessagingActionType.USER_INVITE,
+                to_identity=Identity(email=user.email_address),
+                service_type=self.config_proxy.notifications.notification_service_type,
+                message_body_params=UserInviteBodyParams(
+                    username=user.username, invite_code=invite_code
+                ),
+            )
+        else:
+            logger.debug(
+                "Skipping invitation email, an email messaging provider is not enabled",
+            )
+
+    def perform_login(
+        self,
+        client_id_byte_length: int,
+        client_secret_byte_length: int,
+        user: FidesUser,
+        skip_save: Optional[bool] = False,
+    ) -> ClientDetail:
+        """Performs a login by updating the FidesUser instance and creating and returning
+        an associated ClientDetail.
+
+        If the username or password was bad, skip_save should be True. We still run through
+        parallel operations to keep the timing of operations similar, but should skip
+        saving to the database.
+        """
+
+        client = user.client
+        if not client:
+            logger.info("Creating client for login")
+            client, _ = ClientDetail.create_client_and_secret(
+                self.db,
+                client_id_byte_length,
+                client_secret_byte_length,
+                scopes=[],  # type: ignore
+                roles=user.permissions.roles,  # type: ignore
+                systems=user.system_ids,  # type: ignore
+                user_id=user.id,
+                in_memory=skip_save,  # If login flow has already errored, don't persist this to the database
+            )
+        else:
+            # Refresh the client just in case - for example, scopes and roles were added via the db directly.
+            client.roles = user.permissions.roles  # type: ignore
+            client.systems = user.system_ids  # type: ignore
+            if not skip_save:
+                client.save(self.db)
+
+        if user.permissions and (not user.permissions.roles and not user.systems):  # type: ignore
+            logger.warning("User {} needs roles or systems to login.", user.id)
+            raise AuthorizationError(detail="Not Authorized for this action")
+
+        if not skip_save:
+            user.last_login_at = datetime.utcnow()
+            user.save(self.db)
+
+        return client
+
+    def accept_invite(
+        self, user: FidesUser, new_password: str
+    ) -> Tuple[FidesUser, str]:
+        """
+        Updates the user password and enables the user. Also removes the user invite from the database.
+        Returns a tuple of the updated user and their access code.
+        """
+
+        # update password and enable
+        user.update_password(db=self.db, new_password=new_password)
+        user.update(
+            self.db,
+            data={"disabled": False, "disabled_reason": None},
+        )
+        self.db.refresh(user)
+
+        # delete invite
+        if user.username:
+            invite = FidesUserInvite.get_by(
+                db=self.db, field="username", value=user.username
+            )
+            if invite:
+                invite.delete(self.db)
+        else:
+            logger.warning("Username is missing, skipping invite deletion.")
+
+        client = self.perform_login(
+            self.config.security.oauth_client_id_length_bytes,
+            self.config.security.oauth_client_secret_length_bytes,
+            user,
+        )
+
+        logger.info("Creating login access token")
+        access_code = client.create_access_code_jwe(
+            self.config.security.app_encryption_key
+        )
+
+        return user, access_code