ethyca-fides 2.56.3b1__py2.py3-none-any.whl → 2.57.0__py2.py3-none-any.whl

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -1,6 +1,7 @@
 # pylint: disable=too-many-instance-attributes
 from __future__ import annotations
 
+from copy import deepcopy
 from datetime import datetime
 from itertools import product
 from typing import Any, Dict, List, Literal, Optional, Tuple, TypeVar
@@ -12,7 +13,6 @@ from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.common_exceptions import FidesopsException
-from fides.api.graph.config import ScalarField
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import (
@@ -33,7 +33,12 @@ from fides.api.task.refine_target_path import (
     join_detailed_path,
 )
 from fides.api.util import saas_util
-from fides.api.util.collection_util import Row, merge_dicts
+from fides.api.util.collection_util import (
+    Row,
+    flatten_dict,
+    merge_dicts,
+    unflatten_dict,
+)
 from fides.api.util.saas_util import (
     ALL_OBJECT_FIELDS,
     CUSTOM_PRIVACY_REQUEST_FIELDS,
@@ -45,7 +50,6 @@ from fides.api.util.saas_util import (
     REPLY_TO_TOKEN,
     UUID,
     get_identities,
-    unflatten_dict,
 )
 from fides.common.api.v1.urn_registry import REQUEST_TASK_CALLBACK, V1_URL_PREFIX
 from fides.config.config_proxy import ConfigProxy
@@ -510,22 +514,24 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
 
     def all_value_map(self, row: Row) -> Dict[str, Any]:
         """
-        Takes a row and preserves only the fields that are defined in the Dataset.
+        Takes a row and preserves only the fields that are defined in the collection.
         Used for scenarios when an update endpoint has required fields other than
         just the fields being updated.
         """
+        flattened_row = flatten_dict(deepcopy(row))
 
-        all_value_map: Dict[str, Any] = {}
-        for field_path, field in self.field_map().items():
-            # only map scalar fields
-            if (
-                isinstance(field, ScalarField)
-                and pydash.get(row, field_path.string_path) is not None
-            ):
-                all_value_map[field_path.string_path] = pydash.get(
-                    row, field_path.string_path
-                )
-        return all_value_map
+        # Get root field names defined in the collection
+        collection_fields = {
+            field_path.string_path.split(".")[0]
+            for field_path, _ in self.field_map().items()
+        }
+
+        # Only keep the field values defined in the collection
+        return {
+            path: value
+            for path, value in flattened_row.items()
+            if path.split(".")[0] in collection_fields
+        }
 
     def query_to_str(self, t: T, input_data: Dict[str, List[Any]]) -> str:
         """Convert query to string"""

fides/api/service/storage/storage_uploader_service.py

@@ -41,9 +41,7 @@ def upload(
         logger.warning("Storage type not found: {}", storage_key)
         raise StorageUploadError(f"Storage type not found: {storage_key}")
     uploader: Any = _get_uploader_from_config_type(config.type)  # type: ignore
-    return uploader(
-        db, config, data, privacy_request, data_category_field_mapping, data_use_map
-    )
+    return uploader(db, config, data, privacy_request)
 
 
 def get_extension(resp_format: ResponseFormat) -> str:
@@ -88,14 +86,13 @@ def _s3_uploader(
     config: StorageConfig,
     data: Dict,
     privacy_request: PrivacyRequest,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
 ) -> str:
     """Constructs necessary info needed for s3 before calling upload"""
     file_key: str = _construct_file_key(privacy_request.id, config)
 
     bucket_name = config.details[StorageDetails.BUCKET.value]
     auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+    document = None
 
     return upload_to_s3(
         config.secrets,  # type: ignore
@@ -104,9 +101,8 @@ def _s3_uploader(
         file_key,
         config.format.value,  # type: ignore
         privacy_request,
+        document,
         auth_method,
-        data_category_field_mapping,
-        data_use_map,
     )
 
 
@@ -115,9 +111,7 @@ def _local_uploader(
     config: StorageConfig,
     data: Dict,
     privacy_request: PrivacyRequest,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
 ) -> str:
     """Uploads data to local storage, used for quick-start/demo purposes"""
     file_key: str = _construct_file_key(privacy_request.id, config)
-    return upload_to_local(data, file_key, privacy_request, config.format.value, data_category_field_mapping, data_use_map)  # type: ignore
+    return upload_to_local(data, file_key, privacy_request, config.format.value)  # type: ignore

fides/api/tasks/storage.py

@@ -5,20 +5,20 @@ import os
 import secrets
 import zipfile
 from io import BytesIO
-from typing import Any, Dict, Optional, Set, Union
+from typing import Any, Dict, Optional, Union
 
 import pandas as pd
 from botocore.exceptions import ClientError, ParamValidationError
+from fideslang.validation import AnyHttpUrlString
 from loguru import logger
 
 from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
-from fides.api.graph.graph import DataCategoryFieldMapping
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.storage.storage import ResponseFormat, StorageSecrets
 from fides.api.service.privacy_request.dsr_package.dsr_report_builder import (
     DsrReportBuilder,
 )
-from fides.api.util.aws_util import get_aws_session
+from fides.api.util.aws_util import get_s3_client
 from fides.api.util.cache import get_cache, get_encryption_cache_key
 from fides.api.util.encryption.aes_gcm_encryption_scheme import (
     encrypt_to_bytes_verify_secrets_length,
@@ -101,7 +101,9 @@ def write_to_in_memory_buffer(
     raise NotImplementedError(f"No handling for response format {resp_format}.")
 
 
-def create_presigned_url_for_s3(s3_client: Any, bucket_name: str, file_key: str) -> str:
+def create_presigned_url_for_s3(
+    s3_client: Any, bucket_name: str, file_key: str
+) -> AnyHttpUrlString:
     """ "Generate a presigned URL to share an S3 object
 
     :param s3_client: s3 base client
@@ -119,23 +121,108 @@ def create_presigned_url_for_s3(s3_client: Any, bucket_name: str, file_key: str)
     return response
 
 
+def generic_upload_to_s3(  # pylint: disable=R0913
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+    document: bytes,
+) -> Optional[AnyHttpUrlString]:
+    """Uploads arbitrary data to s3 returned from an access request"""
+    logger.info("Starting S3 Upload of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            s3_client.put_object(Bucket=bucket_name, Key=file_key, Body=document)
+        except Exception as e:
+            logger.error("Encountered error while uploading s3 object: {}", e)
+            raise e
+
+        presigned_url: AnyHttpUrlString = create_presigned_url_for_s3(
+            s3_client, bucket_name, file_key
+        )
+
+        return presigned_url
+    except ClientError as e:
+        logger.error(
+            "Encountered error while uploading and generating link for s3 object: {}", e
+        )
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
+def generic_retrieve_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> Optional[bytes]:
+    """Retrieves arbitrary data from s3"""
+    logger.info("Starting S3 Retrieve of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
+            return response["Body"].read()
+        except Exception as e:
+            logger.error("Encountered error while retrieving s3 object: {}", e)
+            raise e
+    except ClientError as e:
+        logger.error("Encountered error while retrieving s3 object: {}", e)
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
+def generic_delete_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> None:
+    """Deletes arbitrary data from s3"""
+    logger.info("Starting S3 Delete of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            s3_client.delete_object(Bucket=bucket_name, Key=file_key)
+        except Exception as e:
+            logger.error("Encountered error while deleting s3 object: {}", e)
+            raise e
+    except ClientError as e:
+        logger.error("Encountered error while deleting s3 object: {}", e)
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
 def upload_to_s3(  # pylint: disable=R0913
     storage_secrets: Dict[StorageSecrets, Any],
     data: Dict,
     bucket_name: str,
     file_key: str,
     resp_format: str,
-    privacy_request: PrivacyRequest,
+    privacy_request: Optional[PrivacyRequest],
+    document: Optional[bytes],
     auth_method: str,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
-) -> str:
+) -> Optional[AnyHttpUrlString]:
     """Uploads arbitrary data to s3 returned from an access request"""
     logger.info("Starting S3 Upload of {}", file_key)
 
+    if privacy_request is None and document is not None:
+        return generic_upload_to_s3(
+            storage_secrets, bucket_name, file_key, auth_method, document
+        )
+
+    if privacy_request is None:
+        raise ValueError("Privacy request must be provided")
+
     try:
-        my_session = get_aws_session(auth_method, storage_secrets)
-        s3_client = my_session.client("s3")
+        s3_client = get_s3_client(auth_method, storage_secrets)
 
         # handles file chunking
         try:
@@ -148,7 +235,7 @@ def upload_to_s3(  # pylint: disable=R0913
             logger.error("Encountered error while uploading s3 object: {}", e)
             raise e
 
-        presigned_url: str = create_presigned_url_for_s3(
+        presigned_url: AnyHttpUrlString = create_presigned_url_for_s3(
             s3_client, bucket_name, file_key
         )
 
@@ -162,17 +249,21 @@ def upload_to_s3(  # pylint: disable=R0913
         raise ValueError(f"The parameters you provided are incorrect: {e}")
 
 
+def get_local_filename(file_key: str) -> str:
+    """Verifies that the local storage directory exists"""
+    if not os.path.exists(LOCAL_FIDES_UPLOAD_DIRECTORY):
+        os.makedirs(LOCAL_FIDES_UPLOAD_DIRECTORY)
+    return f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{file_key}"
+
+
 def upload_to_local(
     data: Dict,
     file_key: str,
     privacy_request: PrivacyRequest,
     resp_format: str = ResponseFormat.json.value,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
 ) -> str:
     """Uploads access request data to a local folder - for testing/demo purposes only"""
-    if not os.path.exists(LOCAL_FIDES_UPLOAD_DIRECTORY):
-        os.makedirs(LOCAL_FIDES_UPLOAD_DIRECTORY)
+    get_local_filename(file_key)
 
     filename = f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{file_key}"
     in_memory_file = write_to_in_memory_buffer(resp_format, data, privacy_request)

fides/api/util/aws_util.py

@@ -70,3 +70,22 @@ def get_aws_session(
             raise
     else:
         return session
+
+
+def get_s3_client(
+    auth_method: str,
+    storage_secrets: Optional[Dict[StorageSecrets, Any]],
+    assume_role_arn: Optional[str] = None,
+) -> Session:
+    """
+    Abstraction to retrieve an AWS S3 client using secrets.
+
+    If an `assume_role_arn` is provided, the secrets will be used to
+    assume that role and return a Session instantiated with that role.
+    """
+    session = get_aws_session(
+        auth_method=auth_method,
+        storage_secrets=storage_secrets,
+        assume_role_arn=assume_role_arn,
+    )
+    return session.client("s3")

fides/api/util/collection_util.py

@@ -1,3 +1,4 @@
+from collections import deque
 from functools import reduce
 from typing import Any, Callable, Dict, Iterable, List, Optional, TypeVar, Union
 
@@ -119,3 +120,144 @@ def extract_key_for_address(
     request_id_dataset, collection = full_request_id.split(":")
     dataset = request_id_dataset.split("__", number_of_leading_strings_to_exclude)[-1]
     return f"{dataset}:{collection}"
+
+
+def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str, Any]:
+    """
+    Converts a dictionary of paths/values into a nested dictionary
+
+    example:
+
+    {"A.B": "1", "A.C": "2"}
+
+    becomes
+
+    {
+        "A": {
+            "B": "1",
+            "C": "2"
+        }
+    }
+    """
+    output: Dict[Any, Any] = {}
+    queue = deque(flat_dict.items())
+
+    while queue:
+        path, value = queue.popleft()
+        keys = path.split(separator)
+        target = output
+        for i, current_key in enumerate(keys[:-1]):
+            next_key = keys[i + 1]
+            if next_key.isdigit():
+                target = target.setdefault(current_key, [])
+            else:
+                if isinstance(target, dict):
+                    target = target.setdefault(current_key, {})
+                elif isinstance(target, list):
+                    while len(target) <= int(current_key):
+                        target.append({})
+                    target = target[int(current_key)]
+        try:
+            if isinstance(target, list):
+                target.append(value)
+            else:
+                # If the value is a dictionary, add its components to the queue for processing
+                if isinstance(value, dict):
+                    target = target.setdefault(keys[-1], {})
+                    for inner_key, inner_value in value.items():
+                        new_key = f"{path}{separator}{inner_key}"
+                        queue.append((new_key, inner_value))
+                else:
+                    target[keys[-1]] = value
+        except TypeError as exc:
+            raise ValueError(
+                f"Error unflattening dictionary, conflicting levels detected: {exc}"
+            )
+    return output
+
+
+def flatten_dict(data: Any, prefix: str = "", separator: str = ".") -> Dict[str, Any]:
+    """
+    Recursively flatten a dictionary or list into a flat dictionary with dot-notation keys.
+    Handles nested dictionaries and arrays with proper indices.
+
+    example:
+
+    {
+        "A": {
+            "B": "1",
+            "C": "2"
+        },
+        "D": [
+            {"E": "3"},
+            {"E": "4"}
+        ]
+    }
+
+    becomes
+
+    {
+        "A.B": "1",
+        "A.C": "2",
+        "D.0.E": "3",
+        "D.1.E": "4"
+    }
+
+    Args:
+        data: The data to flatten (dict, list, or scalar value)
+        prefix: The current key prefix (used in recursion)
+        separator: The separator to use between key segments (default: ".")
+
+    Returns:
+        A flattened dictionary with dot-notation keys
+    """
+    items = {}
+
+    if isinstance(data, dict):
+        for k, v in data.items():
+            new_key = f"{prefix}{separator}{k}" if prefix else k
+            if isinstance(v, (dict, list)):
+                items.update(flatten_dict(v, new_key, separator))
+            else:
+                items[new_key] = v
+    elif isinstance(data, list):
+        for i, v in enumerate(data):
+            new_key = f"{prefix}{separator}{i}"
+            if isinstance(v, (dict, list)):
+                items.update(flatten_dict(v, new_key, separator))
+            else:
+                items[new_key] = v
+    else:
+        raise ValueError(
+            f"Input to flatten_dict must be a dict or list, got {type(data).__name__}"
+        )
+
+    return items
+
+
+def replace_none_arrays(
+    data: Any,
+) -> Any:
+    """
+    Recursively replace any arrays containing only None values with empty arrays.
+
+    Args:
+        data: Any Python object (dict, list, etc.)
+
+    Returns:
+        The transformed data structure with None-only arrays replaced by empty arrays
+    """
+    if isinstance(data, dict):
+        # Process each key-value pair in dictionaries
+        return {k: replace_none_arrays(v) for k, v in data.items()}
+
+    if isinstance(data, list):
+        # Check if list contains only None values
+        if all(item is None for item in data):
+            return []
+
+        # Otherwise process each item in the list
+        return [replace_none_arrays(item) for item in data]
+
+    # Return other data types unchanged
+    return data

fides/api/util/logger_context_utils.py

@@ -1,3 +1,4 @@
+import inspect
 from abc import abstractmethod
 from enum import Enum
 from functools import wraps
@@ -81,12 +82,46 @@ def log_context(
         def wrapper(*args: Any, **kwargs: Any) -> Any:
             context = dict(additional_context)
 
-            # extract specified param values from kwargs
+            # extract specified param values from kwargs and args
             if capture_args:
+                # First, process kwargs as they're explicitly named
                 for arg_name, context_name in capture_args.items():
                     if arg_name in kwargs:
                         context[context_name.value] = kwargs[arg_name]
 
+                # Process args using signature binding for more robust parameter mapping
+                if args:
+                    try:
+                        # Get the signature and bind the arguments
+                        sig = inspect.signature(func)
+                        # This will map positional args to their parameter names correctly
+                        bound_args = sig.bind_partial(*args, **kwargs)
+
+                        # Now we can iterate through the bound arguments
+                        for param_name, arg_value in bound_args.arguments.items():
+                            # Only process if this parameter is in capture_args and wasn't already found in kwargs
+                            if param_name in capture_args and param_name not in kwargs:
+                                context_name = capture_args[param_name]
+                                context[context_name.value] = arg_value
+                    except TypeError:
+                        # Handle the case where the arguments don't match the signature
+                        pass
+
+                # Handle default parameters that weren't provided in args or kwargs
+                if capture_args:
+                    sig = inspect.signature(func)
+                    for param_name, param in sig.parameters.items():
+                        # Check if parameter has a default value and is in capture_args
+                        # and hasn't been processed yet (not in context)
+                        if (
+                            param.default is not param.empty
+                            and param_name in capture_args
+                            and capture_args[param_name].value not in context
+                        ):
+                            context_name = capture_args[param_name]
+                            context[context_name.value] = param.default
+
+            # Process Contextualizable args
             for arg in args:
                 if isinstance(arg, Contextualizable):
                     arg_context = arg.get_log_context()

fides/api/util/saas_util.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import json
 import re
 import socket
-from collections import defaultdict, deque
+from collections import defaultdict
 from ipaddress import IPv4Address, IPv6Address, ip_address
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
@@ -256,60 +256,6 @@ def merge_datasets(dataset: GraphDataset, config_dataset: GraphDataset) -> Graph
     )
 
 
-def unflatten_dict(flat_dict: Dict[str, Any], separator: str = ".") -> Dict[str, Any]:
-    """
-    Converts a dictionary of paths/values into a nested dictionary
-
-    example:
-
-    {"A.B": "1", "A.C": "2"}
-
-    becomes
-
-    {
-        "A": {
-            "B": "1",
-            "C": "2"
-        }
-    }
-    """
-    output: Dict[Any, Any] = {}
-    queue = deque(flat_dict.items())
-
-    while queue:
-        path, value = queue.popleft()
-        keys = path.split(separator)
-        target = output
-        for i, current_key in enumerate(keys[:-1]):
-            next_key = keys[i + 1]
-            if next_key.isdigit():
-                target = target.setdefault(current_key, [])
-            else:
-                if isinstance(target, dict):
-                    target = target.setdefault(current_key, {})
-                elif isinstance(target, list):
-                    while len(target) <= int(current_key):
-                        target.append({})
-                    target = target[int(current_key)]
-        try:
-            if isinstance(target, list):
-                target.append(value)
-            else:
-                # If the value is a dictionary, add its components to the queue for processing
-                if isinstance(value, dict):
-                    target = target.setdefault(keys[-1], {})
-                    for inner_key, inner_value in value.items():
-                        new_key = f"{path}{separator}{inner_key}"
-                        queue.append((new_key, inner_value))
-                else:
-                    target[keys[-1]] = value
-        except TypeError as exc:
-            raise FidesopsException(
-                f"Error unflattening dictionary, conflicting levels detected: {exc}"
-            )
-    return output
-
-
 def format_body(
     headers: Dict[str, Any],
     body: Optional[str],
@@ -339,7 +285,7 @@ def format_body(
     if content_type == "application/json":
         output = body
     elif content_type == "application/x-www-form-urlencoded":
-        output = multidimensional_urlencode(json.loads(body))
+        output = nullsafe_urlencode(json.loads(body))
     elif content_type == "text/plain":
         output = body
     else:
@@ -470,3 +416,33 @@ def replace_version(saas_config: str, new_version: str) -> str:
         version_pattern, f"version: {new_version}", saas_config, count=1
     )
     return updated_config
+
+
+def nullsafe_urlencode(data: Any) -> str:
+    """
+    Wrapper around multidimensional_urlencode that preserves null values as empty strings.
+
+    This is useful for APIs that expect keys with empty values (e.g., "name=") to represent
+    null values, rather than omitting the field entirely.
+
+    Args:
+        data: The data to encode (can be a dict, list, or other nested structure)
+
+    Returns:
+        URL-encoded string with null values properly handled
+    """
+
+    def prepare_null_values(data: Any) -> Any:
+        """
+        Recursively process data for URL encoding, converting None values to empty strings.
+        """
+        if data is None:
+            return ""
+        if isinstance(data, dict):
+            return {k: prepare_null_values(v) for k, v in data.items()}
+        if isinstance(data, list):
+            return [prepare_null_values(item) for item in data]
+        return data
+
+    processed_data = prepare_null_values(data)
+    return multidimensional_urlencode(processed_data)

fides/data/language/languages.yml

@@ -17,6 +17,8 @@ languages:
   name: Greek
 - id: en
   name: English
+- id: en-GB
+  name: English (UK)
 - id: es
   name: Spanish
 - id: es-MX

fides/service/privacy_request/privacy_request_service.py

@@ -332,11 +332,20 @@ class PrivacyRequestService:
         )
 
         if _manual_approval_required(self.config_proxy, privacy_request):
-            self.approve_privacy_requests(
-                [privacy_request_id],
-                reviewed_by=reviewed_by,
-                suppress_notification=True,
-            )
+            has_webhooks = self.db.query(PreApprovalWebhook).count() > 0
+            # If there are pre-approval webhooks, then we do nothing since the
+            # create_privacy_request method will have already triggered them,
+            # and they will be responsible of approving the request.
+
+            # If there are no pre-approval webhooks, approve the request immediately
+            # since it was originally approved by a person and we don't want them to
+            # have to manually re-approve
+            if not has_webhooks:
+                self.approve_privacy_requests(
+                    [privacy_request_id],
+                    reviewed_by=reviewed_by,
+                    suppress_notification=True,
+                )
 
         return privacy_request