ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/service/storage/util.py

@@ -10,7 +10,7 @@ from fides.api.util.storage_util import format_size
 
 # This is the max file size for downloading the content of an attachment.
 # This is an industry standard used by companies like Google and Microsoft.
-LARGE_FILE_THRESHOLD = 25 * 1024 * 1024  # 25 MB
+LARGE_FILE_THRESHOLD = 2 * 1024 * 1024 * 1024  # 2 GB
 
 
 class AllowedFileType(EnumType):
@@ -172,6 +172,7 @@ def generate_attachment_url_from_storage_path(
     1. Using resolve_attachment_storage_path() to calculate the actual storage path
     2. Handling different directory structures (attachments vs data/dataset/collection)
     3. Generating proper relative paths from HTML template locations to attachment files
+    4. URL-encoding filenames for proper HTML link functionality
 
     Used by:
     - _process_attachment_list() in this file
@@ -191,17 +192,21 @@ def generate_attachment_url_from_storage_path(
         # Calculate the actual storage path
         storage_path = resolve_attachment_storage_path(unique_filename, base_path)
 
+        # URL-encode the filename for proper HTML link functionality
+        # Always encode when streaming is enabled to ensure consistency
+        encoded_filename = quote(unique_filename, safe="")
+
         # Generate relative path from HTML template directory to storage path
         if html_directory == "attachments" and base_path == "attachments":
             # From attachments/index.html to attachments/filename.pdf (same directory)
-            return unique_filename
+            return encoded_filename
         if html_directory.startswith("data/") and base_path.startswith("data/"):
             # From data/dataset/collection/index.html to data/dataset/collection/attachments/filename.pdf
             # Both are in data/ structure, so go to attachments subdirectory
-            return f"attachments/{unique_filename}"
+            return f"attachments/{encoded_filename}"
         # For other cases, calculate relative path
         # This is a simplified approach - in practice, you might need more sophisticated path resolution
-        return f"../{storage_path}"
+        return f"../{storage_path.replace(unique_filename, encoded_filename)}"
     return download_url
 
 
@@ -539,31 +544,9 @@ def _process_attachment_list(
     return processed_attachments_list
 
 
-def extract_storage_key_from_attachment(attachment: dict[str, Any]) -> str:
-    """
-    Extract storage key from attachment data with fallback logic.
-
-    This function provides a consistent way to extract storage keys from
-    attachment dictionaries across different components.
-
-    Args:
-        attachment: The attachment dictionary
-
-    Returns:
-        The storage key (URL or filename) for the attachment
-    """
-    if original_url := attachment.get("original_download_url"):
-        return original_url
-
-    if download_url := attachment.get("download_url"):
-        return download_url
-
-    file_name = attachment.get("file_name")
-    return file_name if file_name is not None else ""
-
-
-def resolve_base_path_from_context(
-    attachment: dict[str, Any], default_base_path: str = "attachments"
+def resolve_path_from_context(
+    attachment: dict[str, Any],
+    default_path: str = "attachments",
 ) -> str:
     """
     Resolve the base path for an attachment based on its context.
@@ -573,93 +556,24 @@ def resolve_base_path_from_context(
 
     Args:
         attachment: The attachment dictionary
-        default_base_path: Default base path if no context is found
+        default_path: Default path if no context is found
 
     Returns:
-        The resolved base path for the attachment
+        The resolved path for the attachment
     """
     if not attachment.get("_context"):
-        return default_base_path
+        return default_path
 
     context = attachment["_context"]
     context_type = context.get("type")
 
-    if context_type == "direct":
-        return f"data/{context['dataset']}/{context['collection']}/attachments"
-    if context_type == "nested":
-        return f"data/{context['dataset']}/{context['collection']}/attachments"
+    if context_type in ["direct", "nested"]:
+        dataset = context.get("dataset", "")
+        collection = context.get("collection", "")
+        return f"data/{dataset}/{collection}/attachments"
     if context_type == "top_level":
         return "attachments"
-    # Handle old context format
     if context.get("key") and context.get("item_id"):
         return f"{context['key']}/{context['item_id']}/attachments"
-    # Fallback for unknown context types
-    return "unknown/unknown/attachments"
-
-
-def resolve_directory_from_context(
-    attachment: dict[str, Any], default_directory: str = "attachments"
-) -> str:
-    """
-    Resolve the directory path for an attachment based on its context.
-
-    This function provides consistent directory resolution logic for DSR report builder.
-
-    Args:
-        attachment: The attachment dictionary
-        default_directory: Default directory if no context is found
-
-    Returns:
-        The resolved directory path for the attachment
-    """
-    if not attachment.get("_context"):
-        return default_directory
-
-    context = attachment["_context"]
-    context_type = context.get("type")
-
-    if context_type == "direct":
-        return f"data/{context['dataset']}/{context['collection']}"
-    if context_type == "nested":
-        return f"data/{context['dataset']}/{context['collection']}"
-    if context_type == "top_level":
-        return "attachments"
-    if context.get("key") and context.get("item_id"):
-        return f"{context['key']}/{context['item_id']}"
-
-    return default_directory
-
-
-def convert_processed_attachments_to_attachment_processing_info(
-    processed_attachments_list: list[dict[str, Any]], validate_attachment_func: Callable
-) -> list[Any]:
-    """
-    Convert processed attachments list to AttachmentProcessingInfo objects.
-
-    This is a shared utility function to avoid duplication between different
-    attachment collection methods.
-
-    Args:
-        processed_attachments_list: List of processed attachment dictionaries
-        validate_attachment_func: Function to validate individual attachments
-                                 Signature: validate_attachment_func(attachment_with_context) -> AttachmentProcessingInfo | None
-
-    Returns:
-        List of validated AttachmentProcessingInfo objects
-    """
-    validated_attachments = []
-
-    for processed_attachment in processed_attachments_list:
-        attachment_data = processed_attachment["attachment"]
-
-        # Add context information to the attachment data
-        attachment_with_context = attachment_data.copy()
-        attachment_with_context["_context"] = processed_attachment["context"]
-
-        # Validate and convert to AttachmentProcessingInfo
-        if validate_attachment_func is not None:
-            validated = validate_attachment_func(attachment_with_context)
-            if validated:
-                validated_attachments.append(validated)
 
-    return validated_attachments
+    return default_path

fides/api/service/strategy.py

@@ -41,13 +41,13 @@ class Strategy(ABC, Generic[C]):
     """Abstract base class for strategies"""
 
     name: str
-    configuration_model: Type[C]
+    configuration_model: Optional[Type[C]] = None
 
     @classmethod
     def get_strategy(
         cls: Type[T],
         strategy_name: str,
-        configuration: Dict[str, Any],
+        configuration: Optional[Dict[str, Any]] = None,
     ) -> T:
         """
         Returns the strategy given the name and configuration.
@@ -64,7 +64,10 @@ class Strategy(ABC, Generic[C]):
                 f"Strategy '{strategy_name}' does not exist. Valid strategies are [{valid_strategies}]"
             )
         try:
-            strategy_config = strategy_class.configuration_model(**configuration)
+            if strategy_class.configuration_model:
+                strategy_config = strategy_class.configuration_model(**configuration)
+            else:
+                return strategy_class()  # type: ignore
         except ValidationError as e:
             raise FidesopsValidationError(message=str(e))
         return strategy_class(strategy_config)  # type: ignore

fides/api/task/conditional_dependencies/privacy_request/convenience_fields.py

@@ -0,0 +1,155 @@
+from typing import Any, Optional
+
+from fides.api.models.location_regulation_selections import get_location_by_id
+from fides.api.models.policy import Policy
+from fides.api.schemas.policy import ActionType
+from fides.api.task.conditional_dependencies.privacy_request.schemas import (
+    ConditionalDependencyFieldInfo,
+    ConditionalDependencyFieldType,
+    PrivacyRequestConvenienceFields,
+    PrivacyRequestLocationConvenienceFields,
+    PrivacyRequestPolicyConvenienceFields,
+)
+
+
+def build_convenience_field_list() -> list[ConditionalDependencyFieldInfo]:
+    """Builds a list of ConditionalDependencyFieldInfo objects for convenience fields."""
+    return [
+        # Policy convenience fields
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.rule_action_types.value,
+            field_type=ConditionalDependencyFieldType.array,
+            description="List of action types from policy rules (e.g., ['access', 'erasure']). Use with list_contains operator.",
+            is_convenience_field=True,
+        ),
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.has_access_rule.value,
+            field_type=ConditionalDependencyFieldType.boolean,
+            description="Whether the policy has at least one rule with action_type 'access'.",
+            is_convenience_field=True,
+        ),
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.has_erasure_rule.value,
+            field_type=ConditionalDependencyFieldType.boolean,
+            description="Whether the policy has at least one rule with action_type 'erasure'.",
+            is_convenience_field=True,
+        ),
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.has_consent_rule.value,
+            field_type=ConditionalDependencyFieldType.boolean,
+            description="Whether the policy has at least one rule with action_type 'consent'.",
+            is_convenience_field=True,
+        ),
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.has_update_rule.value,
+            field_type=ConditionalDependencyFieldType.boolean,
+            description="Whether the policy has at least one rule with action_type 'update'.",
+            is_convenience_field=True,
+        ),
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.rule_count.value,
+            field_type=ConditionalDependencyFieldType.integer,
+            description="Number of rules in the policy.",
+            is_convenience_field=True,
+        ),
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.rule_names.value,
+            field_type=ConditionalDependencyFieldType.array,
+            description="List of rule names from the policy. Use with list_contains operator.",
+            is_convenience_field=True,
+        ),
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.has_storage_destination.value,
+            field_type=ConditionalDependencyFieldType.boolean,
+            description="Whether any rule in the policy has a storage destination configured.",
+            is_convenience_field=True,
+        ),
+        # Location hierarchy convenience fields
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.location_country.value,
+            field_type=ConditionalDependencyFieldType.string,
+            description="The country code for the location (e.g., 'us' for 'us_ca', 'fr' for France). Use with eq operator to check if a location is in a specific country.",
+            is_convenience_field=True,
+        ),
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.location_groups.value,
+            field_type=ConditionalDependencyFieldType.array,
+            description="List of location groups this location belongs to (e.g., ['us'] for US states, ['eea'] for EU countries). Use with list_contains operator.",
+            is_convenience_field=True,
+        ),
+        ConditionalDependencyFieldInfo(
+            field_path=PrivacyRequestConvenienceFields.location_regulations.value,
+            field_type=ConditionalDependencyFieldType.array,
+            description="List of regulations applicable to this location (e.g., ['ccpa'] for California, ['gdpr'] for EU). Use with list_contains operator.",
+            is_convenience_field=True,
+        ),
+    ]
+
+
+def get_policy_convenience_fields(
+    policy: Policy,
+) -> dict[str, Any]:
+    """Gets convenience fields for a policy."""
+    extra_fields: dict[str, Any] = {}
+    extra_fields["id"] = policy.id
+
+    # Get rules for convenience field derivation
+    rules = policy.rules if hasattr(policy, "rules") and policy.rules else []
+    action_types = policy.get_all_action_types()
+
+    # Add convenience fields for rules
+    extra_fields[PrivacyRequestPolicyConvenienceFields.rule_action_types.value] = [
+        action_type.value for action_type in action_types if action_type
+    ]
+    for action_type in ActionType:
+        extra_fields[f"has_{action_type.value}_rule"] = action_type in action_types
+    extra_fields[PrivacyRequestPolicyConvenienceFields.rule_count.value] = (
+        len(rules) if rules else 0
+    )
+    extra_fields[PrivacyRequestPolicyConvenienceFields.rule_names.value] = [
+        rule.name for rule in rules if hasattr(rule, "name") and rule.name
+    ]
+    extra_fields[
+        PrivacyRequestPolicyConvenienceFields.has_storage_destination.value
+    ] = any(
+        hasattr(rule, "storage_destination") and rule.storage_destination is not None
+        for rule in rules
+    )
+    return extra_fields
+
+
+def get_location_convenience_fields(location: Optional[str]) -> dict[str, Any]:
+    """Gets convenience fields for a location to support hierarchy-based conditions."""
+    extra_fields: dict[str, Any] = {
+        PrivacyRequestLocationConvenienceFields.location_country.value: None,
+        PrivacyRequestLocationConvenienceFields.location_groups.value: [],
+        PrivacyRequestLocationConvenienceFields.location_regulations.value: [],
+    }
+
+    if not location:
+        return extra_fields
+
+    location_data = get_location_by_id(location)
+    if not location_data:
+        return extra_fields
+
+    # If location has parent groups, the first one is typically the country
+    # (e.g., us_ca belongs to [us])
+    belongs_to = location_data.belongs_to or []
+    extra_fields[PrivacyRequestLocationConvenienceFields.location_groups.value] = (
+        belongs_to
+    )
+
+    # Set location_country to the first parent if the location is a subdivision
+    # This helps with conditions like "is this a US state?"
+    if belongs_to and not location_data.is_country:
+        extra_fields[PrivacyRequestLocationConvenienceFields.location_country.value] = (
+            belongs_to[0]
+        )
+
+    # Add regulations applicable to this location
+    extra_fields[PrivacyRequestLocationConvenienceFields.location_regulations.value] = (
+        location_data.regulation or []
+    )
+
+    return extra_fields

fides/api/task/conditional_dependencies/privacy_request/privacy_request_data.py

@@ -0,0 +1,140 @@
+from typing import Any, Optional
+
+from pydantic.v1.utils import deep_update
+
+from fides.api.models.policy import Policy
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.schemas.policy import Policy as PolicySchema
+from fides.api.task.conditional_dependencies.privacy_request.convenience_fields import (
+    get_location_convenience_fields,
+    get_policy_convenience_fields,
+)
+from fides.api.task.conditional_dependencies.privacy_request.schemas import (
+    PrivacyRequestLocationConvenienceFields,
+)
+from fides.api.task.conditional_dependencies.util import (
+    extract_nested_field_value,
+    set_nested_value,
+    transform_value_for_evaluation,
+)
+
+
+class PrivacyRequestDataTransformer:
+    """
+    Transforms a PrivacyRequest ORM object into a dictionary structure for condition evaluation.
+
+    - This class extracts the fields referenced in conditions and builds a minimal nested structure
+    for evaluation. Privacy request fields must be prefixed with the "privacy_request" namespace.
+    - This class is used to get the privacy request data for condition evaluation.
+        - The task.conditional_dependencies.evaluator expects the data to be in a nested dictionary structure
+        and uses the operators defined in the task.conditional_dependencies.operators.py which have defined
+        data type compatibility.
+
+    Note: Privacy request relationships are eager loaded in the execute_request_tasks.py file.
+    """
+
+    def __init__(self, privacy_request: PrivacyRequest):
+        self.privacy_request = privacy_request
+        self.policy_data = self._transform_policy_object(privacy_request.policy)
+        self.identity_data = self.privacy_request.get_persisted_identity()
+        self.custom_privacy_request_fields_data = (
+            self.privacy_request.get_persisted_custom_privacy_request_fields()
+        )
+        self.location_convenience_fields = get_location_convenience_fields(
+            privacy_request.location
+        )
+
+    def to_evaluation_data(self, field_addresses: set[str]) -> dict[str, Any]:
+        """
+        Transforms the PrivacyRequest into an evaluation-ready dictionary structure.
+
+        Args:
+            field_addresses: Set of field addresses referenced in conditions
+        """
+        if not field_addresses:
+            return {}
+
+        result: dict[str, Any] = {"privacy_request": {}}
+        field_paths: set[str] = {addr.replace(":", ".") for addr in field_addresses}
+
+        # Extract each field dynamically based on its path
+        for field_path in field_paths:
+            value = self._extract_value_by_path(field_path)
+            privacy_request_data: Optional[dict[str, Any]] = set_nested_value(
+                field_path.split("."), value
+            )
+            if privacy_request_data:
+                result = deep_update(result, privacy_request_data)
+
+        return result
+
+    def _extract_value_by_path(self, field_address: str) -> Any:
+        """
+        Extracts a value from PrivacyRequest by following the field address path.
+
+        Args:
+            field_address: The field address path (e.g., "privacy_request.location", "privacy_request.policy.rules")
+
+        """
+        # first part will always be privacy_request
+        parts: list[str] = field_address.split(".")[1:]
+        current: Any = self.privacy_request
+        is_labeled_field = False
+
+        # Handle location convenience fields - these are derived fields, not direct attributes
+        # Check if we're accessing a location convenience field before trying to extract from privacy_request
+
+        if len(parts) == 1 and parts[0] in [
+            e.value for e in PrivacyRequestLocationConvenienceFields
+        ]:
+            return self.location_convenience_fields.get(parts[0])
+
+        # Route to the appropriate data source based on the first path component
+        if parts[0] == "policy":
+            current = self.policy_data
+            parts.pop(0)
+        elif parts[0] == "identity":
+            current = self.identity_data
+            # Identity fields may contain LabeledIdentity objects with label/value structure
+            is_labeled_field = True
+            parts.pop(0)
+        elif parts[0] == "custom_privacy_request_fields":
+            current = self.custom_privacy_request_fields_data
+            # Custom fields are stored as {"label": ..., "value": ...} dicts
+            is_labeled_field = True
+            parts.pop(0)
+
+        current = extract_nested_field_value(current, parts)
+
+        # If we started with an Identity object and didn't extract any field from it
+        # (current is still the identity object), return None
+        # Otherwise, transform the value (which could be a BaseModel like LabeledIdentity)
+        if current is self.identity_data:
+            return None
+
+        # Transform the value first (converts LabeledIdentity to dict, handles other types)
+        current = transform_value_for_evaluation(current)
+
+        # For labeled fields (custom privacy request fields and identity fields with labels),
+        # extract just the value from the {"label": ..., "value": ...} structure
+        if is_labeled_field and isinstance(current, dict) and "value" in current:
+            current = current["value"]
+
+        return current
+
+    def _transform_policy_object(self, policy: Policy) -> dict[str, Any]:
+        """
+        Transforms a Policy ORM object to a dictionary using existing Pydantic schema
+        and adds convenience fields.
+
+        Args:
+            policy: The Policy ORM object
+        """
+        if not policy:
+            return {}
+        extra_fields = get_policy_convenience_fields(policy)
+
+        # Convert Policy ORM object to dictionary using Pydantic schema
+        policy_dict = PolicySchema.model_validate(policy).model_dump()
+        policy_dict.update(extra_fields)
+        return policy_dict