ethyca-fides 2.70.6rc0__py2.py3-none-any.whl → 2.71.0__py2.py3-none-any.whl

fides/api/service/connectors/http_connector.py

@@ -3,6 +3,9 @@ from typing import Any, Dict, List, Optional
 
 import requests
 from loguru import logger
+from oauthlib.oauth2 import BackendApplicationClient
+from requests.auth import HTTPBasicAuth
+from requests_oauthlib import OAuth2Session
 from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR
 
 from fides.api.common_exceptions import ClientUnsuccessfulException
@@ -41,25 +44,58 @@ class HTTPSConnector(BaseConnector[None]):
     ) -> Optional[Dict[str, Any]]:
         """Calls a client-defined endpoint and returns the data that it responds with"""
         config = HttpsSchema(**self.configuration.secrets or {})
-        headers = self.build_authorization_header()
-        headers.update(additional_headers)
+
+        def _request_with_oauth() -> requests.Response:
+            """Helper function to make a request with OAuth2 authentication"""
+            client_id = oauth_config.client_id
+            client_secret = oauth_config.client_secret
+            scopes = oauth_config.scope.split(" ") or []
+            auth = HTTPBasicAuth(client_id, client_secret)
+            client = BackendApplicationClient(client_id=client_id)
+            session_client = OAuth2Session(client=client, scope=scopes)
+            try:
+                # Fetch the access token from the token URL
+                session_client.fetch_token(token_url=oauth_config.token_url, auth=auth)
+            except Exception as e:
+                logger.error(f"Error fetching OAuth2 token: {e}")
+                raise ClientUnsuccessfulException(
+                    status_code=HTTP_500_INTERNAL_SERVER_ERROR
+                )
+
+            return session_client.post(
+                url=config.url, headers=additional_headers, json=request_body
+            )
+
+        def _request_without_oauth() -> requests.Response:
+            """Helper function to make a request without OAuth2 authentication"""
+            headers = self.build_authorization_header()
+            headers.update(additional_headers)
+            return requests.post(url=config.url, headers=headers, json=request_body)
+
+        oauth_config = self.configuration.oauth_config
 
         try:
-            response = requests.post(url=config.url, headers=headers, json=request_body)
+            response = (
+                _request_with_oauth()
+                if oauth_config is not None
+                else _request_without_oauth()
+            )
+
+            if not response_expected:
+                return {}
+
+            if not response.ok:
+                logger.error("Invalid response received from webhook.")
+                raise ClientUnsuccessfulException(status_code=response.status_code)
+
+            return json.loads(response.text)
+
         except requests.ConnectionError:
-            logger.info("Requests connection error received.")
+            logger.error("HTTPS client received a connection error.")
             raise ClientUnsuccessfulException(
                 status_code=HTTP_500_INTERNAL_SERVER_ERROR
             )
 
-        if not response_expected:
-            return {}
-
-        if not response.ok:
-            logger.error("Invalid response received from webhook.")
-            raise ClientUnsuccessfulException(status_code=response.status_code)
-        return json.loads(response.text)
-
     def test_connection(self) -> Optional[ConnectionTestStatus]:
         """
         Override to skip connection test

fides/api/service/privacy_request/dsr_package/dsr_report_builder.py

@@ -26,7 +26,7 @@ from fides.api.service.storage.util import (
     is_attachment_field,
     process_attachment_naming,
     process_attachments_contextually,
-    resolve_directory_from_context,
+    resolve_path_from_context,
 )
 from fides.api.util.storage_util import StorageJSONEncoder, format_size
 from fides.config import CONFIG
@@ -226,7 +226,7 @@ class DSRReportBuilder:
             file_size = format_attachment_size(attachment.get("file_size"))
 
             # Determine the actual directory for this attachment based on its context
-            actual_directory = resolve_directory_from_context(attachment, directory)
+            actual_directory = resolve_path_from_context(attachment, directory)
 
             # Generate attachment URL using shared utility with actual storage path
             download_url = attachment.get("download_url")

fides/api/service/storage/streaming/smart_open_streaming_storage.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 import csv
 import json
 import time
+from copy import deepcopy
 from datetime import datetime
 from io import BytesIO, StringIO
 from itertools import chain
@@ -36,13 +37,11 @@ from fides.api.service.storage.streaming.schemas import (
 )
 from fides.api.service.storage.streaming.smart_open_client import SmartOpenStorageClient
 from fides.api.service.storage.util import (
-    convert_processed_attachments_to_attachment_processing_info,
     determine_dataset_name_from_path,
-    extract_storage_key_from_attachment,
     get_unique_filename,
     process_attachments_contextually,
     resolve_attachment_storage_path,
-    resolve_base_path_from_context,
+    resolve_path_from_context,
 )
 
 DEFAULT_ATTACHMENT_NAME = "attachment"
@@ -169,7 +168,9 @@ class SmartOpenStreamingStorage:
         """
         try:
             # Extract storage key using shared utility
-            storage_key = extract_storage_key_from_attachment(attachment)
+            storage_key = (
+                attachment.get("download_url") or attachment.get("file_name") or ""
+            )
             if not storage_key:
                 return None
 
@@ -182,7 +183,7 @@ class SmartOpenStreamingStorage:
             )
 
             # Resolve base path using shared utility
-            base_path = resolve_base_path_from_context(attachment)
+            base_path = resolve_path_from_context(attachment)
 
             # Create AttachmentProcessingInfo
             processing_info = AttachmentProcessingInfo(
@@ -281,7 +282,11 @@ class SmartOpenStreamingStorage:
             ) from e
 
     def _collect_and_validate_attachments(
-        self, data: dict
+        self,
+        data: dict,
+        used_filenames_data: set[str],
+        used_filenames_attachments: set[str],
+        processed_attachments: dict[tuple[str, str], str],
     ) -> list[AttachmentProcessingInfo]:
         """Collect and validate attachments using the same contextual approach as DSR report builder.
 
@@ -294,26 +299,30 @@ class SmartOpenStreamingStorage:
         Returns:
             List of validated AttachmentProcessingInfo objects
         """
-        # Initialize tracking structures (similar to DSR report builder)
-        used_filenames_data: set[str] = set()
-        used_filenames_attachments: set[str] = set()
-        processed_attachments: dict[tuple[str, str], str] = {}
+        validated_attachments = []
 
         # Use the shared contextual processing function
         # Note: This method should only be used when DSR report builder is not available
         # For HTML format, use _collect_and_validate_attachments_from_dsr_builder instead
         processed_attachments_list = process_attachments_contextually(
-            data,
-            used_filenames_data,
-            used_filenames_attachments,
-            processed_attachments,
-            enable_streaming=True,  # Always use streaming mode for storage
+            data=data,
+            used_filenames_data=used_filenames_data,
+            used_filenames_attachments=used_filenames_attachments,
+            processed_attachments=processed_attachments,
+            enable_streaming=True,
         )
 
-        # Convert to AttachmentProcessingInfo objects using shared utility
-        return convert_processed_attachments_to_attachment_processing_info(
-            processed_attachments_list, self._validate_attachment
-        )
+        for processed_attachment in processed_attachments_list:
+            # Add context information to the attachment data
+            attachment_with_context = deepcopy(processed_attachment["attachment"])
+            attachment_with_context["_context"] = processed_attachment["context"]
+
+            # Validate and convert to AttachmentProcessingInfo
+            validated = self._validate_attachment(attachment_with_context)
+            if validated:
+                validated_attachments.append(validated)
+
+        return validated_attachments
 
     def _collect_and_validate_attachments_from_dsr_builder(
         self, data: dict, dsr_builder: "DSRReportBuilder"
@@ -342,17 +351,11 @@ class SmartOpenStreamingStorage:
             else:
                 used_filenames_data.update(filenames)
 
-        processed_attachments_list = process_attachments_contextually(
+        return self._collect_and_validate_attachments(
             data,
             used_filenames_data,
             used_filenames_attachments,
             dsr_builder.processed_attachments,
-            enable_streaming=True,  # Always use streaming mode for storage
-        )
-
-        # Convert to AttachmentProcessingInfo objects using shared utility
-        return convert_processed_attachments_to_attachment_processing_info(
-            processed_attachments_list, self._validate_attachment
         )
 
     @retry_cloud_storage_operation(
@@ -374,7 +377,7 @@ class SmartOpenStreamingStorage:
         """Upload data to cloud storage using smart-open streaming for memory efficiency.
 
         This function leverages smart-open's streaming capabilities while maintaining
-        our DSR-specific business logic for package splitting and attachment processing.
+        our DSR-specific business logic for package and attachment processing.
         All data is streamed directly from source to destination without local storage.
 
         Args:
@@ -627,7 +630,12 @@ class SmartOpenStreamingStorage:
             resp_format: Response format (csv, json)
         """
         # Collect and validate all attachments using shared contextual processing
-        all_attachments = self._collect_and_validate_attachments(data)
+        all_attachments = self._collect_and_validate_attachments(
+            data=data,
+            used_filenames_data=set(),
+            used_filenames_attachments=set(),
+            processed_attachments={},
+        )
 
         if not all_attachments:
             # No attachments, just upload the data

fides/api/service/storage/util.py

@@ -544,31 +544,9 @@ def _process_attachment_list(
     return processed_attachments_list
 
 
-def extract_storage_key_from_attachment(attachment: dict[str, Any]) -> str:
-    """
-    Extract storage key from attachment data with fallback logic.
-
-    This function provides a consistent way to extract storage keys from
-    attachment dictionaries across different components.
-
-    Args:
-        attachment: The attachment dictionary
-
-    Returns:
-        The storage key (URL or filename) for the attachment
-    """
-    if original_url := attachment.get("original_download_url"):
-        return original_url
-
-    if download_url := attachment.get("download_url"):
-        return download_url
-
-    file_name = attachment.get("file_name")
-    return file_name if file_name is not None else ""
-
-
-def resolve_base_path_from_context(
-    attachment: dict[str, Any], default_base_path: str = "attachments"
+def resolve_path_from_context(
+    attachment: dict[str, Any],
+    default_path: str = "attachments",
 ) -> str:
     """
     Resolve the base path for an attachment based on its context.
@@ -578,93 +556,24 @@ def resolve_base_path_from_context(
 
     Args:
         attachment: The attachment dictionary
-        default_base_path: Default base path if no context is found
+        default_path: Default path if no context is found
 
     Returns:
-        The resolved base path for the attachment
+        The resolved path for the attachment
     """
     if not attachment.get("_context"):
-        return default_base_path
+        return default_path
 
     context = attachment["_context"]
     context_type = context.get("type")
 
-    if context_type == "direct":
-        return f"data/{context['dataset']}/{context['collection']}/attachments"
-    if context_type == "nested":
-        return f"data/{context['dataset']}/{context['collection']}/attachments"
+    if context_type in ["direct", "nested"]:
+        dataset = context.get("dataset", "")
+        collection = context.get("collection", "")
+        return f"data/{dataset}/{collection}/attachments"
     if context_type == "top_level":
         return "attachments"
-    # Handle old context format
     if context.get("key") and context.get("item_id"):
         return f"{context['key']}/{context['item_id']}/attachments"
-    # Fallback for unknown context types
-    return "unknown/unknown/attachments"
-
-
-def resolve_directory_from_context(
-    attachment: dict[str, Any], default_directory: str = "attachments"
-) -> str:
-    """
-    Resolve the directory path for an attachment based on its context.
-
-    This function provides consistent directory resolution logic for DSR report builder.
-
-    Args:
-        attachment: The attachment dictionary
-        default_directory: Default directory if no context is found
-
-    Returns:
-        The resolved directory path for the attachment
-    """
-    if not attachment.get("_context"):
-        return default_directory
-
-    context = attachment["_context"]
-    context_type = context.get("type")
-
-    if context_type == "direct":
-        return f"data/{context['dataset']}/{context['collection']}"
-    if context_type == "nested":
-        return f"data/{context['dataset']}/{context['collection']}"
-    if context_type == "top_level":
-        return "attachments"
-    if context.get("key") and context.get("item_id"):
-        return f"{context['key']}/{context['item_id']}"
-
-    return default_directory
-
-
-def convert_processed_attachments_to_attachment_processing_info(
-    processed_attachments_list: list[dict[str, Any]], validate_attachment_func: Callable
-) -> list[Any]:
-    """
-    Convert processed attachments list to AttachmentProcessingInfo objects.
-
-    This is a shared utility function to avoid duplication between different
-    attachment collection methods.
-
-    Args:
-        processed_attachments_list: List of processed attachment dictionaries
-        validate_attachment_func: Function to validate individual attachments
-                                 Signature: validate_attachment_func(attachment_with_context) -> AttachmentProcessingInfo | None
-
-    Returns:
-        List of validated AttachmentProcessingInfo objects
-    """
-    validated_attachments = []
-
-    for processed_attachment in processed_attachments_list:
-        attachment_data = processed_attachment["attachment"]
-
-        # Add context information to the attachment data
-        attachment_with_context = attachment_data.copy()
-        attachment_with_context["_context"] = processed_attachment["context"]
-
-        # Validate and convert to AttachmentProcessingInfo
-        if validate_attachment_func is not None:
-            validated = validate_attachment_func(attachment_with_context)
-            if validated:
-                validated_attachments.append(validated)
 
-    return validated_attachments
+    return default_path

fides/api/task/conditional_dependencies/sql_schemas.py

@@ -0,0 +1,301 @@
+import types
+import uuid
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+from sqlalchemy import Column, any_, bindparam
+
+from fides.api.task.conditional_dependencies.schemas import Operator
+
+
+class SQLTranslationError(Exception):
+    """Error raised when SQL translation fails"""
+
+
+def _escape_like_pattern(val: Any) -> str:
+    """
+    Escape LIKE wildcards in user input to prevent pattern injection attacks.
+
+    This prevents users from injecting wildcards (% and _) that could:
+    - Match unintended records through pattern injection
+    - Cause performance issues with expensive wildcard operations
+    - Enable information disclosure through pattern probing
+
+    Note: This function escapes ALL % and _ characters as a security measure.
+    While this may seem aggressive for normal field names, it's necessary because:
+    1. We can't distinguish between intentional and malicious wildcards
+    2. Field values shouldn't typically contain SQL wildcards
+    3. Better to be overly cautious with user input
+
+    Args:
+        val: User input value to escape
+
+    Returns:
+        Escaped string safe for use in LIKE patterns
+
+    Examples:
+        _escape_like_pattern("admin%") -> "admin\\%"
+        _escape_like_pattern("test_user") -> "test\\_user"  # Escapes _ for security
+        _escape_like_pattern("normal") -> "normal"
+    """
+    if val is None:
+        return ""
+
+    val_str = str(val)
+    # Escape backslashes first to prevent double-escaping
+    # Then escape LIKE wildcards (% and _)
+    return val_str.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+
+
+def _validate_and_escape_json_path_component(component: str) -> str:
+    """
+    Validate and escape JSON path components to prevent injection attacks.
+
+    This prevents users from injecting malicious content in JSON path components that could:
+    - Break out of single quotes in PostgreSQL JSON operators
+    - Inject arbitrary SQL through malformed JSON paths
+    - Cause syntax errors or unexpected behavior
+
+    Args:
+        component: JSON path component to validate and escape
+
+    Returns:
+        Validated and escaped component safe for use in PostgreSQL JSON operators
+
+    Raises:
+        SQLTranslationError: If the component contains invalid characters
+
+    Examples:
+        _validate_and_escape_json_path_component("field") -> "field"
+        _validate_and_escape_json_path_component("field'name") -> "field''name"
+        _validate_and_escape_json_path_component("") -> raises SQLTranslationError
+    """
+    if not component or not isinstance(component, str):
+        raise SQLTranslationError(
+            f"Invalid JSON path component: '{component}'. Components must be non-empty strings."
+        )
+
+    # Check for potentially dangerous characters
+    if len(component) > 100:  # Reasonable limit for JSON field names
+        raise SQLTranslationError(
+            f"JSON path component too long: '{component[:50]}...'. Maximum length is 100 characters."
+        )
+
+    # Escape single quotes for PostgreSQL JSON operators (double them)
+    escaped_component = component.replace("'", "''")
+
+    return escaped_component
+
+
+def _handle_list_contains(col: Column, val: Any) -> Column:
+    """
+    Handle list_contains operator for different scenarios:
+    - If val is a list: check if column value is IN the list
+    - If val is a single value: check if column contains the value (for arrays/JSON) or use LIKE (for strings)
+
+    Args:
+        col: SQLAlchemy column
+        val: Value to compare against
+
+    Returns:
+        SQLAlchemy expression for the comparison
+    """
+    if isinstance(val, list):
+        # If value is a list, check if column value is IN the list
+        return col.in_(val)
+
+    # For single values, we need to handle different column types
+    # Check if this is a PostgreSQL array column
+    if hasattr(col.type, "item_type") or str(col.type).startswith("ARRAY"):
+        # This is a PostgreSQL array - use the ANY operator with proper parameter binding
+        # This is safer and prevents SQL injection
+        # Generate unique parameter name to avoid conflicts when multiple list operations exist in same query
+        unique_param_name = f"array_val_{uuid.uuid4().hex[:8]}"
+        param = bindparam(unique_param_name, val)
+        return param == any_(col)
+
+    # Try JSON containment for JSONB/JSON columns
+    try:
+        return col.contains(val)
+    except Exception:
+        # Fallback to LIKE for string columns with escaped wildcards
+        escaped_val = _escape_like_pattern(val)
+        return col.like(f"%{escaped_val}%", escape="\\")
+
+
+OPERATOR_MAP = types.MappingProxyType(
+    {
+        Operator.eq: lambda col, val: col == val,
+        Operator.neq: lambda col, val: col != val,
+        Operator.lt: lambda col, val: col < val,
+        Operator.lte: lambda col, val: col <= val,
+        Operator.gt: lambda col, val: col > val,
+        Operator.gte: lambda col, val: col >= val,
+        Operator.contains: lambda col, val: col.like(
+            f"%{_escape_like_pattern(val)}%", escape="\\"
+        ),
+        Operator.starts_with: lambda col, val: col.like(
+            f"{_escape_like_pattern(val)}%", escape="\\"
+        ),
+        Operator.ends_with: lambda col, val: col.like(
+            f"%{_escape_like_pattern(val)}", escape="\\"
+        ),
+        Operator.exists: lambda col, val: col.isnot(None),
+        Operator.not_exists: lambda col, val: col.is_(None),
+        Operator.list_contains: _handle_list_contains,
+        Operator.not_in_list: lambda col, val: ~_handle_list_contains(col, val),
+    }
+)
+
+
+class FieldAddress(BaseModel):
+    """Parsed field address with table and column information"""
+
+    table_name: str = Field(description="Table name extracted from field address")
+    column_name: str = Field(description="Base column name without JSON path")
+    json_path: Optional[list[str]] = Field(
+        default=None, description="JSON path components if this is a JSON field"
+    )
+    full_address: str = Field(description="Original field address string")
+
+    def __hash__(self) -> int:
+        """Make FieldAddress hashable for use in sets"""
+        return hash(
+            (
+                self.table_name,
+                self.column_name,
+                tuple(self.json_path) if self.json_path else None,
+            )
+        )
+
+    def __eq__(self, other: Any) -> bool:
+        """Enable equality comparison for FieldAddress objects"""
+        if not isinstance(other, FieldAddress):
+            return False
+        return (
+            self.table_name == other.table_name
+            and self.column_name == other.column_name
+            and self.json_path == other.json_path
+        )
+
+    def to_sql_column(self, enable_json_operators: bool = True) -> str:
+        """
+        Convert field address to PostgreSQL column reference
+
+        Args:
+            enable_json_operators: Whether to use PostgreSQL JSON operators for JSON paths
+
+        Returns:
+            SQL column reference string
+        """
+        is_json_path = self.json_path is not None and len(self.json_path) > 0
+        if not is_json_path or not enable_json_operators:
+            return self.column_name
+
+        if self.json_path is None:
+            # This should never happen
+            raise SQLTranslationError(
+                "Field address internal error."
+            )  # pragma: no cover
+
+        # Build PostgreSQL JSON path: column->'path'->'path'->>'final_path'
+        # Validate and escape all components to prevent injection
+        if self.json_path and len(self.json_path) == 1:
+            # Simple case: column->>'field'
+            escaped_component = _validate_and_escape_json_path_component(
+                self.json_path[0]
+            )
+            return f"{self.column_name}->>'{escaped_component}'"
+
+        # Complex case: column->'field'->'field'->>'final_field'
+        path_parts = []
+        # All but last use -> operator
+        for part in self.json_path[:-1]:
+            escaped_part = _validate_and_escape_json_path_component(part)
+            path_parts.append(f"->'{escaped_part}'")
+        # Last part uses ->> operator (returns text)
+        escaped_final = _validate_and_escape_json_path_component(self.json_path[-1])
+        path_parts.append(f"->>'{escaped_final}'")
+
+        return f"{self.column_name}{''.join(path_parts)}"
+
+    @classmethod
+    def _parse_parts_to_components(
+        cls, parts: list[str]
+    ) -> tuple[str, str, Optional[list[str]]]:
+        """
+        Parse a list of parts into table_name, column_name, and json_path components.
+
+        Args:
+            parts: List of address parts (e.g., ["table", "column", "path1", "path2"])
+
+        Returns:
+            Tuple of (table_name, column_name, json_path)
+        """
+        if len(parts) < 2:
+            return "", parts[0] if parts else "", None
+
+        table_name = parts[0]
+        column_name = parts[1]
+        json_path = parts[2:] if len(parts) > 2 else None
+
+        return table_name, column_name, json_path
+
+    @classmethod
+    def parse(cls, field_address: str) -> "FieldAddress":
+        """
+        Parse a field address into components
+
+        Supports formats:
+        - "table.column" -> table="table", column="column"
+        - "table.json_column.path.subpath" -> table="table", column="json_column", json_path=["path", "subpath"]
+        - "table:column" -> table="table", column="column" (alternative format)
+        - "column" -> table="", column="column" (requires default table)
+        """
+        if ":" in field_address:
+            # Format: table:column or dataset:collection:field
+            # Handle colon-separated format first, then check for dots in the remaining parts
+            colon_parts = field_address.split(":")
+            if len(colon_parts) >= 2:
+                table_name = colon_parts[0]
+                remaining = ":".join(colon_parts[1:])  # Join remaining parts
+
+                # Check if the remaining part has dots (JSON path)
+                if "." in remaining:
+                    # Mixed format: table:column.path1.path2
+                    dot_parts = remaining.split(".")
+                    parts = [table_name] + dot_parts
+                else:
+                    # Pure colon format: table:column or table:column:path1:path2
+                    parts = colon_parts
+
+                table_name, column_name, json_path = cls._parse_parts_to_components(
+                    parts
+                )
+
+                return cls(
+                    table_name=table_name,
+                    column_name=column_name,
+                    json_path=json_path,
+                    full_address=field_address,
+                )
+
+        if "." in field_address:
+            # Format: table.column or table.json_column.path.subpath
+            parts = field_address.split(".")
+            table_name, column_name, json_path = cls._parse_parts_to_components(parts)
+
+            return cls(
+                table_name=table_name,
+                column_name=column_name,
+                json_path=json_path,
+                full_address=field_address,
+            )
+
+        # Simple column name without table
+        return cls(
+            table_name="",  # Will need default table
+            column_name=field_address,
+            json_path=None,
+            full_address=field_address,
+        )