ethyca-fides 2.67.2rc0__py2.py3-none-any.whl → 2.67.3b0__py2.py3-none-any.whl

fides/api/models/detection_discovery/core.py

@@ -380,38 +380,51 @@ class StagedResourceAncestor(Base):
     )
 
     @classmethod
-    def create_staged_resource_ancestor_links(
+    def create_all_staged_resource_ancestor_links(
         cls,
         db: Session,
-        resource_urn: str,
-        ancestor_urns: Set[str],
+        ancestor_links: Dict[str, Set[str]],
+        batch_size: int = 10000,  # Conservative batch size
     ) -> None:
         """
-        Bulk inserts entries in the StagedResourceAncestor table
-        based on the provided resource URN and the set of its ancestor URNs.
+        Bulk inserts all entries in the StagedResourceAncestor table
+        based on the provided mapping of descendant URNs to their ancestor URN sets.
 
         We execute the bulk INSERT with the provided (synchronous) db session,
         but the transaction is _not_ committed, so the caller must commit the transaction
         to persist the changes.
+
+        Uses batching to handle large datasets without hitting PostgreSQL parameter limits.
+
+        Args:
+            db: Database session
+            ancestor_links: Dict mapping descendant URNs to sets of ancestor URNs
         """
-        links_to_insert = []
+        stmt_text = text(
+            """
+            INSERT INTO stagedresourceancestor (id, ancestor_urn, descendant_urn)
+            VALUES ('srl_' || gen_random_uuid(), :ancestor_urn, :descendant_urn)
+            ON CONFLICT (ancestor_urn, descendant_urn) DO NOTHING;
+            """
+        )
 
-        for ancestor_urn in ancestor_urns:
-            links_to_insert.append(
-                {"ancestor_urn": ancestor_urn, "descendant_urn": resource_urn}
-            )
+        current_batch = []
 
-        if links_to_insert:
-            # Using raw SQL for ON CONFLICT with parameters for safety
-            stmt_text = text(
-                """
-                INSERT INTO stagedresourceancestor (id, ancestor_urn, descendant_urn)
-                VALUES ('srl_' || gen_random_uuid(), :ancestor_urn, :descendant_urn)
-                ON CONFLICT (ancestor_urn, descendant_urn) DO NOTHING;
-                """
-            )
+        for descendant_urn, ancestor_urns in ancestor_links.items():
+            if ancestor_urns:  # Only create links if there are ancestors
+                for ancestor_urn in ancestor_urns:
+                    current_batch.append(
+                        {"ancestor_urn": ancestor_urn, "descendant_urn": descendant_urn}
+                    )
+
+                    # Execute batch when it reaches the desired size
+                    if len(current_batch) >= batch_size:
+                        db.execute(stmt_text, current_batch)
+                        current_batch = []
 
-            db.execute(stmt_text, links_to_insert)
+        # Execute any remaining items in the final batch
+        if current_batch:
+            db.execute(stmt_text, current_batch)
 
 
 class StagedResource(Base):

fides/api/models/manual_task/manual_task.py

@@ -383,15 +383,13 @@ class ManualTaskInstance(Base):
 
     @property
     def incomplete_fields(self) -> list["ManualTaskConfigField"]:
-        """Get all fields that haven't been completed yet.
-        A field is considered incomplete if:
-        1. It's required and has no submission
+        """Get all fields that have no submission.
         Returns:
             list[ManualTaskConfigField]: List of incomplete fields
         """
         return [
             field
-            for field in self.required_fields
+            for field in self.config.field_definitions
             if not self.get_submission_for_field(field.id)
         ]
 
@@ -401,8 +399,7 @@ class ManualTaskInstance(Base):
         return [
             field
             for field in self.config.field_definitions
-            if field.field_metadata.get("required", False)
-            and self.get_submission_for_field(field.id)
+            if self.get_submission_for_field(field.id)
         ]
 
     def get_submission_for_field(

fides/api/models/privacy_request/privacy_request.py

@@ -47,8 +47,16 @@ from fides.api.models.attachment import (
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.client import ClientDetail
 from fides.api.models.comment import Comment, CommentReference, CommentReferenceType
+from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.field_types import EncryptedLargeDataDescriptor
+from fides.api.models.manual_task import (
+    ManualTask,
+    ManualTaskConfig,
+    ManualTaskConfigurationType,
+    ManualTaskEntityType,
+    ManualTaskInstance,
+)
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.masking_secret import MaskingSecret
 from fides.api.models.policy import (
@@ -200,6 +208,14 @@ class PrivacyRequest(
         viewonly=True,
         uselist=True,
     )
+    manual_task_instances = relationship(
+        "ManualTaskInstance",
+        lazy="select",
+        passive_deletes="all",
+        primaryjoin="and_(ManualTaskInstance.entity_id==foreign(PrivacyRequest.id), "
+        "ManualTaskInstance.entity_type=='privacy_request')",
+        uselist=True,
+    )
     property_id = Column(String, nullable=True)
 
     cancel_reason = Column(String(200))
@@ -1175,6 +1191,68 @@ class PrivacyRequest(
             db, manual_webhook_id, "erasure_manual_webhook"
         )
 
+    def create_manual_task_instances(
+        self, db: Session, connection_configs_with_manual_tasks: list[ConnectionConfig]
+    ) -> list[ManualTaskInstance]:
+        """Create ManualTaskInstance entries for all active manual tasks relevant to a privacy request."""
+        # Early return if no relevant policy rules
+        policy_rules = {
+            ActionType.access: bool(
+                self.policy.get_rules_for_action(action_type=ActionType.access)
+            ),
+            ActionType.erasure: bool(
+                self.policy.get_rules_for_action(action_type=ActionType.erasure)
+            ),
+        }
+
+        if not any(policy_rules.values()):
+            return []
+
+        # Build configuration types using list comprehension
+        config_types = [
+            (
+                ManualTaskConfigurationType.access_privacy_request
+                if action_type == ActionType.access
+                else ManualTaskConfigurationType.erasure_privacy_request
+            )
+            for action_type, has_rules in policy_rules.items()
+            if has_rules
+        ]
+
+        # Get all relevant manual tasks and configs in one query
+        connection_config_ids = [cc.id for cc in connection_configs_with_manual_tasks]
+        manual_tasks_with_configs = (
+            db.query(ManualTask, ManualTaskConfig)
+            .join(ManualTaskConfig, ManualTask.id == ManualTaskConfig.task_id)
+            .filter(
+                ManualTask.parent_entity_id.in_(connection_config_ids),
+                ManualTask.parent_entity_type == "connection_config",
+                ManualTaskConfig.is_current.is_(True),
+                ManualTaskConfig.config_type.in_(config_types),
+            )
+            .all()
+        )
+
+        # Get existing config IDs to avoid duplicates
+        existing_config_ids = {
+            instance.config_id for instance in self.manual_task_instances
+        }
+
+        # Create instances using list comprehension and filter out existing ones
+        return [
+            ManualTaskInstance.create(
+                db=db,
+                data={
+                    "entity_id": self.id,
+                    "entity_type": ManualTaskEntityType.privacy_request,
+                    "task_id": manual_task.id,
+                    "config_id": config.id,
+                },
+            )
+            for manual_task, config in manual_tasks_with_configs
+            if config.id not in existing_config_ids
+        ]
+
     def get_existing_request_task(
         self,
         db: Session,

fides/api/service/privacy_request/dsr_package/dsr_report_builder.py

@@ -13,7 +13,7 @@ from loguru import logger
 
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.policy import ActionType
-from fides.api.util.storage_util import StorageJSONEncoder
+from fides.api.util.storage_util import StorageJSONEncoder, format_size
 
 DSR_DIRECTORY = Path(__file__).parent.resolve()
 
@@ -204,7 +204,7 @@ class DsrReportBuilder:
 
             file_size = attachment.get("file_size")
             if isinstance(file_size, (int, float)):
-                file_size = self._format_size(float(file_size))
+                file_size = format_size(float(file_size))
             else:
                 file_size = "Unknown"
 
@@ -321,22 +321,6 @@ class DsrReportBuilder:
 
         return datasets
 
-    def _format_size(self, size_bytes: float) -> str:
-        """
-        Format size in bytes to human readable format.
-
-        Args:
-            size_bytes: Size in bytes
-
-        Returns:
-            Formatted string with appropriate unit (B, KB, MB, GB)
-        """
-        for unit in ["B", "KB", "MB", "GB"]:
-            if size_bytes < 1024.0:
-                return f"{size_bytes:.1f} {unit}"
-            size_bytes /= 1024.0
-        return f"{size_bytes:.1f} TB"
-
     def generate(self) -> BytesIO:
         """
         Processes the request and DSR data to build zip file containing the DSR report.
@@ -395,7 +379,7 @@ class DsrReportBuilder:
 
         # Calculate time taken and file size
         time_taken = time_module.time() - start_time
-        file_size = self._format_size(float(len(self.baos.getvalue())))
+        file_size = format_size(float(len(self.baos.getvalue())))
 
         logger.bind(time_to_generate=time_taken, dsr_package_size=file_size).info(
             "DSR report generation complete."

fides/api/task/conditional_dependencies/evaluator.py

@@ -1,10 +1,10 @@
-import operator as py_operator
 from typing import Any, Union
 
 from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.graph.config import FieldPath
+from fides.api.task.conditional_dependencies.operators import operator_methods
 from fides.api.task.conditional_dependencies.schemas import (
     Condition,
     ConditionGroup,
@@ -13,18 +13,9 @@ from fides.api.task.conditional_dependencies.schemas import (
     Operator,
 )
 
-operator_methods = {
-    Operator.exists: lambda a, _: a is not None,
-    Operator.not_exists: lambda a, _: a is None,
-    Operator.eq: py_operator.eq,
-    Operator.neq: py_operator.ne,
-    Operator.lt: lambda a, b: a < b if a is not None else False,
-    Operator.lte: lambda a, b: a <= b if a is not None else False,
-    Operator.gt: lambda a, b: a > b if a is not None else False,
-    Operator.gte: lambda a, b: a >= b if a is not None else False,
-    Operator.list_contains: lambda a, b: b in a if isinstance(a, list) else False,
-    Operator.not_in_list: lambda a, b: a not in b if isinstance(b, list) else True,
-}
+
+class ConditionEvaluationError(Exception):
+    """Error raised when a condition evaluation fails"""
 
 
 class ConditionEvaluator:
@@ -44,9 +35,9 @@ class ConditionEvaluator:
         self, condition: ConditionLeaf, data: Union[dict, Any]
     ) -> bool:
         """Evaluate a leaf condition against input data"""
-        actual_value = self._get_nested_value(data, condition.field_address.split("."))
+        data_value = self._get_nested_value(data, condition.field_address.split("."))
         # Apply operator and return result
-        return self._apply_operator(actual_value, condition.operator, condition.value)
+        return self._apply_operator(data_value, condition.operator, condition.value)
 
     def _evaluate_group_condition(
         self, group: ConditionGroup, data: Union[dict, Any]
@@ -96,7 +87,7 @@ class ConditionEvaluator:
         return current if current != {} else None
 
     def _apply_operator(
-        self, actual_value: Any, operator: Operator, expected_value: Any
+        self, data_value: Any, operator: Operator, user_input_value: Any
     ) -> bool:
         """Apply operator to actual and expected values"""
 
@@ -104,6 +95,8 @@ class ConditionEvaluator:
         operator_method = operator_methods.get(operator)
         if operator_method is None:
             logger.warning(f"Unknown operator: {operator}")
-            return False
-
-        return operator_method(actual_value, expected_value)
+            raise ConditionEvaluationError(f"Unknown operator: {operator}")
+        try:
+            return operator_method(data_value, user_input_value)
+        except (TypeError, ValueError) as e:
+            raise ConditionEvaluationError(f"Error evaluating condition: {e}") from e

fides/api/task/conditional_dependencies/operators.py

@@ -0,0 +1,150 @@
+import numbers
+import operator as py_operator
+
+from fides.api.task.conditional_dependencies.schemas import Operator
+
+# Define operator methods for validation
+#
+# None Value Handling:
+# - Basic operators (eq, neq, exists, not_exists) handle None naturally
+#   * Note: eq and neq use Python's built-in behavior, so True == 1 returns True
+# - Numeric operators return False for None values (can't compare None with numbers)
+#   * Note: Boolean values are excluded from numeric comparisons (True < 10 returns False)
+# - String operators return False for None values (can't perform string operations on None)
+# - List operators handle None naturally using Python's built-in behavior:
+#   * None in [None] returns True
+#   * None not in [None] returns False
+#   * None in [] returns False
+#   * None not in [] returns True
+#   * This allows None to be a valid list element for membership testing
+operator_methods = {
+    # Basic operators - handle None naturally using Python's built-in behavior
+    Operator.exists: lambda a, _: a is not None,
+    Operator.not_exists: lambda a, _: a is None,
+    Operator.eq: py_operator.eq,
+    # None == None returns True, None == "anything" returns False
+    # None != None returns False, None != "anything" returns True
+    Operator.neq: py_operator.ne,
+    # Numeric comparison operators - return False for None or non-numeric types
+    # Note: Boolean values are excluded as they are not considered numeric for comparisons
+    # This differs from basic comparison operators (eq, neq) which use Python's built-in behavior
+    Operator.lt: lambda a, b: (
+        a < b
+        if a is not None and isinstance(a, numbers.Number) and not isinstance(a, bool)
+        else False
+    ),
+    Operator.lte: lambda a, b: (
+        a <= b
+        if a is not None and isinstance(a, numbers.Number) and not isinstance(a, bool)
+        else False
+    ),
+    Operator.gt: lambda a, b: (
+        a > b
+        if a is not None and isinstance(a, numbers.Number) and not isinstance(a, bool)
+        else False
+    ),
+    Operator.gte: lambda a, b: (
+        a >= b
+        if a is not None and isinstance(a, numbers.Number) and not isinstance(a, bool)
+        else False
+    ),
+    Operator.list_contains: lambda a, b: (
+        # If user provides a list, check if column value is in it
+        # Note: Handles None values naturally using Python's built-in behavior
+        a in b
+        if isinstance(b, list)
+        # If column value is a list, check if user value is in it
+        else b in a if isinstance(a, list) else False
+    ),
+    Operator.not_in_list: lambda a, b: (
+        # If user provides a list, check if column value is NOT in it
+        # Note: Handles None values naturally using Python's built-in behavior
+        a not in b
+        if isinstance(b, list)
+        # If column value is a list, check if user value is NOT in it
+        else b not in a if isinstance(a, list) else False
+    ),
+    Operator.list_intersects: lambda a, b: (
+        # Check if there are any common elements between the lists
+        # Note: Returns False for None values because both values must be lists
+        bool(set(a) & set(b))
+        if isinstance(a, list) and isinstance(b, list)
+        else False
+    ),
+    Operator.list_subset: lambda a, b: (
+        # Check if column list is a subset of user's list
+        # Note: Returns False for None values because both values must be lists
+        set(a).issubset(set(b))
+        if isinstance(a, list) and isinstance(b, list)
+        else False
+    ),
+    Operator.list_superset: lambda a, b: (
+        # Check if column list is a superset of user's list
+        # Note: Returns False for None values because both values must be lists
+        set(a).issuperset(set(b))
+        if isinstance(a, list) and isinstance(b, list)
+        else False
+    ),
+    Operator.list_disjoint: lambda a, b: (
+        # Check if lists have no common elements
+        # Note: Returns False for None values because both values must be lists
+        set(a).isdisjoint(set(b))
+        if isinstance(a, list) and isinstance(b, list)
+        else False
+    ),
+    Operator.starts_with: lambda a, b: (
+        a.startswith(b) if isinstance(a, str) and isinstance(b, str) else False
+    ),
+    Operator.ends_with: lambda a, b: (
+        a.endswith(b) if isinstance(a, str) and isinstance(b, str) else False
+    ),
+    Operator.contains: lambda a, b: (
+        b in a if isinstance(a, str) and isinstance(b, str) else False
+    ),
+}
+
+# Common operators that work with most data types
+COMMON_OPERATORS = {
+    Operator.eq,
+    Operator.neq,
+    Operator.exists,
+    Operator.not_exists,
+}
+
+# Numeric comparison operators
+NUMERIC_OPERATORS = {Operator.lt, Operator.lte, Operator.gt, Operator.gte}
+
+# String-specific operators
+STRING_OPERATORS = {
+    Operator.contains,
+    Operator.starts_with,
+    Operator.ends_with,
+}
+
+# List operations that work with all data types
+LIST_OPERATORS = {
+    Operator.list_contains,  # Element in list
+    Operator.not_in_list,  # Element not in list
+    Operator.list_intersects,  # Any common elements
+    Operator.list_subset,  # Column list ⊆ user list
+    Operator.list_superset,  # Column list ⊇ user list
+    Operator.list_disjoint,  # No common elements
+}
+
+# Define data type compatibility with operators
+data_type_operator_compatibility = {
+    "integer": {*COMMON_OPERATORS, *NUMERIC_OPERATORS, *LIST_OPERATORS},
+    "float": {*COMMON_OPERATORS, *NUMERIC_OPERATORS, *LIST_OPERATORS},
+    "double": {*COMMON_OPERATORS, *NUMERIC_OPERATORS, *LIST_OPERATORS},
+    "long": {*COMMON_OPERATORS, *NUMERIC_OPERATORS, *LIST_OPERATORS},
+    "boolean": {*COMMON_OPERATORS, *LIST_OPERATORS},
+    "string": {*COMMON_OPERATORS, *STRING_OPERATORS, *LIST_OPERATORS},
+    "text": {
+        *COMMON_OPERATORS,
+        *LIST_OPERATORS,
+    },  # Form input - no string search operations
+    "array": {*COMMON_OPERATORS, *LIST_OPERATORS},
+    "object": {*COMMON_OPERATORS, *LIST_OPERATORS},
+    "object_id": {*COMMON_OPERATORS, *LIST_OPERATORS},
+    "no_op": {*COMMON_OPERATORS, *LIST_OPERATORS},
+}

fides/api/task/conditional_dependencies/schemas.py

@@ -5,16 +5,67 @@ from pydantic import BaseModel, Field, model_validator
 
 
 class Operator(str, Enum):
+    # Basic comparison operators
+    # Column value equals user input (e.g., user.role eq "admin")
     eq = "eq"
+    # Column value not equal to user input (e.g., user.status neq "inactive")
     neq = "neq"
+
+    # Numeric comparison operators
+    # Column value less than user input (e.g., user.age lt 18)
     lt = "lt"
+    # Column value less than or equal to user input (e.g., user.score lte 100)
     lte = "lte"
+    # Column value greater than user input (e.g., user.balance gt 1000)
     gt = "gt"
+    # Column value greater than or equal to user input (e.g., user.rating gte 4.0)
     gte = "gte"
+
+    # Existence operators
+    # Field exists and is not None (e.g., user.email exists)
     exists = "exists"
+    # Field does not exist or is None (e.g., user.middle_name not_exists)
     not_exists = "not_exists"
+
+    # List membership operators (work with both single values and lists)
+    #
+    # Column value is in user's list OR user's value is in column's list
     list_contains = "list_contains"
+    # Examples: user.role list_contains ["admin", "moderator"] (role in list)
+    #          user.permissions list_contains "write" (value in permissions)
+
+    # Column value is NOT in user's list OR user's value is NOT in column's list
     not_in_list = "not_in_list"
+    # Examples: user.role not_in_list ["banned", "suspended"] (role not blocked)
+    #          user.permissions not_in_list "delete" (value not in permissions)
+
+    # List-to-list comparison operators (both values must be lists)
+    # Lists have at least one common element
+    list_intersects = "list_intersects"
+    # Example: user.roles list_intersects ["admin", "moderator"] (any common role)
+
+    # Column list is completely contained within user's list
+    list_subset = "list_subset"
+    # Example: user.permissions list_subset ["read", "write", "delete", "manage"]
+    #         (all user permissions are allowed)
+
+    # Column list completely contains user's list
+    list_superset = "list_superset"
+    # Example: user.tags list_superset ["premium", "verified"]
+    #         (user has all required tags plus extras)
+
+    # Lists have no common elements
+    list_disjoint = "list_disjoint"
+    # Example: user.roles list_disjoint ["banned", "suspended"]
+    #         (user has no restricted roles)
+
+    # String operators
+    # String starts with user input (e.g., user.email starts_with "admin@")
+    starts_with = "starts_with"
+    # String ends with user input (e.g., user.domain ends_with ".com")
+    ends_with = "ends_with"
+    # String contains user input (e.g., user.description contains "verified")
+    contains = "contains"
 
 
 class GroupOperator(str, Enum):

fides/api/task/create_request_tasks.py

@@ -35,7 +35,7 @@ from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
 from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
-    create_manual_task_instances_for_privacy_request,
+    get_connection_configs_with_manual_tasks,
 )
 from fides.api.util.logger_context_utils import log_context
 
@@ -92,7 +92,7 @@ def build_access_networkx_digraph(
     manual_nodes = [
         addr
         for addr in traversal_nodes.keys()
-        if addr.collection == ManualTaskAddress.MANUAL_DATA_COLLECTION
+        if ManualTaskAddress.is_manual_task_address(addr)
     ]
     for manual_node in manual_nodes:
         networkx_graph.add_edge(ROOT_COLLECTION_ADDRESS, manual_node)
@@ -472,7 +472,9 @@ def run_access_request(
             )
 
             # Snapshot manual task field instances for this privacy request
-            create_manual_task_instances_for_privacy_request(session, privacy_request)
+            privacy_request.create_manual_task_instances(
+                session, get_connection_configs_with_manual_tasks(session)
+            )
 
             # Save Access Request Tasks to the database
             ready_tasks = persist_new_access_request_tasks(

fides/api/task/filter_results.py

@@ -39,7 +39,7 @@ def filter_data_categories(
             continue
 
         # Skip manual task data - it doesn't need filtering since it's controlled by field definitions
-        if f":{ManualTaskAddress.MANUAL_DATA_COLLECTION}" in node_address:
+        if ManualTaskAddress.is_manual_task_address(node_address):
             filtered_access_results[node_address].extend(results)
             continue
 
@@ -122,6 +122,10 @@ def select_and_save_field(saved: Any, row: Row, target_path: FieldPath) -> Dict:
         """Helper for building new nested resource - can return an empty dict, empty array or resource itself"""
         return type(resource)() if isinstance(resource, (list, dict)) else resource
 
+    # If we've reached the end of the field path, return the entire current object/array
+    if not target_path.levels:
+        return row
+
     if isinstance(row, list):
         for i, elem in enumerate(row):
             try:

fides/api/task/manual/manual_task_address.py

@@ -1,3 +1,5 @@
+from typing import Union
+
 from fides.api.graph.config import CollectionAddress
 
 
@@ -20,7 +22,7 @@ class ManualTaskAddress:
         return collection_name == ManualTaskAddress.MANUAL_DATA_COLLECTION
 
     @staticmethod
-    def is_manual_task_address(address: CollectionAddress) -> bool:
+    def is_manual_task_address(address: Union[str, CollectionAddress]) -> bool:
         """Check if address represents manual task data"""
         if isinstance(address, str):
             # Handle string format "connection_key:collection_name"
@@ -33,7 +35,7 @@ class ManualTaskAddress:
         return ManualTaskAddress._is_manual_data_collection(address.collection)
 
     @staticmethod
-    def get_connection_key(address: CollectionAddress) -> str:
+    def get_connection_key(address: Union[str, CollectionAddress]) -> str:
         """Extract connection config key from manual task address"""
         if not ManualTaskAddress.is_manual_task_address(address):
             raise ValueError(f"Not a manual task address: {address}")