ethyca-fides 2.67.2b3__py2.py3-none-any.whl → 2.67.2rc0__py2.py3-none-any.whl

fides/api/models/detection_discovery/core.py

@@ -380,51 +380,38 @@ class StagedResourceAncestor(Base):
     )
 
     @classmethod
-    def create_all_staged_resource_ancestor_links(
+    def create_staged_resource_ancestor_links(
         cls,
         db: Session,
-        ancestor_links: Dict[str, Set[str]],
-        batch_size: int = 10000,  # Conservative batch size
+        resource_urn: str,
+        ancestor_urns: Set[str],
     ) -> None:
         """
-        Bulk inserts all entries in the StagedResourceAncestor table
-        based on the provided mapping of descendant URNs to their ancestor URN sets.
+        Bulk inserts entries in the StagedResourceAncestor table
+        based on the provided resource URN and the set of its ancestor URNs.
 
         We execute the bulk INSERT with the provided (synchronous) db session,
         but the transaction is _not_ committed, so the caller must commit the transaction
         to persist the changes.
-
-        Uses batching to handle large datasets without hitting PostgreSQL parameter limits.
-
-        Args:
-            db: Database session
-            ancestor_links: Dict mapping descendant URNs to sets of ancestor URNs
         """
-        stmt_text = text(
-            """
-            INSERT INTO stagedresourceancestor (id, ancestor_urn, descendant_urn)
-            VALUES ('srl_' || gen_random_uuid(), :ancestor_urn, :descendant_urn)
-            ON CONFLICT (ancestor_urn, descendant_urn) DO NOTHING;
-            """
-        )
+        links_to_insert = []
 
-        current_batch = []
-
-        for descendant_urn, ancestor_urns in ancestor_links.items():
-            if ancestor_urns:  # Only create links if there are ancestors
-                for ancestor_urn in ancestor_urns:
-                    current_batch.append(
-                        {"ancestor_urn": ancestor_urn, "descendant_urn": descendant_urn}
-                    )
+        for ancestor_urn in ancestor_urns:
+            links_to_insert.append(
+                {"ancestor_urn": ancestor_urn, "descendant_urn": resource_urn}
+            )
 
-                    # Execute batch when it reaches the desired size
-                    if len(current_batch) >= batch_size:
-                        db.execute(stmt_text, current_batch)
-                        current_batch = []
+        if links_to_insert:
+            # Using raw SQL for ON CONFLICT with parameters for safety
+            stmt_text = text(
+                """
+                INSERT INTO stagedresourceancestor (id, ancestor_urn, descendant_urn)
+                VALUES ('srl_' || gen_random_uuid(), :ancestor_urn, :descendant_urn)
+                ON CONFLICT (ancestor_urn, descendant_urn) DO NOTHING;
+                """
+            )
 
-        # Execute any remaining items in the final batch
-        if current_batch:
-            db.execute(stmt_text, current_batch)
+            db.execute(stmt_text, links_to_insert)
 
 
 class StagedResource(Base):

fides/api/models/manual_task/manual_task.py

@@ -383,13 +383,15 @@ class ManualTaskInstance(Base):
 
     @property
     def incomplete_fields(self) -> list["ManualTaskConfigField"]:
-        """Get all fields that have no submission.
+        """Get all fields that haven't been completed yet.
+        A field is considered incomplete if:
+        1. It's required and has no submission
         Returns:
             list[ManualTaskConfigField]: List of incomplete fields
         """
         return [
             field
-            for field in self.config.field_definitions
+            for field in self.required_fields
             if not self.get_submission_for_field(field.id)
         ]
 
@@ -399,7 +401,8 @@ class ManualTaskInstance(Base):
         return [
             field
             for field in self.config.field_definitions
-            if self.get_submission_for_field(field.id)
+            if field.field_metadata.get("required", False)
+            and self.get_submission_for_field(field.id)
         ]
 
     def get_submission_for_field(

fides/api/models/privacy_request/privacy_request.py

@@ -47,16 +47,8 @@ from fides.api.models.attachment import (
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.client import ClientDetail
 from fides.api.models.comment import Comment, CommentReference, CommentReferenceType
-from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.fides_user import FidesUser
 from fides.api.models.field_types import EncryptedLargeDataDescriptor
-from fides.api.models.manual_task import (
-    ManualTask,
-    ManualTaskConfig,
-    ManualTaskConfigurationType,
-    ManualTaskEntityType,
-    ManualTaskInstance,
-)
 from fides.api.models.manual_webhook import AccessManualWebhook
 from fides.api.models.masking_secret import MaskingSecret
 from fides.api.models.policy import (
@@ -208,14 +200,6 @@ class PrivacyRequest(
         viewonly=True,
         uselist=True,
     )
-    manual_task_instances = relationship(
-        "ManualTaskInstance",
-        lazy="select",
-        passive_deletes="all",
-        primaryjoin="and_(ManualTaskInstance.entity_id==foreign(PrivacyRequest.id), "
-        "ManualTaskInstance.entity_type=='privacy_request')",
-        uselist=True,
-    )
     property_id = Column(String, nullable=True)
 
     cancel_reason = Column(String(200))
@@ -1191,68 +1175,6 @@ class PrivacyRequest(
             db, manual_webhook_id, "erasure_manual_webhook"
         )
 
-    def create_manual_task_instances(
-        self, db: Session, connection_configs_with_manual_tasks: list[ConnectionConfig]
-    ) -> list[ManualTaskInstance]:
-        """Create ManualTaskInstance entries for all active manual tasks relevant to a privacy request."""
-        # Early return if no relevant policy rules
-        policy_rules = {
-            ActionType.access: bool(
-                self.policy.get_rules_for_action(action_type=ActionType.access)
-            ),
-            ActionType.erasure: bool(
-                self.policy.get_rules_for_action(action_type=ActionType.erasure)
-            ),
-        }
-
-        if not any(policy_rules.values()):
-            return []
-
-        # Build configuration types using list comprehension
-        config_types = [
-            (
-                ManualTaskConfigurationType.access_privacy_request
-                if action_type == ActionType.access
-                else ManualTaskConfigurationType.erasure_privacy_request
-            )
-            for action_type, has_rules in policy_rules.items()
-            if has_rules
-        ]
-
-        # Get all relevant manual tasks and configs in one query
-        connection_config_ids = [cc.id for cc in connection_configs_with_manual_tasks]
-        manual_tasks_with_configs = (
-            db.query(ManualTask, ManualTaskConfig)
-            .join(ManualTaskConfig, ManualTask.id == ManualTaskConfig.task_id)
-            .filter(
-                ManualTask.parent_entity_id.in_(connection_config_ids),
-                ManualTask.parent_entity_type == "connection_config",
-                ManualTaskConfig.is_current.is_(True),
-                ManualTaskConfig.config_type.in_(config_types),
-            )
-            .all()
-        )
-
-        # Get existing config IDs to avoid duplicates
-        existing_config_ids = {
-            instance.config_id for instance in self.manual_task_instances
-        }
-
-        # Create instances using list comprehension and filter out existing ones
-        return [
-            ManualTaskInstance.create(
-                db=db,
-                data={
-                    "entity_id": self.id,
-                    "entity_type": ManualTaskEntityType.privacy_request,
-                    "task_id": manual_task.id,
-                    "config_id": config.id,
-                },
-            )
-            for manual_task, config in manual_tasks_with_configs
-            if config.id not in existing_config_ids
-        ]
-
     def get_existing_request_task(
         self,
         db: Session,

fides/api/service/privacy_request/dsr_package/dsr_report_builder.py

@@ -13,7 +13,7 @@ from loguru import logger
 
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.policy import ActionType
-from fides.api.util.storage_util import StorageJSONEncoder, format_size
+from fides.api.util.storage_util import StorageJSONEncoder
 
 DSR_DIRECTORY = Path(__file__).parent.resolve()
 
@@ -204,7 +204,7 @@ class DsrReportBuilder:
 
             file_size = attachment.get("file_size")
             if isinstance(file_size, (int, float)):
-                file_size = format_size(float(file_size))
+                file_size = self._format_size(float(file_size))
             else:
                 file_size = "Unknown"
 
@@ -321,6 +321,22 @@ class DsrReportBuilder:
 
         return datasets
 
+    def _format_size(self, size_bytes: float) -> str:
+        """
+        Format size in bytes to human readable format.
+
+        Args:
+            size_bytes: Size in bytes
+
+        Returns:
+            Formatted string with appropriate unit (B, KB, MB, GB)
+        """
+        for unit in ["B", "KB", "MB", "GB"]:
+            if size_bytes < 1024.0:
+                return f"{size_bytes:.1f} {unit}"
+            size_bytes /= 1024.0
+        return f"{size_bytes:.1f} TB"
+
     def generate(self) -> BytesIO:
         """
         Processes the request and DSR data to build zip file containing the DSR report.
@@ -379,7 +395,7 @@ class DsrReportBuilder:
 
         # Calculate time taken and file size
         time_taken = time_module.time() - start_time
-        file_size = format_size(float(len(self.baos.getvalue())))
+        file_size = self._format_size(float(len(self.baos.getvalue())))
 
         logger.bind(time_to_generate=time_taken, dsr_package_size=file_size).info(
             "DSR report generation complete."

fides/api/task/conditional_dependencies/evaluator.py

@@ -1,10 +1,10 @@
+import operator as py_operator
 from typing import Any, Union
 
 from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.graph.config import FieldPath
-from fides.api.task.conditional_dependencies.operators import operator_methods
 from fides.api.task.conditional_dependencies.schemas import (
     Condition,
     ConditionGroup,
@@ -13,9 +13,18 @@ from fides.api.task.conditional_dependencies.schemas import (
     Operator,
 )
 
-
-class ConditionEvaluationError(Exception):
-    """Error raised when a condition evaluation fails"""
+operator_methods = {
+    Operator.exists: lambda a, _: a is not None,
+    Operator.not_exists: lambda a, _: a is None,
+    Operator.eq: py_operator.eq,
+    Operator.neq: py_operator.ne,
+    Operator.lt: lambda a, b: a < b if a is not None else False,
+    Operator.lte: lambda a, b: a <= b if a is not None else False,
+    Operator.gt: lambda a, b: a > b if a is not None else False,
+    Operator.gte: lambda a, b: a >= b if a is not None else False,
+    Operator.list_contains: lambda a, b: b in a if isinstance(a, list) else False,
+    Operator.not_in_list: lambda a, b: a not in b if isinstance(b, list) else True,
+}
 
 
 class ConditionEvaluator:
@@ -35,9 +44,9 @@ class ConditionEvaluator:
         self, condition: ConditionLeaf, data: Union[dict, Any]
     ) -> bool:
         """Evaluate a leaf condition against input data"""
-        data_value = self._get_nested_value(data, condition.field_address.split("."))
+        actual_value = self._get_nested_value(data, condition.field_address.split("."))
         # Apply operator and return result
-        return self._apply_operator(data_value, condition.operator, condition.value)
+        return self._apply_operator(actual_value, condition.operator, condition.value)
 
     def _evaluate_group_condition(
         self, group: ConditionGroup, data: Union[dict, Any]
@@ -87,7 +96,7 @@ class ConditionEvaluator:
         return current if current != {} else None
 
     def _apply_operator(
-        self, data_value: Any, operator: Operator, user_input_value: Any
+        self, actual_value: Any, operator: Operator, expected_value: Any
     ) -> bool:
         """Apply operator to actual and expected values"""
 
@@ -95,8 +104,6 @@ class ConditionEvaluator:
         operator_method = operator_methods.get(operator)
         if operator_method is None:
             logger.warning(f"Unknown operator: {operator}")
-            raise ConditionEvaluationError(f"Unknown operator: {operator}")
-        try:
-            return operator_method(data_value, user_input_value)
-        except (TypeError, ValueError) as e:
-            raise ConditionEvaluationError(f"Error evaluating condition: {e}") from e
+            return False
+
+        return operator_method(actual_value, expected_value)

fides/api/task/conditional_dependencies/schemas.py

@@ -5,67 +5,16 @@ from pydantic import BaseModel, Field, model_validator
 
 
 class Operator(str, Enum):
-    # Basic comparison operators
-    # Column value equals user input (e.g., user.role eq "admin")
     eq = "eq"
-    # Column value not equal to user input (e.g., user.status neq "inactive")
     neq = "neq"
-
-    # Numeric comparison operators
-    # Column value less than user input (e.g., user.age lt 18)
     lt = "lt"
-    # Column value less than or equal to user input (e.g., user.score lte 100)
     lte = "lte"
-    # Column value greater than user input (e.g., user.balance gt 1000)
     gt = "gt"
-    # Column value greater than or equal to user input (e.g., user.rating gte 4.0)
     gte = "gte"
-
-    # Existence operators
-    # Field exists and is not None (e.g., user.email exists)
     exists = "exists"
-    # Field does not exist or is None (e.g., user.middle_name not_exists)
     not_exists = "not_exists"
-
-    # List membership operators (work with both single values and lists)
-    #
-    # Column value is in user's list OR user's value is in column's list
     list_contains = "list_contains"
-    # Examples: user.role list_contains ["admin", "moderator"] (role in list)
-    #          user.permissions list_contains "write" (value in permissions)
-
-    # Column value is NOT in user's list OR user's value is NOT in column's list
     not_in_list = "not_in_list"
-    # Examples: user.role not_in_list ["banned", "suspended"] (role not blocked)
-    #          user.permissions not_in_list "delete" (value not in permissions)
-
-    # List-to-list comparison operators (both values must be lists)
-    # Lists have at least one common element
-    list_intersects = "list_intersects"
-    # Example: user.roles list_intersects ["admin", "moderator"] (any common role)
-
-    # Column list is completely contained within user's list
-    list_subset = "list_subset"
-    # Example: user.permissions list_subset ["read", "write", "delete", "manage"]
-    #         (all user permissions are allowed)
-
-    # Column list completely contains user's list
-    list_superset = "list_superset"
-    # Example: user.tags list_superset ["premium", "verified"]
-    #         (user has all required tags plus extras)
-
-    # Lists have no common elements
-    list_disjoint = "list_disjoint"
-    # Example: user.roles list_disjoint ["banned", "suspended"]
-    #         (user has no restricted roles)
-
-    # String operators
-    # String starts with user input (e.g., user.email starts_with "admin@")
-    starts_with = "starts_with"
-    # String ends with user input (e.g., user.domain ends_with ".com")
-    ends_with = "ends_with"
-    # String contains user input (e.g., user.description contains "verified")
-    contains = "contains"
 
 
 class GroupOperator(str, Enum):

fides/api/task/create_request_tasks.py

@@ -35,7 +35,7 @@ from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
 from fides.api.task.manual.manual_task_address import ManualTaskAddress
 from fides.api.task.manual.manual_task_utils import (
-    get_connection_configs_with_manual_tasks,
+    create_manual_task_instances_for_privacy_request,
 )
 from fides.api.util.logger_context_utils import log_context
 
@@ -92,7 +92,7 @@ def build_access_networkx_digraph(
     manual_nodes = [
         addr
         for addr in traversal_nodes.keys()
-        if ManualTaskAddress.is_manual_task_address(addr)
+        if addr.collection == ManualTaskAddress.MANUAL_DATA_COLLECTION
     ]
     for manual_node in manual_nodes:
         networkx_graph.add_edge(ROOT_COLLECTION_ADDRESS, manual_node)
@@ -472,9 +472,7 @@ def run_access_request(
             )
 
             # Snapshot manual task field instances for this privacy request
-            privacy_request.create_manual_task_instances(
-                session, get_connection_configs_with_manual_tasks(session)
-            )
+            create_manual_task_instances_for_privacy_request(session, privacy_request)
 
             # Save Access Request Tasks to the database
             ready_tasks = persist_new_access_request_tasks(

fides/api/task/filter_results.py

@@ -39,7 +39,7 @@ def filter_data_categories(
             continue
 
         # Skip manual task data - it doesn't need filtering since it's controlled by field definitions
-        if ManualTaskAddress.is_manual_task_address(node_address):
+        if f":{ManualTaskAddress.MANUAL_DATA_COLLECTION}" in node_address:
             filtered_access_results[node_address].extend(results)
             continue
 

fides/api/task/manual/manual_task_address.py

@@ -1,5 +1,3 @@
-from typing import Union
-
 from fides.api.graph.config import CollectionAddress
 
 
@@ -22,7 +20,7 @@ class ManualTaskAddress:
         return collection_name == ManualTaskAddress.MANUAL_DATA_COLLECTION
 
     @staticmethod
-    def is_manual_task_address(address: Union[str, CollectionAddress]) -> bool:
+    def is_manual_task_address(address: CollectionAddress) -> bool:
         """Check if address represents manual task data"""
         if isinstance(address, str):
             # Handle string format "connection_key:collection_name"
@@ -35,7 +33,7 @@ class ManualTaskAddress:
         return ManualTaskAddress._is_manual_data_collection(address.collection)
 
     @staticmethod
-    def get_connection_key(address: Union[str, CollectionAddress]) -> str:
+    def get_connection_key(address: CollectionAddress) -> str:
         """Extract connection config key from manual task address"""
         if not ManualTaskAddress.is_manual_task_address(address):
             raise ValueError(f"Not a manual task address: {address}")