ethyca-fides 2.64.5rc4__py2.py3-none-any.whl → 2.64.6b1__py2.py3-none-any.whl

fides/api/service/connectors/__init__.py

@@ -39,6 +39,9 @@ from fides.api.service.connectors.google_cloud_postgres_connector import (
     GoogleCloudSQLPostgresConnector as GoogleCloudSQLPostgresConnector,
 )
 from fides.api.service.connectors.http_connector import HTTPSConnector as HTTPSConnector
+from fides.api.service.connectors.manual_task_connector import (
+    ManualTaskConnector as ManualTaskConnector,
+)
 from fides.api.service.connectors.manual_webhook_connector import (
     ManualWebhookConnector as ManualWebhookConnector,
 )
@@ -93,6 +96,7 @@ supported_connectors: Dict[str, Any] = {
     ConnectionType.google_cloud_sql_postgres.value: GoogleCloudSQLPostgresConnector,
     ConnectionType.https.value: HTTPSConnector,
     ConnectionType.manual_webhook.value: ManualWebhookConnector,
+    ConnectionType.manual_task.value: ManualTaskConnector,
     ConnectionType.mariadb.value: MariaDBConnector,
     ConnectionType.mongodb.value: MongoDBConnector,
     ConnectionType.mssql.value: MicrosoftSQLServerConnector,

fides/api/service/connectors/manual_task_connector.py

@@ -0,0 +1,96 @@
+"""
+Manual Task Connector - A minimal connector for manual task operations.
+
+Since manual tasks don't actually connect to external systems, this connector
+provides no-op implementations of the BaseConnector interface.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from fides.api.graph.execution import ExecutionNode
+from fides.api.models.connectionconfig import ConnectionTestStatus
+from fides.api.models.policy import Policy
+from fides.api.models.privacy_request import PrivacyRequest, RequestTask
+from fides.api.service.connectors.base_connector import BaseConnector
+from fides.api.service.connectors.query_configs.query_config import QueryConfig
+from fides.api.util.collection_util import Row
+
+
+class ManualTaskQueryConfig(QueryConfig):
+    """Minimal query config for manual tasks - not actually used"""
+
+    def generate_query(
+        self, input_data: Dict[str, List[Any]], policy: Optional[Policy]
+    ) -> str:
+        return "Manual task: no query needed"
+
+    def dry_run_query(self) -> str:
+        return "Manual task: no query needed"
+
+    def query_to_str(self, t: Any, input_data: Dict[str, List[Any]]) -> str:
+        """Convert query to string - not used for manual tasks"""
+        return "Manual task: no query needed"
+
+    def generate_update_stmt(
+        self, row: Row, policy: Policy, request: PrivacyRequest
+    ) -> Any:
+        """Generate update statement - not used for manual tasks"""
+        return None
+
+
+class ManualTaskConnector(BaseConnector):
+    """
+    Minimal connector for manual tasks.
+
+    This connector provides no-op implementations since manual tasks don't
+    actually connect to external systems. The actual manual task logic
+    is handled by ManualTaskGraphTask.access_request()
+    """
+
+    def query_config(self, node: ExecutionNode) -> QueryConfig[Any]:
+        """Return a minimal query config - not actually used for manual tasks"""
+        return ManualTaskQueryConfig(node)
+
+    def test_connection(self) -> Optional[ConnectionTestStatus]:
+        """Manual tasks don't have connections to test"""
+        return ConnectionTestStatus.succeeded
+
+    def create_client(self) -> None:
+        """Manual tasks don't need database clients"""
+        return None
+
+    def retrieve_data(
+        self,
+        node: ExecutionNode,
+        policy: Policy,
+        privacy_request: PrivacyRequest,
+        request_task: RequestTask,
+        input_data: Dict[str, List[Any]],
+    ) -> List[Row]:
+        """
+        This method is not used for manual tasks.
+        Manual task data retrieval is handled by ManualTaskGraphTask.access_request()
+        """
+        return []
+
+    def mask_data(
+        self,
+        node: ExecutionNode,
+        policy: Policy,
+        privacy_request: PrivacyRequest,
+        request_task: RequestTask,
+        rows: List[Row],
+    ) -> int:
+        """
+        Manual tasks don't support erasure operations.
+        Manual tasks are for data collection, not data modification.
+        """
+        return 0
+
+    def close(self) -> None:
+        """No resources to close for manual tasks"""
+
+    @property
+    def requires_primary_keys(self) -> bool:
+        """Manual tasks don't require primary keys since they don't modify data"""
+        return False

fides/api/service/connectors/query_configs/bigquery_query_config.py

@@ -5,8 +5,9 @@ from fideslang.models import MaskingStrategies
 from loguru import logger
 from sqlalchemy import MetaData, Table, text
 from sqlalchemy.engine import Engine
-from sqlalchemy.sql import Delete, Update  # type: ignore
+from sqlalchemy.sql import Delete, Update
 from sqlalchemy.sql.elements import ColumnElement, TextClause
+from sqlalchemy_bigquery import BigQueryDialect
 
 from fides.api.graph.config import Field, FieldPath
 from fides.api.graph.execution import ExecutionNode
@@ -15,6 +16,8 @@ from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.namespace_meta.bigquery_namespace_meta import (
     BigQueryNamespaceMeta,
 )
+from fides.api.schemas.partitioning import TIME_BASED_REQUIRED_KEYS, combine_partitions
+from fides.api.schemas.partitioning.time_based_partitioning import TimeBasedPartitioning
 from fides.api.service.connectors.query_configs.query_config import (
     QueryStringWithoutTuplesOverrideQueryConfig,
 )
@@ -36,40 +39,59 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
     namespace_meta_schema = BigQueryNamespaceMeta
 
     @property
-    def partitioning(self) -> Optional[Dict]:
-        # Overriden from base implementation to allow for _only_ BQ partitioning, for now
+    def partitioning(
+        self,
+    ) -> Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]]:
+        # Overridden from base implementation to allow for _only_ BQ partitioning, for now
         return self.node.collection.partitioning
 
     def get_partition_clauses(
         self,
     ) -> List[str]:
         """
-        Returns the WHERE clauses specified in the partitioning spec
-
-        Currently, only where-clause based partitioning is supported.
-
-        TODO: derive partitions from a start/end/interval specification
+        Build a list of SQL `WHERE` clause strings for the collection's partitioning
+        configuration.
 
+        Supported modes:
+        1. `Legacy static` - A list of `where_clauses`.
+        2. `Single time-based` - A single time-based partitioning spec
+        3. `Multiple time-based` - A list of time-based partitioning specs, with different intervals
 
-        NOTE: when we deprecate `where_clause` partitioning in favor of a more proper partitioning DSL,
-        we should be sure to still support the existing `where_clause` partition definition on
-        any in-progress DSRs so that they can run through to completion.
+        Any other combination (e.g. mixing modes or missing keys) raises a
+        `ValueError` so that mis-configurations surface early.
         """
-        partition_spec = self.partitioning
+
+        partition_spec: Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]] = (
+            self.partitioning
+        )
         if not partition_spec:
-            logger.error(
-                "Partitioning clauses cannot be retrieved, no partitioning specification found"
+            logger.warning(
+                f"No partitioning specification found for node '{self.node.address}', skipping partition clauses"
             )
             return []
 
-        if where_clauses := partition_spec.get("where_clauses"):
-            return where_clauses
-
-        # TODO: implement more advanced partitioning support!
+        # Legacy mode using `where_clauses`
+        if isinstance(partition_spec, dict) and "where_clauses" in partition_spec:
+            if any(k in partition_spec for k in TIME_BASED_REQUIRED_KEYS):
+                # Mixed mode not allowed
+                raise ValueError(
+                    "Partitioning spec cannot define both `where_clauses` and time-based partitioning."
+                )
+            where = partition_spec.get("where_clauses") or []
+            if not where:
+                raise ValueError("`where_clauses` must be a non-empty list.")
+            return where
+
+        # Combine and de-duplicate boundary rows
+        combined_expressions = combine_partitions(partition_spec)  # type: ignore
+
+        dialect = BigQueryDialect()
+        where_clauses = [
+            str(expr.compile(dialect=dialect, compile_kwargs={"literal_binds": True}))
+            for expr in combined_expressions
+        ]
 
-        raise ValueError(
-            "`where_clauses` must be specified in partitioning specification!"
-        )
+        return where_clauses
 
     def _generate_table_name(self) -> str:
         """
@@ -147,7 +169,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
         nested_result = unflatten_dict(merged_dict)
 
         # 5. Replace any arrays containing only None values with empty arrays
-        nested_result = replace_none_arrays(nested_result)  # type: ignore
+        nested_result = replace_none_arrays(nested_result)
 
         # 6. Only keep top-level keys that are in the update_value_map
         top_level_keys = {key.split(".")[0] for key in update_value_map}

fides/api/service/connectors/query_configs/query_config.py

@@ -1,7 +1,7 @@
 # pylint: disable=too-many-lines
 import re
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar
+from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar, Union
 
 import pydash
 from loguru import logger
@@ -22,6 +22,7 @@ from fides.api.graph.execution import ExecutionNode
 from fides.api.models.policy import Policy, Rule
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.namespace_meta.namespace_meta import NamespaceMeta
+from fides.api.schemas.partitioning.time_based_partitioning import TimeBasedPartitioning
 from fides.api.schemas.policy import ActionType
 from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
 from fides.api.service.masking.strategy.masking_strategy_nullify import (
@@ -46,7 +47,9 @@ class QueryConfig(Generic[T], ABC):
         self.node = node
 
     @property
-    def partitioning(self) -> Optional[Dict]:  # pylint: disable=R1711
+    def partitioning(  # pylint: disable=useless-return
+        self,
+    ) -> Optional[Union[List[TimeBasedPartitioning], Dict[str, Any]]]:
         # decided to de-scope partitioning support to only bigquery as this grew more complex,
         # but keeping more generic support stubbed out feels like a reasonable step.
         if self.node.collection.partitioning:

fides/api/service/privacy_request/dsr_package/templates/collection_index.html

@@ -26,7 +26,15 @@
                   <div class="table-row">
                      <div class="table-cell">{{ field }}</div>
                      <div class="table-cell">
-                        {% if field == "attachments" and value is mapping and value|length > 0 %}
+                        {% set _is_attachment_block = false %}
+                        {% if value is mapping and value|length > 0 %}
+                           {% set _first_key = (value.keys() | list)[0] %}
+                           {% if value[_first_key] is mapping and ('url' in value[_first_key]) %}
+                               {% set _is_attachment_block = true %}
+                           {% endif %}
+                        {% endif %}
+
+                        {% if _is_attachment_block %}
                            <p class="expiration-notice">Note: All download links will expire in 7 days.</p>
                            <div class="table table-hover">
                               <div class="table-row">

fides/api/service/privacy_request/dsr_package/templates/main.css

@@ -23,6 +23,10 @@ h1 {
     color: var(--text-color);
 }
 
+h2 {
+    margin-bottom: 12px;
+}
+
 .container {
     display: flex;
     flex-direction: column;
@@ -109,8 +113,8 @@ h1 {
     width: 100%;
     border-collapse: separate;
     border-spacing: 0;
-    padding-top: 20px;
-    padding-bottom: 100px;
+    padding-top: 0;
+    padding-bottom: 14px;
     font-size: 14px;
 }
 

fides/api/service/privacy_request/request_runner_service.py

@@ -1,3 +1,4 @@
+# pylint: disable=too-many-lines
 import time
 from copy import deepcopy
 from datetime import datetime, timedelta
@@ -71,6 +72,7 @@ from fides.api.task.graph_task import (
     filter_by_enabled_actions,
     get_cached_data_for_erasures,
 )
+from fides.api.task.manual.manual_task_utils import create_manual_task_artificial_graphs
 from fides.api.tasks import DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
 from fides.api.util.collection_util import Row
@@ -450,6 +452,11 @@ def run_privacy_request(
                     for dataset_config in datasets
                     if not dataset_config.connection_config.disabled
                 ]
+
+                # Add manual task artificial graphs to dataset graphs
+                manual_task_graphs = create_manual_task_artificial_graphs(session)
+                dataset_graphs.extend(manual_task_graphs)
+
                 dataset_graph = DatasetGraph(*dataset_graphs)
 
                 # Add success log for dataset configuration

fides/api/task/create_request_tasks.py

@@ -33,6 +33,10 @@ from fides.api.models.worker_task import ExecutionLogStatus
 from fides.api.schemas.policy import ActionType
 from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
 from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
+from fides.api.task.manual.manual_task_utils import (
+    ManualTaskAddress,
+    create_manual_task_instances_for_privacy_request,
+)
 from fides.api.util.logger_context_utils import log_context
 
 
@@ -85,6 +89,14 @@ def build_access_networkx_digraph(
         # Connect the end nodes, those that have no downstream dependencies, to the terminator node
         networkx_graph.add_edge(node, TERMINATOR_ADDRESS)
 
+    manual_nodes = [
+        addr
+        for addr in traversal_nodes.keys()
+        if addr.collection == ManualTaskAddress.MANUAL_DATA_COLLECTION
+    ]
+    for manual_node in manual_nodes:
+        networkx_graph.add_edge(ROOT_COLLECTION_ADDRESS, manual_node)
+
     _add_edge_if_no_nodes(traversal_nodes, networkx_graph)
     return networkx_graph
 
@@ -458,6 +470,10 @@ def run_access_request(
             end_nodes: List[CollectionAddress] = traversal.traverse(
                 traversal_nodes, collect_tasks_fn
             )
+
+            # Snapshot manual task field instances for this privacy request
+            create_manual_task_instances_for_privacy_request(session, privacy_request)
+
             # Save Access Request Tasks to the database
             ready_tasks = persist_new_access_request_tasks(
                 session, privacy_request, traversal, traversal_nodes, end_nodes, graph

fides/api/task/execute_request_tasks.py

@@ -29,6 +29,8 @@ from fides.api.task.graph_task import (
     GraphTask,
     mark_current_and_downstream_nodes_as_failed,
 )
+from fides.api.task.manual.manual_task_graph_task import ManualTaskGraphTask
+from fides.api.task.manual.manual_task_utils import ManualTaskAddress
 from fides.api.task.task_resources import TaskResources
 from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.util.cache import cache_task_tracking_key
@@ -108,7 +110,14 @@ def create_graph_task(
     to begin with - this may be unrecoverable and a new Privacy Request should be created.
     """
     try:
-        graph_task: GraphTask = GraphTask(resources)
+        collection_address = request_task.request_task_address
+
+        # Check if this is a manual task address
+        graph_task: GraphTask
+        if ManualTaskAddress.is_manual_task_address(collection_address):
+            graph_task = ManualTaskGraphTask(resources)
+        else:
+            graph_task = GraphTask(resources)
 
     except Exception as exc:
         logger.debug(

fides/api/task/filter_results.py

@@ -6,6 +6,7 @@ from loguru import logger
 
 from fides.api.graph.config import CollectionAddress, FieldPath
 from fides.api.graph.graph import DatasetGraph
+from fides.api.task.manual.manual_task_utils import ManualTaskAddress
 from fides.api.util.collection_util import Row
 
 
@@ -37,6 +38,11 @@ def filter_data_categories(
         if not results:
             continue
 
+        # Skip manual task data - it doesn't need filtering since it's controlled by field definitions
+        if f":{ManualTaskAddress.MANUAL_DATA_COLLECTION}" in node_address:
+            filtered_access_results[node_address].extend(results)
+            continue
+
         # Results from fides connectors are a special case:
         # they've already been filtered and stored in a dict keyed by rule key.
         # So here, we simply find the results corresponding to our current rule

fides/api/task/graph_task.py

@@ -109,6 +109,7 @@ def retry(
                         method_name,
                         self.execution_node.address,
                     )
+                    # Log the awaiting processing status and exit without retrying.
                     self.log_awaiting_processing(action_type, ex)
                     # Request Task put in "awaiting_processing" status and exited, awaiting Async Callback
                     return None