ethyca-fides 2.56.1b1__tar.gz → 2.56.1b3__tar.gz

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ethyca_fides
-Version: 2.56.1b1
+Version: 2.56.1b3
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b3}/src/ethyca_fides.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ethyca-fides
-Version: 2.56.1b1
+Version: 2.56.1b3
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b3}/src/ethyca_fides.egg-info/SOURCES.txt

@@ -787,8 +787,6 @@ src/fides/ui-build/static/admin/properties.html
 src/fides/ui-build/static/admin/systems.html
 src/fides/ui-build/static/admin/taxonomy.html
 src/fides/ui-build/static/admin/user-management.html
-src/fides/ui-build/static/admin/_next/static/Yp4lTh1cgsDcOfwB95ehL/_buildManifest.js
-src/fides/ui-build/static/admin/_next/static/Yp4lTh1cgsDcOfwB95ehL/_ssgManifest.js
 src/fides/ui-build/static/admin/_next/static/chunks/1241-438935108aed21e7.js
 src/fides/ui-build/static/admin/_next/static/chunks/1288-ba1df63e7fa2234a.js
 src/fides/ui-build/static/admin/_next/static/chunks/1827-2d2f6eb59ebca43f.js
@@ -841,7 +839,7 @@ src/fides/ui-build/static/admin/_next/static/chunks/8986-b955f08aba857064.js
 src/fides/ui-build/static/admin/_next/static/chunks/9111-36f3eb6456817560.js
 src/fides/ui-build/static/admin/_next/static/chunks/9255-2b1bafefb091a640.js
 src/fides/ui-build/static/admin/_next/static/chunks/9278-9b1b5970f0702668.js
-src/fides/ui-build/static/admin/_next/static/chunks/9659-4757cc92c551432f.js
+src/fides/ui-build/static/admin/_next/static/chunks/9659-7a6f150315045fdd.js
 src/fides/ui-build/static/admin/_next/static/chunks/9822-e0b4ee48b9ca9d6c.js
 src/fides/ui-build/static/admin/_next/static/chunks/c78d26b1-88a3e1bacb2a03c2.js
 src/fides/ui-build/static/admin/_next/static/chunks/ea076c0a.84423f606aef37cd.js
@@ -851,7 +849,7 @@ src/fides/ui-build/static/admin/_next/static/chunks/main-app-de538a24cbd2733c.js
 src/fides/ui-build/static/admin/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js
 src/fides/ui-build/static/admin/_next/static/chunks/webpack-f8d056f5e13fe08a.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/404-cf2b98b932c99b1e.js
-src/fides/ui-build/static/admin/_next/static/chunks/pages/_app-817f3fe70282b855.js
+src/fides/ui-build/static/admin/_next/static/chunks/pages/_app-ec8f262156df2273.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/_error-c91b5eabd7ac9382.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/add-systems-500ee461301938a8.js
 src/fides/ui-build/static/admin/_next/static/chunks/pages/ant-poc-833eede89996ded2.js
@@ -930,6 +928,8 @@ src/fides/ui-build/static/admin/_next/static/css/95ac327e5d70148d.css
 src/fides/ui-build/static/admin/_next/static/css/972dc7eef106ee7c.css
 src/fides/ui-build/static/admin/_next/static/css/d2d4e9a6923e6730.css
 src/fides/ui-build/static/admin/_next/static/css/ddd0b958fa57d1b9.css
+src/fides/ui-build/static/admin/_next/static/gfK90uth_1hk3Qh5aYeCe/_buildManifest.js
+src/fides/ui-build/static/admin/_next/static/gfK90uth_1hk3Qh5aYeCe/_ssgManifest.js
 src/fides/ui-build/static/admin/_next/static/media/inter-all-400-normal.2ae8ed37.woff
 src/fides/ui-build/static/admin/_next/static/media/inter-all-500-normal.56c3d87b.woff
 src/fides/ui-build/static/admin/_next/static/media/inter-all-600-normal.d9666969.woff

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b3}/src/fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-02-27T16:33:57+0000",
+ "date": "2025-02-28T18:15:41-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "3c01497d0f655c36d069ca0cd566daeeb0d59d70",
- "version": "2.56.1b1"
+ "full-revisionid": "c51cd945858ae1bbcc1f72c26ab4422f3f019907",
+ "version": "2.56.1b3"
 }
 '''  # END VERSION_JSON
 

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b3}/src/fides/api/db/samples.py

@@ -12,6 +12,7 @@ import yaml
 from expandvars import expandvars  # type: ignore
 from fideslang.models import Taxonomy
 from fideslang.validation import FidesKey
+from loguru import logger as log
 
 from fides.api.schemas.connection_configuration.connection_config import (
     CreateConnectionConfigurationWithSecrets,
@@ -96,6 +97,7 @@ def load_sample_connections_from_project() -> List[SampleConnection]:
         "sample_connections/"
     )
     sample_connections = []
+    log.debug(f"Collecting sample connections from '{sample_connections_path}'/*...")
     for yaml_path in sample_connections_path.iterdir():
         with yaml_path.open("r") as file:
             # Expand ENV vars when reading the YAML, to handle secrets
@@ -106,17 +108,41 @@ def load_sample_connections_from_project() -> List[SampleConnection]:
             )
 
     # Exclude any connections whose "secrets" dict has empty values
+    log.debug(
+        f"Found {len(sample_connections)} sample connections. Skipping any sample connections without configured secrets..."
+    )
     valid_connections = []
     for connection in sample_connections:
-        # If there are no secrets at all, skip this connection
+        # If there are no secrets at all, skip this connection. This shouldn't happen unless sample_connections.yml is misconfigured
         if not connection.secrets:
+            log.debug(
+                f"Skipping sample connection {connection.key}! No secrets exist, cannot create"
+            )
+            continue
+
+        # Check if all secret values exist and are non-empty
+        expected_keys = connection.secrets.keys()  # type: ignore
+        missing_keys = [
+            key
+            for key, value in connection.secrets.items()  # type: ignore
+            if value is None or value == ""
+        ]
+        if len(missing_keys) > 0:
+            log.debug(
+                f"Skipping sample connection {connection.key}. To include this sample, configure ENV secrets for these missing keys: {missing_keys} (see {sample_connections_path}/* for expected ENV vars)"
+            )
             continue
 
-        # Check if all secret values are present and non-empty
-        if all(value and value != "" for value in connection.secrets.values()):  # type: ignore
-            valid_connections.append(connection)
+        # If all secrets are configured, include this connection
+        log.debug(
+            f"Including sample connection {connection.key} because all {len(expected_keys)} required secrets are configured"
+        )
+        valid_connections.append(connection)
 
     # Exclude any invalid connections from the final results
+    log.info(
+        f"Collected {len(valid_connections)} sample connections with configured ENV secrets: {[connection.key for connection in valid_connections]}"
+    )
     return valid_connections
 
 

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b3}/src/fides/api/service/privacy_request/request_service.py

@@ -8,7 +8,6 @@ from typing import Any, Dict, List, Optional, Set
 from httpx import AsyncClient
 from loguru import logger
 from sqlalchemy import text
-from sqlalchemy.orm import Query
 from sqlalchemy.sql.elements import TextClause
 
 from fides.api.common_exceptions import PrivacyRequestNotFound
@@ -16,6 +15,7 @@ from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import (
     EXITED_EXECUTION_LOG_STATUSES,
     PrivacyRequest,
+    RequestTask,
 )
 from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
 from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
@@ -27,9 +27,14 @@ from fides.api.schemas.privacy_request import (
 )
 from fides.api.schemas.redis_cache import Identity
 from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
-from fides.api.tasks import DatabaseTask, celery_app
+from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
 from fides.api.tasks.scheduled.scheduler import scheduler
-from fides.api.util.cache import FidesopsRedis, celery_tasks_in_flight, get_cache
+from fides.api.util.cache import (
+    FidesopsRedis,
+    celery_tasks_in_flight,
+    get_async_task_tracking_cache_key,
+    get_cache,
+)
 from fides.common.api.v1.urn_registry import PRIVACY_REQUESTS, V1_URL_PREFIX
 from fides.config import CONFIG
 
@@ -210,34 +215,51 @@ def poll_for_exited_privacy_request_tasks(self: DatabaseTask) -> Set[str]:
             .order_by(PrivacyRequest.created_at)
         )
 
-        def some_errored(tasks: Query) -> bool:
-            """All statuses have exited and at least one is errored"""
-            statuses: List[ExecutionLogStatus] = [tsk.status for tsk in tasks]
+        # TODO: With this approach, we're making 3*n queries , where n is the number of in-progress privacy requests.
+        # We could optimize this to just get all the RequestTasks for all in-progress privacy requests in one single
+        # query and then process them in-memory. This could be more efficient.
+        def privacy_request_has_errored_tasks(
+            privacy_request: PrivacyRequest, task_type: ActionType
+        ) -> bool:
+            """Check if a privacy request has exited all its tasks of the given type,
+            and at least one of the tasks has errored.
+            We specifically only query for the RequestTask.status column to avoid
+            querying for the entire RequestTask row.
+            """
+            tasks_statuses_query = db.query(RequestTask.status).filter(
+                RequestTask.privacy_request_id == privacy_request.id,
+                RequestTask.action_type == task_type,
+            )
+
+            statuses = set(status for status, in tasks_statuses_query.all())
             all_exited = all(
                 status in EXITED_EXECUTION_LOG_STATUSES for status in statuses
             )
-            return all_exited and ExecutionLogStatus.error in statuses
+            if all_exited and ExecutionLogStatus.error in statuses:
+                logger.info(
+                    f"Marking {task_type.value} step of {privacy_request.id} as error"
+                )
+                return True
+
+            return False
 
         marked_as_errored: Set[str] = set()
         for pr in in_progress_privacy_requests.all():
             if pr.consent_tasks.count():
                 # Consent propagation tasks - these are not created until access and erasure steps are complete.
-                if some_errored(pr.consent_tasks):
-                    logger.info(f"Marking consent step of {pr.id} as error")
+                if privacy_request_has_errored_tasks(pr, ActionType.consent):
                     pr.error_processing(db)
                     marked_as_errored.add(pr.id)
 
             if pr.erasure_tasks.count():
                 # Erasure tasks are created at the same time as access tasks but if any are errored, this means
                 # we made it to the erasure section
-                if some_errored(pr.erasure_tasks):
-                    logger.info(f"Marking erasure step of {pr.id} as error")
+                if privacy_request_has_errored_tasks(pr, ActionType.erasure):
                     pr.error_processing(db)
                     marked_as_errored.add(pr.id)
 
             if pr.access_tasks.count():
-                if some_errored(pr.access_tasks):
-                    logger.info(f"Marking access step of {pr.id} as error")
+                if privacy_request_has_errored_tasks(pr, ActionType.access):
                     pr.error_processing(db)
                     marked_as_errored.add(pr.id)
 
@@ -347,6 +369,50 @@ def initiate_interrupted_task_requeue_poll() -> None:
     )
 
 
+def get_cached_task_id(entity_id: str) -> Optional[str]:
+    """Gets the cached task ID for a privacy request or request task by ID."""
+    cache: FidesopsRedis = get_cache()
+    task_id = cache.get(get_async_task_tracking_cache_key(entity_id))
+    return task_id
+
+
+REQUEUE_INTERRUPTED_TASKS_LOCK = "requeue_interrupted_tasks_lock"
+
+
+def _get_task_ids_from_dsr_queue(
+    redis_client: FidesopsRedis, chunk_size: int = 100
+) -> Set[str]:
+    """
+    Get all task IDs from a Redis queue in a memory-efficient way.
+
+    Args:
+        redis_client: Redis client
+        chunk_size: Size of chunks to process
+
+    Returns:
+        Set of task IDs
+    """
+    queued_tasks_ids = set()
+    queue_length = redis_client.llen(DSR_QUEUE_NAME)
+
+    # Process the queue in chunks to avoid memory issues with large queues
+    for offset in range(0, queue_length, chunk_size):
+        # Get a chunk of tasks (at most chunk_size elements)
+        tasks_chunk = redis_client.lrange(
+            DSR_QUEUE_NAME, offset, offset + chunk_size - 1
+        )
+
+        # Extract task IDs from the chunk
+        for task in tasks_chunk:
+            try:
+                queued_tasks_ids.add(json.loads(task)["headers"]["id"])
+            except (json.JSONDecodeError, KeyError) as e:
+                logger.warning(f"Failed to parse task from queue: {e}")
+                continue
+
+    return queued_tasks_ids
+
+
 # pylint: disable=too-many-branches
 @celery_app.task(base=DatabaseTask, bind=True)
 def requeue_interrupted_tasks(self: DatabaseTask) -> None:
@@ -364,12 +430,11 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
     3. Checks each privacy request to determine if its tasks are still active.
     4. Requeues the privacy request if any of its tasks are found to be interrupted.
     """
-
     redis_conn: FidesopsRedis = get_cache()
 
     # Create a lock with a timeout of 10 minutes (600 seconds)
     # This ensures the lock is eventually released even if the process crashes
-    lock = redis_conn.lock("requeue_interrupted_tasks_lock", timeout=600)
+    lock = redis_conn.lock(REQUEUE_INTERRUPTED_TASKS_LOCK, timeout=600)
 
     # Try to acquire the lock without blocking
     if not lock.acquire(blocking=False):
@@ -406,30 +471,14 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
             )
 
             # Get task IDs from the queue in a memory-efficient way
-            queued_tasks_ids = set()
-            chunk_size = 100
-            queue_length = redis_conn.llen("fides.dsr")
-
-            # Process the queue in chunks to avoid memory issues with large queues
-            for offset in range(0, queue_length, chunk_size):
-                # Get a chunk of tasks (at most chunk_size elements)
-                tasks_chunk = redis_conn.lrange(
-                    "fides.dsr", offset, offset + chunk_size - 1
-                )
-
-                # Extract task IDs from the chunk
-                for task in tasks_chunk:
-                    try:
-                        queued_tasks_ids.add(json.loads(task)["headers"]["id"])
-                    except (json.JSONDecodeError, KeyError) as e:
-                        logger.warning(f"Failed to parse task from queue: {e}")
-                        continue
+            queued_tasks_ids = _get_task_ids_from_dsr_queue(redis_conn)
 
             # Check each privacy request
             for privacy_request in in_progress_requests:
                 should_requeue = False
                 logger.debug(f"Checking tasks for privacy request {privacy_request.id}")
-                task_id = privacy_request.get_cached_task_id()
+
+                task_id = get_cached_task_id(privacy_request.id)
 
                 # If the task ID is not cached, we can't check if it's running
                 if not task_id:
@@ -439,33 +488,51 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
                 if task_id not in queued_tasks_ids and not celery_tasks_in_flight(
                     [task_id]
                 ):
-                    if privacy_request.request_tasks.count() == 0:
+                    request_tasks_count = (
+                        db.query(RequestTask)
+                        .filter(RequestTask.privacy_request_id == privacy_request.id)
+                        .count()
+                    )
+                    if request_tasks_count == 0:
                         logger.warning(
                             f"The task for privacy request {privacy_request.id} was terminated before it could schedule any request tasks, requeueing privacy request"
                         )
                         should_requeue = True
 
+                    request_tasks_in_progress = (
+                        db.query(RequestTask.id)
+                        .filter(RequestTask.privacy_request_id == privacy_request.id)
+                        .filter(
+                            RequestTask.status.in_(
+                                [
+                                    ExecutionLogStatus.in_processing,
+                                    ExecutionLogStatus.pending,
+                                ]
+                            )
+                        )
+                        .all()
+                    )
+                    request_task_ids_in_progress = [
+                        task[0] for task in request_tasks_in_progress
+                    ]
+
                     # Check each individual request task
-                    for request_task in privacy_request.request_tasks:
-                        if request_task.status in [
-                            ExecutionLogStatus.in_processing,
-                            ExecutionLogStatus.pending,
-                        ]:
-                            subtask_id = request_task.get_cached_task_id()
-
-                            # If the task ID is not cached, we can't check if it's running
-                            if not subtask_id:
-                                continue
-
-                            if (
-                                subtask_id not in queued_tasks_ids
-                                and not celery_tasks_in_flight([subtask_id])
-                            ):
-                                logger.warning(
-                                    f"Request task {request_task.id} is not in the queue or running, requeueing privacy request"
-                                )
-                                should_requeue = True
-                                break
+                    for request_task_id in request_task_ids_in_progress:
+                        subtask_id = get_cached_task_id(request_task_id)
+
+                        # If the task ID is not cached, we can't check if it's running
+                        if not subtask_id:
+                            continue
+
+                        if (
+                            subtask_id not in queued_tasks_ids
+                            and not celery_tasks_in_flight([subtask_id])
+                        ):
+                            logger.warning(
+                                f"Request task {request_task_id} is not in the queue or running, requeueing privacy request"
+                            )
+                            should_requeue = True
+                            break
 
                 # Requeue the privacy request if needed
                 if should_requeue:

{ethyca_fides-2.56.1b1 → ethyca_fides-2.56.1b3}/src/fides/ui-build/static/admin/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/ddd0b958fa57d1b9.css" as="style"/><link rel="stylesheet" href="/_next/static/css/ddd0b958fa57d1b9.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-f8d056f5e13fe08a.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-6880a3888740c515.js" defer=""></script><script src="/_next/static/chunks/pages/_app-817f3fe70282b855.js" defer=""></script><script src="/_next/static/chunks/pages/404-cf2b98b932c99b1e.js" defer=""></script><script src="/_next/static/Yp4lTh1cgsDcOfwB95ehL/_buildManifest.js" defer=""></script><script src="/_next/static/Yp4lTh1cgsDcOfwB95ehL/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"Yp4lTh1cgsDcOfwB95ehL","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link data-next-font="" rel="preconnect" href="/" crossorigin="anonymous"/><link rel="preload" href="/_next/static/css/ddd0b958fa57d1b9.css" as="style"/><link rel="stylesheet" href="/_next/static/css/ddd0b958fa57d1b9.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-f8d056f5e13fe08a.js" defer=""></script><script src="/_next/static/chunks/framework-c92fc3344e6fd165.js" defer=""></script><script src="/_next/static/chunks/main-6880a3888740c515.js" defer=""></script><script src="/_next/static/chunks/pages/_app-ec8f262156df2273.js" defer=""></script><script src="/_next/static/chunks/pages/404-cf2b98b932c99b1e.js" defer=""></script><script src="/_next/static/gfK90uth_1hk3Qh5aYeCe/_buildManifest.js" defer=""></script><script src="/_next/static/gfK90uth_1hk3Qh5aYeCe/_ssgManifest.js" defer=""></script><style>.data-ant-cssinjs-cache-path{content:"";}</style></head><body><div id="__next"><div style="height:100%;display:flex"></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/404","query":{},"buildId":"gfK90uth_1hk3Qh5aYeCe","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>