ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0rc0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-07-29T12:24:25-0400",
+ "date": "2025-07-30T16:22:05-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "0d085cb0c09afcc5309f42f6c24de184d443af13",
- "version": "2.66.2rc0"
+ "full-revisionid": "1b5a5cfbc8dfd15b989a5fc4778c2155aef1a36d",
+ "version": "2.67.0rc0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/7e9a2b52f498_adding_masking_secrets.py

@@ -0,0 +1,60 @@
+"""adding masking secrets
+
+Revision ID: 7e9a2b52f498
+Revises: d0031087eacb
+Create Date: 2024-12-23 23:29:53.557951
+
+"""
+
+import sqlalchemy as sa
+import sqlalchemy_utils
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "7e9a2b52f498"
+down_revision = "d0031087eacb"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "masking_secret",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("privacy_request_id", sa.String(), nullable=False),
+        sa.Column(
+            "secret",
+            sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(),
+            nullable=False,
+        ),
+        sa.Column("masking_strategy", sa.String(), nullable=False),
+        sa.Column(
+            "secret_type",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["privacy_request_id"], ["privacyrequest.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_masking_secret_id"), "masking_secret", ["id"], unique=False
+    )
+
+
+def downgrade():
+    op.drop_index(op.f("ix_masking_secret_id"), table_name="masking_secret")
+    op.drop_table("masking_secret")

fides/api/alembic/migrations/versions/a7065df4dcf1_add_finalized_fields_to_privacy_request.py

@@ -0,0 +1,65 @@
+"""add finalized fields to privacy request
+
+Revision ID: a7065df4dcf1
+Revises: 7e9a2b52f498
+Create Date: 2025-07-01 14:07:36.779437
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a7065df4dcf1"
+down_revision = "7e9a2b52f498"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "privacyrequest",
+        sa.Column("finalized_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.add_column(
+        "privacyrequest", sa.Column("finalized_by", sa.String(), nullable=True)
+    )
+    op.create_foreign_key(
+        "privacyrequest_fidesuser_id_fkey",
+        "privacyrequest",
+        "fidesuser",
+        ["finalized_by"],
+        ["id"],
+        ondelete="SET NULL",
+    )
+    op.execute(
+        "alter type privacyrequeststatus add value 'requires_manual_finalization'"
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "privacyrequest_fidesuser_id_fkey", "privacyrequest", type_="foreignkey"
+    )
+    op.drop_column("privacyrequest", "finalized_by")
+    op.drop_column("privacyrequest", "finalized_at")
+
+    op.execute(
+        "delete from privacyrequest where status in ('requires_manual_finalization')"
+    )
+
+    op.execute("alter type privacyrequeststatus rename to privacyrequeststatus_old")
+    op.execute(
+        "create type privacyrequeststatus as enum('identity_unverified', 'requires_input', 'pending', 'in_processing', 'complete', 'pending', 'error', 'paused', 'approved', 'denied', 'canceled', 'awaiting_email_send')"
+    )
+    op.execute(
+        (
+            "alter table privacyrequest alter column status type privacyrequeststatus using "
+            "status::text::privacyrequeststatus"
+        )
+    )
+    op.execute("drop type privacyrequeststatus_old")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/d0031087eacb_create_manualtaskconditionaldependency_.py

@@ -0,0 +1,106 @@
+"""create ManualTaskConditionalDependency table
+
+Revision ID: d0031087eacb
+Revises: 9e3dab26f75f
+Create Date: 2025-07-17 21:28:20.937945
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "d0031087eacb"
+down_revision = "9e3dab26f75f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.create_table(
+        "manual_task_conditional_dependency",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column("manual_task_id", sa.String(), nullable=False),
+        sa.Column("parent_id", sa.String(), nullable=True),
+        sa.Column(
+            "condition_type",
+            sa.String(),  # Use String instead of Enum to avoid auto-creation
+            nullable=False,
+        ),
+        sa.Column("field_address", sa.String(), nullable=True),
+        sa.Column("operator", sa.String(), nullable=True),
+        sa.Column("value", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+        sa.Column("logical_operator", sa.String(), nullable=True),
+        sa.Column("sort_order", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["manual_task_id"], ["manual_task.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["parent_id"], ["manual_task_conditional_dependency.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+    op.create_index(
+        "ix_manual_task_conditional_dependency_condition_type",
+        "manual_task_conditional_dependency",
+        ["condition_type"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_manual_task_conditional_dependency_manual_task_id",
+        "manual_task_conditional_dependency",
+        ["manual_task_id"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_manual_task_conditional_dependency_parent_id",
+        "manual_task_conditional_dependency",
+        ["parent_id"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_manual_task_conditional_dependency_sort_order",
+        "manual_task_conditional_dependency",
+        ["sort_order"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        "ix_manual_task_conditional_dependency_sort_order",
+        table_name="manual_task_conditional_dependency",
+    )
+    op.drop_index(
+        "ix_manual_task_conditional_dependency_parent_id",
+        table_name="manual_task_conditional_dependency",
+    )
+    op.drop_index(
+        "ix_manual_task_conditional_dependency_manual_task_id",
+        table_name="manual_task_conditional_dependency",
+    )
+    op.drop_index(
+        "ix_manual_task_conditional_dependency_condition_type",
+        table_name="manual_task_conditional_dependency",
+    )
+
+    op.drop_table("manual_task_conditional_dependency")
+    # ### end Alembic commands ###

fides/api/api/v1/endpoints/dataset_config_endpoints.py

@@ -1,7 +1,7 @@
 from typing import Annotated, Any, Dict, List, Optional
 
 import yaml
-from fastapi import Depends, HTTPException, Request
+from fastapi import Depends, HTTPException, Query, Request
 from fastapi.encoders import jsonable_encoder
 from fastapi.params import Security
 from fastapi_pagination import Page, Params
@@ -68,10 +68,18 @@ from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort:
 )
 
 X_YAML = "application/x-yaml"
+MAX_DATASET_CONFIGS_FOR_INTEGRATION_FORM = 1000
 
 router = APIRouter(tags=["Dataset Configs"], prefix=V1_URL_PREFIX)
 
 
+# Custom Params class with higher limit for dataset configs
+class DatasetConfigParams(Params):
+    size: int = Query(
+        50, ge=1, le=MAX_DATASET_CONFIGS_FOR_INTEGRATION_FORM, description="Page size"
+    )
+
+
 # Helper method to inject the parent ConnectionConfig into these child routes
 def _get_connection_config(
     connection_key: FidesKey, db: Session = Depends(deps.get_db)
@@ -131,7 +139,7 @@ def validate_dataset(
     response_model=BulkPutDataset,
 )
 def put_dataset_configs(
-    dataset_pairs: Annotated[List[DatasetConfigCtlDataset], Field(max_length=50)],  # type: ignore
+    dataset_pairs: Annotated[List[DatasetConfigCtlDataset], Field(max_length=MAX_DATASET_CONFIGS_FOR_INTEGRATION_FORM)],  # type: ignore
     db: Session = Depends(deps.get_db),
     dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
@@ -181,7 +189,7 @@ def put_dataset_configs(
     response_model=BulkPutDataset,
 )
 def patch_dataset_configs(
-    dataset_pairs: Annotated[List[DatasetConfigCtlDataset], Field(max_length=50)],  # type: ignore
+    dataset_pairs: Annotated[List[DatasetConfigCtlDataset], Field(max_length=MAX_DATASET_CONFIGS_FOR_INTEGRATION_FORM)],  # type: ignore
     dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
 ) -> BulkPutDataset:
@@ -217,7 +225,7 @@ def patch_dataset_configs(
     response_model=BulkPutDataset,
 )
 def patch_datasets(
-    datasets: Annotated[List[FideslangDataset], Field(max_length=50)],  # type: ignore
+    datasets: Annotated[List[FideslangDataset], Field(max_length=1000)],  # type: ignore
     dataset_config_service: DatasetConfigService = Depends(get_dataset_config_service),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
 ) -> BulkPutDataset:
@@ -367,7 +375,7 @@ def get_dataset(
 )
 def get_dataset_configs(
     db: Session = Depends(deps.get_db),
-    params: Params = Depends(),
+    params: DatasetConfigParams = Depends(),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
 ) -> AbstractPage[DatasetConfig]:
     """Returns all Dataset Configs attached to current Connection Config."""

fides/api/api/v1/endpoints/drp_endpoints.py

@@ -125,7 +125,13 @@ async def create_drp_privacy_request(
             "Decrypting identity for DRP privacy request {}", privacy_request.id
         )
 
-        cache_data(privacy_request, policy, mapped_identity, None, data)
+        cache_data(privacy_request, mapped_identity, None, data)
+
+        if masking_secrets := policy.generate_masking_secrets():
+            logger.info(
+                "Caching masking secrets for privacy request {}", privacy_request.id
+            )
+            privacy_request.persist_masking_secrets(masking_secrets)
 
         queue_privacy_request(privacy_request.id)
 

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -4,7 +4,7 @@ import csv
 import io
 import json
 from collections import defaultdict
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import (
     Annotated,
     Any,
@@ -146,6 +146,7 @@ from fides.common.api.v1.urn_registry import (
     PRIVACY_REQUEST_BULK_SOFT_DELETE,
     PRIVACY_REQUEST_DENY,
     PRIVACY_REQUEST_FILTERED_RESULTS,
+    PRIVACY_REQUEST_FINALIZE,
     PRIVACY_REQUEST_MANUAL_WEBHOOK_ACCESS_INPUT,
     PRIVACY_REQUEST_MANUAL_WEBHOOK_ERASURE_INPUT,
     PRIVACY_REQUEST_NOTIFICATIONS,
@@ -1862,6 +1863,48 @@ def resume_privacy_request_from_requires_input(
     return privacy_request  # type: ignore[return-value]
 
 
+@router.post(
+    PRIVACY_REQUEST_FINALIZE,
+    status_code=HTTP_200_OK,
+    response_model=PrivacyRequestResponse,
+    dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_REVIEW])],
+)
+def finalize_privacy_request(
+    privacy_request_id: str,
+    *,
+    db: Session = Depends(deps.get_db),
+    client: ClientDetail = Security(
+        verify_oauth_client,
+        scopes=[PRIVACY_REQUEST_REVIEW],
+    ),
+) -> PrivacyRequestResponse:
+    """
+    Finalizes a privacy request, moving it from the 'requires_finalization' state to 'complete'.
+    This is done by re-queueing the request, which will then hit the finalization logic in the
+    request runner service. This logic marks the privacy request as complete
+    and sends out any configured messaging to the user.
+    """
+    privacy_request = get_privacy_request_or_error(db, privacy_request_id)
+
+    if privacy_request.status != PrivacyRequestStatus.requires_manual_finalization:
+        raise HTTPException(
+            status_code=HTTP_400_BAD_REQUEST,
+            detail=f"Cannot manually finalize privacy request '{privacy_request_id}': status is {privacy_request.status}, not requires_manual_finalization.",
+        )
+
+    # Set finalized_by and finalized_at here, so the request runner service knows not to
+    # put the request back into the requires_finalization state.
+    privacy_request.finalized_at = datetime.now(timezone.utc)
+    privacy_request.finalized_by = client.user_id
+    privacy_request.save(db=db)
+
+    queue_privacy_request(
+        privacy_request_id=privacy_request_id,
+    )
+
+    return privacy_request  # type: ignore[return-value]
+
+
 @router.get(
     REQUEST_TASKS,
     dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_READ])],

fides/api/api/v1/endpoints/user_endpoints.py

@@ -6,6 +6,7 @@ from typing import List, Optional
 
 import jose.exceptions
 from fastapi import Depends, HTTPException, Security
+from fastapi.security import SecurityScopes
 from fastapi_pagination import Page, Params
 from fastapi_pagination.bases import AbstractPage
 from fastapi_pagination.ext.sqlalchemy import paginate
@@ -39,7 +40,9 @@ from fides.api.oauth.roles import APPROVER, VIEWER
 from fides.api.oauth.utils import (
     create_temporary_user_for_login_flow,
     extract_payload,
+    extract_token_and_load_client,
     get_current_user,
+    has_permissions,
     oauth2_scheme,
     verify_oauth_client,
 )
@@ -65,6 +68,7 @@ from fides.common.api.scope_registry import (
     USER_DELETE,
     USER_PASSWORD_RESET,
     USER_READ,
+    USER_READ_OWN,
     USER_UPDATE,
 )
 from fides.common.api.v1 import urn_registry as urls
@@ -107,6 +111,37 @@ def _validate_current_user(user_id: str, user_from_token: FidesUser) -> None:
         )
 
 
+def verify_user_read_scopes(
+    authorization: str = Security(oauth2_scheme),
+    db: Session = Depends(get_db),
+) -> ClientDetail:
+    """
+    Custom dependency that verifies the user has either USER_READ or USER_READ_OWN scope.
+    Returns the client if authorized.
+    """
+    token_data, client = extract_token_and_load_client(authorization, db)
+
+    # Try USER_READ first
+    if has_permissions(
+        token_data=token_data,
+        client=client,
+        endpoint_scopes=SecurityScopes([USER_READ]),
+    ):
+        return client
+
+    if has_permissions(
+        token_data=token_data,
+        client=client,
+        endpoint_scopes=SecurityScopes([USER_READ_OWN]),
+    ):
+        return client
+
+    raise HTTPException(
+        status_code=HTTP_403_FORBIDDEN,
+        detail="Not authorized.",
+    )
+
+
 @router.put(
     urls.USER_DETAIL,
     dependencies=[Security(verify_oauth_client)],
@@ -498,14 +533,38 @@ def delete_user(
 
 @router.get(
     urls.USER_DETAIL,
-    dependencies=[Security(verify_oauth_client, scopes=[USER_READ])],
+    dependencies=[Security(verify_user_read_scopes)],
     response_model=UserResponse,
 )
-def get_user(*, db: Session = Depends(get_db), user_id: str) -> FidesUser:
-    """Returns a User based on an Id"""
+def get_user(
+    *,
+    db: Session = Depends(get_db),
+    user_id: str,
+    client: ClientDetail = Security(verify_user_read_scopes),
+    authorization: str = Security(oauth2_scheme),
+) -> FidesUser:
+    """Returns a User based on an Id. Users with USER_READ_OWN scope can only access their own data."""
     user: Optional[FidesUser] = FidesUser.get_by_key_or_id(db, data={"id": user_id})
     if user is None:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="User not found")
+    token_data, _ = extract_token_and_load_client(authorization, db)
+    # Check if user has USER_READ_OWN scope and is trying to access someone else's data
+    # The verify_user_read_scopes dependency already verified the user has either USER_READ or USER_READ_OWN
+    # We need to check if they have USER_READ_OWN and are accessing their own data
+    if has_permissions(
+        token_data=token_data,
+        client=client,
+        endpoint_scopes=SecurityScopes([USER_READ]),
+    ):
+        logger.debug("Returning user with id: '{}'.", user_id)
+        return user
+
+    # User has USER_READ_OWN scope, check if they're accessing their own data
+    if user.id != client.user_id:
+        raise HTTPException(
+            status_code=HTTP_403_FORBIDDEN,
+            detail="You can only access your own user data with USER_READ_OWN scope.",
+        )
 
     logger.debug("Returning user with id: '{}'.", user_id)
     return user
@@ -513,7 +572,7 @@ def get_user(*, db: Session = Depends(get_db), user_id: str) -> FidesUser:
 
 @router.get(
     urls.USERS,
-    dependencies=[Security(verify_oauth_client, scopes=[USER_READ])],
+    dependencies=[Security(verify_user_read_scopes)],
     response_model=Page[UserResponse],
 )
 def get_users(
@@ -521,11 +580,28 @@ def get_users(
     db: Session = Depends(get_db),
     params: Params = Depends(),
     username: Optional[str] = None,
+    client: ClientDetail = Security(verify_user_read_scopes),
+    authorization: str = Security(oauth2_scheme),
 ) -> AbstractPage[FidesUser]:
-    """Returns a paginated list of all users"""
+    """Returns a paginated list of users. Users with USER_READ_OWN scope only see their own data."""
     query = FidesUser.query(db)
-    if username:
-        query = query.filter(FidesUser.username.ilike(f"%{escape_like(username)}%"))
+
+    # Check if user has USER_READ_OWN scope and filter accordingly
+    # The verify_user_read_scopes dependency already verified the user has either USER_READ or USER_READ_OWN
+    token_data, _ = extract_token_and_load_client(authorization, db)
+    if has_permissions(
+        token_data=token_data,
+        client=client,
+        endpoint_scopes=SecurityScopes([USER_READ]),
+    ):
+        # User has USER_READ scope, can see all users
+        if username:
+            query = query.filter(FidesUser.username.ilike(f"%{escape_like(username)}%"))
+    else:
+        # User has USER_READ_OWN scope, only show their own data
+        query = query.filter(FidesUser.id == client.user_id)
+        if username:
+            query = query.filter(FidesUser.username.ilike(f"%{escape_like(username)}%"))
 
     logger.debug("Returning a paginated list of users.")
 

fides/api/app_setup.py

@@ -171,8 +171,9 @@ async def run_database_startup(app: FastAPI) -> None:
     if CONFIG.database.automigrate:
         try:
             configure_db(CONFIG.database.sync_database_uri)
-            with get_autoclose_db_session() as session:
-                seed_db(session)
+            if not CONFIG.test_mode:
+                with get_autoclose_db_session() as session:
+                    seed_db(session)
             if CONFIG.database.load_samples:
                 async with get_async_autoclose_db_session() as async_session:
                     await seed.load_samples(async_session)

fides/api/common_exceptions.py

@@ -283,6 +283,10 @@ class MalisciousUrlException(Exception):
     """Fides has detected a potentially maliscious URL."""
 
 
+class MaskingSecretsExpired(BaseException):
+    """The cached masking secrets have expired for the given privacy request."""
+
+
 class AuthenticationError(HTTPException):
     """To be raised when attempting to fetch an access token using
     invalid credentials.

fides/api/db/base.py

@@ -33,6 +33,7 @@ from fides.api.models.identity_salt import IdentitySalt
 from fides.api.models.location_regulation_selections import LocationRegulationSelections
 from fides.api.models.manual_task import (
     ManualTask,
+    ManualTaskConditionalDependency,
     ManualTaskConfig,
     ManualTaskConfigField,
     ManualTaskInstance,

fides/api/db/database.py

@@ -123,7 +123,7 @@ def get_db_health(
 
 
 def seed_db(session: Session) -> None:
-    """Load default resources into the database, and optionally load samples."""
+    """Load default resources into the database"""
     load_default_resources(session)
 
 

fides/api/graph/execution.py

@@ -1,6 +1,7 @@
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple
 
 from fideslang.validation import FidesKey
+from loguru import logger
 
 from fides.api.graph.config import (
     Collection,
@@ -117,6 +118,8 @@ class ExecutionNode(Contextualizable):  # pylint: disable=too-many-instance-attr
         The values are cast based on field types, if those types are specified.
         """
         out = {}
+        failed_conversions: Dict[str, Dict[str, Any]] = {}
+
         for key, values in input_data.items():
             path: FieldPath = FieldPath.parse(key)
             field: Optional[Field] = self.collection.field(path)
@@ -126,4 +129,31 @@ class ExecutionNode(Contextualizable):  # pylint: disable=too-many-instance-attr
                 filtered = list(filter(lambda x: x is not None, cast_values))
                 if filtered:
                     out[key] = filtered
+                elif values:  # Had input values but all failed conversion
+                    # Track conversion failures for logging
+                    failed_conversions[key] = {"field": field, "input_values": values}
+
+        # Log conversion failures if any occurred
+        if failed_conversions:
+            field_details = []
+            for field_name, failure_info in failed_conversions.items():
+                field = failure_info["field"]
+                values = failure_info["input_values"]
+
+                expected_type = field.data_type() if field else "unknown"
+                actual_types = {type(v).__name__ for v in values if v is not None}
+                actual_type_str = (
+                    ", ".join(sorted(actual_types)) if actual_types else "NoneType"
+                )
+
+                field_details.append(
+                    f"{field_name} (expected: {expected_type}, got: {actual_type_str})"
+                )
+
+            logger.warning(
+                "Type conversion failures for {}: {}",
+                self.address,
+                "; ".join(field_details),
+            )
+
         return out

fides/api/graph/graph.py

@@ -180,8 +180,19 @@ class DatasetGraph:
         this is a destructive operation on the input datasets, as it
         will alter references within them"""
 
-        # build nodes
-        nodes = [Node(dr, ds) for dr in datasets for ds in dr.collections]
+        # build nodes, filtering out collections with skip_processing=True
+        nodes = []
+        for dr in datasets:
+            for ds in dr.collections:
+                if ds.skip_processing:
+                    logger.debug(
+                        "Skipping collection {} on dataset {} marked with skip_processing",
+                        ds.name,
+                        dr.name,
+                    )
+                    continue
+                nodes.append(Node(dr, ds))
+
         self.nodes: dict[CollectionAddress, Node] = {
             node.address: node for node in nodes
         }