ethyca-fides 2.69.3b0__py2.py3-none-any.whl → 2.69.3b1__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-04T14:43:27-0600",
+ "date": "2025-09-08T12:08:59-0300",
  "dirty": false,
  "error": null,
- "full-revisionid": "a95760a6703aeb620d8d3df8a983363d1595279f",
- "version": "2.69.3b0"
+ "full-revisionid": "4e4ef38fde9cbb24f1b053bb67af1945cf409477",
+ "version": "2.69.3b1"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/30369bb8ae01_add_webmonitorgroupjob_model.py

@@ -0,0 +1,67 @@
+"""Add WebMonitorGroupJob model
+
+Revision ID: 30369bb8ae01
+Revises: 67d0e389b003
+Create Date: 2025-09-04 13:18:41.306744
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "30369bb8ae01"
+down_revision = "67d0e389b003"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "web_monitor_group_job",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("group_id", sa.String(), nullable=False, unique=True),
+        sa.Column(
+            "raw_experiences_data",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+        sa.Column(
+            "processed_experiences_data",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+        sa.Column("is_test_run", sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("group_id"),
+    )
+    op.create_index(
+        op.f("ix_web_monitor_group_job_id"),
+        "web_monitor_group_job",
+        ["id"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_web_monitor_group_job_id"), table_name="web_monitor_group_job"
+    )
+    op.drop_table("web_monitor_group_job")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/67d0e389b003_update_consentstatus_enum.py

@@ -0,0 +1,47 @@
+"""Update ConsentStatus enum
+
+Revision ID: 67d0e389b003
+Revises: f36ce1bde293
+Create Date: 2025-09-02 19:32:05.025508
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "67d0e389b003"
+down_revision = "f36ce1bde293"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute("ALTER TYPE consentstatus ADD VALUE 'pre_consent'")
+    op.execute("ALTER TYPE consentstatus ADD VALUE 'cmp_error'")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # Remove any records that have the new enum values before dropping them
+    op.execute(
+        "UPDATE asset SET consent_status = 'unknown' WHERE consent_status IN ('pre_consent', 'cmp_error')"
+    )
+
+    # Recreate the enum without the new values
+    op.execute("ALTER TYPE consentstatus RENAME TO consentstatus_old")
+    op.execute(
+        "CREATE TYPE consentstatus AS ENUM ('with_consent', 'without_consent', 'exempt', 'unknown')"
+    )
+
+    # Drop default, change type, and set default back
+    op.execute("ALTER TABLE asset ALTER COLUMN consent_status DROP DEFAULT")
+    op.execute(
+        "ALTER TABLE asset ALTER COLUMN consent_status TYPE consentstatus USING "
+        "consent_status::text::consentstatus"
+    )
+    op.execute("ALTER TABLE asset ALTER COLUMN consent_status SET DEFAULT 'unknown'")
+    op.execute("DROP TYPE consentstatus_old")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/f36ce1bde293_add_system_groups.py

@@ -0,0 +1,132 @@
+"""add system groups
+
+Revision ID: f36ce1bde293
+Revises: 78dbe23d8204
+Create Date: 2025-08-18 21:36:19.279969
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from citext import CIText
+
+# revision identifiers, used by Alembic.
+revision = "f36ce1bde293"
+down_revision = "78dbe23d8204"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "system_group",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("fides_key", sa.String(), nullable=False),
+        sa.Column(
+            "label_color",
+            sa.Enum(
+                "taxonomy_white",
+                "taxonomy_red",
+                "taxonomy_orange",
+                "taxonomy_yellow",
+                "taxonomy_green",
+                "taxonomy_blue",
+                "taxonomy_purple",
+                "sandstone",
+                "minos",
+                name="customtaxonomycolor",
+            ),
+            nullable=True,
+        ),
+        sa.Column("data_steward_username", CIText(), nullable=True),
+        sa.Column(
+            "data_uses", sa.ARRAY(sa.String()), server_default="{}", nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["data_steward_username"], ["fidesuser.username"], ondelete="SET NULL"
+        ),
+        sa.ForeignKeyConstraint(
+            ["fides_key"], ["taxonomy_element.fides_key"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_system_group_data_steward_username"),
+        "system_group",
+        ["data_steward_username"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_system_group_fides_key"), "system_group", ["fides_key"], unique=True
+    )
+    op.create_index(op.f("ix_system_group_id"), "system_group", ["id"], unique=False)
+    op.create_table(
+        "system_group_member",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("system_group_id", sa.String(length=255), nullable=True),
+        sa.Column("system_id", sa.String(length=255), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["system_group_id"], ["system_group.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(["system_id"], ["ctl_systems.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "system_group_id", "system_id", name="uq_system_group_member_group_system"
+        ),
+    )
+    op.create_index(
+        op.f("ix_system_group_member_id"), "system_group_member", ["id"], unique=False
+    )
+    op.create_index(
+        op.f("ix_system_group_member_system_group_id"),
+        "system_group_member",
+        ["system_group_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_system_group_member_system_id"),
+        "system_group_member",
+        ["system_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_system_group_member_system_id"), table_name="system_group_member"
+    )
+    op.drop_index(
+        op.f("ix_system_group_member_system_group_id"), table_name="system_group_member"
+    )
+    op.drop_index(op.f("ix_system_group_member_id"), table_name="system_group_member")
+    op.drop_table("system_group_member")
+    op.drop_index(op.f("ix_system_group_id"), table_name="system_group")
+    op.drop_index(op.f("ix_system_group_fides_key"), table_name="system_group")
+    op.drop_index(
+        op.f("ix_system_group_data_steward_username"), table_name="system_group"
+    )
+    op.drop_table("system_group")

fides/api/api/v1/endpoints/system.py

@@ -1,29 +1,27 @@
-import datetime
 from typing import Annotated, Dict, List, Optional, Union
 
 from fastapi import Depends, HTTPException, Query, Response, Security
 from fastapi_pagination import Page, Params
 from fastapi_pagination.bases import AbstractPage
-from fastapi_pagination.ext.async_sqlalchemy import paginate as async_paginate
 from fastapi_pagination.ext.sqlalchemy import paginate
 from fideslang.models import System as SystemSchema
 from fideslang.validation import FidesKey
 from loguru import logger
 from pydantic import Field
-from sqlalchemy import or_
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 from sqlalchemy.orm import Session
 from starlette import status
-from starlette.status import HTTP_200_OK, HTTP_204_NO_CONTENT, HTTP_404_NOT_FOUND
+from starlette.status import (
+    HTTP_200_OK,
+    HTTP_204_NO_CONTENT,
+    HTTP_400_BAD_REQUEST,
+    HTTP_404_NOT_FOUND,
+)
 
 from fides.api.api import deps
 from fides.api.api.v1.endpoints.saas_config_endpoints import instantiate_connection
-from fides.api.db.crud import (
-    get_resource,
-    get_resource_with_custom_fields,
-    list_resource,
-)
+from fides.api.db.crud import get_resource, get_resource_with_custom_fields
 from fides.api.db.ctl_session import get_async_db
 from fides.api.db.system import (
     create_system,
@@ -34,10 +32,8 @@ from fides.api.db.system import (
 )
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
 from fides.api.models.fides_user import FidesUser
-from fides.api.models.sql_models import (  # type:ignore[attr-defined]
-    PrivacyDeclaration,
-    System,
-)
+from fides.api.models.sql_models import System  # type:ignore[attr-defined]
+from fides.api.oauth.roles import APPROVER
 from fides.api.oauth.system_manager_oauth_util import (
     verify_oauth_client_for_system_from_fides_key,
     verify_oauth_client_for_system_from_request_body_cli,
@@ -56,8 +52,12 @@ from fides.api.schemas.connection_configuration.connection_secrets import (
 from fides.api.schemas.connection_configuration.saas_config_template_values import (
     SaasConnectionTemplateValues,
 )
-from fides.api.schemas.filter_params import FilterParams
-from fides.api.schemas.system import BasicSystemResponse, SystemResponse
+from fides.api.schemas.system import (
+    AssignStewardRequest,
+    BasicSystemResponse,
+    SystemResponse,
+)
+from fides.api.service.deps import get_system_service
 from fides.api.util.api_router import APIRouter
 from fides.api.util.connection_util import (
     connection_status,
@@ -66,7 +66,6 @@ from fides.api.util.connection_util import (
     patch_connection_configs,
     validate_secrets,
 )
-from fides.api.util.filter_utils import apply_filters_to_query
 from fides.common.api.scope_registry import (
     CONNECTION_CREATE_OR_UPDATE,
     CONNECTION_DELETE,
@@ -82,6 +81,7 @@ from fides.common.api.v1.urn_registry import (
     SYSTEM_CONNECTIONS,
     V1_URL_PREFIX,
 )
+from fides.service.system.system_service import SystemService
 
 SYSTEM_ROUTER = APIRouter(tags=["System"], prefix=f"{V1_URL_PREFIX}/system")
 SYSTEM_CONNECTIONS_ROUTER = APIRouter(
@@ -349,6 +349,123 @@ async def delete(
     }
 
 
+@SYSTEM_ROUTER.post(
+    "/bulk-delete",
+    dependencies=[
+        Security(
+            verify_oauth_client_prod,
+            scopes=[SYSTEM_DELETE],
+        )
+    ],
+    status_code=status.HTTP_200_OK,
+)
+async def system_bulk_delete(
+    fides_keys: List[str],
+    db: AsyncSession = Depends(get_async_db),
+) -> Dict:
+    """Delete multiple systems by their fides_keys."""
+
+    deleted: List[Dict] = []
+
+    async with db.begin():
+        # Retrieve all systems within the same transactional context
+        stmt = select(System).filter(System.fides_key.in_(fides_keys))
+        result = await db.execute(stmt)
+        systems_to_delete = result.scalars().all()
+
+        for system in systems_to_delete:
+            await db.delete(system)
+            deleted.append(SystemSchema.model_validate(system).model_dump(mode="json"))
+
+    return {
+        "message": f"Deleted {len(deleted)} system(s)",
+        "deleted": deleted,
+    }
+
+
+@SYSTEM_ROUTER.post(
+    "/assign-steward",
+    dependencies=[
+        Security(
+            verify_oauth_client_prod,
+            scopes=[SYSTEM_UPDATE],
+        )
+    ],
+)
+async def bulk_assign_steward(
+    data: AssignStewardRequest,
+    db: Session = Depends(deps.get_db),
+) -> Dict:
+    """Assign the given `data_steward` (username) as a system manager for the list of `system_keys`.
+
+    This mirrors the behavior of the user permissions endpoint that assigns systems to a given user but
+    from the perspective of the System API. It validates that:
+
+    1. The provided user exists.
+    2. The user already has permissions (cannot be None).
+    3. The user is not an approver (approvers are explicitly disallowed from being system managers).
+    4. All provided `system_keys` resolve to existing System records.
+    5. There are no duplicate keys in the payload.
+    """
+
+    data_steward = data.data_steward
+    system_keys = data.system_keys
+
+    user: Optional[FidesUser] = FidesUser.get_by(
+        db, field="username", value=data_steward
+    )
+    if not user:
+        raise HTTPException(
+            status_code=HTTP_404_NOT_FOUND,
+            detail=f"No user found with username {data_steward}.",
+        )
+
+    # Validate user has permissions assigned
+    if not (user.permissions and user.permissions.roles):  # type: ignore[attr-defined]
+        raise HTTPException(
+            status_code=HTTP_400_BAD_REQUEST,
+            detail=f"User {data_steward} needs permissions before they can be assigned as system manager.",
+        )
+
+    # Approvers are not allowed to be system managers
+    if APPROVER in user.permissions.roles:  # type: ignore[attr-defined]
+        raise HTTPException(
+            status_code=HTTP_400_BAD_REQUEST,
+            detail=f"User {data_steward} is an {APPROVER} and cannot be assigned as a system manager.",
+        )
+
+    # Check for duplicate system keys
+    if len(set(system_keys)) != len(system_keys):
+        raise HTTPException(
+            status_code=HTTP_400_BAD_REQUEST,
+            detail=f"Cannot add user {data_steward} as system manager. Duplicate systems in request body.",
+        )
+
+    # Retrieve systems and validate existence
+    systems_query = db.query(System).filter(System.fides_key.in_(system_keys))
+    if systems_query.count() != len(system_keys):
+        raise HTTPException(
+            status_code=HTTP_404_NOT_FOUND,
+            detail=f"Cannot add user {data_steward} as system manager. System(s) not found.",
+        )
+
+    systems = systems_query.all()
+
+    logger.info(
+        "Assigning user {} as data steward for {} systems", data_steward, len(systems)
+    )
+
+    updated_count = 0
+    for system in systems:
+        if user not in system.data_stewards:
+            user.set_as_system_manager(db, system)
+            updated_count += 1
+
+    return {
+        "message": f"User {data_steward} assigned as data steward to {updated_count} system(s)",
+    }
+
+
 @SYSTEM_ROUTER.post(
     "/",
     response_model=SystemResponse,
@@ -384,62 +501,30 @@ async def create(
     name="List systems (optionally paginated)",
 )
 async def ls(  # pylint: disable=invalid-name
-    db: AsyncSession = Depends(get_async_db),
+    system_service: SystemService = Depends(get_system_service),
     size: Optional[int] = Query(None, ge=1, le=100),
     page: Optional[int] = Query(None, ge=1),
     search: Optional[str] = None,
     data_uses: Optional[List[FidesKey]] = Query(None),
     data_categories: Optional[List[FidesKey]] = Query(None),
     data_subjects: Optional[List[FidesKey]] = Query(None),
-    show_deleted: Optional[bool] = Query(False),
-) -> List:
+    show_deleted: bool = Query(False),
+) -> Union[List[System], Page[System]]:
     """Get a list of all of the Systems.
     If any parameters or filters are provided the response will be paginated and/or filtered.
     Otherwise all Systems will be returned (this may be a slow operation if there are many systems,
     so using the pagination parameters is recommended).
     """
-    if not (size or page or search or data_uses or data_categories or data_subjects):
-        # if no advanced parameters are passed, we return a very basic list of all System resources
-        # to maintain backward compatibility of the original API, which backs some important client usages, e.g. the fides CLI
 
-        return await list_resource(System, db)
-
-    query = select(System)
-
-    pagination_params = Params(page=page or 1, size=size or 50)
-    # Need to join with PrivacyDeclaration in order to be able to filter
-    # by data use, data category, and data subject
-    if any([data_uses, data_categories, data_subjects]):
-        query = query.outerjoin(
-            PrivacyDeclaration, System.id == PrivacyDeclaration.system_id
-        )
-
-    # Filter out any vendor deleted systems, unless explicitly asked for
-    if not show_deleted:
-        query = query.filter(
-            or_(
-                System.vendor_deleted_date.is_(None),
-                System.vendor_deleted_date >= datetime.datetime.now(),
-            )
-        )
-
-    filter_params = FilterParams(
+    return await system_service.get_systems(
         search=search,
         data_uses=data_uses,
         data_categories=data_categories,
         data_subjects=data_subjects,
+        size=size,
+        page=page,
+        show_deleted=show_deleted,
     )
-    filtered_query = apply_filters_to_query(
-        query=query,
-        filter_params=filter_params,
-        search_model=System,
-        taxonomy_model=PrivacyDeclaration,
-    )
-
-    # Add a distinct so we only get one row per system
-    duplicates_removed = filtered_query.distinct(System.id)
-
-    return await async_paginate(db, duplicates_removed, pagination_params)
 
 
 @SYSTEM_ROUTER.patch(

fides/api/db/base.py

@@ -21,6 +21,7 @@ from fides.api.models.detection_discovery.monitor_task import (
     MonitorTask,
     MonitorTaskExecutionLog,
 )
+from fides.api.models.detection_discovery.web_monitor import WebMonitorGroupJob
 from fides.api.models.experience_notices import ExperienceNotices
 from fides.api.models.fides_cloud import FidesCloud
 from fides.api.models.fides_user import FidesUser
@@ -77,6 +78,7 @@ from fides.api.models.property import (
 from fides.api.models.registration import UserRegistration
 from fides.api.models.storage import StorageConfig
 from fides.api.models.system_compass_sync import SystemCompassSync
+from fides.api.models.system_group import SystemGroup, SystemGroupMember
 from fides.api.models.system_history import SystemHistory
 from fides.api.models.system_manager import SystemManager
 from fides.api.models.taxonomy import (

fides/api/main.py

@@ -45,6 +45,7 @@ from fides.api.service.privacy_request.email_batch_service import (
     initiate_scheduled_batch_email_send,
 )
 from fides.api.service.privacy_request.request_service import (
+    initiate_async_tasks_status_polling,
     initiate_interrupted_task_requeue_poll,
     initiate_poll_for_exited_privacy_request_tasks,
     initiate_scheduled_dsr_data_removal,
@@ -102,6 +103,7 @@ async def lifespan(wrapped_app: FastAPI) -> AsyncGenerator[None, None]:
     initiate_poll_for_exited_privacy_request_tasks()
     initiate_scheduled_dsr_data_removal()
     initiate_interrupted_task_requeue_poll()
+    initiate_async_tasks_status_polling()
     initiate_bcrypt_migration_task()
     initiate_post_upgrade_index_creation()
 

fides/api/models/asset.py

@@ -29,6 +29,8 @@ class ConsentStatus(str, Enum):
     Consent status of the asset
     """
 
+    cmp_error = "cmp_error"
+    pre_consent = "pre_consent"
     with_consent = "with_consent"
     without_consent = "without_consent"
     exempt = "exempt"

fides/api/models/detection_discovery/web_monitor.py

@@ -0,0 +1,61 @@
+from typing import Any, Optional
+
+from sqlalchemy import Boolean, Column, String, insert, select
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.ext.declarative import declared_attr
+
+from fides.api.db.base_class import Base
+
+
+class WebMonitorGroupJob(Base):
+    """SQL model for storing data returned from a web monitor group job"""
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "web_monitor_group_job"
+
+    # The id for the web monitor group on the web monitor service
+    group_id = Column(String, nullable=False, unique=True)
+
+    # The raw experiences data obtained for that group from the web monitor service
+    raw_experiences_data = Column(JSONB, nullable=True)
+    # The processed experiences data, used to calculate an StagedResource's consent status
+    processed_experiences_data = Column(JSONB, nullable=True)
+
+    # Whether this data is from a test_website monitor rather than a real one
+    is_test_run = Column(Boolean, nullable=False, default=False)
+
+    @classmethod
+    async def create_async(
+        cls,
+        async_session: AsyncSession,
+        *,
+        group_id: str,
+        is_test_run: Optional[bool] = False,
+        raw_experiences_data: Optional[dict[str, Any]] = None,
+        processed_experiences_data: Optional[dict[str, Any]] = None,
+    ) -> "WebMonitorGroupJob":
+        """Create a new WebMonitorGroupJob"""
+        values = {
+            "group_id": group_id,
+            "is_test_run": is_test_run,
+            "raw_experiences_data": raw_experiences_data,
+            "processed_experiences_data": processed_experiences_data,
+        }
+        stmt = insert(cls.__table__).values(**values).returning(*cls.__table__.columns)
+        result = await async_session.execute(stmt)
+        row = result.first()
+        return cls(**dict(row))
+
+    @classmethod
+    async def get_by_group_id_async(
+        cls, async_session: AsyncSession, group_id: str
+    ) -> Optional["WebMonitorGroupJob"]:
+        """Get a WebMonitorGroupJob by group_id"""
+        stmt = select(*cls.__table__.columns).where(
+            cls.__table__.c.group_id == group_id
+        )
+        result = await async_session.execute(stmt)
+        row = result.first()
+        return cls(**dict(row)) if row else None

fides/api/models/sql_models.py

@@ -461,6 +461,14 @@ class System(Base, FidesBase):
         "Asset", back_populates="system", lazy="selectin", uselist=True, viewonly=True
     )
 
+    system_groups = relationship(
+        "SystemGroup",
+        secondary="system_group_member",
+        back_populates="systems",
+        lazy="selectin",
+        viewonly=True,
+    )
+
     @classmethod
     def get_data_uses(
         cls: Type[System], systems: List[System], include_parents: bool = True