ethyca-fides 2.70.5rc0__py2.py3-none-any.whl → 2.71.0rc0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-24T16:39:08-0400",
+ "date": "2025-09-24T16:33:32-0400",
  "dirty": false,
  "error": null,
- "full-revisionid": "b6ea73ff2f2245e67ca50d550cd9560fb3072f44",
- "version": "2.70.5rc0"
+ "full-revisionid": "32f4216c8a85464d318fc4088c1f99d8e5e8d86e",
+ "version": "2.71.0rc0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/0eef0016cf06_adding_oauth2_config_for_.py

@@ -0,0 +1,69 @@
+"""Adding OAuth2 config for ConnectionConfig
+
+Revision ID: 0eef0016cf06
+Revises: 9e0dcbf67b9f
+Create Date: 2025-09-12 01:06:26.838560
+
+"""
+
+import sqlalchemy as sa
+import sqlalchemy_utils
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0eef0016cf06"
+down_revision = "9e0dcbf67b9f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "oauth_config",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("connection_config_id", sa.String(), nullable=False),
+        sa.Column(
+            "grant_type",
+            sa.Enum(
+                "authorization_code",
+                "client_credentials",
+                "password",
+                "implicit",
+                name="oauthgranttype",
+            ),
+            nullable=False,
+        ),
+        sa.Column("token_url", sa.String(), nullable=False),
+        sa.Column("scope", sa.String(), nullable=True),
+        sa.Column("client_id", sa.String(), nullable=False),
+        sa.Column(
+            "client_secret", sqlalchemy_utils.types.StringEncryptedType, nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["connection_config_id"],
+            ["connectionconfig.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_oauth_config_id"), "oauth_config", ["id"], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_oauth_config_id"), table_name="oauth_config")
+    op.drop_table("oauth_config")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/9e0dcbf67b9f_add_digest_config.py

@@ -0,0 +1,84 @@
+"""add digest config
+
+Revision ID: 9e0dcbf67b9f
+Revises: cd8649be3a2b
+Create Date: 2025-09-22 20:23:48.258509
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "9e0dcbf67b9f"
+down_revision = "cd8649be3a2b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "digest_config",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("digest_type", sa.String(length=255), nullable=False),
+        sa.Column("name", sa.String(length=255), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("enabled", sa.Boolean(), nullable=False),
+        sa.Column("messaging_service_type", sa.String(length=255), nullable=False),
+        sa.Column("cron_expression", sa.String(length=100), nullable=False),
+        sa.Column("timezone", sa.String(length=50), nullable=False),
+        sa.Column("last_sent_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("next_scheduled_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "config_metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_digest_config_digest_type"),
+        "digest_config",
+        ["digest_type"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_config_enabled"), "digest_config", ["enabled"], unique=False
+    )
+    op.create_index(op.f("ix_digest_config_id"), "digest_config", ["id"], unique=False)
+    op.create_index(
+        op.f("ix_digest_config_messaging_service_type"),
+        "digest_config",
+        ["messaging_service_type"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_digest_config_next_scheduled_at"),
+        "digest_config",
+        ["next_scheduled_at"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_digest_config_next_scheduled_at"), table_name="digest_config"
+    )
+    op.drop_index(
+        op.f("ix_digest_config_messaging_service_type"), table_name="digest_config"
+    )
+    op.drop_index(op.f("ix_digest_config_id"), table_name="digest_config")
+    op.drop_index(op.f("ix_digest_config_enabled"), table_name="digest_config")
+    op.drop_index(op.f("ix_digest_config_digest_type"), table_name="digest_config")
+    op.drop_table("digest_config")

fides/api/alembic/migrations/versions/a8e0c016afd_add_classification_benchmark_table.py

@@ -0,0 +1,126 @@
+"""Add classification benchmark table
+
+Revision ID: a8e0c016afd
+Revises: 4d8c0fcc5771
+Create Date: 2024-12-19 12:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "a8e0c016afd"
+down_revision = "4d8c0fcc5771"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "classification_benchmark",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "monitor_config_key",
+            sa.String(),
+            sa.ForeignKey("monitorconfig.key", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "dataset_fides_key",
+            sa.String(),
+            sa.ForeignKey("ctl_datasets.fides_key", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "resource_urns",
+            postgresql.ARRAY(sa.String()),
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.Column(
+            "overall_metrics",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+            server_default=None,
+        ),
+        sa.Column(
+            "field_accuracy_details",
+            postgresql.ARRAY(postgresql.JSONB(astext_type=sa.Text())),
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.Column(
+            "status",
+            sa.String(length=50),
+            nullable=False,
+            server_default="pending",
+        ),
+        sa.Column(
+            "messages",
+            postgresql.ARRAY(sa.String()),
+            nullable=False,
+            server_default="{}",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_classification_benchmark_id"),
+        "classification_benchmark",
+        ["id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_classification_benchmark_monitor_config_key"),
+        "classification_benchmark",
+        ["monitor_config_key"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_classification_benchmark_dataset_fides_key"),
+        "classification_benchmark",
+        ["dataset_fides_key"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_classification_benchmark_status"),
+        "classification_benchmark",
+        ["status"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_classification_benchmark_status"),
+        table_name="classification_benchmark",
+    )
+    op.drop_index(
+        op.f("ix_classification_benchmark_dataset_fides_key"),
+        table_name="classification_benchmark",
+    )
+    op.drop_index(
+        op.f("ix_classification_benchmark_monitor_config_key"),
+        table_name="classification_benchmark",
+    )
+    op.drop_index(
+        op.f("ix_classification_benchmark_id"), table_name="classification_benchmark"
+    )
+    op.drop_table("classification_benchmark")
+    # ### end Alembic commands ###

fides/api/alembic/migrations/versions/cd8649be3a2b_add_classifications_and_user_assigned_.py

@@ -0,0 +1,74 @@
+"""add classifications and user_assigned_data_categories indices
+
+Revision ID: cd8649be3a2b
+Revises: a8e0c016afd
+Create Date: 2025-09-19 13:49:02.762672
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from loguru import logger
+
+# revision identifiers, used by Alembic.
+revision = "cd8649be3a2b"
+down_revision = "a8e0c016afd"
+branch_labels = None
+depends_on = None
+
+"""
+WARNING - Conditional migration based on table size
+"""
+
+
+def upgrade():
+    # Check stagedresource table size
+    connection = op.get_bind()
+
+    # Check stagedresource table size to decide if we should create indexes immediately
+    table_size = connection.execute(
+        sa.text("SELECT COUNT(*) FROM stagedresource")
+    ).scalar()
+
+    if table_size < 1000000:
+        logger.info(
+            f"stagedresource table has {table_size} rows, creating GIN indices directly"
+        )
+
+        # Create GIN index for user_assigned_data_categories array operations (&&, @>, <@)
+        op.execute(
+            "CREATE INDEX idx_stagedresource_user_categories_gin "
+            "ON stagedresource USING GIN (user_assigned_data_categories)"
+        )
+
+        # Create GIN index for classifications array operations (&&, @>, <@)
+        op.execute(
+            "CREATE INDEX idx_stagedresource_classifications_gin "
+            "ON stagedresource USING GIN (classifications)"
+        )
+
+        logger.info("GIN indices created successfully")
+
+    else:
+        logger.warning(
+            "The stagedresource table has more than 1 million rows, "
+            "skipping index creation. Indexes will be created during application startup "
+            "via post_upgrade_index_creation.py"
+        )
+
+
+def downgrade():
+    # Drop indices in reverse order
+    logger.info("Dropping GIN indices for stagedresource arrays")
+
+    try:
+        op.drop_index("idx_stagedresource_classifications_gin", "stagedresource")
+        logger.info("Dropped classifications GIN index")
+    except Exception as e:
+        logger.warning(f"Could not drop classifications index: {e}")
+
+    try:
+        op.drop_index("idx_stagedresource_user_categories_gin", "stagedresource")
+        logger.info("Dropped user_assigned_data_categories GIN index")
+    except Exception as e:
+        logger.warning(f"Could not drop user_categories index: {e}")

fides/api/alembic/migrations/versions/e2cda8d6abc3_add_dismissed_in_activity_view_to_.py

@@ -0,0 +1,29 @@
+"""add_dismissed_to_monitor_task
+
+Revision ID: e2cda8d6abc3
+Revises: 0eef0016cf06
+Create Date: 2025-09-17 23:37:49.678071
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "e2cda8d6abc3"
+down_revision = "0eef0016cf06"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Add the dismissed column to monitortask table
+    op.add_column(
+        "monitortask",
+        sa.Column("dismissed", sa.Boolean(), nullable=False, server_default="f"),
+    )
+
+
+def downgrade():
+    # Remove the dismissed column from monitortask table
+    op.drop_column("monitortask", "dismissed")

fides/api/api/v1/endpoints/connection_endpoints.py

@@ -2,7 +2,8 @@ from __future__ import annotations
 
 from typing import Annotated, Any, Dict, List, Optional
 
-from fastapi import Depends
+import sqlalchemy
+from fastapi import Depends, HTTPException
 from fastapi.params import Query, Security
 from fastapi_pagination import Page, Params
 from fastapi_pagination.bases import AbstractPage
@@ -13,9 +14,15 @@ from pydantic import Field
 from sqlalchemy import null, or_
 from sqlalchemy.orm import Session
 from sqlalchemy_utils import escape_like
-from starlette.status import HTTP_200_OK, HTTP_204_NO_CONTENT
+from starlette.status import (
+    HTTP_200_OK,
+    HTTP_204_NO_CONTENT,
+    HTTP_404_NOT_FOUND,
+    HTTP_422_UNPROCESSABLE_ENTITY,
+)
 
 from fides.api.api import deps
+from fides.api.models.connection_oauth_credentials import OAuthConfig
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType
 from fides.api.oauth.utils import verify_oauth_client
 from fides.api.schemas.connection_configuration import connection_secrets_schemas
@@ -25,6 +32,9 @@ from fides.api.schemas.connection_configuration.connection_config import (
     ConnectionConfigurationResponseWithSystemKey,
     CreateConnectionConfigurationWithSecrets,
 )
+from fides.api.schemas.connection_configuration.connection_oauth_config import (
+    OAuthConfigSchema,
+)
 from fides.api.schemas.connection_configuration.connection_secrets import (
     TestStatusMessage,
 )
@@ -45,6 +55,7 @@ from fides.common.api.scope_registry import (
 )
 from fides.common.api.v1.urn_registry import (
     CONNECTION_BY_KEY,
+    CONNECTION_OAUTH,
     CONNECTION_SECRETS,
     CONNECTION_TEST,
     CONNECTIONS,
@@ -312,3 +323,143 @@ def test_connection_config_secrets(
     connection_config = get_connection_config_or_error(db, connection_key)
     msg = f"Test completed for ConnectionConfig with key: {connection_key}."
     return connection_status(connection_config, msg, db)
+
+
+@router.put(
+    CONNECTION_OAUTH,
+    dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_CREATE_OR_UPDATE])],
+    status_code=HTTP_200_OK,
+)
+def put_connection_oauth_config(
+    connection_key: FidesKey,
+    *,
+    oauth_config: OAuthConfigSchema,
+    db: Session = Depends(deps.get_db),
+    verify: Optional[bool] = True,
+) -> TestStatusMessage:
+    """
+    Create or update the OAuth2 configuration for a given connection.
+    """
+    connection_config = get_connection_config_or_error(db, connection_key)
+
+    if connection_config.connection_type != ConnectionType.https:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="OAuth2 configuration can only be set for HTTPS connections.",
+        )
+
+    logger.info("Updating OAuth2 config for '{}'", connection_key)
+    connection_config.oauth_config = OAuthConfig(
+        **oauth_config.model_dump(mode="json")  # type: ignore[arg-type]
+    )
+    connection_config.save(db=db)
+
+    msg = (
+        f"OAuth2 configuration updated for ConnectionConfig with key: {connection_key}."
+    )
+
+    if verify:
+        return connection_status(connection_config, msg, db)
+
+    return TestStatusMessage(msg=msg, test_status=None)
+
+
+@router.delete(
+    CONNECTION_OAUTH,
+    dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_CREATE_OR_UPDATE])],
+    status_code=HTTP_204_NO_CONTENT,
+    response_model=None,
+)
+def delete_connection_oauth_config(
+    connection_key: FidesKey,
+    *,
+    db: Session = Depends(deps.get_db),
+) -> None:
+    """
+    Delete the OAuth2 configuration for a given connection.
+    """
+    connection_config = get_connection_config_or_error(db, connection_key)
+
+    if connection_config.connection_type != ConnectionType.https:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="OAuth2 configuration can only be deleted for HTTPS connections",
+        )
+
+    if not connection_config.oauth_config:
+        raise HTTPException(
+            status_code=HTTP_404_NOT_FOUND,
+            detail="No OAuth2 configuration found to delete.",
+        )
+
+    logger.info("Deleting OAuth2 config for '{}'", connection_key)
+    connection_config.oauth_config = None
+    connection_config.save(db=db)
+
+
+@router.patch(
+    CONNECTION_OAUTH,
+    dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_CREATE_OR_UPDATE])],
+    status_code=HTTP_200_OK,
+)
+def patch_connection_oauth_config(
+    connection_key: FidesKey,
+    *,
+    oauth_config: OAuthConfigSchema,
+    db: Session = Depends(deps.get_db),
+    verify: Optional[bool] = True,
+) -> TestStatusMessage:
+    """
+    Partially update the OAuth2 configuration for a given connection.
+    """
+    connection_config = get_connection_config_or_error(db, connection_key)
+
+    if connection_config.connection_type != ConnectionType.https:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="OAuth2 configuration can only be set for HTTPS connections.",
+        )
+
+    existing_oauth_config: Optional[Dict[str, Any]] = (
+        {
+            "grant_type": connection_config.oauth_config.grant_type.value,
+            "token_url": connection_config.oauth_config.token_url,
+            "scope": connection_config.oauth_config.scope,
+            "client_id": connection_config.oauth_config.client_id,
+            "client_secret": connection_config.oauth_config.client_secret,
+        }
+        if connection_config.oauth_config
+        else None
+    )
+
+    # Combine the existing config with the new config overlaid
+    patched_oauth_config = {}
+    if existing_oauth_config:
+        patched_oauth_config = {**existing_oauth_config}
+
+    patched_oauth_config = {
+        **patched_oauth_config,
+        **{k: v for (k, v) in oauth_config.model_dump(mode="json").items() if v is not None},  # type: ignore[dict-item]
+    }
+
+    logger.info("Patching OAuth2 config for '{}'", connection_key)
+    validated_config = OAuthConfigSchema.model_validate(patched_oauth_config)
+    try:
+        connection_config.oauth_config = OAuthConfig(
+            **validated_config.model_dump(mode="json")
+        )
+        connection_config.save(db=db)
+    except sqlalchemy.exc.IntegrityError as exc:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Invalid OAuth2 configuration.",
+        ) from exc
+
+    msg = (
+        f"OAuth2 configuration updated for ConnectionConfig with key: {connection_key}."
+    )
+
+    if verify:
+        return connection_status(connection_config, msg, db)
+
+    return TestStatusMessage(msg=msg, test_status=None)

fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -4,7 +4,7 @@ import csv
 import io
 import json
 from collections import defaultdict
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import (
     Annotated,
     Any,
@@ -285,15 +285,19 @@ def privacy_request_csv_download(
             "Status",
             "Request Type",
             "Time Received",
+            "Deadline",
             "Reviewed By",
             "Request ID",
             "Time Approved/Denied",
             "Denial Reason",
+            "Last Updated",
+            "Completed On",
         ]
         + identity_columns
         + with_prefix("Custom Field", list(custom_field_columns.values()))
     )
 
+    pr: PrivacyRequest
     for pr in privacy_request_query:
         denial_reason = (
             denial_audit_logs[pr.id]
@@ -301,14 +305,26 @@ def privacy_request_csv_download(
             else None
         )
 
+        action_types: List[str] = []
+        if pr and pr.policy:
+            for rule in pr.policy.rules or []:  # type: ignore
+                action_types.append(rule.action_type)
+
+        deadline: Optional[datetime] = None
+        if pr.days_left and pr.created_at:
+            deadline = pr.created_at + timedelta(days=pr.days_left)
+
         static_cells = [
             pr.status.value if pr.status else None,
-            pr.policy.rules[0].action_type if len(pr.policy.rules) > 0 else None,
+            ("+".join(action_types)),
             pr.created_at,
+            deadline,
             pr.reviewed_by,
             pr.id,
             pr.reviewed_at,
             denial_reason,
+            pr.updated_at,
+            pr.finalized_at,
         ]
 
         identity_cells = extract_identity_cells(identity_columns, pr)

fides/api/db/base.py

@@ -9,6 +9,7 @@ from fides.api.models.audit_log import AuditLog
 from fides.api.models.authentication_request import AuthenticationRequest
 from fides.api.models.client import ClientDetail
 from fides.api.models.comment import Comment, CommentReference
+from fides.api.models.connection_oauth_credentials import OAuthConfig
 from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.consent_automation import ConsentAutomation
 from fides.api.models.custom_asset import CustomAsset
@@ -22,6 +23,7 @@ from fides.api.models.detection_discovery.monitor_task import (
     MonitorTaskExecutionLog,
 )
 from fides.api.models.detection_discovery.web_monitor import WebMonitorGroupJob
+from fides.api.models.digest import DigestConfig
 from fides.api.models.event_audit import EventAudit
 from fides.api.models.experience_notices import ExperienceNotices
 from fides.api.models.fides_cloud import FidesCloud