ethyca-fides 2.73.1rc2__py2.py3-none-any.whl → 2.73.2b1__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-30T16:26:15-0400",
+ "date": "2025-10-31T14:51:49+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "34dad3dac5891167e4a1b6cd743199971220da57",
- "version": "2.73.1rc2"
+ "full-revisionid": "3a382cb972e2ea964f078259f93ed9b6ea0b1d02",
+ "version": "2.73.2b1"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/xx_2025_10_29_1659_80d28dea3b6b_added_duplicate_group_table.py

@@ -0,0 +1,79 @@
+"""added duplicate group table
+
+Revision ID: 80d28dea3b6b
+Revises: c09e76282dd1
+Create Date: 2025-10-29 16:59:09.551676
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "80d28dea3b6b"
+down_revision = "c09e76282dd1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "duplicate_group",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("rule_version", sa.Text(), nullable=False),
+        sa.Column("is_active", sa.Boolean(), server_default="t", nullable=False),
+        sa.Column("dedup_key", sa.Text(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.drop_index(
+        "ix_privacyrequest_duplicate_request_group_id", table_name="privacyrequest"
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+    op.add_column(
+        "privacyrequest",
+        sa.Column(
+            "duplicate_request_group_id", postgresql.UUID(as_uuid=True), nullable=True
+        ),
+    )
+    op.create_foreign_key(
+        "fk_privacyrequest_duplicate_group",
+        "privacyrequest",
+        "duplicate_group",
+        ["duplicate_request_group_id"],
+        ["id"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "fk_privacyrequest_duplicate_group", "privacyrequest", type_="foreignkey"
+    )
+    op.drop_column("privacyrequest", "duplicate_request_group_id")
+    op.add_column(
+        "privacyrequest",
+        sa.Column("duplicate_request_group_id", sa.String(), nullable=True),
+    )
+    op.create_index(
+        "ix_privacyrequest_duplicate_request_group_id",
+        "privacyrequest",
+        ["duplicate_request_group_id"],
+        unique=False,
+    )
+    op.drop_table("duplicate_group")
+    # ### end Alembic commands ###

fides/api/api/v1/endpoints/generic_overrides.py

@@ -368,7 +368,7 @@ async def create_data_use(
     """
     try:
         return taxonomy_service.create_or_update_element(
-            "data_uses", data_use.model_dump()
+            "data_use", data_use.model_dump()
         )
     except KeyOrNameAlreadyExists:
         raise HTTPException(
@@ -399,7 +399,7 @@ async def create_data_category(
 
     try:
         return taxonomy_service.create_or_update_element(
-            "data_categories", data_category.model_dump()
+            "data_category", data_category.model_dump()
         )
     except KeyOrNameAlreadyExists:
         raise HTTPException(
@@ -430,7 +430,7 @@ async def create_data_subject(
 
     try:
         return taxonomy_service.create_or_update_element(
-            "data_subjects", data_subject.model_dump()
+            "data_subject", data_subject.model_dump()
         )
     except KeyOrNameAlreadyExists:
         raise HTTPException(
@@ -465,7 +465,7 @@ async def update_data_use(
         )
     try:
         result = taxonomy_service.update_element(
-            "data_uses", data_use.fides_key, data_use.model_dump()
+            "data_use", data_use.fides_key, data_use.model_dump()
         )
         if not result:
             raise HTTPException(
@@ -501,7 +501,7 @@ async def update_data_category(
         )
     try:
         result = taxonomy_service.update_element(
-            "data_categories", data_category.fides_key, data_category.model_dump()
+            "data_category", data_category.fides_key, data_category.model_dump()
         )
         if not result:
             raise HTTPException(
@@ -534,7 +534,7 @@ async def update_data_category_tagging_instructions(
     """
     try:
         result = taxonomy_service.update_element(
-            "data_categories",
+            "data_category",
             fides_key,
             {"tagging_instructions": tagging_instructions},
         )
@@ -568,7 +568,7 @@ async def delete_data_category_tagging_instructions(
     """
     try:
         result = taxonomy_service.update_element(
-            "data_categories",
+            "data_category",
             fides_key,
             {"tagging_instructions": None},
         )

fides/api/models/privacy_request/duplicate_group.py

@@ -0,0 +1,84 @@
+import hashlib
+import json
+import uuid
+from typing import TYPE_CHECKING, Any
+
+from sqlalchemy import Boolean, Column, Text
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base  # type: ignore[attr-defined]
+from fides.config.duplicate_detection_settings import DuplicateDetectionSettings
+
+if TYPE_CHECKING:
+    from fides.api.models.privacy_request.privacy_request import PrivacyRequest
+
+
+def generate_deterministic_uuid(rule_version: str, dedup_key: str) -> uuid.UUID:
+    """Overrides the base class method to generate a deterministic group id.
+
+    The actual rule version is a hash of the duplicate detection settings config,
+    using simple examples here for illustration:
+    - under rule "email", duplicate@example.com → group A,
+    - under rule "email|phone", duplicate@example.com|1234567890 → group B.
+    - ... and so on.
+
+    No collisions, no overlap.
+    """
+    # Combine the rule version and the key into a stable hash
+    hash_input = f"{rule_version}:{dedup_key}".encode("utf-8")
+    return uuid.UUID(hashlib.md5(hash_input).hexdigest())
+
+
+def generate_rule_version(config: DuplicateDetectionSettings) -> str:
+    """Generate a stable short hash for the dedup rule config."""
+    normalized = json.dumps(config.model_dump(), sort_keys=True)
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:8]
+
+
+class DuplicateGroup(Base):
+    """
+    A table for storing duplicate request group information.
+    """
+
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "duplicate_group"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    rule_version = Column(Text, nullable=False)
+    # TODO: The is_active column is not used yet since we are defaulting to just email for
+    # duplicate detection. When we allow users to configure the duplicate detection settings,
+    # we will need to add a way to activate/deactivate duplicate groups.
+    # This should be done by getting the previous rule version and deactivating all groups with
+    # that rule version. New groups will be created with the new rule version.
+    is_active = Column(Boolean, default=True, nullable=False)
+    dedup_key = Column(Text, nullable=False)
+    privacy_requests = relationship(
+        "PrivacyRequest",
+        back_populates="duplicate_group",
+        lazy="dynamic",
+        primaryjoin="and_(DuplicateGroup.id == PrivacyRequest.duplicate_request_group_id)",
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> "DuplicateGroup":
+        """Create a new duplicate group.
+        Override the base class method to generate a deterministic group id.
+        If the group already exists, update the is_active column to True and return the group.
+        """
+        group_id = generate_deterministic_uuid(data["rule_version"], data["dedup_key"])
+        # If the group already exists, update the is_active column to True and return the group
+        duplicate_group = (
+            db.query(DuplicateGroup).filter(DuplicateGroup.id == group_id).one_or_none()
+        )
+        if duplicate_group:
+            duplicate_group.update(db, data={"is_active": True})
+            return duplicate_group
+
+        # If the group does not exist, create a new one
+        data["id"] = group_id
+        return super().create(db, data=data, check_name=check_name)

fides/api/models/privacy_request/privacy_request.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union
 from celery.result import AsyncResult
 from loguru import logger
 from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String
-from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.mutable import MutableDict, MutableList
 from sqlalchemy.orm import Query, RelationshipProperty, Session, backref, relationship
@@ -69,6 +69,7 @@ from fides.api.models.pre_approval_webhook import (
     PreApprovalWebhook,
     PreApprovalWebhookReply,
 )
+from fides.api.models.privacy_request.duplicate_group import DuplicateGroup
 from fides.api.models.privacy_request.execution_log import (
     COMPLETED_EXECUTION_LOG_STATUSES,
     EXITED_EXECUTION_LOG_STATUSES,
@@ -225,7 +226,16 @@ class PrivacyRequest(
     location = Column(String, nullable=True)
 
     # Duplicate detection - group ID to link duplicates together
-    duplicate_request_group_id = Column(String, index=True, nullable=True)
+    duplicate_request_group_id = Column(
+        UUID(as_uuid=True),
+        ForeignKey("duplicate_group.id"),
+        nullable=True,
+    )
+    duplicate_group = relationship(
+        DuplicateGroup,
+        back_populates="privacy_requests",
+        uselist=False,
+    )
 
     # A PrivacyRequest can be soft deleted, so we store when it was deleted
     deleted_at = Column(DateTime(timezone=True), nullable=True)

fides/api/models/taxonomy.py

@@ -21,7 +21,15 @@ from fides.api.common_exceptions import ValidationError
 from fides.api.db.base_class import Base
 from fides.api.models.sql_models import FidesBase  # type: ignore[attr-defined]
 
-LEGACY_TAXONOMIES = {"data_categories", "data_uses", "data_subjects"}
+# Legacy Fideslang taxonomy keys
+LEGACY_TAXONOMY_KEYS = {
+    "data_category",
+    "data_use",
+    "data_subject",
+}
+
+# Taxonomies that are managed by Fides (legacy taxonomies and system group)
+MANAGED_TAXONOMY_KEYS = {"data_category", "data_use", "data_subject", "system_group"}
 
 
 class TargetType(str, Enum):
@@ -81,7 +89,9 @@ class Taxonomy(Base, FidesBase):
         """Create a new Taxonomy with proper handling of applies_to."""
         # Disallow creating taxonomies that represent legacy types
         fides_key = data.get("fides_key")
-        if fides_key in LEGACY_TAXONOMIES:
+        # Specifically exclude system_group to allow for creation during tests
+        # We already have unique constraints on fides_key to prevent duplicates on DB level
+        if fides_key in MANAGED_TAXONOMY_KEYS:
             raise ValidationError(
                 f"Cannot create taxonomy '{fides_key}'. This is a taxonomy managed by the system."
             )
@@ -166,7 +176,7 @@ class TaxonomyAllowedUsage(Base):
 
     target_type can be either:
     - A generic type: "system", "privacy_declaration", "taxonomy"
-    - A taxonomy key: "data_categories", "data_uses", etc.
+    - A taxonomy key: "data_category", "data_use", etc.
     """
 
     @declared_attr
@@ -193,7 +203,7 @@ class TaxonomyAllowedUsage(Base):
     )
     target_type: Column[str] = Column(
         String, primary_key=True
-    )  # Can be "system", "dataset", OR a taxonomy key like "data_categories"
+    )  # Can be "system", "dataset", OR a taxonomy key like "data_category"
 
 
 class TaxonomyElement(Base, FidesBase):

fides/api/schemas/privacy_request.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from enum import Enum as EnumType
 from typing import Any, Dict, List, Optional, Type, Union
+from uuid import UUID
 
 from fideslang.validation import FidesKey
 from pydantic import ConfigDict, Field, field_serializer, field_validator
@@ -343,7 +344,7 @@ class PrivacyRequestResponse(FidesSchema):
     deleted_by: Optional[str] = None
     finalized_at: Optional[datetime] = None
     finalized_by: Optional[str] = None
-    duplicate_request_group_id: Optional[str] = None
+    duplicate_request_group_id: Optional[UUID] = None
 
     model_config = ConfigDict(from_attributes=True, use_enum_values=True)
 

fides/api/service/async_dsr/strategies/async_dsr_strategy_polling.py

@@ -551,13 +551,20 @@ class AsyncPollingStrategy(AsyncDSRStrategy):
                 self.result_request.result_path,
             )
 
-        # Ensure we have the expected polling result type
-        if not isinstance(polling_result, PollingResult):
-            raise PrivacyRequestError("Polling result must be PollingResult instance")
-
-        # Store results on the sub-request
-        self._store_sub_request_result(polling_result, sub_request, polling_task)
+        # Checks if we have a polling result, response could be empty in case there was no data to access
+        if polling_result:
+            # Ensure we have the expected polling result type
+            if not isinstance(polling_result, PollingResult):
+                raise PrivacyRequestError(
+                    "Polling result must be PollingResult instance"
+                )
 
+            # Store results on the sub-request
+            self._store_sub_request_result(polling_result, sub_request, polling_task)
+        else:
+            logger.info(
+                f"No result response for sub-request {sub_request.id} for task {polling_task.id}"
+            )
         # Mark as complete using existing method
         sub_request.update_status(self.session, ExecutionLogStatus.complete.value)