ethyca-fides 2.72.3rc1__py2.py3-none-any.whl → 2.73.0__py2.py3-none-any.whl

fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-21T10:52:42-0300",
+ "date": "2025-10-24T16:42:20-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "9a96fc10ec86ded899e833ed49eabea81a1b04a2",
- "version": "2.72.3rc1"
+ "full-revisionid": "ca872f4fade7f59c7c49c558ecb610e1f0b41128",
+ "version": "2.73.0"
 }
 '''  # END VERSION_JSON
 

fides/api/alembic/migrations/versions/a55e12c2c2df_add_tagging_instructions_to_data_.py

@@ -0,0 +1,29 @@
+"""add_tagging_instructions_to_data_category
+
+Revision ID: a55e12c2c2df
+Revises: 65a1bc82ae09
+Create Date: 2025-10-13 21:14:00.723464
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a55e12c2c2df"
+down_revision = "65a1bc82ae09"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Add tagging_instructions column to ctl_data_categories table."""
+    op.add_column(
+        "ctl_data_categories",
+        sa.Column("tagging_instructions", sa.Text(), nullable=True),
+    )
+
+
+def downgrade():
+    """Remove tagging_instructions column from ctl_data_categories table."""
+    op.drop_column("ctl_data_categories", "tagging_instructions")

fides/api/api/v1/endpoints/generic_overrides.py

@@ -1,7 +1,7 @@
 from functools import partial
 from typing import Dict, List, Optional, Union
 
-from fastapi import Depends, HTTPException, Query, Response, Security
+from fastapi import Body, Depends, HTTPException, Query, Response, Security
 from fastapi.concurrency import run_in_threadpool
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse
@@ -516,6 +516,75 @@ async def update_data_category(
         )
 
 
+@data_category_router.patch(
+    "/data_category/{fides_key}/tagging_instructions",
+    dependencies=[Security(verify_oauth_client, scopes=[DATA_CATEGORY_UPDATE])],
+    response_model=DataCategory,
+    status_code=status.HTTP_200_OK,
+    name="Update tagging instructions",
+)
+async def update_data_category_tagging_instructions(
+    fides_key: str,
+    tagging_instructions: str = Body(..., embed=True),
+    taxonomy_service: TaxonomyService = Depends(get_taxonomy_service),
+) -> Dict:
+    """
+    Update or add tagging_instructions for a data category.
+    This is used by LLM-based classification to provide custom instructions for tagging.
+    """
+    try:
+        result = taxonomy_service.update_element(
+            "data_categories",
+            fides_key,
+            {"tagging_instructions": tagging_instructions},
+        )
+        if not result:
+            raise HTTPException(
+                status_code=HTTP_404_NOT_FOUND,
+                detail=f"Data category not found with key: {fides_key}",
+            )
+        return result
+    except (ValidationError, PydanticValidationError) as e:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
+        )
+
+
+@data_category_router.delete(
+    "/data_category/{fides_key}/tagging_instructions",
+    dependencies=[Security(verify_oauth_client, scopes=[DATA_CATEGORY_UPDATE])],
+    response_model=DataCategory,
+    status_code=status.HTTP_200_OK,
+    name="Delete tagging instructions",
+)
+async def delete_data_category_tagging_instructions(
+    fides_key: str,
+    taxonomy_service: TaxonomyService = Depends(get_taxonomy_service),
+) -> Dict:
+    """
+    Remove tagging_instructions from a data category.
+    Sets the tagging_instructions field to None.
+    """
+    try:
+        result = taxonomy_service.update_element(
+            "data_categories",
+            fides_key,
+            {"tagging_instructions": None},
+        )
+        if not result:
+            raise HTTPException(
+                status_code=HTTP_404_NOT_FOUND,
+                detail=f"Data category not found with key: {fides_key}",
+            )
+        return result
+    except (ValidationError, PydanticValidationError) as e:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
+        )
+
+
 GENERIC_OVERRIDES_ROUTER = APIRouter()
 GENERIC_OVERRIDES_ROUTER.include_router(dataset_router)
 GENERIC_OVERRIDES_ROUTER.include_router(data_use_router)

fides/api/models/privacy_preference.py

@@ -58,6 +58,7 @@ class ConsentMethod(Enum):
     gpc = "gpc"
     individual_notice = "individual_notice"
     acknowledge = "acknowledge"
+    external_provider = "external_provider"
 
 
 class ServingComponent(Enum):

fides/api/models/sql_models.py

@@ -184,6 +184,9 @@ class DataCategory(Base, FidesBase):
     version_deprecated = Column(Text)
     replaced_by = Column(Text)
 
+    # LLM Classification
+    tagging_instructions = Column(Text, nullable=True)
+
     children: "RelationshipProperty[List[DataCategory]]" = relationship(
         "DataCategory",
         back_populates="parent",

fides/api/schemas/taxonomy_extensions.py

@@ -23,6 +23,10 @@ class DataUse(BaseDataUse):
 
 class DataCategory(BaseDataCategory):
     active: bool = active_field
+    tagging_instructions: Optional[str] = Field(
+        default=None,
+        description="Instructions for LLM-based data classification tagging.",
+    )
 
 
 class DataSubject(BaseDataSubject):
@@ -44,6 +48,10 @@ class DataUseCreateOrUpdate(TaxonomyCreateOrUpdateBase):
 
 class DataCategoryCreateOrUpdate(TaxonomyCreateOrUpdateBase):
     parent_key: Optional[FidesKey] = None
+    tagging_instructions: Optional[str] = Field(
+        default=None,
+        description="Instructions for LLM-based data classification tagging.",
+    )
 
 
 class DataSubjectCreateOrUpdate(TaxonomyCreateOrUpdateBase):

fides/api/service/async_dsr/strategies/async_dsr_strategy_polling.py

@@ -157,6 +157,7 @@ class AsyncPollingStrategy(AsyncDSRStrategy):
             logger.info(
                 f"Creating initial polling sub-requests for task {request_task.id}"
             )
+
             self._handle_polling_initial_request(
                 request_task,
                 query_config,

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -83,12 +83,10 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         """
         Returns the appropriate request configs based on the current collection and identity
         """
-        collection_name = self.node.address.collection
-
         try:
-            requests = self.endpoints[collection_name].requests
+            requests = self.endpoints[self.collection_name].requests
         except KeyError:
-            logger.error("The '{}' endpoint is not defined", collection_name)
+            logger.error("The '{}' endpoint is not defined", self.collection_name)
             return []
 
         if not requests.read:
@@ -282,6 +280,111 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
             name: value for name, value in self.secrets.items() if name in param_names
         }
 
+    def _safe_extract_value(self, value_list: Optional[List[Any]]) -> Optional[Any]:
+        """
+        Safely extract the first value from a list
+        """
+        return value_list[0] if value_list else None
+
+    def _process_param_values_for_query(
+        self,
+        request: SaaSRequest,
+        input_data: Dict[str, List[Any]],
+        param_values: Dict[str, Any],
+    ) -> None:
+        """
+        Process param_values for query generation on access requests.
+        """
+        for param_value in request.param_values or []:
+            if param_value.references or param_value.identity:
+                input_list = input_data.get(param_value.name)
+                if input_list:
+                    param_values[param_value.name] = self._safe_extract_value(
+                        input_list
+                    )
+            elif param_value.connector_param:
+                connector_value = pydash.get(input_data, param_value.connector_param)
+                param_values[param_value.name] = self._safe_extract_value(
+                    connector_value
+                )
+
+    def _process_param_values_for_update(
+        self,
+        request: SaaSRequest,
+        collection_values: Dict[str, Row],
+        identity_data: Dict[str, Any],
+        input_data: Optional[Dict[str, List[Any]]] = None,
+        param_values: Dict[str, Any] = {},
+    ) -> None:
+        """
+        Process param_values for update generation (erasure requests).
+        Modifies param_values in place.
+
+        Args:
+            request: The SaaS request configuration
+            collection_values: Collection data for reference resolution
+            identity_data: Identity data for parameter resolution
+            input_data: Optional upstream data from other collections
+        """
+        for param_value in request.param_values or []:
+            if param_value.references:
+                # we resolve the param reference here for consistency,
+                # i.e. as if it may be a pointer to an `external_reference`.
+                # Cross-collection references are now supported by looking up data
+                # from upstream collections passed via input_data.
+                reference: FidesDatasetReference = SaaSConfig.resolve_param_reference(
+                    param_value.references[0], self.secrets
+                )
+                param_values[param_value.name] = pydash.get(
+                    collection_values, reference.field
+                )
+                if param_values[param_value.name] is None and input_data:
+                    logger.info(
+                        "Reference not found in collection, checking input data from upstream tasks"
+                    )
+                    # If the reference is not found in the collection (None), check the input data from upstream tasks
+                    input_list = input_data.get(param_value.name)
+                    if input_list:
+                        param_values[param_value.name] = self._safe_extract_value(
+                            input_list
+                        )
+
+            elif param_value.identity:
+                param_values[param_value.name] = pydash.get(
+                    identity_data, param_value.identity
+                )
+            elif param_value.connector_param:
+                param_values[param_value.name] = pydash.get(
+                    self.secrets, param_value.connector_param
+                )
+
+    def _add_standard_parameters(
+        self,
+        param_values: Dict[str, Any],
+    ) -> None:
+        """
+        Add standard parameters that are common across query and update operations.
+        """
+
+        # common parameters for read and update
+        if self.privacy_request:
+            param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
+        param_values[UUID] = str(uuid4())
+        param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
+        param_values[FIELD_LIST] = ",".join(
+            [
+                field.name
+                for field in self.top_level_field_map().values()
+                if field.data_type() != "None"
+            ]
+        )
+
+        if self.request_task and self.request_task.id:
+            param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
+                self.request_task
+            )
+            param_values[REPLY_TO] = V1_URL_PREFIX + REQUEST_TASK_CALLBACK
+
     @staticmethod
     def _generate_product_list(*args: Dict[str, Any]) -> List[Dict[str, Any]]:
         """
@@ -331,22 +434,11 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
                 f"The 'read' action is not defined for the '{self.collection_name}' "
                 f"endpoint in {self.node.connection_key}"
             )
-
-        # create the source of param values to populate the various placeholders
-        # in the path, headers, query_params, and body
         param_values: Dict[str, Any] = {}
-        for param_value in self.current_request.param_values or []:
-            if param_value.references or param_value.identity:
-                input_list = input_data.get(param_value.name)
-                if input_list:
-                    param_values[param_value.name] = input_list[0]
-            elif param_value.connector_param:
-                param_values[param_value.name] = pydash.get(
-                    input_data, param_value.connector_param
-                )[0]
-
-        if self.privacy_request:
-            param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
+        # Process param values using utility method
+        self._process_param_values_for_query(
+            self.current_request, input_data, param_values
+        )
 
         privacy_request_object = input_data.get(PRIVACY_REQUEST_OBJECT)
         if (
@@ -365,21 +457,9 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
                 0
             ]
 
-        param_values[UUID] = str(uuid4())
-        # Use full ISO-8601 timestamp including time component (UTC, seconds precision)
-        param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
-        param_values[FIELD_LIST] = ",".join(
-            [
-                field.name
-                for field in self.top_level_field_map().values()
-                if field.data_type() != "None"
-            ]
-        )
-        if self.request_task and self.request_task.id:
-            param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
-                self.request_task
-            )
-            param_values[REPLY_TO] = V1_URL_PREFIX + REQUEST_TASK_CALLBACK
+        # Add standard parameters
+        # For generate_query, we don't have a PrivacyRequest object, so we use self.privacy_request
+        self._add_standard_parameters(param_values)
 
         # map param values to placeholders in path, headers, and query params
         saas_request_params: SaaSRequestParams = saas_util.map_param_values(
@@ -391,16 +471,26 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         return saas_request_params
 
     def generate_update_stmt(
-        self, row: Row, policy: Policy, request: PrivacyRequest
+        self,
+        row: Row,
+        policy: Policy,
+        request: PrivacyRequest,
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> SaaSRequestParams:
         """
         This returns the method, path, header, query, and body params needed to make an API call.
         The fields in the row are masked according to the policy and added to the request body
         if specified by the body field of the masking request.
+
+        Args:
+            row: The current row data from this collection
+            policy: The privacy policy being applied
+            request: The privacy request being processed
+            input_data: Optional upstream data from other collections for cross-collection references
         """
         current_request: SaaSRequest = self.get_masking_request()  # type: ignore
         param_values: Dict[str, Any] = self.generate_update_param_values(
-            row, policy, request, current_request
+            row, policy, request, current_request, input_data
         )
 
         return self.generate_update_request_params(param_values, current_request)
@@ -429,6 +519,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         policy: Policy,
         privacy_request: PrivacyRequest,
         saas_request: SaaSRequest,
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> Dict[str, Any]:
         """
         A utility that generates the update request param values
@@ -439,10 +530,16 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         the fields in the provided row that have been masked according
         to provided policy. The `all_object_fields` key maps to a JSON structure
         that holds all values, including those that have not been masked.
+
+        Args:
+            row: The current row data from this collection
+            policy: The privacy policy being applied
+            privacy_request: The privacy request being processed
+            saas_request: The SaaS request configuration
+            input_data: Optional upstream data from other collections for cross-collection references
         """
 
-        collection_name: str = self.node.address.collection
-        collection_values: Dict[str, Row] = {collection_name: row}
+        collection_values: Dict[str, Row] = {self.collection_name: row}
         identity_data: Dict[str, Any] = privacy_request.get_cached_identity_data()
         custom_privacy_request_fields: Dict[str, Any] = (
             privacy_request.get_cached_custom_privacy_request_fields()
@@ -451,48 +548,15 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         # create the source of param values to populate the various placeholders
         # in the path, headers, query_params, and body
         param_values: Dict[str, Any] = {}
-        for param_value in saas_request.param_values or []:
-            if param_value.references:
-                # we resolve the param reference here for consistency,
-                # i.e. as if it may be a pointer to an `external_reference`.
-                # however, `references` in update requests can, currently, only reference
-                # the same collection the same collection, and so it is highly unlikely
-                # that this would be an external reference at this point.
-                reference: FidesDatasetReference = SaaSConfig.resolve_param_reference(
-                    param_value.references[0], self.secrets
-                )
-                param_values[param_value.name] = pydash.get(
-                    collection_values, reference.field
-                )
-            elif param_value.identity:
-                param_values[param_value.name] = pydash.get(
-                    identity_data, param_value.identity
-                )
-            elif param_value.connector_param:
-                param_values[param_value.name] = pydash.get(
-                    self.secrets, param_value.connector_param
-                )
-
-        if self.privacy_request:
-            param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
+        # Process param values using utility method
+        self._process_param_values_for_update(
+            saas_request, collection_values, identity_data, input_data, param_values
+        )
 
         param_values[PRIVACY_REQUEST_OBJECT] = privacy_request.to_safe_dict()
-
         param_values[CUSTOM_PRIVACY_REQUEST_FIELDS] = custom_privacy_request_fields
-        param_values[UUID] = str(uuid4())
-        param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
-        param_values[FIELD_LIST] = ",".join(
-            [
-                field.name
-                for field in self.top_level_field_map().values()
-                if field.data_type() != "None"
-            ]
-        )
-        if self.request_task and self.request_task.id:
-            param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
-                self.request_task
-            )
-            param_values[REPLY_TO] = V1_URL_PREFIX + REQUEST_TASK_CALLBACK
+        # Add standard parameters
+        self._add_standard_parameters(param_values)
         # remove any row values for fields marked as read-only, these will be omitted from all update maps
         for field_path, field in self.field_map().items():
             if field.read_only:
@@ -518,7 +582,6 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
 
         param_values[MASKED_OBJECT_FIELDS] = masked_object
         param_values[ALL_OBJECT_FIELDS] = complete_object
-
         return param_values
 
     def generate_update_request_params(

fides/api/service/connectors/saas_connector.py

@@ -71,6 +71,7 @@ from fides.api.util.saas_util import (
     CUSTOM_PRIVACY_REQUEST_FIELDS,
     PRIVACY_REQUEST_OBJECT,
     assign_placeholders,
+    check_dataset_missing_reference_values,
     map_param_values,
 )
 
@@ -229,21 +230,8 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
 
         # pylint: disable=too-many-branches
         self.set_privacy_request_state(privacy_request, node, request_task)
-
         query_config: SaaSQueryConfig = self.query_config(node)
 
-        # Delegate async requests
-        with get_db() as db:
-            if async_dsr_strategy := _get_async_dsr_strategy(
-                db, request_task, query_config, ActionType.access
-            ):
-                return async_dsr_strategy.async_retrieve_data(
-                    client=self.create_client(),
-                    request_task_id=request_task.id,
-                    query_config=query_config,
-                    input_data=input_data,
-                )
-
         # generate initial set of requests if read request is defined, otherwise raise an exception
         # An endpoint can be defined with multiple 'read' requests if the data for a single
         # collection can be accessed in multiple ways for example:
@@ -280,16 +268,31 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
             input_data[CUSTOM_PRIVACY_REQUEST_FIELDS] = [custom_privacy_request_fields]
 
         input_data[PRIVACY_REQUEST_OBJECT] = [privacy_request.to_safe_dict()]
-
-        rows: List[Row] = []
+        # check all the values specified by param_values are provided in input_data
         for read_request in read_requests:
-            self.set_saas_request_state(read_request)
-            # check all the values specified by param_values are provided in input_data
             if self._missing_dataset_reference_values(
                 input_data, read_request.param_values
             ):
                 return []
 
+        # Delegate async requests
+        with get_db() as db:
+            # Guard clause to ensure we only run async access requests for access requests
+            if self.guard_access_request(policy):
+                if async_dsr_strategy := _get_async_dsr_strategy(
+                    db, request_task, query_config, ActionType.access
+                ):
+                    return async_dsr_strategy.async_retrieve_data(
+                        client=self.create_client(),
+                        request_task_id=request_task.id,
+                        query_config=query_config,
+                        input_data=input_data,
+                    )
+
+        rows: List[Row] = []
+        for read_request in read_requests:
+            self.set_saas_request_state(read_request)
+
             # hook for user-provided request override functions
             if read_request.request_override:
                 return self._invoke_read_request_override(
@@ -347,6 +350,19 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         self.unset_connector_state()
         return rows
 
+    def guard_access_request(self, policy: Policy) -> bool:
+        """
+        Guard clause to ensure we only run async access requests
+        if the access request is enabled and we are in an Access Request
+        """
+        if (
+            self.configuration.enabled_actions is None
+            or ActionType.access in self.configuration.enabled_actions
+        ):
+            if policy.get_rules_for_action(ActionType.access):
+                return True
+        return False
+
     def _apply_output_template(
         self,
         param_value_maps: List[Dict[str, Any]],
@@ -376,29 +392,15 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         return result
 
     def _missing_dataset_reference_values(
-        self, input_data: Dict[str, Any], param_values: Optional[List[ParamValue]]
+        self, input_data: Dict[str, List[Any]], param_values: Optional[List[ParamValue]]
     ) -> List[str]:
         """Return a list of dataset reference values that are not found in the input_data map"""
-
-        # get the list of param_value references
-        required_param_value_references = [
-            param_value.name
-            for param_value in param_values or []
-            if param_value.references
-        ]
-
-        # extract the keys from inside the fides_grouped_inputs and append them the other input_data keys
-        provided_input_keys = (
-            list(input_data.get("fidesops_grouped_inputs")[0].keys())  # type: ignore
-            if input_data.get("fidesops_grouped_inputs")
-            else []
-        ) + list(input_data.keys())
-
-        # find the missing values
-        missing_dataset_reference_values = list(
-            set(required_param_value_references) - set(provided_input_keys)
+        missing_dataset_reference_values = check_dataset_missing_reference_values(
+            input_data, param_values
+        )
+        logger.info(
+            f"Missing dataset reference values: {missing_dataset_reference_values}"
         )
-
         if missing_dataset_reference_values:
             logger.info(
                 "The '{}' request of {} is missing the following dataset reference values [{}], skipping traversal",
@@ -545,7 +547,6 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
                     query_config=query_config,
                     rows=rows,
                 )
-
         masking_request = query_config.get_masking_request()
         rows_updated = 0
 
@@ -591,7 +592,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
         for row in rows:
             try:
                 prepared_request = query_config.generate_update_stmt(
-                    row, policy, privacy_request
+                    row, policy, privacy_request, input_data
                 )
             except ValueError as exc:
                 if masking_request.skip_missing_param_values:
@@ -786,10 +787,11 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
 
         else:
             # follow the basic (global opt-in/out) SaaS consent flow
-            should_opt_in, filtered_preferences = (
-                build_user_consent_and_filtered_preferences_for_service(
-                    self.configuration.system, privacy_request, session, False
-                )
+            (
+                should_opt_in,
+                filtered_preferences,
+            ) = build_user_consent_and_filtered_preferences_for_service(
+                self.configuration.system, privacy_request, session, False
             )
 
             if should_opt_in is None:

fides/api/service/privacy_request/request_service.py

@@ -320,7 +320,7 @@ def initiate_polling_task_requeue() -> None:
         id=ASYNC_TASKS_STATUS_POLLING,
         coalesce=True,
         replace_existing=True,
-        seconds=20,
+        seconds=CONFIG.execution.async_polling_interval_hours * 3600,
     )