ethyca-fides 2.69.1rc0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -10,7 +10,6 @@ from uuid import uuid4
 import pydash
 from fideslang.models import FidesDatasetReference
 from loguru import logger
-from sqlalchemy.orm import Session
 
 from fides.api.common_exceptions import FidesopsException
 from fides.api.graph.execution import ExecutionNode
@@ -47,6 +46,7 @@ from fides.api.util.saas_util import (
     ISO_8601_DATETIME,
     MASKED_OBJECT_FIELDS,
     PRIVACY_REQUEST_ID,
+    PRIVACY_REQUEST_OBJECT,
     REPLY_TO,
     REPLY_TO_TOKEN,
     UUID,
@@ -83,12 +83,10 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         """
         Returns the appropriate request configs based on the current collection and identity
         """
-        collection_name = self.node.address.collection
-
         try:
-            requests = self.endpoints[collection_name].requests
+            requests = self.endpoints[self.collection_name].requests
         except KeyError:
-            logger.error("The '{}' endpoint is not defined", collection_name)
+            logger.error("The '{}' endpoint is not defined", self.collection_name)
             return []
 
         if not requests.read:
@@ -140,7 +138,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
             )
         return request
 
-    def get_masking_request(self, db: Session) -> Optional[SaaSRequest]:
+    def get_masking_request(self) -> Optional[SaaSRequest]:
         """
         Returns a tuple of the preferred action and SaaSRequest to use for masking.
         An update request is preferred, but we can use a gdpr delete endpoint or
@@ -212,6 +210,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         input_data: Dict[str, List[Any]],
         policy: Optional[Policy],
         read_request: SaaSRequest,
+        privacy_request: Optional[PrivacyRequest] = None,
     ) -> List[Tuple[SaaSRequestParams, Dict[str, Any]]]:
         """
         Takes the identity and reference values from input_data and combines them
@@ -224,6 +223,10 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         self.current_request = read_request
 
         request_params = []
+
+        if privacy_request:
+            input_data[PRIVACY_REQUEST_OBJECT] = [privacy_request.to_safe_dict()]
+
         param_value_maps = self.generate_param_value_maps(input_data, read_request)
 
         for param_value_map in param_value_maps:
@@ -277,6 +280,114 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
             name: value for name, value in self.secrets.items() if name in param_names
         }
 
+    def _safe_extract_value(self, value_list: Optional[List[Any]]) -> Optional[Any]:
+        """
+        Safely extract the first value from a list
+        """
+        return value_list[0] if value_list else None
+
+    def _process_param_values_for_query(
+        self,
+        request: SaaSRequest,
+        input_data: Dict[str, List[Any]],
+        param_values: Dict[str, Any],
+    ) -> None:
+        """
+        Process param_values for query generation on access requests.
+        """
+        for param_value in request.param_values or []:
+            if param_value.references or param_value.identity:
+                input_list = input_data.get(param_value.name)
+                if input_list:
+                    param_values[param_value.name] = self._safe_extract_value(
+                        input_list
+                    )
+            elif param_value.connector_param:
+                connector_value = pydash.get(input_data, param_value.connector_param)
+                param_values[param_value.name] = self._safe_extract_value(
+                    connector_value
+                )
+
+    def _process_param_values_for_update(
+        self,
+        request: SaaSRequest,
+        collection_values: Dict[str, Row],
+        identity_data: Dict[str, Any],
+        input_data: Optional[Dict[str, List[Any]]] = None,
+        param_values: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """
+        Process param_values for update generation (erasure requests).
+        Modifies param_values in place.
+
+        Args:
+            request: The SaaS request configuration
+            collection_values: Collection data for reference resolution
+            identity_data: Identity data for parameter resolution
+            input_data: Optional upstream data from other collections
+        """
+        if param_values is None:
+            param_values = {}
+
+        for param_value in request.param_values or []:
+            if param_value.references:
+                # we resolve the param reference here for consistency,
+                # i.e. as if it may be a pointer to an `external_reference`.
+                # Cross-collection references are now supported by looking up data
+                # from upstream collections passed via input_data.
+                reference: FidesDatasetReference = SaaSConfig.resolve_param_reference(
+                    param_value.references[0], self.secrets
+                )
+                param_values[param_value.name] = pydash.get(
+                    collection_values, reference.field
+                )
+                if param_values[param_value.name] is None and input_data:
+                    logger.info(
+                        "Reference not found in collection, checking input data from upstream tasks"
+                    )
+                    # If the reference is not found in the collection (None), check the input data from upstream tasks
+                    input_list = input_data.get(param_value.name)
+                    if input_list:
+                        param_values[param_value.name] = self._safe_extract_value(
+                            input_list
+                        )
+
+            elif param_value.identity:
+                param_values[param_value.name] = pydash.get(
+                    identity_data, param_value.identity
+                )
+            elif param_value.connector_param:
+                param_values[param_value.name] = pydash.get(
+                    self.secrets, param_value.connector_param
+                )
+
+    def _add_standard_parameters(
+        self,
+        param_values: Dict[str, Any],
+    ) -> None:
+        """
+        Add standard parameters that are common across query and update operations.
+        """
+
+        # common parameters for read and update
+        if self.privacy_request:
+            param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
+        param_values[UUID] = str(uuid4())
+        param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
+        param_values[FIELD_LIST] = ",".join(
+            [
+                field.name
+                for field in self.top_level_field_map().values()
+                if field.data_type() != "None"
+            ]
+        )
+
+        if self.request_task and self.request_task.id:
+            param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
+                self.request_task
+            )
+            param_values[REPLY_TO] = V1_URL_PREFIX + REQUEST_TASK_CALLBACK
+
     @staticmethod
     def _generate_product_list(*args: Dict[str, Any]) -> List[Dict[str, Any]]:
         """
@@ -326,22 +437,19 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
                 f"The 'read' action is not defined for the '{self.collection_name}' "
                 f"endpoint in {self.node.connection_key}"
             )
-
-        # create the source of param values to populate the various placeholders
-        # in the path, headers, query_params, and body
         param_values: Dict[str, Any] = {}
-        for param_value in self.current_request.param_values or []:
-            if param_value.references or param_value.identity:
-                input_list = input_data.get(param_value.name)
-                if input_list:
-                    param_values[param_value.name] = input_list[0]
-            elif param_value.connector_param:
-                param_values[param_value.name] = pydash.get(
-                    input_data, param_value.connector_param
-                )[0]
+        # Process param values using utility method
+        self._process_param_values_for_query(
+            self.current_request, input_data, param_values
+        )
 
-        if self.privacy_request:
-            param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
+        privacy_request_object = input_data.get(PRIVACY_REQUEST_OBJECT)
+        if (
+            isinstance(privacy_request_object, list)
+            and len(privacy_request_object) > 0
+            and isinstance(privacy_request_object[0], dict)
+        ):
+            param_values[PRIVACY_REQUEST_OBJECT] = privacy_request_object[0]
 
         custom_privacy_request_fields = input_data.get(CUSTOM_PRIVACY_REQUEST_FIELDS)
         if (
@@ -352,20 +460,9 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
                 0
             ]
 
-        param_values[UUID] = str(uuid4())
-        param_values[ISO_8601_DATETIME] = datetime.now().date().isoformat()
-        param_values[FIELD_LIST] = ",".join(
-            [
-                field.name
-                for field in self.top_level_field_map().values()
-                if field.data_type() != "None"
-            ]
-        )
-        if self.request_task and self.request_task.id:
-            param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
-                self.request_task
-            )
-            param_values[REPLY_TO] = V1_URL_PREFIX + REQUEST_TASK_CALLBACK
+        # Add standard parameters
+        # For generate_query, we don't have a PrivacyRequest object, so we use self.privacy_request
+        self._add_standard_parameters(param_values)
 
         # map param values to placeholders in path, headers, and query params
         saas_request_params: SaaSRequestParams = saas_util.map_param_values(
@@ -377,17 +474,26 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         return saas_request_params
 
     def generate_update_stmt(
-        self, row: Row, policy: Policy, request: PrivacyRequest
+        self,
+        row: Row,
+        policy: Policy,
+        request: PrivacyRequest,
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> SaaSRequestParams:
         """
         This returns the method, path, header, query, and body params needed to make an API call.
         The fields in the row are masked according to the policy and added to the request body
         if specified by the body field of the masking request.
+
+        Args:
+            row: The current row data from this collection
+            policy: The privacy policy being applied
+            request: The privacy request being processed
+            input_data: Optional upstream data from other collections for cross-collection references
         """
-        session = Session.object_session(request)
-        current_request: SaaSRequest = self.get_masking_request(session)  # type: ignore
+        current_request: SaaSRequest = self.get_masking_request()  # type: ignore
         param_values: Dict[str, Any] = self.generate_update_param_values(
-            row, policy, request, current_request
+            row, policy, request, current_request, input_data
         )
 
         return self.generate_update_request_params(param_values, current_request)
@@ -416,6 +522,7 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         policy: Policy,
         privacy_request: PrivacyRequest,
         saas_request: SaaSRequest,
+        input_data: Optional[Dict[str, List[Any]]] = None,
     ) -> Dict[str, Any]:
         """
         A utility that generates the update request param values
@@ -426,10 +533,16 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         the fields in the provided row that have been masked according
         to provided policy. The `all_object_fields` key maps to a JSON structure
         that holds all values, including those that have not been masked.
+
+        Args:
+            row: The current row data from this collection
+            policy: The privacy policy being applied
+            privacy_request: The privacy request being processed
+            saas_request: The SaaS request configuration
+            input_data: Optional upstream data from other collections for cross-collection references
         """
 
-        collection_name: str = self.node.address.collection
-        collection_values: Dict[str, Row] = {collection_name: row}
+        collection_values: Dict[str, Row] = {self.collection_name: row}
         identity_data: Dict[str, Any] = privacy_request.get_cached_identity_data()
         custom_privacy_request_fields: Dict[str, Any] = (
             privacy_request.get_cached_custom_privacy_request_fields()
@@ -438,46 +551,15 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
         # create the source of param values to populate the various placeholders
         # in the path, headers, query_params, and body
         param_values: Dict[str, Any] = {}
-        for param_value in saas_request.param_values or []:
-            if param_value.references:
-                # we resolve the param reference here for consistency,
-                # i.e. as if it may be a pointer to an `external_reference`.
-                # however, `references` in update requests can, currently, only reference
-                # the same collection the same collection, and so it is highly unlikely
-                # that this would be an external reference at this point.
-                reference: FidesDatasetReference = SaaSConfig.resolve_param_reference(
-                    param_value.references[0], self.secrets
-                )
-                param_values[param_value.name] = pydash.get(
-                    collection_values, reference.field
-                )
-            elif param_value.identity:
-                param_values[param_value.name] = pydash.get(
-                    identity_data, param_value.identity
-                )
-            elif param_value.connector_param:
-                param_values[param_value.name] = pydash.get(
-                    self.secrets, param_value.connector_param
-                )
-
-        if self.privacy_request:
-            param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
+        # Process param values using utility method
+        self._process_param_values_for_update(
+            saas_request, collection_values, identity_data, input_data, param_values
+        )
 
+        param_values[PRIVACY_REQUEST_OBJECT] = privacy_request.to_safe_dict()
         param_values[CUSTOM_PRIVACY_REQUEST_FIELDS] = custom_privacy_request_fields
-        param_values[UUID] = str(uuid4())
-        param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
-        param_values[FIELD_LIST] = ",".join(
-            [
-                field.name
-                for field in self.top_level_field_map().values()
-                if field.data_type() != "None"
-            ]
-        )
-        if self.request_task and self.request_task.id:
-            param_values[REPLY_TO_TOKEN] = generate_request_task_callback_jwe(
-                self.request_task
-            )
-            param_values[REPLY_TO] = V1_URL_PREFIX + REQUEST_TASK_CALLBACK
+        # Add standard parameters
+        self._add_standard_parameters(param_values)
         # remove any row values for fields marked as read-only, these will be omitted from all update maps
         for field_path, field in self.field_map().items():
             if field.read_only:
@@ -503,7 +585,6 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
 
         param_values[MASKED_OBJECT_FIELDS] = masked_object
         param_values[ALL_OBJECT_FIELDS] = complete_object
-
         return param_values
 
     def generate_update_request_params(

fides/api/service/connectors/saas/connector_registry_service.py

@@ -1,25 +1,23 @@
 # pylint: disable=protected-access
 import os
 from abc import ABC, abstractmethod
-from typing import Dict, Iterable, List, Optional, Type
+from typing import Dict, List, Optional, Type
 from zipfile import ZipFile
 
 from fideslang.models import Dataset
 from loguru import logger
-from packaging.version import Version
 from packaging.version import parse as parse_version
 from sqlalchemy.orm import Session
 
 from fides.api.api.deps import get_api_session
 from fides.api.common_exceptions import ValidationError
 from fides.api.cryptography.cryptographic_util import str_to_b64_str
-from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.custom_connector_template import CustomConnectorTemplate
-from fides.api.models.datasetconfig import DatasetConfig
-from fides.api.schemas.connection_configuration.saas_config_template_values import (
-    SaasConnectionTemplateValues,
+from fides.api.models.saas_template_dataset import SaasTemplateDataset
+from fides.api.schemas.saas.connector_template import (
+    ConnectorTemplate,
+    ConnectorTemplateListResponse,
 )
-from fides.api.schemas.saas.connector_template import ConnectorTemplate
 from fides.api.schemas.saas.saas_config import SaaSConfig
 from fides.api.service.authentication.authentication_strategy_oauth2_authorization_code import (
     OAuth2AuthorizationCodeAuthenticationStrategy,
@@ -31,8 +29,6 @@ from fides.api.util.saas_util import (
     load_config_from_string,
     load_dataset_from_string,
     load_yaml_as_string,
-    replace_config_placeholders,
-    replace_dataset_placeholders,
     replace_version,
 )
 from fides.api.util.unsafe_file_util import verify_svg, verify_zip
@@ -126,6 +122,11 @@ class CustomConnectorTemplateLoader(ConnectorTemplateLoader):
                     f"Replacing {template.key} connector template with newer version."
                 )
                 template.delete(db=db)
+                template_dataset = SaasTemplateDataset.get_by(
+                    db=db, field="connection_type", value=template.key
+                )
+                if template_dataset:
+                    template_dataset.delete(db=db)
                 continue
             try:
                 CustomConnectorTemplateLoader._register_template(template)
@@ -292,6 +293,15 @@ class CustomConnectorTemplateLoader(ConnectorTemplateLoader):
             },
         )
 
+        # Store the original template dataset (with placeholders) instead of the modified version
+        template_dataset_json = load_dataset_from_string(template.dataset)
+
+        SaasTemplateDataset.get_or_create(
+            db=db,
+            connector_type=connector_type,
+            dataset_json=template_dataset_json,
+        )
+
 
 class ConnectorRegistry:
     @classmethod
@@ -318,131 +328,28 @@ class ConnectorRegistry:
         """
         return cls._get_combined_templates().get(connector_type)
 
-
-def create_connection_config_from_template_no_save(
-    db: Session,
-    template: ConnectorTemplate,
-    template_values: SaasConnectionTemplateValues,
-    system_id: Optional[str] = None,
-) -> ConnectionConfig:
-    """Creates a SaaS connection config from a template without saving it."""
-    # Load SaaS config from template and replace every instance of "<instance_fides_key>" with the fides_key
-    # the user has chosen
-    config_from_template: Dict = replace_config_placeholders(
-        template.config, "<instance_fides_key>", template_values.instance_key
-    )
-
-    data = template_values.generate_config_data_from_template(
-        config_from_template=config_from_template
-    )
-
-    if system_id:
-        data["system_id"] = system_id
-
-    # Create SaaS ConnectionConfig
-    connection_config = ConnectionConfig.create_without_saving(db, data=data)
-
-    return connection_config
-
-
-def upsert_dataset_config_from_template(
-    db: Session,
-    connection_config: ConnectionConfig,
-    template: ConnectorTemplate,
-    template_values: SaasConnectionTemplateValues,
-) -> DatasetConfig:
-    """
-    Creates a `DatasetConfig` from a template
-    and associates it with a ConnectionConfig.
-    If the `DatasetConfig` already exists in the db,
-    then the existing record is updated.
-    """
-    # Load the dataset config from template and replace every instance of "<instance_fides_key>" with the fides_key
-    # the user has chosen
-    dataset_from_template: Dict = replace_dataset_placeholders(
-        template.dataset, "<instance_fides_key>", template_values.instance_key
-    )
-    data = {
-        "connection_config_id": connection_config.id,
-        "fides_key": template_values.instance_key,
-        "dataset": dataset_from_template,  # Currently used for upserting a CTL Dataset
-    }
-    dataset_config = DatasetConfig.create_or_update(db, data=data)
-    return dataset_config
-
-
-def update_saas_configs(db: Session) -> None:
-    """
-    Updates SaaS config instances currently in the DB if to the
-    corresponding template in the registry are found.
-
-    Effectively an "update script" for SaaS config instances,
-    to be run on server bootstrap.
-    """
-    for connector_type in ConnectorRegistry.connector_types():
-        logger.debug(
-            "Determining if any updates are needed for connectors of type {} based on templates...",
-            connector_type,
-        )
-        template: ConnectorTemplate = ConnectorRegistry.get_connector_template(  # type: ignore
-            connector_type
-        )
-        saas_config = SaaSConfig(**load_config_from_string(template.config))
-        template_version: Version = parse_version(saas_config.version)
-
-        connection_configs: Iterable[ConnectionConfig] = ConnectionConfig.filter(
-            db=db,
-            conditions=(ConnectionConfig.saas_config["type"].astext == connector_type),
-        ).all()
-        for connection_config in connection_configs:
-            saas_config_instance = SaaSConfig.model_validate(
-                connection_config.saas_config
-            )
-            if parse_version(saas_config_instance.version) < template_version:
-                logger.info(
-                    "Updating SaaS config instance '{}' of type '{}' as its version, {}, was found to be lower than the template version {}",
-                    saas_config_instance.fides_key,
-                    connector_type,
-                    saas_config_instance.version,
-                    template_version,
+    @classmethod
+    def get_all_connector_templates_summary(cls) -> List[ConnectorTemplateListResponse]:
+        """
+        Returns summary information for all connector templates.
+        Includes connector_type, name, supported_actions, category, and whether it's custom.
+        """
+        custom_templates = CustomConnectorTemplateLoader.get_connector_templates()  # type: ignore
+        combined_templates = cls._get_combined_templates()
+
+        summaries: List[ConnectorTemplateListResponse] = []
+        for connector_type, template in combined_templates.items():
+            # Determine if the template is custom by checking if it exists in CustomConnectorTemplateLoader
+            is_custom = connector_type in custom_templates
+
+            summaries.append(
+                ConnectorTemplateListResponse(
+                    type=connector_type,
+                    name=template.human_readable,
+                    supported_actions=template.supported_actions,
+                    category=template.category,
+                    custom=is_custom,
                 )
-                try:
-                    update_saas_instance(
-                        db,
-                        connection_config,
-                        template,
-                        saas_config_instance,
-                    )
-                except Exception:
-                    logger.exception(
-                        "Encountered error attempting to update SaaS config instance {}",
-                        saas_config_instance.fides_key,
-                    )
-
-
-def update_saas_instance(
-    db: Session,
-    connection_config: ConnectionConfig,
-    template: ConnectorTemplate,
-    saas_config_instance: SaaSConfig,
-) -> None:
-    """
-    Replace in the DB the existing SaaS instance configuration data
-    (SaaSConfig, DatasetConfig) associated with the given ConnectionConfig
-    with new instance configuration data based on the given ConnectorTemplate
-    """
-    template_vals = SaasConnectionTemplateValues(
-        name=connection_config.name,
-        key=connection_config.key,
-        description=connection_config.description,
-        secrets=connection_config.secrets,
-        instance_key=saas_config_instance.fides_key,
-    )
-
-    config_from_template: Dict = replace_config_placeholders(
-        template.config, "<instance_fides_key>", template_vals.instance_key
-    )
-
-    connection_config.update_saas_config(db, SaaSConfig(**config_from_template))
+            )
 
-    upsert_dataset_config_from_template(db, connection_config, template, template_vals)
+        return summaries