ethyca-fides 2.57.1b9__py2.py3-none-any.whl → 2.57.1rc0__py2.py3-none-any.whl

fides/api/models/sql_models.py

@@ -295,38 +295,15 @@ class Dataset(Base, FidesBase):
     collections = Column(JSON)
     fides_meta = Column(JSON)
 
-    @classmethod
-    def create(
-        cls: Type["Dataset"],
-        db: Session,
-        *,
-        data: dict[str, Any],
-        check_name: bool = False,
-    ) -> "Dataset":
-        """
-        Override create to check for existing datasets with the same fides_key.
-        Duplicate names are allowed.
-        """
-        # Check if dataset with same fides_key already exists
-        if "fides_key" in data:
-            existing_by_key = (
-                db.query(cls).filter(cls.fides_key == data["fides_key"]).first()
-            )
-            if existing_by_key:
-                raise KeyOrNameAlreadyExists(
-                    f'Dataset with fides_key "{data["fides_key"]}" already exists.'
-                )
-
-        # Create the dataset using the parent class's method
-        db_obj = cls(**data)
-        return cls.persist_obj(db, db_obj)
-
     @classmethod
     def create_from_dataset_dict(cls, db: Session, dataset: dict) -> "Dataset":
         """Add a method to create directly using a synchronous session"""
         validated_dataset: FideslangDataset = FideslangDataset(**dataset)
-        data_dict = validated_dataset.model_dump(mode="json")
-        return cls.create(db=db, data=data_dict)
+        ctl_dataset = cls(**validated_dataset.model_dump(mode="json"))
+        db.add(ctl_dataset)
+        db.commit()
+        db.refresh(ctl_dataset)
+        return ctl_dataset
 
     @property
     def field_data_categories(self) -> Set[str]:

fides/api/oauth/jwt.py

@@ -14,21 +14,3 @@ def generate_jwe(payload: str, encryption_key: str, encoding: str = "UTF-8") ->
         encryption_key,
         encryption=_JWT_ENCRYPTION_ALGORITHM,
     ).decode(encoding)
-
-
-def decrypt_jwe(token: str, encryption_key: str, encoding: str = "UTF-8") -> str:
-    """Decrypts a JWE token and returns the payload as a string.
-
-    Args:
-        token: The JWE token to decrypt.
-        encryption_key: The key used to decrypt the token.
-        encoding: The encoding to use for the decrypted payload.
-
-    Returns:
-        The decrypted payload as a string.
-    """
-    decrypted_payload = jwe.decrypt(
-        token,
-        encryption_key,
-    )
-    return decrypted_payload.decode(encoding)

fides/api/oauth/utils.py

@@ -339,16 +339,7 @@ def has_permissions(
     has_role: bool = _has_scope_via_role(
         token_data=token_data, client=client, endpoint_scopes=endpoint_scopes
     )
-
-    has_required_permissions = has_direct_scope or has_role
-    if not has_required_permissions:
-        scopes_required = ",".join(endpoint_scopes.scopes)
-        logger.debug(
-            "Authorization failed. Missing required scopes: {}. Neither direct scopes nor role-derived scopes were sufficient.",
-            scopes_required,
-        )
-
-    return has_required_permissions
+    return has_direct_scope or has_role
 
 
 def _has_scope_via_role(
@@ -394,7 +385,16 @@ def _has_direct_scopes(
 
 def has_scope_subset(user_scopes: List[str], endpoint_scopes: SecurityScopes) -> bool:
     """Are the required scopes a subset of the scopes belonging to the user?"""
-    return set(endpoint_scopes.scopes).issubset(user_scopes)
+    if not set(endpoint_scopes.scopes).issubset(user_scopes):
+        scopes_required = ",".join(endpoint_scopes.scopes)
+        scopes_provided = ",".join(user_scopes)
+        logger.debug(
+            "Auth token missing required scopes: {}. Scopes provided: {}.",
+            scopes_required,
+            scopes_provided,
+        )
+        return False
+    return True
 
 
 def create_temporary_user_for_login_flow(config: FidesConfig) -> FidesUser:

fides/api/schemas/dataset.py

@@ -1,8 +1,8 @@
 from typing import Any, Dict, List, Optional, Union
 
-from fideslang.models import Dataset, DatasetCollection
+from fideslang.models import Dataset
 from fideslang.validation import FidesKey
-from pydantic import ConfigDict, Field
+from pydantic import ConfigDict
 
 from fides.api.schemas.api import BulkResponse, BulkUpdateFailed
 from fides.api.schemas.base_class import FidesSchema
@@ -72,23 +72,3 @@ class DatasetReachability(FidesSchema):
 
     reachable: bool
     details: Optional[Union[str, List[Dict[str, Any]]]]
-
-
-class DatasetResponse(Dataset):
-    """
-    Dataset response model for API endpoints.
-
-    Note: This class extends the Dataset model from fideslang rather than having a proper
-    dedicated API response model. We had to make the collections field
-    Optional and allow null values to support the minimal=true parameter in API responses,
-    even though collections is required in the base fideslang Dataset model.
-    """
-
-    collections: Optional[List[DatasetCollection]] = Field(  # type: ignore
-        description="An array of objects that describe the Dataset's collections.",
-        default=None,
-    )
-
-    model_config = ConfigDict(
-        extra="ignore", from_attributes=False, coerce_numbers_to_str=True
-    )

fides/api/schemas/saas/strategy_configuration.py

@@ -89,7 +89,6 @@ class CursorPaginationConfiguration(StrategyConfiguration):
 
     cursor_param: str
     field: str
-    has_next: Optional[str] = None
 
 
 class ApiKeyAuthenticationConfiguration(StrategyConfiguration):

fides/api/service/pagination/pagination_strategy_cursor.py

@@ -1,7 +1,6 @@
 from typing import Any, Dict, Optional
 
 import pydash
-from loguru import logger
 from requests import Response
 
 from fides.api.schemas.saas.shared_schemas import SaaSRequestParams
@@ -16,7 +15,6 @@ class CursorPaginationStrategy(PaginationStrategy):
     def __init__(self, configuration: CursorPaginationConfiguration):
         self.cursor_param = configuration.cursor_param
         self.field = configuration.field
-        self.has_next = configuration.has_next
 
     def get_next_request(
         self,
@@ -29,15 +27,11 @@ class CursorPaginationStrategy(PaginationStrategy):
 
         # get the last object in the array and read the cursor value
         cursor = None
-        has_next = None
         object_list = (
             pydash.get(response.json(), data_path) if data_path else response.json()
         )
         if object_list and isinstance(object_list, list):
-            last_object = object_list.pop()
-            cursor = pydash.get(last_object, self.field)
-            if self.has_next:
-                has_next = pydash.get(last_object, self.has_next)
+            cursor = pydash.get(object_list.pop(), self.field)
 
         # If the cursor value isn't found, try to find the value in response.json() using the field value
         if cursor is None:
@@ -47,14 +41,6 @@ class CursorPaginationStrategy(PaginationStrategy):
         if cursor is None:
             return None
 
-        if self.has_next and has_next is None:
-            has_next = pydash.get(response.json(), self.has_next)
-
-        if self.has_next:
-            logger.debug(f"The {self.has_next} field has a value of {has_next}")
-            if str(has_next).lower() != "true":
-                return None
-
         # add or replace cursor_param with new cursor value
         request_params.query_params[self.cursor_param] = cursor
 

fides/api/service/pagination/pagination_strategy_link.py

@@ -43,7 +43,7 @@ class LinkPaginationStrategy(PaginationStrategy):
 
         if self.has_next:
             has_next = pydash.get(response.json(), self.has_next)
-            logger.debug(f"The {self.has_next} field has a value of {has_next}")
+            logger.info(f"The {self.has_next} field has a value of {has_next}")
             if str(has_next).lower() != "true":
                 return None
 

fides/api/tasks/storage.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 import json
+import os
 import secrets
 import zipfile
 from io import BytesIO
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+from typing import Any, Dict, Optional, Union
 
 import pandas as pd
 from botocore.exceptions import ClientError, ParamValidationError
@@ -12,18 +13,11 @@ from fideslang.validation import AnyHttpUrlString
 from loguru import logger
 
 from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
+from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.storage.storage import ResponseFormat, StorageSecrets
 from fides.api.service.privacy_request.dsr_package.dsr_report_builder import (
     DsrReportBuilder,
 )
-from fides.api.service.storage.s3 import (
-    create_presigned_url_for_s3,
-    generic_upload_to_s3,
-)
-from fides.api.service.storage.util import (
-    LOCAL_FIDES_UPLOAD_DIRECTORY,
-    get_local_filename,
-)
 from fides.api.util.aws_util import get_s3_client
 from fides.api.util.cache import get_cache, get_encryption_cache_key
 from fides.api.util.encryption.aes_gcm_encryption_scheme import (
@@ -32,8 +26,7 @@ from fides.api.util.encryption.aes_gcm_encryption_scheme import (
 from fides.api.util.storage_util import storage_json_encoder
 from fides.config import CONFIG
 
-if TYPE_CHECKING:
-    from fides.api.models.privacy_request import PrivacyRequest
+LOCAL_FIDES_UPLOAD_DIRECTORY = "fides_uploads"
 
 
 def encrypt_access_request_results(data: Union[str, bytes], request_id: str) -> str:
@@ -108,6 +101,105 @@ def write_to_in_memory_buffer(
     raise NotImplementedError(f"No handling for response format {resp_format}.")
 
 
+def create_presigned_url_for_s3(
+    s3_client: Any, bucket_name: str, file_key: str
+) -> AnyHttpUrlString:
+    """ "Generate a presigned URL to share an S3 object
+
+    :param s3_client: s3 base client
+    :param bucket_name: string
+    :param file_key: string
+    :return: Presigned URL as string.
+    """
+    response = s3_client.generate_presigned_url(
+        "get_object",
+        Params={"Bucket": bucket_name, "Key": file_key},
+        ExpiresIn=CONFIG.security.subject_request_download_link_ttl_seconds,
+    )
+
+    # The response contains the presigned URL
+    return response
+
+
+def generic_upload_to_s3(  # pylint: disable=R0913
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+    document: bytes,
+) -> Optional[AnyHttpUrlString]:
+    """Uploads arbitrary data to s3 returned from an access request"""
+    logger.info("Starting S3 Upload of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            s3_client.put_object(Bucket=bucket_name, Key=file_key, Body=document)
+        except Exception as e:
+            logger.error("Encountered error while uploading s3 object: {}", e)
+            raise e
+
+        presigned_url: AnyHttpUrlString = create_presigned_url_for_s3(
+            s3_client, bucket_name, file_key
+        )
+
+        return presigned_url
+    except ClientError as e:
+        logger.error(
+            "Encountered error while uploading and generating link for s3 object: {}", e
+        )
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
+def generic_retrieve_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> Optional[bytes]:
+    """Retrieves arbitrary data from s3"""
+    logger.info("Starting S3 Retrieve of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
+            return response["Body"].read()
+        except Exception as e:
+            logger.error("Encountered error while retrieving s3 object: {}", e)
+            raise e
+    except ClientError as e:
+        logger.error("Encountered error while retrieving s3 object: {}", e)
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
+def generic_delete_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> None:
+    """Deletes arbitrary data from s3"""
+    logger.info("Starting S3 Delete of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            s3_client.delete_object(Bucket=bucket_name, Key=file_key)
+        except Exception as e:
+            logger.error("Encountered error while deleting s3 object: {}", e)
+            raise e
+    except ClientError as e:
+        logger.error("Encountered error while deleting s3 object: {}", e)
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
 def upload_to_s3(  # pylint: disable=R0913
     storage_secrets: Dict[StorageSecrets, Any],
     data: Dict,
@@ -115,7 +207,7 @@ def upload_to_s3(  # pylint: disable=R0913
     file_key: str,
     resp_format: str,
     privacy_request: Optional[PrivacyRequest],
-    document: Optional[BytesIO],
+    document: Optional[bytes],
     auth_method: str,
 ) -> Optional[AnyHttpUrlString]:
     """Uploads arbitrary data to s3 returned from an access request"""
@@ -157,6 +249,13 @@ def upload_to_s3(  # pylint: disable=R0913
         raise ValueError(f"The parameters you provided are incorrect: {e}")
 
 
+def get_local_filename(file_key: str) -> str:
+    """Verifies that the local storage directory exists"""
+    if not os.path.exists(LOCAL_FIDES_UPLOAD_DIRECTORY):
+        os.makedirs(LOCAL_FIDES_UPLOAD_DIRECTORY)
+    return f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{file_key}"
+
+
 def upload_to_local(
     data: Dict,
     file_key: str,

fides/config/__init__.py

@@ -133,7 +133,7 @@ def censor_config(config: Union[FidesConfig, Dict[str, Any]]) -> Dict[str, Any]:
             data = as_dict[key]
             filtered[key] = {}
             for field in value:
-                if data and field in data:
+                if field in data:
                     filtered[key][field] = data[field]
 
     return filtered

fides/data/language/languages.yml

@@ -45,8 +45,6 @@ languages:
   name: Italian
 - id: ja
   name: Japanese
-- id: ko
-  name: Korean
 - id: lt
   name: Lithuanian
 - id: lv