ethyca-fides 2.57.1rc0__py2.py3-none-any.whl → 2.58.0rc0__py2.py3-none-any.whl

fides/api/service/pagination/pagination_strategy_cursor.py

@@ -1,6 +1,7 @@
 from typing import Any, Dict, Optional
 
 import pydash
+from loguru import logger
 from requests import Response
 
 from fides.api.schemas.saas.shared_schemas import SaaSRequestParams
@@ -15,6 +16,7 @@ class CursorPaginationStrategy(PaginationStrategy):
     def __init__(self, configuration: CursorPaginationConfiguration):
         self.cursor_param = configuration.cursor_param
         self.field = configuration.field
+        self.has_next = configuration.has_next
 
     def get_next_request(
         self,
@@ -27,11 +29,15 @@ class CursorPaginationStrategy(PaginationStrategy):
 
         # get the last object in the array and read the cursor value
         cursor = None
+        has_next = None
         object_list = (
             pydash.get(response.json(), data_path) if data_path else response.json()
         )
         if object_list and isinstance(object_list, list):
-            cursor = pydash.get(object_list.pop(), self.field)
+            last_object = object_list.pop()
+            cursor = pydash.get(last_object, self.field)
+            if self.has_next:
+                has_next = pydash.get(last_object, self.has_next)
 
         # If the cursor value isn't found, try to find the value in response.json() using the field value
         if cursor is None:
@@ -41,6 +47,14 @@ class CursorPaginationStrategy(PaginationStrategy):
         if cursor is None:
             return None
 
+        if self.has_next and has_next is None:
+            has_next = pydash.get(response.json(), self.has_next)
+
+        if self.has_next:
+            logger.debug(f"The {self.has_next} field has a value of {has_next}")
+            if str(has_next).lower() != "true":
+                return None
+
         # add or replace cursor_param with new cursor value
         request_params.query_params[self.cursor_param] = cursor
 

fides/api/service/pagination/pagination_strategy_link.py

@@ -43,7 +43,7 @@ class LinkPaginationStrategy(PaginationStrategy):
 
         if self.has_next:
             has_next = pydash.get(response.json(), self.has_next)
-            logger.info(f"The {self.has_next} field has a value of {has_next}")
+            logger.debug(f"The {self.has_next} field has a value of {has_next}")
             if str(has_next).lower() != "true":
                 return None
 

fides/api/service/storage/s3.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+from typing import IO, Any, Dict, Optional, Tuple
+
+from boto3.s3.transfer import TransferConfig
+from botocore.exceptions import ClientError, ParamValidationError
+from fideslang.validation import AnyHttpUrlString
+from loguru import logger
+
+from fides.api.schemas.storage.storage import StorageSecrets
+from fides.api.util.aws_util import get_s3_client
+from fides.config import CONFIG
+
+LARGE_FILE_THRESHOLD = 5 * 1024 * 1024  # 5 MB threshold for large files
+LARGE_FILE_WARNING_TEXT = (
+    "File is too large to display directly. Please use the provided link to download."
+)
+
+
+def maybe_get_s3_client(
+    auth_method: str, storage_secrets: Dict[StorageSecrets, Any]
+) -> Any:
+    """
+    Returns an S3 client if the client can be created successfully, otherwise raises an exception.
+    """
+    try:
+        return get_s3_client(auth_method, storage_secrets)
+    except ClientError as e:
+        logger.error(
+            "Encountered error while uploading and generating link for S3 object: {}", e
+        )
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
+def create_presigned_url_for_s3(
+    s3_client: Any, bucket_name: str, file_key: str
+) -> AnyHttpUrlString:
+    """
+    Generates a presigned URL to share an S3 object
+
+    :param s3_client: s3 base client
+    :param bucket_name: string
+    :param file_key: string
+    :return: Presigned URL as string.
+    """
+    response = s3_client.generate_presigned_url(
+        "get_object",
+        Params={"Bucket": bucket_name, "Key": file_key},
+        ExpiresIn=CONFIG.security.subject_request_download_link_ttl_seconds,
+    )
+
+    # The response contains the presigned URL
+    return response
+
+
+def generic_upload_to_s3(  # pylint: disable=R0913
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+    document: IO[bytes],
+    size_threshold: int = LARGE_FILE_THRESHOLD,  # 5 MB threshold
+) -> Optional[AnyHttpUrlString]:
+    """
+    Uploads arbitrary data to S3 returned from an access request.
+    Handles both small and large uploads.
+
+    :param storage_secrets: S3 storage secrets
+    :param bucket_name: Name of the S3 bucket
+    :param file_key: Key of the file in the bucket
+    :param auth_method: Authentication method for S3
+    :param document: File contents to upload
+    """
+    logger.info("Starting S3 Upload of {}", file_key)
+
+    s3_client = maybe_get_s3_client(auth_method, storage_secrets)
+
+    # Define a transfer configuration for multipart uploads
+    transfer_config = TransferConfig(
+        multipart_threshold=size_threshold,
+        multipart_chunksize=size_threshold,
+    )
+
+    # Use upload_fileobj for efficient uploads (handles both small and large files)
+    try:
+        # Use upload_fileobj for efficient uploads (handles both small and large files)
+        s3_client.upload_fileobj(
+            Fileobj=document,
+            Bucket=bucket_name,
+            Key=file_key,
+            Config=transfer_config,
+        )
+    except Exception as e:
+        logger.error(f"Failed to upload file {file_key} to bucket {bucket_name}: {e}")
+        raise e  # Re-raise the exception if you want it to propagate
+
+    # Generate a presigned URL for the uploaded file
+    logger.info(f"Successfully uploaded file {file_key} to bucket {bucket_name}")
+    return create_presigned_url_for_s3(s3_client, bucket_name, file_key)
+
+
+def generic_download_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> Optional[AnyHttpUrlString]:
+    """
+    Returns a presigned URL for downloading an S3 object.
+
+    :param storage_secrets: S3 storage secrets
+    :param bucket_name: Name of the S3 bucket
+    :param file_key: Key of the file in the bucket
+    :param auth_method: Authentication method for S3
+    :return: A presigned URL (str) for the S3 object
+    """
+    logger.info("Generating presigned URL for downloading file: {}", file_key)
+
+    s3_client = maybe_get_s3_client(auth_method, storage_secrets)
+
+    # Generate a presigned URL for the file
+    return create_presigned_url_for_s3(s3_client, bucket_name, file_key)
+
+
+def generic_retrieve_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+    size_threshold: int = LARGE_FILE_THRESHOLD,  # 5 MB threshold
+) -> Tuple[str, AnyHttpUrlString]:
+    """
+    Retrieves arbitrary data from S3.
+    Returns the file contents and presigned url to download the file if the file is small,
+    or a warning message with the presigned URL to download the file if it is large.
+
+    :param storage_secrets: S3 storage secrets
+    :param bucket_name: Name of the S3 bucket
+    :param file_key: Key of the file in the bucket
+    :param auth_method: Authentication method for S3
+    :param size_threshold: Size threshold in bytes to determine small vs large files
+    :return: File contents (BytesIO) for small files or a presigned URL (str) for large files
+    """
+    logger.info("Starting S3 Retrieve of {}", file_key)
+
+    s3_client = maybe_get_s3_client(auth_method, storage_secrets)
+    try:
+        # Get the object metadata to check its size
+        head_response = s3_client.head_object(Bucket=bucket_name, Key=file_key)
+        file_size = head_response["ContentLength"]
+
+        if file_size <= size_threshold:
+            # File is small, return its contents and presigned url
+            response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
+            return response["Body"].read(), create_presigned_url_for_s3(
+                s3_client, bucket_name, file_key
+            )
+
+        # File is large, return warning text and a presigned URL
+        return (
+            LARGE_FILE_WARNING_TEXT,
+            create_presigned_url_for_s3(s3_client, bucket_name, file_key),
+        )
+
+    except Exception as e:
+        logger.error("Encountered error while retrieving S3 object: {}", e)
+        raise e
+
+
+def generic_delete_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> None:
+    """
+    Deletes arbitrary data from s3
+
+    :param storage_secrets: S3 storage secrets
+    :param bucket_name: Name of the S3 bucket
+    :param file_key: Key of the file in the bucket
+    :param auth_method: Authentication method for S3
+    """
+    logger.info("Starting S3 Delete of {}", file_key)
+
+    s3_client = maybe_get_s3_client(auth_method, storage_secrets)
+    try:
+        s3_client.delete_object(Bucket=bucket_name, Key=file_key)
+    except Exception as e:
+        logger.error("Encountered error while deleting s3 object: {}", e)
+        raise e

fides/api/service/storage/util.py

@@ -0,0 +1,10 @@
+import os
+
+LOCAL_FIDES_UPLOAD_DIRECTORY = "fides_uploads"
+
+
+def get_local_filename(file_key: str) -> str:
+    """Verifies that the local storage directory exists"""
+    if not os.path.exists(LOCAL_FIDES_UPLOAD_DIRECTORY):
+        os.makedirs(LOCAL_FIDES_UPLOAD_DIRECTORY)
+    return f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{file_key}"

fides/api/tasks/storage.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
 
 import json
-import os
 import secrets
 import zipfile
 from io import BytesIO
-from typing import Any, Dict, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
 
 import pandas as pd
 from botocore.exceptions import ClientError, ParamValidationError
@@ -13,11 +12,18 @@ from fideslang.validation import AnyHttpUrlString
 from loguru import logger
 
 from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
-from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.storage.storage import ResponseFormat, StorageSecrets
 from fides.api.service.privacy_request.dsr_package.dsr_report_builder import (
     DsrReportBuilder,
 )
+from fides.api.service.storage.s3 import (
+    create_presigned_url_for_s3,
+    generic_upload_to_s3,
+)
+from fides.api.service.storage.util import (
+    LOCAL_FIDES_UPLOAD_DIRECTORY,
+    get_local_filename,
+)
 from fides.api.util.aws_util import get_s3_client
 from fides.api.util.cache import get_cache, get_encryption_cache_key
 from fides.api.util.encryption.aes_gcm_encryption_scheme import (
@@ -26,7 +32,8 @@ from fides.api.util.encryption.aes_gcm_encryption_scheme import (
 from fides.api.util.storage_util import storage_json_encoder
 from fides.config import CONFIG
 
-LOCAL_FIDES_UPLOAD_DIRECTORY = "fides_uploads"
+if TYPE_CHECKING:
+    from fides.api.models.privacy_request import PrivacyRequest
 
 
 def encrypt_access_request_results(data: Union[str, bytes], request_id: str) -> str:
@@ -101,105 +108,6 @@ def write_to_in_memory_buffer(
     raise NotImplementedError(f"No handling for response format {resp_format}.")
 
 
-def create_presigned_url_for_s3(
-    s3_client: Any, bucket_name: str, file_key: str
-) -> AnyHttpUrlString:
-    """ "Generate a presigned URL to share an S3 object
-
-    :param s3_client: s3 base client
-    :param bucket_name: string
-    :param file_key: string
-    :return: Presigned URL as string.
-    """
-    response = s3_client.generate_presigned_url(
-        "get_object",
-        Params={"Bucket": bucket_name, "Key": file_key},
-        ExpiresIn=CONFIG.security.subject_request_download_link_ttl_seconds,
-    )
-
-    # The response contains the presigned URL
-    return response
-
-
-def generic_upload_to_s3(  # pylint: disable=R0913
-    storage_secrets: Dict[StorageSecrets, Any],
-    bucket_name: str,
-    file_key: str,
-    auth_method: str,
-    document: bytes,
-) -> Optional[AnyHttpUrlString]:
-    """Uploads arbitrary data to s3 returned from an access request"""
-    logger.info("Starting S3 Upload of {}", file_key)
-
-    try:
-        s3_client = get_s3_client(auth_method, storage_secrets)
-        try:
-            s3_client.put_object(Bucket=bucket_name, Key=file_key, Body=document)
-        except Exception as e:
-            logger.error("Encountered error while uploading s3 object: {}", e)
-            raise e
-
-        presigned_url: AnyHttpUrlString = create_presigned_url_for_s3(
-            s3_client, bucket_name, file_key
-        )
-
-        return presigned_url
-    except ClientError as e:
-        logger.error(
-            "Encountered error while uploading and generating link for s3 object: {}", e
-        )
-        raise e
-    except ParamValidationError as e:
-        raise ValueError(f"The parameters you provided are incorrect: {e}")
-
-
-def generic_retrieve_from_s3(
-    storage_secrets: Dict[StorageSecrets, Any],
-    bucket_name: str,
-    file_key: str,
-    auth_method: str,
-) -> Optional[bytes]:
-    """Retrieves arbitrary data from s3"""
-    logger.info("Starting S3 Retrieve of {}", file_key)
-
-    try:
-        s3_client = get_s3_client(auth_method, storage_secrets)
-        try:
-            response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
-            return response["Body"].read()
-        except Exception as e:
-            logger.error("Encountered error while retrieving s3 object: {}", e)
-            raise e
-    except ClientError as e:
-        logger.error("Encountered error while retrieving s3 object: {}", e)
-        raise e
-    except ParamValidationError as e:
-        raise ValueError(f"The parameters you provided are incorrect: {e}")
-
-
-def generic_delete_from_s3(
-    storage_secrets: Dict[StorageSecrets, Any],
-    bucket_name: str,
-    file_key: str,
-    auth_method: str,
-) -> None:
-    """Deletes arbitrary data from s3"""
-    logger.info("Starting S3 Delete of {}", file_key)
-
-    try:
-        s3_client = get_s3_client(auth_method, storage_secrets)
-        try:
-            s3_client.delete_object(Bucket=bucket_name, Key=file_key)
-        except Exception as e:
-            logger.error("Encountered error while deleting s3 object: {}", e)
-            raise e
-    except ClientError as e:
-        logger.error("Encountered error while deleting s3 object: {}", e)
-        raise e
-    except ParamValidationError as e:
-        raise ValueError(f"The parameters you provided are incorrect: {e}")
-
-
 def upload_to_s3(  # pylint: disable=R0913
     storage_secrets: Dict[StorageSecrets, Any],
     data: Dict,
@@ -207,7 +115,7 @@ def upload_to_s3(  # pylint: disable=R0913
     file_key: str,
     resp_format: str,
     privacy_request: Optional[PrivacyRequest],
-    document: Optional[bytes],
+    document: Optional[BytesIO],
     auth_method: str,
 ) -> Optional[AnyHttpUrlString]:
     """Uploads arbitrary data to s3 returned from an access request"""
@@ -249,13 +157,6 @@ def upload_to_s3(  # pylint: disable=R0913
         raise ValueError(f"The parameters you provided are incorrect: {e}")
 
 
-def get_local_filename(file_key: str) -> str:
-    """Verifies that the local storage directory exists"""
-    if not os.path.exists(LOCAL_FIDES_UPLOAD_DIRECTORY):
-        os.makedirs(LOCAL_FIDES_UPLOAD_DIRECTORY)
-    return f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{file_key}"
-
-
 def upload_to_local(
     data: Dict,
     file_key: str,

fides/config/__init__.py

@@ -133,7 +133,7 @@ def censor_config(config: Union[FidesConfig, Dict[str, Any]]) -> Dict[str, Any]:
             data = as_dict[key]
             filtered[key] = {}
             for field in value:
-                if field in data:
+                if data and field in data:
                     filtered[key][field] = data[field]
 
     return filtered

fides/data/language/languages.yml

@@ -45,6 +45,8 @@ languages:
   name: Italian
 - id: ja
   name: Japanese
+- id: ko
+  name: Korean
 - id: lt
   name: Lithuanian
 - id: lv