ethyca-fides 2.63.1b0__tar.gz → 2.63.1b1__tar.gz

{ethyca_fides-2.63.1b0/src/ethyca_fides.egg-info → ethyca_fides-2.63.1b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ethyca_fides
-Version: 2.63.1b0
+Version: 2.63.1b1
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.63.1b0 → ethyca_fides-2.63.1b1/src/ethyca_fides.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ethyca_fides
-Version: 2.63.1b0
+Version: 2.63.1b1
 Summary: Open-source ecosystem for data privacy as code.
 Home-page: https://github.com/ethyca/fides
 Author: Ethyca, Inc.

{ethyca_fides-2.63.1b0 → ethyca_fides-2.63.1b1}/src/ethyca_fides.egg-info/SOURCES.txt

@@ -626,15 +626,16 @@ src/fides/api/service/pagination/pagination_strategy_factory.py
 src/fides/api/service/pagination/pagination_strategy_link.py
 src/fides/api/service/pagination/pagination_strategy_offset.py
 src/fides/api/service/privacy_request/__init__.py
+src/fides/api/service/privacy_request/attachment_handling.py
 src/fides/api/service/privacy_request/email_batch_service.py
 src/fides/api/service/privacy_request/request_runner_service.py
 src/fides/api/service/privacy_request/request_service.py
 src/fides/api/service/privacy_request/dsr_package/__init__.py
 src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py
 src/fides/api/service/privacy_request/dsr_package/assets/back.svg
+src/fides/api/service/privacy_request/dsr_package/templates/attachments_index.html
 src/fides/api/service/privacy_request/dsr_package/templates/collection_index.html
 src/fides/api/service/privacy_request/dsr_package/templates/dataset_index.html
-src/fides/api/service/privacy_request/dsr_package/templates/item.html
 src/fides/api/service/privacy_request/dsr_package/templates/main.css
 src/fides/api/service/privacy_request/dsr_package/templates/welcome.html
 src/fides/api/service/processors/__init__.py
@@ -666,6 +667,8 @@ src/fides/api/task/graph_task.py
 src/fides/api/task/refine_target_path.py
 src/fides/api/task/task_resources.py
 src/fides/api/tasks/__init__.py
+src/fides/api/tasks/csv_utils.py
+src/fides/api/tasks/encryption_utils.py
 src/fides/api/tasks/storage.py
 src/fides/api/tasks/scheduled/__init__.py
 src/fides/api/tasks/scheduled/scheduler.py
@@ -824,6 +827,8 @@ src/fides/ui-build/static/admin/properties.html
 src/fides/ui-build/static/admin/systems.html
 src/fides/ui-build/static/admin/taxonomy.html
 src/fides/ui-build/static/admin/user-management.html
+src/fides/ui-build/static/admin/_next/static/74KgkHM2cEVIXGgJPlTZ3/_buildManifest.js
+src/fides/ui-build/static/admin/_next/static/74KgkHM2cEVIXGgJPlTZ3/_ssgManifest.js
 src/fides/ui-build/static/admin/_next/static/chunks/1099-667e84655846e78c.js
 src/fides/ui-build/static/admin/_next/static/chunks/1100-0b01f500b1eaf60d.js
 src/fides/ui-build/static/admin/_next/static/chunks/1817-951f58ddd5d46081.js
@@ -988,8 +993,6 @@ src/fides/ui-build/static/admin/_next/static/css/ab65b8cc3144bfc8.css
 src/fides/ui-build/static/admin/_next/static/css/d2d4e9a6923e6730.css
 src/fides/ui-build/static/admin/_next/static/css/e458b5f1afdbb7fc.css
 src/fides/ui-build/static/admin/_next/static/css/eb46b504bb5d980d.css
-src/fides/ui-build/static/admin/_next/static/j0v5qPG9TaezfK2WMkHhI/_buildManifest.js
-src/fides/ui-build/static/admin/_next/static/j0v5qPG9TaezfK2WMkHhI/_ssgManifest.js
 src/fides/ui-build/static/admin/_next/static/media/inter-all-400-normal.2ae8ed37.woff
 src/fides/ui-build/static/admin/_next/static/media/inter-all-500-normal.56c3d87b.woff
 src/fides/ui-build/static/admin/_next/static/media/inter-all-600-normal.d9666969.woff

{ethyca_fides-2.63.1b0 → ethyca_fides-2.63.1b1}/src/fides/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-06-06T15:23:03-0400",
+ "date": "2025-06-09T15:18:50-0400",
  "dirty": true,
  "error": null,
- "full-revisionid": "b43cc283abf5f561e6346db6b0c0995c12133749",
- "version": "2.63.1b0"
+ "full-revisionid": "015433003992aa2bacc15f1730dc22591866809b",
+ "version": "2.63.1b1"
 }
 '''  # END VERSION_JSON
 

{ethyca_fides-2.63.1b0 → ethyca_fides-2.63.1b1}/src/fides/api/alembic/migrations/versions/bf713b5a021d_staged_resource_ancestor_link_data_.py

@@ -74,6 +74,11 @@ def upgrade():
             children = result.children
             if children:
                 resource_children[urn] = children
+            else:
+                # even if no children, we still add the resource to the map
+                # so that we maintain a record of all resources and can
+                # check for orphaned descendant records later
+                resource_children[urn] = []
 
     # Build list of ancestor-descendant pairs
     ancestor_links = []
@@ -86,17 +91,21 @@ def upgrade():
         for child_urn in children:
             if child_urn not in visited:
                 visited.add(child_urn)
-                # Add direct ancestor link
-                ancestor_links.append(
-                    {
-                        "id": f"srl_{uuid.uuid4()}",
-                        "ancestor_urn": ancestor_urn,
-                        "descendant_urn": child_urn,
-                    }
-                )
-
-                # Recursively process this child's children
-                if child_urn in resource_children:
+                if child_urn not in resource_children:
+                    logger.warning(
+                        f"Found orphaned descendant URN: {child_urn}, not adding ancestor link"
+                    )
+                else:
+                    # Add direct ancestor link
+                    ancestor_links.append(
+                        {
+                            "id": f"srl_{uuid.uuid4()}",
+                            "ancestor_urn": ancestor_urn,
+                            "descendant_urn": child_urn,
+                        }
+                    )
+
+                    # Recursively process this child's children
                     process_children(
                         ancestor_urn, resource_children[child_urn], visited
                     )

{ethyca_fides-2.63.1b0 → ethyca_fides-2.63.1b1}/src/fides/api/migrations/post_upgrade_index_creation.py

@@ -182,7 +182,7 @@ def create_object(db: Session, object_statement: str, object_name: str) -> None:
 def check_and_create_objects(
     db: Session, table_object_map: Dict[str, List[Dict[str, str]]], lock: Lock
 ) -> Dict[str, str]:
-    """Returns a dictionary of any indices or constraints that are in the process of being created."""
+    """Returns a dictionary of any indices or constraints that were created."""
     object_info: Dict[str, str] = {}
     for _, objects in table_object_map.items():
         for object_data in objects:
@@ -203,7 +203,7 @@ def check_and_create_objects(
                             continue
 
                 create_object(db, object_statement, object_name)
-                object_info[object_name] = "in progress"
+                object_info[object_name] = "created"
             else:
                 logger.debug(
                     f"Object {object_name} already exists, skipping index/constraint creation"
@@ -248,7 +248,7 @@ def post_upgrade_index_creation_task() -> None:
                     f"Post upgrade index creation output: {json.dumps(object_info)}"
                 )
             else:
-                logger.debug("All indices and constraints created")
+                logger.debug("All indices and constraints already created")
 
 
 def initiate_post_upgrade_index_creation() -> None:

{ethyca_fides-2.63.1b0 → ethyca_fides-2.63.1b1}/src/fides/api/models/attachment.py

@@ -1,5 +1,6 @@
 import os
 from enum import Enum as EnumType
+from io import BytesIO
 from typing import IO, TYPE_CHECKING, Any, Tuple
 
 from fideslang.validation import AnyHttpUrlString
@@ -19,7 +20,6 @@ from fides.api.service.storage.s3 import (
     generic_upload_to_s3,
 )
 from fides.api.service.storage.util import AllowedFileType, get_local_filename
-from fides.config import CONFIG
 
 if TYPE_CHECKING:
     from fides.api.models.comment import Comment
@@ -28,6 +28,11 @@ if TYPE_CHECKING:
     from fides.api.models.storage import StorageConfig
 
 
+# This is 7 days in seconds and is currently the max allowed
+# configurable expiration time for presigned URLs for both s3 and gcs.
+MAX_TTL_SECONDS = 604800
+
+
 class AttachmentType(str, EnumType):
     """
     Enum for attachment types. Indicates attachment usage.
@@ -122,6 +127,11 @@ class Attachment(Base):
         """Returns the content type of the attachment."""
         return AllowedFileType[self.file_name.split(".")[-1]].value
 
+    @property
+    def file_key(self) -> str:
+        """Returns the file key of the attachment."""
+        return f"{self.id}/{self.file_name}"
+
     def upload(self, attachment: IO[bytes]) -> None:
         """Uploads an attachment to S3, GCS, or local storage."""
         if self.config.type == StorageType.s3:
@@ -130,7 +140,7 @@ class Attachment(Base):
             generic_upload_to_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 document=attachment,
                 auth_method=auth_method,
             )
@@ -142,7 +152,7 @@ class Attachment(Base):
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(f"{self.id}/{self.file_name}")
+            blob = bucket.blob(self.file_key)
 
             # Reset the file pointer to the beginning
             try:
@@ -155,7 +165,7 @@ class Attachment(Base):
             return
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(f"{self.id}/{self.file_name}")
+            filename = get_local_filename(self.file_key)
 
             # Validate that attachment is a file-like object
             if not hasattr(attachment, "read"):
@@ -205,9 +215,10 @@ class Attachment(Base):
             size, url = generic_retrieve_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 auth_method=auth_method,
                 get_content=False,
+                ttl_seconds=MAX_TTL_SECONDS,
             )
             return size, url
 
@@ -216,19 +227,20 @@ class Attachment(Base):
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(f"{self.id}/{self.file_name}")
+            blob = bucket.blob(self.file_key)
 
             # Ensure we have the blob metadata
             blob.reload()
+            # Expiration is set to 7 days
             url = blob.generate_signed_url(
                 version="v4",
-                expiration=CONFIG.security.subject_request_download_link_ttl_seconds,
+                expiration=MAX_TTL_SECONDS,
                 method="GET",
             )
             return blob.size, url
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(f"{self.id}/{self.file_name}")
+            filename = get_local_filename(self.file_key)
             size = os.path.getsize(filename)
             return size, filename
 
@@ -236,7 +248,7 @@ class Attachment(Base):
 
     def retrieve_attachment_content(
         self,
-    ) -> Tuple[int, bytes]:
+    ) -> Tuple[int, IO[bytes]]:
         """
         Retrieves the size of the attachment and its actual content.
         - For s3:
@@ -252,30 +264,33 @@ class Attachment(Base):
         if self.config.type == StorageType.s3:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
-            size, content = generic_retrieve_from_s3(
+            size, fileobj = generic_retrieve_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 auth_method=auth_method,
                 get_content=True,
             )
-            return size, content
+            return size, fileobj
 
         if self.config.type == StorageType.gcs:
             bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
             auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
             storage_client = get_gcs_client(auth_method, self.config.secrets)
             bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(f"{self.id}/{self.file_name}")
+            blob = bucket.blob(self.file_key)
 
-            content = blob.download_as_bytes()
-            return len(content), content
+            fileobj = BytesIO()
+            blob.download_to_file(fileobj)
+            fileobj.seek(0)  # Reset pointer to beginning after download
+            return blob.size, fileobj
 
         if self.config.type == StorageType.local:
-            filename = get_local_filename(f"{self.id}/{self.file_name}")
-            with open(filename, "rb") as file:
-                content = file.read()
-                size = len(content)
+            filename = get_local_filename(self.file_key)
+            size = os.path.getsize(filename)
+            with open(filename, "rb") as fileobj:
+                content = BytesIO(fileobj.read())
+                content.seek(0)  # Reset pointer to beginning
                 return size, content
 
         raise ValueError(f"Unsupported storage type: {self.config.type}")
@@ -288,7 +303,7 @@ class Attachment(Base):
             generic_delete_from_s3(
                 storage_secrets=self.config.secrets,
                 bucket_name=bucket_name,
-                file_key=f"{self.id}/{self.file_name}",
+                file_key=self.file_key,
                 auth_method=auth_method,
             )
             return
@@ -307,9 +322,7 @@ class Attachment(Base):
             return
 
         if self.config.type == StorageType.local:
-            folder_path = os.path.dirname(
-                get_local_filename(f"{self.id}/{self.file_name}")
-            )
+            folder_path = os.path.dirname(get_local_filename(self.file_key))
             if os.path.exists(folder_path):
                 import shutil
 

ethyca_fides-2.63.1b1/src/fides/api/service/privacy_request/attachment_handling.py

@@ -0,0 +1,132 @@
+import time as time_module
+from dataclasses import dataclass
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+from loguru import logger
+
+from fides.api.models.attachment import Attachment, AttachmentType
+from fides.api.schemas.storage.storage import StorageDetails
+
+
+@dataclass
+class AttachmentData:
+    """Data structure for attachment metadata and content.
+    Using a dataclass rather than a Pydantic model here for the following reasons:
+    - The data structure is simple and doesn't need complex validation.
+    - The fields being used have already been validated and are properly typed.
+    - The class is used internally for data transfer, not for API serialization.
+    - Performance is important since this is used in a data processing pipeline.
+    """
+
+    file_name: str
+    file_size: Optional[int]
+    download_url: Optional[str]
+    content_type: str
+    bucket_name: str
+    file_key: str
+    storage_key: str
+
+    def to_upload_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for upload, including presigned URL."""
+        return {
+            "file_name": self.file_name,
+            "file_size": self.file_size,
+            "download_url": self.download_url,
+            "content_type": self.content_type,
+        }
+
+    def to_storage_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for storage, including the elements needed to recreated the presigned URL."""
+        return {
+            "file_name": self.file_name,
+            "file_size": self.file_size,
+            "content_type": self.content_type,
+            "bucket_name": self.bucket_name,
+            "file_key": self.file_key,
+            "storage_key": self.storage_key,
+        }
+
+
+def get_attachments_content(
+    loaded_attachments: List[Attachment],
+) -> Iterator[AttachmentData]:
+    """
+    Retrieves all attachments associated with a privacy request that are marked to be included with the access package.
+    Yields AttachmentData objects containing attachment metadata and download urls.
+    Uses generators to minimize memory usage.
+
+    Args:
+        loaded_attachments: List of Attachment objects to process
+
+    Yields:
+        AttachmentData object containing attachment metadata and url
+    """
+    start_time = time_module.time()
+    processed_count = 0
+    skipped_count = 0
+    error_count = 0
+    total_size = 0
+
+    for attachment in loaded_attachments:
+        if attachment.attachment_type != AttachmentType.include_with_access_package:
+            skipped_count += 1
+            continue
+
+        try:
+            # Get size and download URL using retrieve_attachment
+            size, url = attachment.retrieve_attachment()
+            total_size += size if size else 0
+            if url is None:
+                logger.warning(
+                    "No download URL retrieved for attachment {}", attachment.file_name
+                )
+                skipped_count += 1
+                continue
+
+            processed_count += 1
+            yield AttachmentData(
+                file_name=attachment.file_name,
+                file_size=size,
+                download_url=str(url) if url else None,
+                content_type=attachment.content_type,
+                bucket_name=attachment.config.details[StorageDetails.BUCKET.value],
+                file_key=attachment.file_key,
+                storage_key=attachment.storage_key,
+            )
+
+        except Exception as e:
+            error_count += 1
+            logger.error(
+                "Error processing attachment {}: {}", attachment.file_name, str(e)
+            )
+            continue
+
+    # Log final metrics
+    time_taken = time_module.time() - start_time
+    logger.bind(
+        time_to_process=time_taken,
+        total_attachments=len(loaded_attachments),
+        processed_attachments=processed_count,
+        skipped_attachments=skipped_count,
+        error_attachments=error_count,
+        total_size_bytes=total_size,
+    ).info("Attachment processing complete")
+
+
+def process_attachments_for_upload(
+    attachments: Iterator[AttachmentData],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    """
+    Process attachments into separate upload and storage formats.
+    Returns both formats:
+    - upload_attachments: Used for uploading to access packages
+    - storage_attachments: Used for saving filtered access results
+    """
+    upload_attachments = []
+    storage_attachments = []
+
+    for attachment in attachments:
+        storage_attachments.append(attachment.to_storage_dict())
+        upload_attachments.append(attachment.to_upload_dict())
+
+    return upload_attachments, storage_attachments