ethyca-fides 2.56.3b1__py2.py3-none-any.whl → 2.57.0rc0__py2.py3-none-any.whl

fides/api/alembic/migrations/versions/6ea2171c544f_change_attachment_storage_key_to_.py

@@ -0,0 +1,77 @@
+"""Change Attachment.storage_key to foreign key
+
+Revision ID: 6ea2171c544f
+Revises: 1152c1717849
+Create Date: 2025-03-10 17:36:31.504831
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "6ea2171c544f"
+down_revision = "1152c1717849"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+
+    # Alter the column type and add the new foreign key constraint
+    op.alter_column(
+        "attachment",
+        "storage_key",
+        existing_type=sa.String(),
+        type_=sa.String(),
+        nullable=False,
+    )
+    op.create_foreign_key(
+        "fk_attachment_storage_key",
+        "attachment",
+        "storageconfig",
+        ["storage_key"],
+        ["key"],
+        ondelete="CASCADE",
+    )
+
+    # Add index on attachment_reference.reference_id
+    op.create_index(
+        "ix_attachment_reference_reference_id",
+        "attachment_reference",
+        ["reference_id"],
+    )
+
+    # Add index on attachment_reference.reference_type
+    op.create_index(
+        "ix_attachment_reference_reference_type",
+        "attachment_reference",
+        ["reference_type"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # Drop the index on attachment_reference.reference_id
+    op.drop_index(
+        "ix_attachment_reference_reference_id", table_name="attachment_reference"
+    )
+
+    # Drop the index on attachment_reference.reference_type
+    op.drop_index(
+        "ix_attachment_reference_reference_type", table_name="attachment_reference"
+    )
+
+    # Drop the new foreign key constraint
+    op.drop_constraint("fk_attachment_storage_key", "attachment", type_="foreignkey")
+
+    # Revert the column type change
+    op.alter_column(
+        "attachment",
+        "storage_key",
+        existing_type=sa.String(),
+        type_=sa.String(),
+        nullable=True,
+    )
+    # ### end Alembic commands ###

fides/api/models/attachment.py

@@ -1,6 +1,8 @@
+import os
 from enum import Enum as EnumType
 from typing import Any, Optional
 
+from loguru import logger as log
 from sqlalchemy import Column
 from sqlalchemy import Enum as EnumColumn
 from sqlalchemy import ForeignKey, String, UniqueConstraint
@@ -9,6 +11,15 @@ from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import Base
 from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+from fides.api.models.storage import StorageConfig  # pylint: disable=unused-import
+from fides.api.schemas.storage.storage import StorageDetails, StorageType
+from fides.api.tasks.storage import (
+    LOCAL_FIDES_UPLOAD_DIRECTORY,
+    generic_delete_from_s3,
+    generic_retrieve_from_s3,
+    get_local_filename,
+    upload_to_s3,
+)
 
 
 class AttachmentType(str, EnumType):
@@ -74,7 +85,9 @@ class Attachment(Base):
     )
     file_name = Column(String, nullable=False)
     attachment_type = Column(EnumColumn(AttachmentType), nullable=False)
-    storage_key = Column(String, nullable=False)
+    storage_key = Column(
+        String, ForeignKey("storageconfig.key", ondelete="CASCADE"), nullable=False
+    )
 
     user = relationship(
         "FidesUser",
@@ -90,64 +103,111 @@ class Attachment(Base):
         uselist=True,
     )
 
-    async def upload_attachment_to_s3(self, attachment: bytes) -> None:
-        """Upload an attachment to S3 to the storage_url."""
-        raise NotImplementedError("This method is not yet implemented")
-        # AuditLog.create(
-        #     db=db,
-        #     data={
-        #         "user_id": "system",
-        #         "privacy_request_id": privacy_request.id,
-        #         "action": AuditLogAction.attachment_uploaded,
-        #         "message": "",
-        #     },
-        # )
-
-    async def retrieve_attachment_from_s3(self) -> bytes:
-        """Retrieve an attachment from S3."""
-        raise NotImplementedError("This method is not yet implemented")
-        # AuditLog.create(
-        #     db=db,
-        #     data={
-        #         "user_id": "system",
-        #         "privacy_request_id": privacy_request.id,
-        #         "action": AuditLogAction.attachment_retrieved,
-        #         "message": "",
-        #     },
-        # )
-
-    async def delete_attachment_from_s3(self) -> None:
-        """Delete an attachment from S3."""
-        raise NotImplementedError("This method is not yet implemented")
-        # AuditLog.create(
-        #     db=db,
-        #     data={
-        #         "user_id": "system",
-        #         "privacy_request_id": privacy_request.id,
-        #         "action": AuditLogAction.attachment_deleted,
-        #         "message": "",
-        #     },
-        # )
+    config = relationship(
+        "StorageConfig",
+        lazy="selectin",
+        uselist=False,
+    )
+
+    def upload(self, attachment: bytes) -> None:
+        """Uploads an attachment to S3 or local storage."""
+        if self.config.type == StorageType.s3:
+            bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
+            auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
+            upload_to_s3(
+                storage_secrets=self.config.secrets,
+                data={},
+                bucket_name=bucket_name,
+                file_key=self.id,
+                resp_format=self.config.format,
+                privacy_request=None,
+                document=attachment,
+                auth_method=auth_method,
+            )
+            log.info(f"Uploaded {self.file_name} to S3 bucket {bucket_name}/{self.id}")
+            return
+
+        if self.config.type == StorageType.local:
+            filename = get_local_filename(self.id)
+            with open(filename, "wb") as file:
+                file.write(attachment)
+            return
+
+        raise ValueError(f"Unsupported storage type: {self.config.type}")
+
+    def retrieve_attachment(self) -> Optional[bytes]:
+        """Returns the attachment from S3 in bytes form."""
+        if self.config.type == StorageType.s3:
+            bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
+            auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
+            return generic_retrieve_from_s3(
+                storage_secrets=self.config.secrets,
+                bucket_name=bucket_name,
+                file_key=self.id,
+                auth_method=auth_method,
+            )
+
+        if self.config.type == StorageType.local:
+            filename = f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{self.id}"
+            with open(filename, "rb") as file:
+                return file.read()
+
+        raise ValueError(f"Unsupported storage type: {self.config.type}")
+
+    def delete_attachment_from_storage(self) -> None:
+        """Deletes an attachment from S3 or local storage."""
+        if self.config.type == StorageType.s3:
+            bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
+            auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
+            generic_delete_from_s3(
+                storage_secrets=self.config.secrets,
+                bucket_name=bucket_name,
+                file_key=self.id,
+                auth_method=auth_method,
+            )
+            return
+
+        if self.config.type == StorageType.local:
+            filename = f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{self.id}"
+            os.remove(filename)
+            return
+
+        raise ValueError(f"Unsupported storage type: {self.config.type}")
 
     @classmethod
-    def create(
+    def create_and_upload(
         cls,
         db: Session,
         *,
         data: dict[str, Any],
+        attachment_file: bytes,
         check_name: bool = False,
-        attachment: Optional[
-            bytes
-        ] = None,  # This will not be optional once the upload method is implemented.
     ) -> "Attachment":
         """Creates a new attachment record in the database and uploads the attachment to S3."""
-        # attachment_record.upload_attachment_to_s3(db, attachment)
-        # log.info(f"Uploaded attachment {attachment_record.id} to S3")
-        return super().create(db=db, data=data, check_name=check_name)
+        if attachment_file is None:
+            raise ValueError("Attachment is required")
+        attachment_model = super().create(db=db, data=data, check_name=check_name)
+
+        try:
+            attachment_model.upload(attachment_file)
+            return attachment_model
+        except Exception as e:
+            log.error(f"Failed to upload attachment: {e}")
+            attachment_model.delete(db)
+            raise e
+
+    @classmethod
+    def create(
+        cls,
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = False,
+    ) -> "Attachment":
+        """Raises Error, provides information for user to create with upload instead."""
+        raise NotImplementedError("Please use create_and_upload method for Attachment")
 
     def delete(self, db: Session) -> None:
         """Deletes an attachment record from the database and deletes the attachment from S3."""
-        # attachment_record = cls.get(db, id)
-        # attachment_record.delete_attachment_from_s3(db)
-        # log.info(f"Deleted attachment {attachment_record.id} from S3")
+        self.delete_attachment_from_storage()
         super().delete(db=db)

fides/api/models/comment.py

@@ -0,0 +1,109 @@
+from enum import Enum as EnumType
+from typing import Any
+
+from sqlalchemy import Column
+from sqlalchemy import Enum as EnumColumn
+from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.models.attachment import Attachment, AttachmentReference
+from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+
+
+class CommentType(str, EnumType):
+    """
+    Enum for comment types. Indicates comment usage.
+
+    - notes are internal comments.
+    - reply comments are public and may cause an email or other communciation to be sent
+    """
+
+    note = "note"
+    reply = "reply"
+
+
+class CommentReferenceType(str, EnumType):
+    """
+    Enum for comment reference types. Indicates where the comment is referenced.
+    """
+
+    manual_step = "manual_step"
+    privacy_request = "privacy_request"
+
+
+class CommentReference(Base):
+    """
+    Stores information about a comment and any other element which may reference that comment.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "comment_reference"
+
+    comment_id = Column(String, ForeignKey("comment.id"), nullable=False)
+    reference_id = Column(String, nullable=False)
+    reference_type = Column(EnumColumn(CommentReferenceType), nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint("comment_id", "reference_id", name="comment_reference_uc"),
+    )
+
+    comment = relationship(
+        "Comment",
+        back_populates="references",
+        uselist=False,
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> "CommentReference":
+        """Creates a new comment reference record in the database."""
+        return super().create(db=db, data=data, check_name=check_name)
+
+
+class Comment(Base):
+    """
+    Stores information about a Comment.
+    """
+
+    user_id = Column(
+        String, ForeignKey("fidesuser.id", ondelete="SET NULL"), nullable=True
+    )
+    comment_text = Column(String, nullable=False)
+    comment_type = Column(EnumColumn(CommentType), nullable=False)
+
+    user = relationship(
+        "FidesUser",
+        backref="comments",
+        lazy="selectin",
+        uselist=False,
+    )
+
+    references = relationship(
+        "CommentReference",
+        back_populates="comment",
+        cascade="all, delete",
+        uselist=True,
+    )
+
+    def get_attachments(self, db: Session) -> list[Attachment]:
+        """Retrieve all attachments associated with this comment."""
+        stmt = (
+            db.query(Attachment)
+            .join(
+                AttachmentReference, Attachment.id == AttachmentReference.attachment_id
+            )
+            .where(AttachmentReference.reference_id == self.id)
+        )
+        return db.execute(stmt).scalars().all()
+
+    def delete(self, db: Session) -> None:
+        """Delete the comment and all associated references."""
+        attachments = self.get_attachments(db)
+        for attachment in attachments:
+            attachment.delete(db)
+        db.delete(self)

fides/api/service/connectors/query_configs/bigquery_query_config.py

@@ -1,5 +1,6 @@
 from typing import Any, Dict, List, Optional, Union, cast
 
+import pydash
 from fideslang.models import MaskingStrategies
 from loguru import logger
 from sqlalchemy import MetaData, Table, text
@@ -17,7 +18,14 @@ from fides.api.schemas.namespace_meta.bigquery_namespace_meta import (
 from fides.api.service.connectors.query_configs.query_config import (
     QueryStringWithoutTuplesOverrideQueryConfig,
 )
-from fides.api.util.collection_util import Row, filter_nonempty_values
+from fides.api.util.collection_util import (
+    Row,
+    filter_nonempty_values,
+    flatten_dict,
+    merge_dicts,
+    replace_none_arrays,
+    unflatten_dict,
+)
 
 
 class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
@@ -120,18 +128,45 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
         A List of multiple Update objects are returned for partitioned tables; for a non-partitioned table,
         a single Update object is returned in a List for consistent typing.
 
-        TODO: DRY up this method and `generate_delete` a bit
+        This implementation handles nested fields by grouping them as JSON objects rather than
+        individual field updates.
+
+        See the README.md in this directory for a detailed example of how nested data is handled.
         """
+
+        # 1. Take update_value_map as-is (already flattened)
         update_value_map: Dict[str, Any] = self.update_value_map(row, policy, request)
+
+        # 2. Flatten the row
+        flattened_row = flatten_dict(row)
+
+        # 3. Merge flattened_row with update_value_map (update_value_map takes precedence)
+        merged_dict = merge_dicts(flattened_row, update_value_map)
+
+        # 4. Unflatten the merged dictionary
+        nested_result = unflatten_dict(merged_dict)
+
+        # 5. Replace any arrays containing only None values with empty arrays
+        nested_result = replace_none_arrays(nested_result)  # type: ignore
+
+        # 6. Only keep top-level keys that are in the update_value_map
+        top_level_keys = {key.split(".")[0] for key in update_value_map}
+
+        # Filter the nested result to only include those top-level keys
+        final_update_map = {
+            k: v for k, v in nested_result.items() if k in top_level_keys
+        }
+
+        # Use existing non-empty reference fields mechanism for WHERE clause
         non_empty_reference_field_keys: Dict[str, Field] = filter_nonempty_values(
             {
-                fpath.string_path: fld.cast(row[fpath.string_path])
+                fpath.string_path: fld.cast(pydash.get(row, fpath.string_path))
                 for fpath, fld in self.reference_field_paths.items()
-                if fpath.string_path in row
+                if pydash.get(row, fpath.string_path) is not None
             }
         )
 
-        valid = len(non_empty_reference_field_keys) > 0 and update_value_map
+        valid = len(non_empty_reference_field_keys) > 0 and final_update_map
         if not valid:
             logger.warning(
                 "There is not enough data to generate a valid update statement for {}",
@@ -154,12 +189,12 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
                 partitioned_queries.append(
                     table.update()
                     .where(*(where_clauses + [text(partition_clause)]))
-                    .values(**update_value_map)
+                    .values(**final_update_map)
                 )
 
             return partitioned_queries
 
-        return [table.update().where(*where_clauses).values(**update_value_map)]
+        return [table.update().where(*where_clauses).values(**final_update_map)]
 
     def generate_delete(self, row: Row, client: Engine) -> List[Delete]:
         """Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
@@ -213,18 +248,14 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
         self,
         field_paths: List[FieldPath],
     ) -> List[str]:
-        """Returns field paths in a format they can be added into SQL queries.
-
+        """
+        Returns field paths in a format they can be added into SQL queries.
         Only returns non-nested fields (fields with exactly one level).
-        Nested fields are skipped with a warning log.
         """
+
         formatted_fields = []
         for field_path in field_paths:
-            if len(field_path.levels) > 1:
-                logger.warning(
-                    f"Skipping nested field '{'.'.join(field_path.levels)}' as nested fields are not supported"
-                )
-            else:
+            if len(field_path.levels) == 1:
                 formatted_fields.append(field_path.levels[0])
         return formatted_fields
 

fides/api/service/connectors/query_configs/query_config.py

@@ -144,6 +144,45 @@ class QueryConfig(Generic[T], ABC):
 
         return data
 
+    def _is_child_of_masked_parent(
+        self, path: str, masked_parent_paths: List[str]
+    ) -> bool:
+        """
+        Check if the given path is a child of any already masked parent object.
+        """
+        for parent_path in masked_parent_paths:
+            if path == parent_path or path.startswith(parent_path + "."):
+                return True
+        return False
+
+    def _is_object_or_array_with_data_category(
+        self, field_val: Any, field: Optional[Field]
+    ) -> bool:
+        """
+        Check if field is an object or array of objects with data categories
+        """
+        # First check if the field has data categories
+        has_data_category = (
+            field and field.data_categories and len(field.data_categories) > 0
+        )
+
+        if not has_data_category:
+            return False
+
+        # Check if it's an object
+        is_object = isinstance(field_val, dict)
+
+        # Check if it's a non-empty array of objects
+        is_array_of_objects = False
+        if (
+            isinstance(field_val, list) and field_val
+        ):  # Check if it's a list and not empty
+            # Only check first element if the list is not empty
+            is_array_of_objects = isinstance(field_val[0], dict)
+
+        # Return true if it's either an object or an array of objects and has data categories
+        return is_object or is_array_of_objects
+
     def update_value_map(  # pylint: disable=R0914
         self, row: Row, policy: Policy, request: PrivacyRequest
     ) -> Dict[str, Any]:
@@ -154,27 +193,44 @@ class QueryConfig(Generic[T], ABC):
         In this example, a Null Masking Strategy was used to determine that the name/ccn/code fields, nested
         workplace_info.employer field, and the first element in 'children' for a given customer_id will be replaced
         with null values.
-
         """
         rule_to_collection_field_paths: Dict[Rule, List[FieldPath]] = (
             self.build_rule_target_field_paths(policy)
         )
 
         value_map: Dict[str, Any] = {}
+        # Track which parent paths have been masked as whole objects/arrays
+        masked_parent_paths: List[str] = []
+
         for rule, field_paths in rule_to_collection_field_paths.items():
             strategy_config = rule.masking_strategy
             if not strategy_config:
                 continue
+
             for rule_field_path in field_paths:
+                # Check if field is read-only before processing
+                field = self.field_map().get(rule_field_path)
+                if field and field.read_only:
+                    logger.debug(
+                        f"Skipping read-only field: {rule_field_path.string_path}"
+                    )
+                    continue
+
+                # Skip if this field is a child of an already masked parent object
+                if self._is_child_of_masked_parent(
+                    rule_field_path.string_path, masked_parent_paths
+                ):
+                    logger.debug(
+                        f"Skipping field {rule_field_path.string_path} because its parent object has already been masked"
+                    )
+                    continue
+
+                # Determine masking strategy
                 strategy: MaskingStrategy = MaskingStrategy.get_strategy(
                     strategy_config["strategy"], strategy_config["configuration"]
                 )
-                truncation: MaskingTruncation = [
-                    MaskingTruncation(field.data_type_converter, field.length)
-                    for field_path, field in self.field_map().items()
-                    if field_path == rule_field_path
-                ][0]
-                field = self.field_map().get(rule_field_path)
+
+                # Apply field-level masking strategy override if present
                 if field and field.masking_strategy_override:
                     masking_strategy_override = field.masking_strategy_override
                     strategy = MaskingStrategy.get_strategy(
@@ -184,7 +240,14 @@ class QueryConfig(Generic[T], ABC):
                     logger.warning(
                         f"Using field-level masking override of type '{strategy.name}' for {rule_field_path.string_path}"
                     )
+
                 null_masking: bool = strategy.name == NullMaskingStrategy.name
+                truncation: MaskingTruncation = [
+                    MaskingTruncation(field.data_type_converter, field.length)
+                    for field_path, field in self.field_map().items()
+                    if field_path == rule_field_path
+                ][0]
+
                 if not self._supported_data_type(truncation, null_masking, strategy):
                     logger.warning(
                         "Unable to generate a query for field {}: data_type is either not present on the field or not supported for the {} masking strategy. Received data type: {}",
@@ -194,21 +257,55 @@ class QueryConfig(Generic[T], ABC):
                     )
                     continue
 
-                paths_to_mask: List[str] = [
-                    join_detailed_path(path)
-                    for path in build_refined_target_paths(
-                        row, query_paths={rule_field_path: None}
-                    )
-                ]
-                for detailed_path in paths_to_mask:
-                    value_map[detailed_path] = self._generate_masked_value(
-                        request_id=request.id,
-                        strategy=strategy,
-                        val=pydash.objects.get(row, detailed_path),
-                        masking_truncation=truncation,
-                        null_masking=null_masking,
-                        str_field_path=detailed_path,
+                # Get the actual field value to determine its type
+                field_val = pydash.objects.get(row, rule_field_path.string_path)
+
+                # Handle objects and arrays with data categories - mask as whole entities
+                if self._is_object_or_array_with_data_category(field_val, field):
+                    logger.info(
+                        f"Field {rule_field_path.string_path} is an object or array of objects with data category. Masking entire field."
                     )
+                    if field_val is not None:
+                        value_map[rule_field_path.string_path] = (
+                            self._generate_masked_value(
+                                request_id=request.id,
+                                strategy=strategy,
+                                val=field_val,
+                                masking_truncation=truncation,
+                                null_masking=null_masking,
+                                str_field_path=rule_field_path.string_path,
+                            )
+                        )
+                        # Add this path to masked parent paths to skip its children later
+                        masked_parent_paths.append(rule_field_path.string_path)
+                else:
+                    # Standard approach: build refined target paths for individual fields
+                    paths_to_mask = [
+                        join_detailed_path(path)
+                        for path in build_refined_target_paths(
+                            row, query_paths={rule_field_path: None}
+                        )
+                    ]
+
+                    # Process each detailed path, skipping those that are children of masked parents
+                    for detailed_path in paths_to_mask:
+                        if self._is_child_of_masked_parent(
+                            detailed_path, masked_parent_paths
+                        ):
+                            logger.debug(
+                                f"Skipping detailed path {detailed_path} because its parent object has already been masked"
+                            )
+                            continue
+
+                        value_map[detailed_path] = self._generate_masked_value(
+                            request_id=request.id,
+                            strategy=strategy,
+                            val=pydash.objects.get(row, detailed_path),
+                            masking_truncation=truncation,
+                            null_masking=null_masking,
+                            str_field_path=detailed_path,
+                        )
+
         return value_map
 
     @staticmethod