ethyca-fides 2.56.3b0__py2.py3-none-any.whl → 2.56.3b2__py2.py3-none-any.whl

fides/api/alembic/migrations/versions/6ea2171c544f_change_attachment_storage_key_to_.py

@@ -0,0 +1,77 @@
+"""Change Attachment.storage_key to foreign key
+
+Revision ID: 6ea2171c544f
+Revises: 1152c1717849
+Create Date: 2025-03-10 17:36:31.504831
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "6ea2171c544f"
+down_revision = "1152c1717849"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+
+    # Alter the column type and add the new foreign key constraint
+    op.alter_column(
+        "attachment",
+        "storage_key",
+        existing_type=sa.String(),
+        type_=sa.String(),
+        nullable=False,
+    )
+    op.create_foreign_key(
+        "fk_attachment_storage_key",
+        "attachment",
+        "storageconfig",
+        ["storage_key"],
+        ["key"],
+        ondelete="CASCADE",
+    )
+
+    # Add index on attachment_reference.reference_id
+    op.create_index(
+        "ix_attachment_reference_reference_id",
+        "attachment_reference",
+        ["reference_id"],
+    )
+
+    # Add index on attachment_reference.reference_type
+    op.create_index(
+        "ix_attachment_reference_reference_type",
+        "attachment_reference",
+        ["reference_type"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # Drop the index on attachment_reference.reference_id
+    op.drop_index(
+        "ix_attachment_reference_reference_id", table_name="attachment_reference"
+    )
+
+    # Drop the index on attachment_reference.reference_type
+    op.drop_index(
+        "ix_attachment_reference_reference_type", table_name="attachment_reference"
+    )
+
+    # Drop the new foreign key constraint
+    op.drop_constraint("fk_attachment_storage_key", "attachment", type_="foreignkey")
+
+    # Revert the column type change
+    op.alter_column(
+        "attachment",
+        "storage_key",
+        existing_type=sa.String(),
+        type_=sa.String(),
+        nullable=True,
+    )
+    # ### end Alembic commands ###

fides/api/custom_types.py

@@ -116,6 +116,23 @@ def validate_path_of_url(value: AnyUrl) -> str:
 
     We perform the basic URL validation, but also prevent URLs with paths,
     as paths are not part of an origin.
+
+    As this is meant to be used as a validator _after_ the AnyUrl validator,
+    we implicitly disallow `*` to be specified as an origin value, since it is
+    rejected by the AnyUrl validator.
+
+    Note that `*` _is_ considered a valid origin value
+    (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin),
+    but we specifically disallow it to be set on the `cors_origins` security setting
+    in our application. This is because non-owner users (contributors)
+    are able to edit the `cors_origins` security setting (via API), and we do not
+    want to allow them to set a wildcard origin.
+
+    Instead, `.*` can be set via the `cors_origin_regex` security setting
+    to effectively allow all origins. This is roughly equivalent to setting
+    `"*"` as a `cors_origin`, but there is a slight behavioral difference:
+    the `Access-Control-Allow-Origin` in the response will be set to the
+    `Access-Control-Allow-Origin` specified in the request rather than `"*"`.
     """
     if value.path and value.path != "/":
         raise ValueError("URL origin values cannot contain a path.")
@@ -125,7 +142,13 @@ def validate_path_of_url(value: AnyUrl) -> str:
     return str(value).rstrip("/")
 
 
-URLOriginString = Annotated[AnyUrl, AfterValidator(validate_path_of_url)]
+# This custom type is used to validate HTTP origins, e.g. CORS origins
+# used in SecuritySettings.  See docstring for `validate_path_of_url`
+# for more details.
+URLOriginString = Annotated[
+    AnyUrl,
+    AfterValidator(validate_path_of_url),
+]
 
 
 def validate_css_str(value: str) -> str:

fides/api/db/base.py

@@ -4,6 +4,7 @@
 from fides.api.db.base_class import Base
 from fides.api.models.application_config import ApplicationConfig
 from fides.api.models.asset import Asset
+from fides.api.models.attachment import Attachment, AttachmentReference
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.authentication_request import AuthenticationRequest
 from fides.api.models.client import ClientDetail

fides/api/models/attachment.py

@@ -1,6 +1,8 @@
+import os
 from enum import Enum as EnumType
 from typing import Any, Optional
 
+from loguru import logger as log
 from sqlalchemy import Column
 from sqlalchemy import Enum as EnumColumn
 from sqlalchemy import ForeignKey, String, UniqueConstraint
@@ -9,6 +11,15 @@ from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import Base
 from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+from fides.api.models.storage import StorageConfig  # pylint: disable=unused-import
+from fides.api.schemas.storage.storage import StorageDetails, StorageType
+from fides.api.tasks.storage import (
+    LOCAL_FIDES_UPLOAD_DIRECTORY,
+    generic_delete_from_s3,
+    generic_retrieve_from_s3,
+    get_local_filename,
+    upload_to_s3,
+)
 
 
 class AttachmentType(str, EnumType):
@@ -74,7 +85,9 @@ class Attachment(Base):
     )
     file_name = Column(String, nullable=False)
     attachment_type = Column(EnumColumn(AttachmentType), nullable=False)
-    storage_key = Column(String, nullable=False)
+    storage_key = Column(
+        String, ForeignKey("storageconfig.key", ondelete="CASCADE"), nullable=False
+    )
 
     user = relationship(
         "FidesUser",
@@ -90,64 +103,111 @@ class Attachment(Base):
         uselist=True,
     )
 
-    async def upload_attachment_to_s3(self, attachment: bytes) -> None:
-        """Upload an attachment to S3 to the storage_url."""
-        raise NotImplementedError("This method is not yet implemented")
-        # AuditLog.create(
-        #     db=db,
-        #     data={
-        #         "user_id": "system",
-        #         "privacy_request_id": privacy_request.id,
-        #         "action": AuditLogAction.attachment_uploaded,
-        #         "message": "",
-        #     },
-        # )
-
-    async def retrieve_attachment_from_s3(self) -> bytes:
-        """Retrieve an attachment from S3."""
-        raise NotImplementedError("This method is not yet implemented")
-        # AuditLog.create(
-        #     db=db,
-        #     data={
-        #         "user_id": "system",
-        #         "privacy_request_id": privacy_request.id,
-        #         "action": AuditLogAction.attachment_retrieved,
-        #         "message": "",
-        #     },
-        # )
-
-    async def delete_attachment_from_s3(self) -> None:
-        """Delete an attachment from S3."""
-        raise NotImplementedError("This method is not yet implemented")
-        # AuditLog.create(
-        #     db=db,
-        #     data={
-        #         "user_id": "system",
-        #         "privacy_request_id": privacy_request.id,
-        #         "action": AuditLogAction.attachment_deleted,
-        #         "message": "",
-        #     },
-        # )
+    config = relationship(
+        "StorageConfig",
+        lazy="selectin",
+        uselist=False,
+    )
+
+    def upload(self, attachment: bytes) -> None:
+        """Uploads an attachment to S3 or local storage."""
+        if self.config.type == StorageType.s3:
+            bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
+            auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
+            upload_to_s3(
+                storage_secrets=self.config.secrets,
+                data={},
+                bucket_name=bucket_name,
+                file_key=self.id,
+                resp_format=self.config.format,
+                privacy_request=None,
+                document=attachment,
+                auth_method=auth_method,
+            )
+            log.info(f"Uploaded {self.file_name} to S3 bucket {bucket_name}/{self.id}")
+            return
+
+        if self.config.type == StorageType.local:
+            filename = get_local_filename(self.id)
+            with open(filename, "wb") as file:
+                file.write(attachment)
+            return
+
+        raise ValueError(f"Unsupported storage type: {self.config.type}")
+
+    def retrieve_attachment(self) -> Optional[bytes]:
+        """Returns the attachment from S3 in bytes form."""
+        if self.config.type == StorageType.s3:
+            bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
+            auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
+            return generic_retrieve_from_s3(
+                storage_secrets=self.config.secrets,
+                bucket_name=bucket_name,
+                file_key=self.id,
+                auth_method=auth_method,
+            )
+
+        if self.config.type == StorageType.local:
+            filename = f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{self.id}"
+            with open(filename, "rb") as file:
+                return file.read()
+
+        raise ValueError(f"Unsupported storage type: {self.config.type}")
+
+    def delete_attachment_from_storage(self) -> None:
+        """Deletes an attachment from S3 or local storage."""
+        if self.config.type == StorageType.s3:
+            bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
+            auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
+            generic_delete_from_s3(
+                storage_secrets=self.config.secrets,
+                bucket_name=bucket_name,
+                file_key=self.id,
+                auth_method=auth_method,
+            )
+            return
+
+        if self.config.type == StorageType.local:
+            filename = f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{self.id}"
+            os.remove(filename)
+            return
+
+        raise ValueError(f"Unsupported storage type: {self.config.type}")
 
     @classmethod
-    def create(
+    def create_and_upload(
         cls,
         db: Session,
         *,
         data: dict[str, Any],
+        attachment_file: bytes,
         check_name: bool = False,
-        attachment: Optional[
-            bytes
-        ] = None,  # This will not be optional once the upload method is implemented.
     ) -> "Attachment":
         """Creates a new attachment record in the database and uploads the attachment to S3."""
-        # attachment_record.upload_attachment_to_s3(db, attachment)
-        # log.info(f"Uploaded attachment {attachment_record.id} to S3")
-        return super().create(db=db, data=data, check_name=check_name)
+        if attachment_file is None:
+            raise ValueError("Attachment is required")
+        attachment_model = super().create(db=db, data=data, check_name=check_name)
+
+        try:
+            attachment_model.upload(attachment_file)
+            return attachment_model
+        except Exception as e:
+            log.error(f"Failed to upload attachment: {e}")
+            attachment_model.delete(db)
+            raise e
+
+    @classmethod
+    def create(
+        cls,
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = False,
+    ) -> "Attachment":
+        """Raises Error, provides information for user to create with upload instead."""
+        raise NotImplementedError("Please use create_and_upload method for Attachment")
 
     def delete(self, db: Session) -> None:
         """Deletes an attachment record from the database and deletes the attachment from S3."""
-        # attachment_record = cls.get(db, id)
-        # attachment_record.delete_attachment_from_s3(db)
-        # log.info(f"Deleted attachment {attachment_record.id} from S3")
+        self.delete_attachment_from_storage()
         super().delete(db=db)

fides/api/models/comment.py

@@ -0,0 +1,109 @@
+from enum import Enum as EnumType
+from typing import Any
+
+from sqlalchemy import Column
+from sqlalchemy import Enum as EnumColumn
+from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.models.attachment import Attachment, AttachmentReference
+from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+
+
+class CommentType(str, EnumType):
+    """
+    Enum for comment types. Indicates comment usage.
+
+    - notes are internal comments.
+    - reply comments are public and may cause an email or other communciation to be sent
+    """
+
+    note = "note"
+    reply = "reply"
+
+
+class CommentReferenceType(str, EnumType):
+    """
+    Enum for comment reference types. Indicates where the comment is referenced.
+    """
+
+    manual_step = "manual_step"
+    privacy_request = "privacy_request"
+
+
+class CommentReference(Base):
+    """
+    Stores information about a comment and any other element which may reference that comment.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "comment_reference"
+
+    comment_id = Column(String, ForeignKey("comment.id"), nullable=False)
+    reference_id = Column(String, nullable=False)
+    reference_type = Column(EnumColumn(CommentReferenceType), nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint("comment_id", "reference_id", name="comment_reference_uc"),
+    )
+
+    comment = relationship(
+        "Comment",
+        back_populates="references",
+        uselist=False,
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> "CommentReference":
+        """Creates a new comment reference record in the database."""
+        return super().create(db=db, data=data, check_name=check_name)
+
+
+class Comment(Base):
+    """
+    Stores information about a Comment.
+    """
+
+    user_id = Column(
+        String, ForeignKey("fidesuser.id", ondelete="SET NULL"), nullable=True
+    )
+    comment_text = Column(String, nullable=False)
+    comment_type = Column(EnumColumn(CommentType), nullable=False)
+
+    user = relationship(
+        "FidesUser",
+        backref="comments",
+        lazy="selectin",
+        uselist=False,
+    )
+
+    references = relationship(
+        "CommentReference",
+        back_populates="comment",
+        cascade="all, delete",
+        uselist=True,
+    )
+
+    def get_attachments(self, db: Session) -> list[Attachment]:
+        """Retrieve all attachments associated with this comment."""
+        stmt = (
+            db.query(Attachment)
+            .join(
+                AttachmentReference, Attachment.id == AttachmentReference.attachment_id
+            )
+            .where(AttachmentReference.reference_id == self.id)
+        )
+        return db.execute(stmt).scalars().all()
+
+    def delete(self, db: Session) -> None:
+        """Delete the comment and all associated references."""
+        attachments = self.get_attachments(db)
+        for attachment in attachments:
+            attachment.delete(db)
+        db.delete(self)

fides/api/models/detection_discovery.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from datetime import datetime, timezone
+from datetime import datetime
 from enum import Enum
 from typing import Any, Dict, Iterable, List, Optional, Type
 
@@ -388,7 +388,9 @@ class MonitorExecution(Base):
     )
     status = Column(String, nullable=True)
     started = Column(
-        DateTime(timezone=True), nullable=True, default=datetime.now(timezone.utc)
+        DateTime(timezone=True),
+        nullable=True,
+        server_default=func.now(),
     )
     completed = Column(DateTime(timezone=True), nullable=True)
     classification_instances = Column(

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -1,6 +1,7 @@
 # pylint: disable=too-many-instance-attributes
 from __future__ import annotations
 
+from copy import deepcopy
 from datetime import datetime
 from itertools import product
 from typing import Any, Dict, List, Literal, Optional, Tuple, TypeVar
@@ -12,7 +13,6 @@ from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.common_exceptions import FidesopsException
-from fides.api.graph.config import ScalarField
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import (
@@ -33,7 +33,12 @@ from fides.api.task.refine_target_path import (
     join_detailed_path,
 )
 from fides.api.util import saas_util
-from fides.api.util.collection_util import Row, merge_dicts
+from fides.api.util.collection_util import (
+    Row,
+    flatten_dict,
+    merge_dicts,
+    unflatten_dict,
+)
 from fides.api.util.saas_util import (
     ALL_OBJECT_FIELDS,
     CUSTOM_PRIVACY_REQUEST_FIELDS,
@@ -45,7 +50,6 @@ from fides.api.util.saas_util import (
     REPLY_TO_TOKEN,
     UUID,
     get_identities,
-    unflatten_dict,
 )
 from fides.common.api.v1.urn_registry import REQUEST_TASK_CALLBACK, V1_URL_PREFIX
 from fides.config.config_proxy import ConfigProxy
@@ -510,22 +514,24 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
 
     def all_value_map(self, row: Row) -> Dict[str, Any]:
         """
-        Takes a row and preserves only the fields that are defined in the Dataset.
+        Takes a row and preserves only the fields that are defined in the collection.
         Used for scenarios when an update endpoint has required fields other than
         just the fields being updated.
         """
+        flattened_row = flatten_dict(deepcopy(row))
 
-        all_value_map: Dict[str, Any] = {}
-        for field_path, field in self.field_map().items():
-            # only map scalar fields
-            if (
-                isinstance(field, ScalarField)
-                and pydash.get(row, field_path.string_path) is not None
-            ):
-                all_value_map[field_path.string_path] = pydash.get(
-                    row, field_path.string_path
-                )
-        return all_value_map
+        # Get root field names defined in the collection
+        collection_fields = {
+            field_path.string_path.split(".")[0]
+            for field_path, _ in self.field_map().items()
+        }
+
+        # Only keep the field values defined in the collection
+        return {
+            path: value
+            for path, value in flattened_row.items()
+            if path.split(".")[0] in collection_fields
+        }
 
     def query_to_str(self, t: T, input_data: Dict[str, List[Any]]) -> str:
         """Convert query to string"""

fides/api/service/storage/storage_uploader_service.py

@@ -41,9 +41,7 @@ def upload(
         logger.warning("Storage type not found: {}", storage_key)
         raise StorageUploadError(f"Storage type not found: {storage_key}")
     uploader: Any = _get_uploader_from_config_type(config.type)  # type: ignore
-    return uploader(
-        db, config, data, privacy_request, data_category_field_mapping, data_use_map
-    )
+    return uploader(db, config, data, privacy_request)
 
 
 def get_extension(resp_format: ResponseFormat) -> str:
@@ -88,14 +86,13 @@ def _s3_uploader(
     config: StorageConfig,
     data: Dict,
     privacy_request: PrivacyRequest,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
 ) -> str:
     """Constructs necessary info needed for s3 before calling upload"""
     file_key: str = _construct_file_key(privacy_request.id, config)
 
     bucket_name = config.details[StorageDetails.BUCKET.value]
     auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+    document = None
 
     return upload_to_s3(
         config.secrets,  # type: ignore
@@ -104,9 +101,8 @@ def _s3_uploader(
         file_key,
         config.format.value,  # type: ignore
         privacy_request,
+        document,
         auth_method,
-        data_category_field_mapping,
-        data_use_map,
     )
 
 
@@ -115,9 +111,7 @@ def _local_uploader(
     config: StorageConfig,
     data: Dict,
     privacy_request: PrivacyRequest,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
 ) -> str:
     """Uploads data to local storage, used for quick-start/demo purposes"""
     file_key: str = _construct_file_key(privacy_request.id, config)
-    return upload_to_local(data, file_key, privacy_request, config.format.value, data_category_field_mapping, data_use_map)  # type: ignore
+    return upload_to_local(data, file_key, privacy_request, config.format.value)  # type: ignore