ethyca-fides 2.56.3b1__py2.py3-none-any.whl → 2.56.3b2__py2.py3-none-any.whl

fides/api/models/attachment.py

@@ -1,6 +1,8 @@
+import os
 from enum import Enum as EnumType
 from typing import Any, Optional
 
+from loguru import logger as log
 from sqlalchemy import Column
 from sqlalchemy import Enum as EnumColumn
 from sqlalchemy import ForeignKey, String, UniqueConstraint
@@ -9,6 +11,15 @@ from sqlalchemy.orm import Session, relationship
 
 from fides.api.db.base_class import Base
 from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+from fides.api.models.storage import StorageConfig  # pylint: disable=unused-import
+from fides.api.schemas.storage.storage import StorageDetails, StorageType
+from fides.api.tasks.storage import (
+    LOCAL_FIDES_UPLOAD_DIRECTORY,
+    generic_delete_from_s3,
+    generic_retrieve_from_s3,
+    get_local_filename,
+    upload_to_s3,
+)
 
 
 class AttachmentType(str, EnumType):
@@ -74,7 +85,9 @@ class Attachment(Base):
     )
     file_name = Column(String, nullable=False)
     attachment_type = Column(EnumColumn(AttachmentType), nullable=False)
-    storage_key = Column(String, nullable=False)
+    storage_key = Column(
+        String, ForeignKey("storageconfig.key", ondelete="CASCADE"), nullable=False
+    )
 
     user = relationship(
         "FidesUser",
@@ -90,64 +103,111 @@ class Attachment(Base):
         uselist=True,
     )
 
-    async def upload_attachment_to_s3(self, attachment: bytes) -> None:
-        """Upload an attachment to S3 to the storage_url."""
-        raise NotImplementedError("This method is not yet implemented")
-        # AuditLog.create(
-        #     db=db,
-        #     data={
-        #         "user_id": "system",
-        #         "privacy_request_id": privacy_request.id,
-        #         "action": AuditLogAction.attachment_uploaded,
-        #         "message": "",
-        #     },
-        # )
-
-    async def retrieve_attachment_from_s3(self) -> bytes:
-        """Retrieve an attachment from S3."""
-        raise NotImplementedError("This method is not yet implemented")
-        # AuditLog.create(
-        #     db=db,
-        #     data={
-        #         "user_id": "system",
-        #         "privacy_request_id": privacy_request.id,
-        #         "action": AuditLogAction.attachment_retrieved,
-        #         "message": "",
-        #     },
-        # )
-
-    async def delete_attachment_from_s3(self) -> None:
-        """Delete an attachment from S3."""
-        raise NotImplementedError("This method is not yet implemented")
-        # AuditLog.create(
-        #     db=db,
-        #     data={
-        #         "user_id": "system",
-        #         "privacy_request_id": privacy_request.id,
-        #         "action": AuditLogAction.attachment_deleted,
-        #         "message": "",
-        #     },
-        # )
+    config = relationship(
+        "StorageConfig",
+        lazy="selectin",
+        uselist=False,
+    )
+
+    def upload(self, attachment: bytes) -> None:
+        """Uploads an attachment to S3 or local storage."""
+        if self.config.type == StorageType.s3:
+            bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
+            auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
+            upload_to_s3(
+                storage_secrets=self.config.secrets,
+                data={},
+                bucket_name=bucket_name,
+                file_key=self.id,
+                resp_format=self.config.format,
+                privacy_request=None,
+                document=attachment,
+                auth_method=auth_method,
+            )
+            log.info(f"Uploaded {self.file_name} to S3 bucket {bucket_name}/{self.id}")
+            return
+
+        if self.config.type == StorageType.local:
+            filename = get_local_filename(self.id)
+            with open(filename, "wb") as file:
+                file.write(attachment)
+            return
+
+        raise ValueError(f"Unsupported storage type: {self.config.type}")
+
+    def retrieve_attachment(self) -> Optional[bytes]:
+        """Returns the attachment from S3 in bytes form."""
+        if self.config.type == StorageType.s3:
+            bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
+            auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
+            return generic_retrieve_from_s3(
+                storage_secrets=self.config.secrets,
+                bucket_name=bucket_name,
+                file_key=self.id,
+                auth_method=auth_method,
+            )
+
+        if self.config.type == StorageType.local:
+            filename = f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{self.id}"
+            with open(filename, "rb") as file:
+                return file.read()
+
+        raise ValueError(f"Unsupported storage type: {self.config.type}")
+
+    def delete_attachment_from_storage(self) -> None:
+        """Deletes an attachment from S3 or local storage."""
+        if self.config.type == StorageType.s3:
+            bucket_name = f"{self.config.details[StorageDetails.BUCKET.value]}"
+            auth_method = self.config.details[StorageDetails.AUTH_METHOD.value]
+            generic_delete_from_s3(
+                storage_secrets=self.config.secrets,
+                bucket_name=bucket_name,
+                file_key=self.id,
+                auth_method=auth_method,
+            )
+            return
+
+        if self.config.type == StorageType.local:
+            filename = f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{self.id}"
+            os.remove(filename)
+            return
+
+        raise ValueError(f"Unsupported storage type: {self.config.type}")
 
     @classmethod
-    def create(
+    def create_and_upload(
         cls,
         db: Session,
         *,
         data: dict[str, Any],
+        attachment_file: bytes,
         check_name: bool = False,
-        attachment: Optional[
-            bytes
-        ] = None,  # This will not be optional once the upload method is implemented.
     ) -> "Attachment":
         """Creates a new attachment record in the database and uploads the attachment to S3."""
-        # attachment_record.upload_attachment_to_s3(db, attachment)
-        # log.info(f"Uploaded attachment {attachment_record.id} to S3")
-        return super().create(db=db, data=data, check_name=check_name)
+        if attachment_file is None:
+            raise ValueError("Attachment is required")
+        attachment_model = super().create(db=db, data=data, check_name=check_name)
+
+        try:
+            attachment_model.upload(attachment_file)
+            return attachment_model
+        except Exception as e:
+            log.error(f"Failed to upload attachment: {e}")
+            attachment_model.delete(db)
+            raise e
+
+    @classmethod
+    def create(
+        cls,
+        db: Session,
+        *,
+        data: dict[str, Any],
+        check_name: bool = False,
+    ) -> "Attachment":
+        """Raises Error, provides information for user to create with upload instead."""
+        raise NotImplementedError("Please use create_and_upload method for Attachment")
 
     def delete(self, db: Session) -> None:
         """Deletes an attachment record from the database and deletes the attachment from S3."""
-        # attachment_record = cls.get(db, id)
-        # attachment_record.delete_attachment_from_s3(db)
-        # log.info(f"Deleted attachment {attachment_record.id} from S3")
+        self.delete_attachment_from_storage()
         super().delete(db=db)

fides/api/models/comment.py

@@ -0,0 +1,109 @@
+from enum import Enum as EnumType
+from typing import Any
+
+from sqlalchemy import Column
+from sqlalchemy import Enum as EnumColumn
+from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import Session, relationship
+
+from fides.api.db.base_class import Base
+from fides.api.models.attachment import Attachment, AttachmentReference
+from fides.api.models.fides_user import FidesUser  # pylint: disable=unused-import
+
+
+class CommentType(str, EnumType):
+    """
+    Enum for comment types. Indicates comment usage.
+
+    - notes are internal comments.
+    - reply comments are public and may cause an email or other communciation to be sent
+    """
+
+    note = "note"
+    reply = "reply"
+
+
+class CommentReferenceType(str, EnumType):
+    """
+    Enum for comment reference types. Indicates where the comment is referenced.
+    """
+
+    manual_step = "manual_step"
+    privacy_request = "privacy_request"
+
+
+class CommentReference(Base):
+    """
+    Stores information about a comment and any other element which may reference that comment.
+    """
+
+    @declared_attr
+    def __tablename__(cls) -> str:
+        """Overriding base class method to set the table name."""
+        return "comment_reference"
+
+    comment_id = Column(String, ForeignKey("comment.id"), nullable=False)
+    reference_id = Column(String, nullable=False)
+    reference_type = Column(EnumColumn(CommentReferenceType), nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint("comment_id", "reference_id", name="comment_reference_uc"),
+    )
+
+    comment = relationship(
+        "Comment",
+        back_populates="references",
+        uselist=False,
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: dict[str, Any], check_name: bool = False
+    ) -> "CommentReference":
+        """Creates a new comment reference record in the database."""
+        return super().create(db=db, data=data, check_name=check_name)
+
+
+class Comment(Base):
+    """
+    Stores information about a Comment.
+    """
+
+    user_id = Column(
+        String, ForeignKey("fidesuser.id", ondelete="SET NULL"), nullable=True
+    )
+    comment_text = Column(String, nullable=False)
+    comment_type = Column(EnumColumn(CommentType), nullable=False)
+
+    user = relationship(
+        "FidesUser",
+        backref="comments",
+        lazy="selectin",
+        uselist=False,
+    )
+
+    references = relationship(
+        "CommentReference",
+        back_populates="comment",
+        cascade="all, delete",
+        uselist=True,
+    )
+
+    def get_attachments(self, db: Session) -> list[Attachment]:
+        """Retrieve all attachments associated with this comment."""
+        stmt = (
+            db.query(Attachment)
+            .join(
+                AttachmentReference, Attachment.id == AttachmentReference.attachment_id
+            )
+            .where(AttachmentReference.reference_id == self.id)
+        )
+        return db.execute(stmt).scalars().all()
+
+    def delete(self, db: Session) -> None:
+        """Delete the comment and all associated references."""
+        attachments = self.get_attachments(db)
+        for attachment in attachments:
+            attachment.delete(db)
+        db.delete(self)

fides/api/service/connectors/query_configs/saas_query_config.py

@@ -1,6 +1,7 @@
 # pylint: disable=too-many-instance-attributes
 from __future__ import annotations
 
+from copy import deepcopy
 from datetime import datetime
 from itertools import product
 from typing import Any, Dict, List, Literal, Optional, Tuple, TypeVar
@@ -12,7 +13,6 @@ from loguru import logger
 from sqlalchemy.orm import Session
 
 from fides.api.common_exceptions import FidesopsException
-from fides.api.graph.config import ScalarField
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_request import (
@@ -33,7 +33,12 @@ from fides.api.task.refine_target_path import (
     join_detailed_path,
 )
 from fides.api.util import saas_util
-from fides.api.util.collection_util import Row, merge_dicts
+from fides.api.util.collection_util import (
+    Row,
+    flatten_dict,
+    merge_dicts,
+    unflatten_dict,
+)
 from fides.api.util.saas_util import (
     ALL_OBJECT_FIELDS,
     CUSTOM_PRIVACY_REQUEST_FIELDS,
@@ -45,7 +50,6 @@ from fides.api.util.saas_util import (
     REPLY_TO_TOKEN,
     UUID,
     get_identities,
-    unflatten_dict,
 )
 from fides.common.api.v1.urn_registry import REQUEST_TASK_CALLBACK, V1_URL_PREFIX
 from fides.config.config_proxy import ConfigProxy
@@ -510,22 +514,24 @@ class SaaSQueryConfig(QueryConfig[SaaSRequestParams]):
 
     def all_value_map(self, row: Row) -> Dict[str, Any]:
         """
-        Takes a row and preserves only the fields that are defined in the Dataset.
+        Takes a row and preserves only the fields that are defined in the collection.
         Used for scenarios when an update endpoint has required fields other than
         just the fields being updated.
         """
+        flattened_row = flatten_dict(deepcopy(row))
 
-        all_value_map: Dict[str, Any] = {}
-        for field_path, field in self.field_map().items():
-            # only map scalar fields
-            if (
-                isinstance(field, ScalarField)
-                and pydash.get(row, field_path.string_path) is not None
-            ):
-                all_value_map[field_path.string_path] = pydash.get(
-                    row, field_path.string_path
-                )
-        return all_value_map
+        # Get root field names defined in the collection
+        collection_fields = {
+            field_path.string_path.split(".")[0]
+            for field_path, _ in self.field_map().items()
+        }
+
+        # Only keep the field values defined in the collection
+        return {
+            path: value
+            for path, value in flattened_row.items()
+            if path.split(".")[0] in collection_fields
+        }
 
     def query_to_str(self, t: T, input_data: Dict[str, List[Any]]) -> str:
         """Convert query to string"""

fides/api/service/storage/storage_uploader_service.py

@@ -41,9 +41,7 @@ def upload(
         logger.warning("Storage type not found: {}", storage_key)
         raise StorageUploadError(f"Storage type not found: {storage_key}")
     uploader: Any = _get_uploader_from_config_type(config.type)  # type: ignore
-    return uploader(
-        db, config, data, privacy_request, data_category_field_mapping, data_use_map
-    )
+    return uploader(db, config, data, privacy_request)
 
 
 def get_extension(resp_format: ResponseFormat) -> str:
@@ -88,14 +86,13 @@ def _s3_uploader(
     config: StorageConfig,
     data: Dict,
     privacy_request: PrivacyRequest,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
 ) -> str:
     """Constructs necessary info needed for s3 before calling upload"""
     file_key: str = _construct_file_key(privacy_request.id, config)
 
     bucket_name = config.details[StorageDetails.BUCKET.value]
     auth_method = config.details[StorageDetails.AUTH_METHOD.value]
+    document = None
 
     return upload_to_s3(
         config.secrets,  # type: ignore
@@ -104,9 +101,8 @@ def _s3_uploader(
         file_key,
         config.format.value,  # type: ignore
         privacy_request,
+        document,
         auth_method,
-        data_category_field_mapping,
-        data_use_map,
     )
 
 
@@ -115,9 +111,7 @@ def _local_uploader(
     config: StorageConfig,
     data: Dict,
     privacy_request: PrivacyRequest,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
 ) -> str:
     """Uploads data to local storage, used for quick-start/demo purposes"""
     file_key: str = _construct_file_key(privacy_request.id, config)
-    return upload_to_local(data, file_key, privacy_request, config.format.value, data_category_field_mapping, data_use_map)  # type: ignore
+    return upload_to_local(data, file_key, privacy_request, config.format.value)  # type: ignore

fides/api/tasks/storage.py

@@ -5,20 +5,20 @@ import os
 import secrets
 import zipfile
 from io import BytesIO
-from typing import Any, Dict, Optional, Set, Union
+from typing import Any, Dict, Optional, Union
 
 import pandas as pd
 from botocore.exceptions import ClientError, ParamValidationError
+from fideslang.validation import AnyHttpUrlString
 from loguru import logger
 
 from fides.api.cryptography.cryptographic_util import bytes_to_b64_str
-from fides.api.graph.graph import DataCategoryFieldMapping
 from fides.api.models.privacy_request import PrivacyRequest
 from fides.api.schemas.storage.storage import ResponseFormat, StorageSecrets
 from fides.api.service.privacy_request.dsr_package.dsr_report_builder import (
     DsrReportBuilder,
 )
-from fides.api.util.aws_util import get_aws_session
+from fides.api.util.aws_util import get_s3_client
 from fides.api.util.cache import get_cache, get_encryption_cache_key
 from fides.api.util.encryption.aes_gcm_encryption_scheme import (
     encrypt_to_bytes_verify_secrets_length,
@@ -101,7 +101,9 @@ def write_to_in_memory_buffer(
     raise NotImplementedError(f"No handling for response format {resp_format}.")
 
 
-def create_presigned_url_for_s3(s3_client: Any, bucket_name: str, file_key: str) -> str:
+def create_presigned_url_for_s3(
+    s3_client: Any, bucket_name: str, file_key: str
+) -> AnyHttpUrlString:
     """ "Generate a presigned URL to share an S3 object
 
     :param s3_client: s3 base client
@@ -119,23 +121,108 @@ def create_presigned_url_for_s3(s3_client: Any, bucket_name: str, file_key: str)
     return response
 
 
+def generic_upload_to_s3(  # pylint: disable=R0913
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+    document: bytes,
+) -> Optional[AnyHttpUrlString]:
+    """Uploads arbitrary data to s3 returned from an access request"""
+    logger.info("Starting S3 Upload of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            s3_client.put_object(Bucket=bucket_name, Key=file_key, Body=document)
+        except Exception as e:
+            logger.error("Encountered error while uploading s3 object: {}", e)
+            raise e
+
+        presigned_url: AnyHttpUrlString = create_presigned_url_for_s3(
+            s3_client, bucket_name, file_key
+        )
+
+        return presigned_url
+    except ClientError as e:
+        logger.error(
+            "Encountered error while uploading and generating link for s3 object: {}", e
+        )
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
+def generic_retrieve_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> Optional[bytes]:
+    """Retrieves arbitrary data from s3"""
+    logger.info("Starting S3 Retrieve of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
+            return response["Body"].read()
+        except Exception as e:
+            logger.error("Encountered error while retrieving s3 object: {}", e)
+            raise e
+    except ClientError as e:
+        logger.error("Encountered error while retrieving s3 object: {}", e)
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
+def generic_delete_from_s3(
+    storage_secrets: Dict[StorageSecrets, Any],
+    bucket_name: str,
+    file_key: str,
+    auth_method: str,
+) -> None:
+    """Deletes arbitrary data from s3"""
+    logger.info("Starting S3 Delete of {}", file_key)
+
+    try:
+        s3_client = get_s3_client(auth_method, storage_secrets)
+        try:
+            s3_client.delete_object(Bucket=bucket_name, Key=file_key)
+        except Exception as e:
+            logger.error("Encountered error while deleting s3 object: {}", e)
+            raise e
+    except ClientError as e:
+        logger.error("Encountered error while deleting s3 object: {}", e)
+        raise e
+    except ParamValidationError as e:
+        raise ValueError(f"The parameters you provided are incorrect: {e}")
+
+
 def upload_to_s3(  # pylint: disable=R0913
     storage_secrets: Dict[StorageSecrets, Any],
     data: Dict,
     bucket_name: str,
     file_key: str,
     resp_format: str,
-    privacy_request: PrivacyRequest,
+    privacy_request: Optional[PrivacyRequest],
+    document: Optional[bytes],
     auth_method: str,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
-) -> str:
+) -> Optional[AnyHttpUrlString]:
     """Uploads arbitrary data to s3 returned from an access request"""
     logger.info("Starting S3 Upload of {}", file_key)
 
+    if privacy_request is None and document is not None:
+        return generic_upload_to_s3(
+            storage_secrets, bucket_name, file_key, auth_method, document
+        )
+
+    if privacy_request is None:
+        raise ValueError("Privacy request must be provided")
+
     try:
-        my_session = get_aws_session(auth_method, storage_secrets)
-        s3_client = my_session.client("s3")
+        s3_client = get_s3_client(auth_method, storage_secrets)
 
         # handles file chunking
         try:
@@ -148,7 +235,7 @@ def upload_to_s3(  # pylint: disable=R0913
             logger.error("Encountered error while uploading s3 object: {}", e)
             raise e
 
-        presigned_url: str = create_presigned_url_for_s3(
+        presigned_url: AnyHttpUrlString = create_presigned_url_for_s3(
             s3_client, bucket_name, file_key
         )
 
@@ -162,17 +249,21 @@ def upload_to_s3(  # pylint: disable=R0913
         raise ValueError(f"The parameters you provided are incorrect: {e}")
 
 
+def get_local_filename(file_key: str) -> str:
+    """Verifies that the local storage directory exists"""
+    if not os.path.exists(LOCAL_FIDES_UPLOAD_DIRECTORY):
+        os.makedirs(LOCAL_FIDES_UPLOAD_DIRECTORY)
+    return f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{file_key}"
+
+
 def upload_to_local(
     data: Dict,
     file_key: str,
     privacy_request: PrivacyRequest,
     resp_format: str = ResponseFormat.json.value,
-    data_category_field_mapping: Optional[DataCategoryFieldMapping] = None,
-    data_use_map: Optional[Dict[str, Set[str]]] = None,
 ) -> str:
     """Uploads access request data to a local folder - for testing/demo purposes only"""
-    if not os.path.exists(LOCAL_FIDES_UPLOAD_DIRECTORY):
-        os.makedirs(LOCAL_FIDES_UPLOAD_DIRECTORY)
+    get_local_filename(file_key)
 
     filename = f"{LOCAL_FIDES_UPLOAD_DIRECTORY}/{file_key}"
     in_memory_file = write_to_in_memory_buffer(resp_format, data, privacy_request)

fides/api/util/aws_util.py

@@ -70,3 +70,22 @@ def get_aws_session(
             raise
     else:
         return session
+
+
+def get_s3_client(
+    auth_method: str,
+    storage_secrets: Optional[Dict[StorageSecrets, Any]],
+    assume_role_arn: Optional[str] = None,
+) -> Session:
+    """
+    Abstraction to retrieve an AWS S3 client using secrets.
+
+    If an `assume_role_arn` is provided, the secrets will be used to
+    assume that role and return a Session instantiated with that role.
+    """
+    session = get_aws_session(
+        auth_method=auth_method,
+        storage_secrets=storage_secrets,
+        assume_role_arn=assume_role_arn,
+    )
+    return session.client("s3")