ethyca-fides 2.68.1b3__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/METADATA +1 -1
- {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/RECORD +249 -243
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/b1a2c3d4e5f6_add_location_to_privacy_request.py +26 -0
- fides/api/api/v1/api.py +2 -0
- fides/api/api/v1/endpoints/dsr_package_link.py +169 -0
- fides/api/api/v1/endpoints/oauth_endpoints.py +1 -0
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +32 -1
- fides/api/api/v1/endpoints/user_endpoints.py +4 -0
- fides/api/models/privacy_request/privacy_request.py +1 -0
- fides/api/models/privacy_request/webhook.py +33 -1
- fides/api/oauth/utils.py +118 -50
- fides/api/schemas/application_config.py +7 -0
- fides/api/schemas/connection_configuration/connection_type_system_map.py +6 -0
- fides/api/schemas/enums/__init__.py +0 -0
- fides/api/schemas/enums/connection_category.py +20 -0
- fides/api/schemas/enums/integration_feature.py +26 -0
- fides/api/schemas/external_https.py +9 -0
- fides/api/schemas/privacy_request.py +16 -0
- fides/api/schemas/saas/connector_template.py +5 -0
- fides/api/schemas/saas/display_info.py +19 -0
- fides/api/schemas/saas/saas_config.py +2 -0
- fides/api/schemas/storage/storage.py +2 -0
- fides/api/service/connectors/saas/connector_registry_service.py +7 -0
- fides/api/service/privacy_request/dsr_package/dsr_report_builder.py +253 -71
- fides/api/service/privacy_request/dsr_package/templates/attachments_index.html +4 -2
- fides/api/service/privacy_request/dsr_package/templates/collection_index.html +3 -1
- fides/api/service/privacy_request/dsr_package/templates/dataset_index.html +1 -1
- fides/api/service/privacy_request/request_runner_service.py +52 -6
- fides/api/service/privacy_request/request_service.py +18 -1
- fides/api/service/storage/storage_uploader_service.py +1 -21
- fides/api/service/storage/streaming/dsr_storage.py +1 -5
- fides/api/service/storage/streaming/s3/s3_storage_client.py +78 -40
- fides/api/service/storage/streaming/s3/streaming_s3.py +9 -21
- fides/api/service/storage/streaming/smart_open_client.py +8 -7
- fides/api/service/storage/streaming/smart_open_streaming_storage.py +109 -196
- fides/api/service/storage/streaming/storage_client_factory.py +7 -3
- fides/api/service/storage/util.py +579 -0
- fides/api/task/manual/manual_task_graph_task.py +11 -9
- fides/api/util/connection_type.py +3 -0
- fides/api/util/saas_util.py +12 -1
- fides/api/util/text.py +51 -0
- fides/common/api/v1/urn_registry.py +3 -0
- fides/service/privacy_request/privacy_request_service.py +83 -0
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/Pc_eOxj5LbY3XOShbrjSX/_buildManifest.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1099-79646e64f26d62fa.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{1345-ab756811e19ff4fc.js → 1345-5e1c5b66e25c566e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{1817-fd21f1f5ef0faffa.js → 1817-3d9e110e007853f0.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{1975.16126463309143e3.js → 1975.78e719130cfe3fd6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{2921-0e5cc63a82e31830.js → 2921-52328140bc420d0f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3620-6cceae71bae5b531.js → 3620-31ebb43dba84cbbd.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/3729-a1ca1608efc11ac4.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3847-230bf61b053bc706.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3855-64541570e2f838fb.js → 3855-ef5194cdb228beb6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3872-7a18d18a5e287e4e.js → 3872-a91143aa35fa8ef8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3923-5c87b3d7f1626678.js → 3923-bb2417b8dcade7a4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{401-3902e3e98790d401.js → 401-4af0a912e249d30f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4121-64ef70ef906bbdd0.js → 4121-c8d5d717e31899e1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4164-355644b916ae0094.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4608-70521532195124de.js → 4608-23bbd4c3c4a59f42.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4786-0827aae7aceadd22.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4808-78ca630f2d2503cd.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4844-351f99b6644b654e.js → 4844-46324c3d848b8b6a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{5258-c6f96dc740eb5fb1.js → 5258-b0de22a8521686ab.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{5487-338800277d36b8d7.js → 5487-8c635883dcaa9c2a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/549-38ea1d91ee2addaa.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6084-da63f20d9416a982.js → 6084-0096d7de64ef8015.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{6853-1d947b75eb07188c.js → 6853-b17673391117c976.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{6954-24f9a4f27d67b732.js → 6954-9d46e2276c461c26.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{7476-a0dd03bfccf60d0c.js → 7476-d1b0af9ade392e5b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{7630-9fbe06cfb98266fe.js → 7630-da0a7ce4e3a0d62c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{787-3dd31844cf7fec55.js → 787-3499983fa346b380.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{79-dcd20e8b09501c17.js → 79-f197fc4db8d530e5.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{796-8773e04b64ce2260.js → 796-db1e30119ea973c7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/8002-971e29181f72edd1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9046-57eab238570b8bf4.js → 9046-712156d461165f56.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9676.bf0a8a6ff6dfd2af.js → 9676.9fd9552ef744c717.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9826-756c958aecab59a2.js → 9826-b0b3d3cfb13bfbc1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9951-cdf73904a3adb27b.js → 9951-a88367a129b724ba.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{404-dd625a559ada46ca.js → 404-471a6b18e712f050.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-b6b09b2878b77b21.js → _app-ef8e1c986bc5b795.js} +12 -12
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-92cf5e313be1f9e2.js → manual-9dc7e70ab5b05723.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-d6c525ee731a2993.js → multiple-4b79a1652297ed9a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-5664a3ea796e5ffb.js → add-systems-1632a59203fe8eab.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-78f13de90111fd80.js → add-vendors-1ca9df7ca91bd101.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-0fc678f3d6d2fcec.js → configure-07bdbc9ae4137db4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-126db59dc25ca326.js → [id]-f80cf2d3966816fd.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-289605267d6cce7e.js → privacy-experience-2795cd4115a77c94.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-e9fd9b28ac9705af.js → [id]-e02921dc82dccbb1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-28c003b6043bd16c.js → new-98f9e4ba3610628a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-c643eff04525298e.js → privacy-notices-17ed82777810d1c6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-3ef5d01779a26455.js → properties-226efa1dcd41437f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-28b192e2c074b0f3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{consent-8d4be9e7ec7d2a35.js → consent-09610b10923d9268.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-f27ec4578c674181.js → [resourceUrn]-da1a48336daff6f8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-27b6c255bd9e73b6.js → [projectUrn]-d8e776f1e64e4ba8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-0f66dac32040519c.js → projects-75b9629b0d9cdf96.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-3b938562df81c4b0.js → [resourceUrn]-470da05db63767cd.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-6984c033b8fe3a13.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-2f0a33ef9ba1f1da.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-e9d4f25b20ff6781.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-9c428d3ef0985915.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-21c141279e66237a.js → activity-b6ae7adb8ef0b525.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-3bc6a207693fd175.js → [resourceUrn]-c3a97e6721ca0abe.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-da16e73df395ad1d.js → detection-a0a7de552ef71f5b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-04b242632a114405.js → [resourceUrn]-109754fec0755339.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-900fe50183a40d72.js → discovery-88654783b06b3b21.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{datamap-4f1f7c3a9531a8f4.js → datamap-89136e6800dc9369.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-343294dcb10d9532.js → [...subfieldNames]-8f58192dcb54883d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-1c097a0809fa5b6f.js → [collectionName]-dcb4ab380a77aa1e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-b47fa2498b534719.js → [datasetId]-6f16d43071fb9c11.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-a31f881cab25704a.js → new-97f06e21580f1f6a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-858c59c9e67e318d.js → dataset-674bb3940f088ecc.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-16c28d272225afb6.js → [id]-6f77d8647fca71e0.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/new-821dd1269834cfa2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-23e4caf79faa8106.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{index-fec557d99211f577.js → index-23eb64eed81dcb69.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-3a4cd3fe9094fba3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-57e618d7b16ac69a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-4a08ca7762a19700.js → [id]-c9a323eb6a929476.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-343a965dcdb3d11e.js → add-template-b9bb09e46921a590.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{messaging-3ade4c54b1c8a11e.js → messaging-82c631a12b5a008c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-9103bfb854f71410.js → ant-components-bc0e2adf6e0d3ac7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-3b97029bd4d3c3ea.js → AntForm-86ffcc1ad3fa912e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-9d9beb8f0d8a278c.js → FormikAntFormItem-ec04f595465bdf69.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-84a4d8fc60f839ed.js → FormikControlled-41d309754ff0c1de.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-1fccf542ab2e33bf.js → FormikField-cab1f78cec7808f9.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-aa75263ae1ba67bb.js → forms-eb6058221403b156.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-db334a1cbb102255.js → table-migration-38360083348c3d6c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{[id]-11c1e4545c8f528c.js → [id]-0d0bb9eb004a3336.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{messaging-192a986f61c23268.js → messaging-f9320a58f489f5b7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{storage-9216ac993d71387e.js → storage-d0cfa8aeddd43a40.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-e55ec84d5380401d.js → configure-72ca94ec5ed85733.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-5a5edc8a4aa7c30a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-a74b51b704b80cb2.js → [id]-5ec775c4904fdbfe.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-8d23f0c55ff6510a.js → add-property-a6812c0916f2949e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{properties-77acceac4f99e7af.js → properties-da734840e4f9d04b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-6903f42a0412bfa6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-6aad3f563ed03b3f.js → alpha-3e72e9f91991c119.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-c1b8f3606d160bb1.js → about-6aab092f4871cecb.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-d9f7f78810d58d08.js → [purpose_id]-9495e2eb506606c7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-ee2c7dde99b1dafb.js → consent-be47008304106395.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-ae1b57589da7b175.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-31c270d228e00581.js → domain-records-23a6d7a921150188.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-996b3f250dd3ea1f.js → domains-2a9e8859ab4d9de6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-ee94981326ddcbf4.js → email-templates-4f9f0fdf9925ae90.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-0b831c58966782b8.js → locations-46f7af35cee4a8bb.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-94271ba4a224a353.js → organization-a596a96cb8d0aa8e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-41b8136e50320fd3.js → regulations-6ed5fc2410e00857.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-52b45569cbc82e60.js → test-datasets-86811e3cda277e77.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-36d74e93e54aabaf.js → [id]-5a43f108d8047d5b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{systems-24dfc8e2279ced2e.js → systems-045a841e22e85ea8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{taxonomy-d9675cf5e6083b27.js → taxonomy-1b3f2d4bcb0e164d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/{[id]-866826d7959df487.js → [id]-05d61c80a556b2d5.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-e63b61a8f99ccd57.js → user-management-2cab41659f1ee7da.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{webpack-6d0a487039bcf30c.js → webpack-678e89d68dbcd94f.js} +1 -1
- fides/ui-build/static/admin/_next/static/css/650df9c348000a26.css +1 -0
- fides/ui-build/static/admin/_next/static/css/{dbcf63488933a4d5.css → 98fab0b3e6aa43ed.css} +1 -1
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +2 -2
- fides/ui-build/static/admin/lib/fides.js +2 -2
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/_BLI2ArqQzY5XnXbrcxa2/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1099-7b2085a3931da9e4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1138-0d846ffef62c580f.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3729-7d2d52400f1f7413.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4786-53ef1662f2d0d98c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4808-8713433c84a62efe.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/549-e6453a3526023e85.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/602-80d113e801d7407d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/8002-dcd02da6e5649a1c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-baa4a2f8f08ac224.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-ebf5e7fa4e2ffb49.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-b27c660039d951c9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-8ce5d24af470888e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-8e35e33928abbcdc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/new-68f502d8b0b5792c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-1eb9acb17b133fd1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-e613543818d6cbd2.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-8069f7c33695fd45.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-48f447b31c786b80.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-e60d398e255f4e00.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-a4dad8ca9de2d07b.js +0 -1
- fides/ui-build/static/admin/_next/static/css/92441453b27e9c34.css +0 -1
- {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{_BLI2ArqQzY5XnXbrcxa2 → Pc_eOxj5LbY3XOShbrjSX}/_ssgManifest.js +0 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class IntegrationFeature(str, Enum):
|
|
5
|
+
"""
|
|
6
|
+
Features that can be enabled for different integration types.
|
|
7
|
+
These control which tabs and functionality are available in the integration detail view.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
# Enables data discovery and monitoring functionality - shows "Data discovery" tab
|
|
11
|
+
DATA_DISCOVERY = "DATA_DISCOVERY"
|
|
12
|
+
|
|
13
|
+
# Enables data synchronization to external systems - shows "Data sync" tab
|
|
14
|
+
DATA_SYNC = "DATA_SYNC"
|
|
15
|
+
|
|
16
|
+
# Enables task/workflow management for manual processes - shows "Tasks" tab
|
|
17
|
+
TASKS = "TASKS"
|
|
18
|
+
|
|
19
|
+
# Indicates integration doesn't require connection testing - shows "Details" tab instead of "Connection" tab
|
|
20
|
+
WITHOUT_CONNECTION = "WITHOUT_CONNECTION"
|
|
21
|
+
|
|
22
|
+
# Enables Data Subject Request automation for SAAS integrations
|
|
23
|
+
DSR_AUTOMATION = "DSR_AUTOMATION"
|
|
24
|
+
|
|
25
|
+
# Enables conditions configuration for manual task creation - shows "Conditions" tab
|
|
26
|
+
CONDITIONS = "CONDITIONS"
|
|
@@ -39,3 +39,12 @@ class RequestTaskJWE(BaseModel):
|
|
|
39
39
|
request_task_id: str
|
|
40
40
|
scopes: List[str]
|
|
41
41
|
iat: str
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class DownloadTokenJWE(BaseModel):
|
|
45
|
+
"""Describes JWE that is given to users to access their privacy request download package"""
|
|
46
|
+
|
|
47
|
+
privacy_request_id: str
|
|
48
|
+
scopes: List[str]
|
|
49
|
+
iat: str
|
|
50
|
+
exp: str
|
|
@@ -18,6 +18,7 @@ from fides.api.schemas.user import PrivacyRequestReviewer
|
|
|
18
18
|
from fides.api.util.collection_util import Row
|
|
19
19
|
from fides.api.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key
|
|
20
20
|
from fides.api.util.enums import ColumnSort
|
|
21
|
+
from fides.api.util.text import normalize_location_code
|
|
21
22
|
from fides.config import CONFIG
|
|
22
23
|
|
|
23
24
|
|
|
@@ -103,6 +104,7 @@ class PrivacyRequestCreate(FidesSchema):
|
|
|
103
104
|
property_id: Optional[str] = None
|
|
104
105
|
consent_preferences: Optional[List[Consent]] = None # TODO Slated for deprecation
|
|
105
106
|
source: Optional[PrivacyRequestSource] = None
|
|
107
|
+
location: Optional[str] = None
|
|
106
108
|
|
|
107
109
|
@field_validator("encryption_key")
|
|
108
110
|
@classmethod
|
|
@@ -114,6 +116,18 @@ class PrivacyRequestCreate(FidesSchema):
|
|
|
114
116
|
verify_encryption_key(value.encode(CONFIG.security.encoding))
|
|
115
117
|
return value
|
|
116
118
|
|
|
119
|
+
@field_validator("location")
|
|
120
|
+
@classmethod
|
|
121
|
+
def validate_location(
|
|
122
|
+
cls: Type["PrivacyRequestCreate"], value: Optional[str] = None
|
|
123
|
+
) -> Optional[str]:
|
|
124
|
+
"""Validate and normalize location to ISO 3166 format"""
|
|
125
|
+
if value is None:
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
# nuance here is the validator will coalesce values from the less strict format to ISO 3166 (i.e. "us_ca" -> "US-CA")
|
|
129
|
+
return normalize_location_code(value)
|
|
130
|
+
|
|
117
131
|
|
|
118
132
|
class PrivacyRequestResubmit(PrivacyRequestCreate):
|
|
119
133
|
"""Schema used to copy a privacy request for resubmission"""
|
|
@@ -321,6 +335,7 @@ class PrivacyRequestResponse(FidesSchema):
|
|
|
321
335
|
custom_privacy_request_fields_approved_by: Optional[str] = None
|
|
322
336
|
custom_privacy_request_fields_approved_at: Optional[datetime] = None
|
|
323
337
|
source: Optional[PrivacyRequestSource] = None
|
|
338
|
+
location: Optional[str] = None
|
|
324
339
|
deleted_at: Optional[datetime] = None
|
|
325
340
|
deleted_by: Optional[str] = None
|
|
326
341
|
finalized_at: Optional[datetime] = None
|
|
@@ -437,6 +452,7 @@ class PrivacyRequestFilter(FidesSchema):
|
|
|
437
452
|
errored_lt: Optional[datetime] = None
|
|
438
453
|
errored_gt: Optional[datetime] = None
|
|
439
454
|
external_id: Optional[str] = None
|
|
455
|
+
location: Optional[str] = None
|
|
440
456
|
action_type: Optional[ActionType] = None
|
|
441
457
|
verbose: Optional[bool] = False
|
|
442
458
|
include_identities: Optional[bool] = False
|
|
@@ -4,6 +4,8 @@ from fideslang.models import Dataset
|
|
|
4
4
|
from pydantic import BaseModel, field_validator
|
|
5
5
|
|
|
6
6
|
from fides.api.models.datasetconfig import validate_masking_strategy_override
|
|
7
|
+
from fides.api.schemas.enums.connection_category import ConnectionCategory
|
|
8
|
+
from fides.api.schemas.enums.integration_feature import IntegrationFeature
|
|
7
9
|
from fides.api.schemas.policy import ActionType
|
|
8
10
|
from fides.api.schemas.saas.saas_config import SaaSConfig
|
|
9
11
|
from fides.api.util.saas_util import load_config_from_string, load_dataset_from_string
|
|
@@ -22,6 +24,9 @@ class ConnectorTemplate(BaseModel):
|
|
|
22
24
|
authorization_required: bool
|
|
23
25
|
user_guide: Optional[str] = None
|
|
24
26
|
supported_actions: List[ActionType]
|
|
27
|
+
category: Optional[ConnectionCategory] = None
|
|
28
|
+
tags: Optional[List[str]] = None
|
|
29
|
+
enabled_features: Optional[List[IntegrationFeature]] = None
|
|
25
30
|
|
|
26
31
|
@field_validator("config")
|
|
27
32
|
@classmethod
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from typing import List, Optional
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, ConfigDict
|
|
4
|
+
|
|
5
|
+
from fides.api.schemas.enums.connection_category import ConnectionCategory
|
|
6
|
+
from fides.api.schemas.enums.integration_feature import IntegrationFeature
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class SaaSDisplayInfo(BaseModel):
|
|
10
|
+
"""
|
|
11
|
+
Optional display information for SAAS integrations to enhance frontend presentation.
|
|
12
|
+
When not provided, smart defaults will be inferred based on the integration type.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
category: Optional[ConnectionCategory] = None
|
|
16
|
+
tags: Optional[List[str]] = None
|
|
17
|
+
enabled_features: Optional[List[IntegrationFeature]] = None
|
|
18
|
+
|
|
19
|
+
model_config = ConfigDict(use_enum_values=True)
|
|
@@ -17,6 +17,7 @@ from fides.api.graph.config import (
|
|
|
17
17
|
from fides.api.schemas.base_class import FidesSchema
|
|
18
18
|
from fides.api.schemas.limiter.rate_limit_config import RateLimitConfig
|
|
19
19
|
from fides.api.schemas.policy import ActionType
|
|
20
|
+
from fides.api.schemas.saas.display_info import SaaSDisplayInfo
|
|
20
21
|
from fides.api.schemas.saas.shared_schemas import HTTPMethod
|
|
21
22
|
from fides.api.service.saas_request.saas_request_override_factory import (
|
|
22
23
|
SaaSRequestOverrideFactory,
|
|
@@ -428,6 +429,7 @@ class SaaSConfig(SaaSConfigBase):
|
|
|
428
429
|
rate_limit_config: Optional[RateLimitConfig] = None
|
|
429
430
|
consent_requests: Optional[ConsentRequestMap] = None
|
|
430
431
|
user_guide: Optional[str] = None
|
|
432
|
+
display_info: Optional[SaaSDisplayInfo] = None
|
|
431
433
|
|
|
432
434
|
@property
|
|
433
435
|
def top_level_endpoint_dict(self) -> Dict[str, Endpoint]:
|
|
@@ -38,6 +38,7 @@ class StorageDetails(Enum):
|
|
|
38
38
|
MAX_RETRIES = "max_retries"
|
|
39
39
|
AUTH_METHOD = "auth_method"
|
|
40
40
|
ENABLE_STREAMING = "enable_streaming"
|
|
41
|
+
ENABLE_ACCESS_PACKAGE_REDIRECT = "enable_access_package_redirect"
|
|
41
42
|
model_config = ConfigDict(extra="forbid")
|
|
42
43
|
|
|
43
44
|
|
|
@@ -60,6 +61,7 @@ class StorageDetailsS3(FileBasedStorageDetails):
|
|
|
60
61
|
bucket: str
|
|
61
62
|
max_retries: Optional[int] = 0
|
|
62
63
|
enable_streaming: Optional[bool] = False
|
|
64
|
+
enable_access_package_redirect: Optional[bool] = False
|
|
63
65
|
model_config = ConfigDict(use_enum_values=True)
|
|
64
66
|
|
|
65
67
|
|
|
@@ -26,6 +26,7 @@ from fides.api.service.authentication.authentication_strategy_oauth2_authorizati
|
|
|
26
26
|
)
|
|
27
27
|
from fides.api.util.saas_util import (
|
|
28
28
|
encode_file_contents,
|
|
29
|
+
extract_display_info_from_config,
|
|
29
30
|
load_config,
|
|
30
31
|
load_config_from_string,
|
|
31
32
|
load_dataset_from_string,
|
|
@@ -78,6 +79,8 @@ class FileConnectorTemplateLoader(ConnectorTemplateLoader):
|
|
|
78
79
|
== OAuth2AuthorizationCodeAuthenticationStrategy.name
|
|
79
80
|
)
|
|
80
81
|
|
|
82
|
+
display_info = extract_display_info_from_config(config)
|
|
83
|
+
|
|
81
84
|
try:
|
|
82
85
|
icon = encode_file_contents(f"data/saas/icon/{connector_type}.svg")
|
|
83
86
|
except FileNotFoundError:
|
|
@@ -100,6 +103,7 @@ class FileConnectorTemplateLoader(ConnectorTemplateLoader):
|
|
|
100
103
|
authorization_required=authorization_required,
|
|
101
104
|
user_guide=config.user_guide,
|
|
102
105
|
supported_actions=config.supported_actions,
|
|
106
|
+
**display_info,
|
|
103
107
|
)
|
|
104
108
|
except Exception:
|
|
105
109
|
logger.exception("Unable to load {} connector", connector_type)
|
|
@@ -165,6 +169,8 @@ class CustomConnectorTemplateLoader(ConnectorTemplateLoader):
|
|
|
165
169
|
== OAuth2AuthorizationCodeAuthenticationStrategy.name
|
|
166
170
|
)
|
|
167
171
|
|
|
172
|
+
display_info = extract_display_info_from_config(config)
|
|
173
|
+
|
|
168
174
|
connector_template = ConnectorTemplate(
|
|
169
175
|
config=template.config,
|
|
170
176
|
dataset=template.dataset,
|
|
@@ -173,6 +179,7 @@ class CustomConnectorTemplateLoader(ConnectorTemplateLoader):
|
|
|
173
179
|
authorization_required=authorization_required,
|
|
174
180
|
user_guide=config.user_guide,
|
|
175
181
|
supported_actions=config.supported_actions,
|
|
182
|
+
**display_info,
|
|
176
183
|
)
|
|
177
184
|
|
|
178
185
|
# register the template in the loader's template dictionary
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import copy
|
|
1
2
|
import json
|
|
2
3
|
import os
|
|
3
4
|
import time as time_module
|
|
4
5
|
import zipfile
|
|
5
|
-
from collections import defaultdict
|
|
6
6
|
from io import BytesIO
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from typing import TYPE_CHECKING, Any, Optional
|
|
@@ -12,7 +12,18 @@ from jinja2 import Environment, FileSystemLoader
|
|
|
12
12
|
from loguru import logger
|
|
13
13
|
|
|
14
14
|
from fides.api.schemas.policy import ActionType
|
|
15
|
+
from fides.api.service.storage.util import (
|
|
16
|
+
_get_datasets_from_dsr_data,
|
|
17
|
+
create_attachment_info_dict,
|
|
18
|
+
format_attachment_size,
|
|
19
|
+
generate_attachment_url_from_storage_path,
|
|
20
|
+
is_attachment_field,
|
|
21
|
+
process_attachment_naming,
|
|
22
|
+
process_attachments_contextually,
|
|
23
|
+
resolve_directory_from_context,
|
|
24
|
+
)
|
|
15
25
|
from fides.api.util.storage_util import StorageJSONEncoder, format_size
|
|
26
|
+
from fides.config import CONFIG
|
|
16
27
|
|
|
17
28
|
DSR_DIRECTORY = Path(__file__).parent.resolve()
|
|
18
29
|
|
|
@@ -46,6 +57,7 @@ class DsrReportBuilder:
|
|
|
46
57
|
self,
|
|
47
58
|
privacy_request: "PrivacyRequest",
|
|
48
59
|
dsr_data: dict[str, Any],
|
|
60
|
+
enable_streaming: bool = False,
|
|
49
61
|
):
|
|
50
62
|
"""
|
|
51
63
|
Initializes the DSR report builder.
|
|
@@ -54,7 +66,6 @@ class DsrReportBuilder:
|
|
|
54
66
|
jinja2.filters.FILTERS["pretty_print"] = lambda value, indent=4: json.dumps(
|
|
55
67
|
value, indent=indent, cls=StorageJSONEncoder
|
|
56
68
|
)
|
|
57
|
-
|
|
58
69
|
# Initialize instance zip file variables
|
|
59
70
|
self.baos = BytesIO()
|
|
60
71
|
|
|
@@ -70,15 +81,29 @@ class DsrReportBuilder:
|
|
|
70
81
|
"text_color": TEXT_COLOR,
|
|
71
82
|
"header_color": HEADER_COLOR,
|
|
72
83
|
"border_color": BORDER_COLOR,
|
|
84
|
+
"download_link_ttl_days": self._get_download_link_ttl_days(),
|
|
85
|
+
"enable_streaming": enable_streaming,
|
|
73
86
|
}
|
|
74
87
|
self.main_links: dict[str, Any] = {} # used to track the generated pages
|
|
75
88
|
|
|
76
89
|
# report data to populate the templates
|
|
77
90
|
self.request_data = _map_privacy_request(privacy_request)
|
|
78
91
|
self.dsr_data = dsr_data
|
|
92
|
+
self.enable_streaming = enable_streaming
|
|
93
|
+
|
|
94
|
+
# Track used filenames per dataset to prevent conflicts within the same dataset
|
|
95
|
+
# Maps dataset_name -> set of used filenames
|
|
96
|
+
self.used_filenames_per_dataset: dict[str, set[str]] = {}
|
|
97
|
+
|
|
98
|
+
# Track attachments by their unique identifier to prevent duplicate processing
|
|
99
|
+
# Maps (download_url, file_name) -> unique_filename
|
|
100
|
+
self.processed_attachments: dict[tuple[str, str], str] = {}
|
|
101
|
+
# Track which attachments were processed as dataset attachments (not top-level)
|
|
102
|
+
self.dataset_processed_attachments: set[tuple[str, str]] = set()
|
|
79
103
|
|
|
80
|
-
|
|
81
|
-
|
|
104
|
+
def _get_download_link_ttl_days(self) -> int:
|
|
105
|
+
"""Get the download link TTL in days from the security configuration."""
|
|
106
|
+
return int(CONFIG.security.subject_request_download_link_ttl_seconds / 86400)
|
|
82
107
|
|
|
83
108
|
def _populate_template(
|
|
84
109
|
self,
|
|
@@ -148,34 +173,12 @@ class DsrReportBuilder:
|
|
|
148
173
|
),
|
|
149
174
|
)
|
|
150
175
|
|
|
151
|
-
|
|
152
|
-
"""
|
|
153
|
-
Generates a unique filename by appending a counter if the file already exists.
|
|
154
|
-
Now tracks filenames across all directories to ensure global uniqueness.
|
|
155
|
-
|
|
156
|
-
Args:
|
|
157
|
-
filename: The original filename
|
|
158
|
-
|
|
159
|
-
Returns:
|
|
160
|
-
A unique filename that won't conflict with existing files
|
|
161
|
-
"""
|
|
162
|
-
base_name, extension = os.path.splitext(filename)
|
|
163
|
-
counter = 1
|
|
164
|
-
unique_filename = filename
|
|
165
|
-
|
|
166
|
-
# Check if file exists in used_filenames set
|
|
167
|
-
while unique_filename in self.used_filenames:
|
|
168
|
-
unique_filename = f"{base_name}_{counter}{extension}"
|
|
169
|
-
counter += 1
|
|
170
|
-
|
|
171
|
-
# Add the new filename to the set
|
|
172
|
-
self.used_filenames.add(unique_filename)
|
|
173
|
-
return unique_filename
|
|
174
|
-
|
|
176
|
+
# pylint: disable=too-many-branches
|
|
175
177
|
def _write_attachment_content(
|
|
176
178
|
self,
|
|
177
179
|
attachments: list[dict[str, Any]],
|
|
178
180
|
directory: str,
|
|
181
|
+
dataset_name: str = "attachments",
|
|
179
182
|
) -> dict[str, dict[str, str]]:
|
|
180
183
|
"""
|
|
181
184
|
Processes attachments and returns a dictionary mapping filenames to their download URLs and sizes.
|
|
@@ -194,33 +197,133 @@ class DsrReportBuilder:
|
|
|
194
197
|
if not isinstance(attachment, dict):
|
|
195
198
|
continue
|
|
196
199
|
|
|
197
|
-
|
|
198
|
-
if not
|
|
199
|
-
|
|
200
|
+
# Get or create the used_filenames set for this dataset
|
|
201
|
+
if dataset_name not in self.used_filenames_per_dataset:
|
|
202
|
+
self.used_filenames_per_dataset[dataset_name] = set()
|
|
203
|
+
used_filenames = self.used_filenames_per_dataset[dataset_name]
|
|
204
|
+
|
|
205
|
+
# Process attachment naming using shared utility
|
|
206
|
+
result = process_attachment_naming(
|
|
207
|
+
attachment, used_filenames, self.processed_attachments, dataset_name
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
if result is None: # Skip if processing failed
|
|
200
211
|
continue
|
|
201
212
|
|
|
213
|
+
unique_filename, attachment_key = result
|
|
214
|
+
# Track that this attachment was processed as a dataset attachment
|
|
215
|
+
self.dataset_processed_attachments.add(attachment_key)
|
|
216
|
+
|
|
217
|
+
# Format file size using shared utility
|
|
218
|
+
file_size = format_attachment_size(attachment.get("file_size"))
|
|
219
|
+
|
|
220
|
+
# Determine the actual directory for this attachment based on its context
|
|
221
|
+
actual_directory = resolve_directory_from_context(attachment, directory)
|
|
222
|
+
|
|
223
|
+
# Generate attachment URL using shared utility with actual storage path
|
|
202
224
|
download_url = attachment.get("download_url")
|
|
203
225
|
if not download_url:
|
|
204
|
-
logger.warning("Skipping attachment with no download URL")
|
|
205
226
|
continue
|
|
206
227
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
228
|
+
attachment_url = generate_attachment_url_from_storage_path(
|
|
229
|
+
download_url,
|
|
230
|
+
unique_filename,
|
|
231
|
+
actual_directory, # This is the base_path where the file will be stored
|
|
232
|
+
actual_directory, # This is the HTML template directory
|
|
233
|
+
self.enable_streaming,
|
|
234
|
+
)
|
|
212
235
|
|
|
213
|
-
#
|
|
214
|
-
|
|
236
|
+
# Create attachment info dictionary using shared utility
|
|
237
|
+
file_name = attachment.get("file_name")
|
|
238
|
+
if not file_name:
|
|
239
|
+
continue
|
|
215
240
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
(unique_filename, {"url": download_url, "size": file_size})
|
|
241
|
+
attachment_info = create_attachment_info_dict(
|
|
242
|
+
attachment_url, file_size, file_name
|
|
219
243
|
)
|
|
220
244
|
|
|
245
|
+
processed_attachments.append((unique_filename, attachment_info))
|
|
246
|
+
|
|
221
247
|
# Convert list of tuples to dictionary
|
|
222
248
|
return dict(processed_attachments)
|
|
223
249
|
|
|
250
|
+
def _get_processed_attachments_list(
|
|
251
|
+
self, data: dict[str, Any]
|
|
252
|
+
) -> list[dict[str, Any]]:
|
|
253
|
+
"""Get all processed attachments using shared contextual logic.
|
|
254
|
+
|
|
255
|
+
Args:
|
|
256
|
+
data: The DSR data dictionary
|
|
257
|
+
|
|
258
|
+
Returns:
|
|
259
|
+
List of processed attachment dictionaries
|
|
260
|
+
"""
|
|
261
|
+
# Create temporary sets for compatibility with the shared function
|
|
262
|
+
used_filenames_data = set()
|
|
263
|
+
used_filenames_attachments = set()
|
|
264
|
+
|
|
265
|
+
# Populate the temporary sets from our per-dataset tracking
|
|
266
|
+
for dataset_name, filenames in self.used_filenames_per_dataset.items():
|
|
267
|
+
if dataset_name == "attachments":
|
|
268
|
+
used_filenames_attachments.update(filenames)
|
|
269
|
+
else:
|
|
270
|
+
used_filenames_data.update(filenames)
|
|
271
|
+
|
|
272
|
+
processed_attachments_list = process_attachments_contextually(
|
|
273
|
+
data,
|
|
274
|
+
used_filenames_data,
|
|
275
|
+
used_filenames_attachments,
|
|
276
|
+
self.processed_attachments,
|
|
277
|
+
enable_streaming=self.enable_streaming,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
# Trust the contextual processing completely - it already correctly determines
|
|
281
|
+
# context based on the attachment's location in the DSR data structure
|
|
282
|
+
filtered_list = processed_attachments_list
|
|
283
|
+
|
|
284
|
+
return filtered_list
|
|
285
|
+
|
|
286
|
+
def _generate_attachment_url_from_index(
|
|
287
|
+
self, context: dict[str, Any], unique_filename: str
|
|
288
|
+
) -> str:
|
|
289
|
+
"""Generate the correct URL from attachments/index.html to an attachment file.
|
|
290
|
+
|
|
291
|
+
Args:
|
|
292
|
+
context: The attachment context information
|
|
293
|
+
unique_filename: The unique filename of the attachment
|
|
294
|
+
|
|
295
|
+
Returns:
|
|
296
|
+
The relative URL from attachments/index.html to the attachment file
|
|
297
|
+
"""
|
|
298
|
+
if context.get("type") == "top_level":
|
|
299
|
+
# Top-level attachments are in the same directory as the index
|
|
300
|
+
return unique_filename
|
|
301
|
+
if context.get("type") in ["direct", "nested"]:
|
|
302
|
+
# Dataset attachments are in data/dataset/collection/attachments/
|
|
303
|
+
# From attachments/index.html, we need to go to ../data/dataset/collection/attachments/filename
|
|
304
|
+
dataset = context.get("dataset", "unknown")
|
|
305
|
+
collection = context.get("collection", "unknown")
|
|
306
|
+
return f"../data/{dataset}/{collection}/attachments/{unique_filename}"
|
|
307
|
+
# Fallback for other cases - return just the filename
|
|
308
|
+
return unique_filename
|
|
309
|
+
|
|
310
|
+
def _create_attachment_info_with_corrected_url(
|
|
311
|
+
self, attachment_info: dict[str, str], correct_url: str
|
|
312
|
+
) -> dict[str, str]:
|
|
313
|
+
"""Create attachment info with corrected URL.
|
|
314
|
+
|
|
315
|
+
Args:
|
|
316
|
+
attachment_info: The original attachment info
|
|
317
|
+
correct_url: The corrected URL
|
|
318
|
+
|
|
319
|
+
Returns:
|
|
320
|
+
New attachment info with corrected URL and safe_url
|
|
321
|
+
"""
|
|
322
|
+
corrected_attachment_info = attachment_info.copy()
|
|
323
|
+
corrected_attachment_info["url"] = correct_url
|
|
324
|
+
corrected_attachment_info["safe_url"] = correct_url
|
|
325
|
+
return corrected_attachment_info
|
|
326
|
+
|
|
224
327
|
def _add_collection(
|
|
225
328
|
self, rows: list[dict[str, Any]], dataset_name: str, collection_name: str
|
|
226
329
|
) -> None:
|
|
@@ -235,10 +338,11 @@ class DsrReportBuilder:
|
|
|
235
338
|
items_content = []
|
|
236
339
|
|
|
237
340
|
for index, collection_item in enumerate(rows, 1):
|
|
238
|
-
# Create a copy of the item data to avoid modifying the original
|
|
239
|
-
|
|
341
|
+
# Create a deep copy of the item data to avoid modifying the original DSR data
|
|
342
|
+
# This ensures the comprehensive attachments index can access unmodified attachments
|
|
343
|
+
item_data = copy.deepcopy(collection_item)
|
|
240
344
|
|
|
241
|
-
# Process any attachments in the item
|
|
345
|
+
# Process any attachments in the item - First check for direct attachments key
|
|
242
346
|
if "attachments" in item_data and isinstance(
|
|
243
347
|
item_data["attachments"], list
|
|
244
348
|
):
|
|
@@ -246,9 +350,32 @@ class DsrReportBuilder:
|
|
|
246
350
|
attachment_links = self._write_attachment_content(
|
|
247
351
|
item_data["attachments"],
|
|
248
352
|
f"data/{dataset_name}/{collection_name}",
|
|
353
|
+
dataset_name,
|
|
249
354
|
)
|
|
250
355
|
# Add the attachment URLs to the item data
|
|
251
356
|
item_data["attachments"] = attachment_links
|
|
357
|
+
else:
|
|
358
|
+
# Check for nested attachment fields (ManualTask format)
|
|
359
|
+
attachment_fields_found = []
|
|
360
|
+
for field_name, field_value in item_data.items():
|
|
361
|
+
if isinstance(field_value, list) and field_value:
|
|
362
|
+
# Check if this field contains attachment-like data
|
|
363
|
+
first_item = field_value[0]
|
|
364
|
+
if isinstance(first_item, dict) and all(
|
|
365
|
+
key in first_item
|
|
366
|
+
for key in ["file_name", "download_url", "file_size"]
|
|
367
|
+
):
|
|
368
|
+
attachment_fields_found.append(field_name)
|
|
369
|
+
|
|
370
|
+
# Process attachments and get their URLs
|
|
371
|
+
attachment_links = self._write_attachment_content(
|
|
372
|
+
field_value,
|
|
373
|
+
f"data/{dataset_name}/{collection_name}",
|
|
374
|
+
dataset_name,
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
# Replace the field value with processed attachment links
|
|
378
|
+
item_data[field_name] = attachment_links
|
|
252
379
|
|
|
253
380
|
# Add item content to the list
|
|
254
381
|
items_content.append(
|
|
@@ -294,34 +421,67 @@ class DsrReportBuilder:
|
|
|
294
421
|
),
|
|
295
422
|
)
|
|
296
423
|
|
|
297
|
-
def
|
|
424
|
+
def _add_comprehensive_attachments_index(self) -> None:
|
|
298
425
|
"""
|
|
299
|
-
|
|
426
|
+
Creates a comprehensive attachments index that includes ALL attachments
|
|
427
|
+
from all datasets and top-level attachments, with links pointing to their
|
|
428
|
+
actual storage locations.
|
|
300
429
|
"""
|
|
301
|
-
#
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
430
|
+
# Get all processed attachments using shared logic on original DSR data
|
|
431
|
+
processed_attachments_list = self._get_processed_attachments_list(self.dsr_data)
|
|
432
|
+
|
|
433
|
+
# Create a comprehensive attachment links dictionary with deduplication
|
|
434
|
+
all_attachment_links = {}
|
|
435
|
+
seen_attachment_keys = set()
|
|
436
|
+
|
|
437
|
+
for processed_attachment in processed_attachments_list:
|
|
438
|
+
unique_filename = processed_attachment["unique_filename"]
|
|
439
|
+
attachment_info = processed_attachment["attachment_info"]
|
|
440
|
+
context = processed_attachment["context"]
|
|
441
|
+
attachment = processed_attachment["attachment"]
|
|
442
|
+
|
|
443
|
+
# Create a unique key based on download_url to avoid duplicates
|
|
444
|
+
attachment_key = attachment.get("download_url")
|
|
445
|
+
if attachment_key in seen_attachment_keys:
|
|
307
446
|
continue
|
|
447
|
+
seen_attachment_keys.add(attachment_key)
|
|
308
448
|
|
|
309
|
-
|
|
310
|
-
if
|
|
311
|
-
|
|
449
|
+
# Generate the correct URL based on streaming settings
|
|
450
|
+
if self.enable_streaming:
|
|
451
|
+
# For streaming mode, use local attachment references
|
|
452
|
+
correct_url = self._generate_attachment_url_from_index(
|
|
453
|
+
context, unique_filename
|
|
454
|
+
)
|
|
312
455
|
else:
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
456
|
+
# For non-streaming mode, use original download URLs
|
|
457
|
+
correct_url = attachment.get("download_url", unique_filename)
|
|
458
|
+
|
|
459
|
+
# Create a descriptive key that includes the source location
|
|
460
|
+
if context.get("type") == "top_level":
|
|
461
|
+
key = f"Top-level: {unique_filename}"
|
|
462
|
+
elif context.get("type") in ["direct", "nested"]:
|
|
463
|
+
dataset = context.get("dataset", "unknown")
|
|
464
|
+
collection = context.get("collection", "unknown")
|
|
465
|
+
key = f"{dataset}/{collection}: {unique_filename}"
|
|
466
|
+
else:
|
|
467
|
+
key = unique_filename
|
|
321
468
|
|
|
322
|
-
|
|
469
|
+
# Create new attachment info with the correct URL
|
|
470
|
+
corrected_attachment_info = self._create_attachment_info_with_corrected_url(
|
|
471
|
+
attachment_info, correct_url
|
|
472
|
+
)
|
|
473
|
+
all_attachment_links[key] = corrected_attachment_info
|
|
323
474
|
|
|
324
|
-
|
|
475
|
+
# Generate comprehensive attachments index page
|
|
476
|
+
self._add_file(
|
|
477
|
+
"attachments/index.html",
|
|
478
|
+
self._populate_template(
|
|
479
|
+
"templates/attachments_index.html",
|
|
480
|
+
"All Attachments",
|
|
481
|
+
"All files attached to this privacy request",
|
|
482
|
+
all_attachment_links,
|
|
483
|
+
),
|
|
484
|
+
)
|
|
325
485
|
|
|
326
486
|
def generate(self) -> BytesIO:
|
|
327
487
|
"""
|
|
@@ -343,7 +503,7 @@ class DsrReportBuilder:
|
|
|
343
503
|
)
|
|
344
504
|
|
|
345
505
|
# pre-process data to split the dataset:collection keys
|
|
346
|
-
datasets: dict[str, Any] = self.
|
|
506
|
+
datasets: dict[str, Any] = _get_datasets_from_dsr_data(self.dsr_data)
|
|
347
507
|
|
|
348
508
|
# Sort datasets alphabetically, excluding special cases
|
|
349
509
|
regular_datasets = [
|
|
@@ -360,10 +520,32 @@ class DsrReportBuilder:
|
|
|
360
520
|
self._add_dataset("dataset", datasets["dataset"])
|
|
361
521
|
self.main_links["Additional Data"] = "data/dataset/index.html"
|
|
362
522
|
|
|
363
|
-
# Add
|
|
364
|
-
if
|
|
365
|
-
|
|
366
|
-
self.
|
|
523
|
+
# Add comprehensive attachments index that includes ALL attachments
|
|
524
|
+
# Check if there are any attachments at all (top-level or in datasets)
|
|
525
|
+
has_top_level_attachments = (
|
|
526
|
+
"attachments" in self.dsr_data and self.dsr_data["attachments"]
|
|
527
|
+
)
|
|
528
|
+
has_dataset_attachments = any(
|
|
529
|
+
any(
|
|
530
|
+
"attachments" in item
|
|
531
|
+
or any(
|
|
532
|
+
is_attachment_field(field_value)
|
|
533
|
+
for field_value in item.values()
|
|
534
|
+
if isinstance(field_value, list)
|
|
535
|
+
)
|
|
536
|
+
for item in collection_items
|
|
537
|
+
if isinstance(item, dict)
|
|
538
|
+
)
|
|
539
|
+
for collection in datasets.values()
|
|
540
|
+
if isinstance(collection, dict)
|
|
541
|
+
for collection_items in collection.values()
|
|
542
|
+
if isinstance(collection_items, list)
|
|
543
|
+
)
|
|
544
|
+
has_attachments = has_top_level_attachments or has_dataset_attachments
|
|
545
|
+
|
|
546
|
+
if has_attachments:
|
|
547
|
+
self._add_comprehensive_attachments_index()
|
|
548
|
+
self.main_links["All Attachments"] = "attachments/index.html"
|
|
367
549
|
|
|
368
550
|
# create the main index once all the datasets have been added
|
|
369
551
|
self._add_file(
|
|
@@ -14,7 +14,9 @@
|
|
|
14
14
|
</a>
|
|
15
15
|
</div>
|
|
16
16
|
<h1>Attachments</h1>
|
|
17
|
-
|
|
17
|
+
{% if not enable_streaming %}
|
|
18
|
+
<p class="expiration-notice">Note: All download links will expire in {{ download_link_ttl_days }} days.</p>
|
|
19
|
+
{% endif %}
|
|
18
20
|
<div class="table table-hover">
|
|
19
21
|
<div class="table-row">
|
|
20
22
|
<div class="table-cell" style="text-align: left;">File Name</div>
|
|
@@ -22,7 +24,7 @@
|
|
|
22
24
|
</div>
|
|
23
25
|
{% for name, info in data.items() %}
|
|
24
26
|
<a href="{{ info.url }}" class="table-row" target="_blank">
|
|
25
|
-
<div class="table-cell" style="text-align: left;">{{ name }}</div>
|
|
27
|
+
<div class="table-cell" style="text-align: left;">{% if enable_streaming %}{{ name }}{% else %}{{ info.original_name }}{% endif %}</div>
|
|
26
28
|
<div class="table-cell" style="text-align: left;">{{ info.size }}</div>
|
|
27
29
|
</a>
|
|
28
30
|
{% endfor %}
|