ethyca-fides 2.68.1b3__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ethyca-fides might be problematic. Click here for more details.

Files changed (271) hide show
  1. {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/METADATA +1 -1
  2. {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/RECORD +249 -243
  3. fides/_version.py +3 -3
  4. fides/api/alembic/migrations/versions/b1a2c3d4e5f6_add_location_to_privacy_request.py +26 -0
  5. fides/api/api/v1/api.py +2 -0
  6. fides/api/api/v1/endpoints/dsr_package_link.py +169 -0
  7. fides/api/api/v1/endpoints/oauth_endpoints.py +1 -0
  8. fides/api/api/v1/endpoints/privacy_request_endpoints.py +32 -1
  9. fides/api/api/v1/endpoints/user_endpoints.py +4 -0
  10. fides/api/models/privacy_request/privacy_request.py +1 -0
  11. fides/api/models/privacy_request/webhook.py +33 -1
  12. fides/api/oauth/utils.py +118 -50
  13. fides/api/schemas/application_config.py +7 -0
  14. fides/api/schemas/connection_configuration/connection_type_system_map.py +6 -0
  15. fides/api/schemas/enums/__init__.py +0 -0
  16. fides/api/schemas/enums/connection_category.py +20 -0
  17. fides/api/schemas/enums/integration_feature.py +26 -0
  18. fides/api/schemas/external_https.py +9 -0
  19. fides/api/schemas/privacy_request.py +16 -0
  20. fides/api/schemas/saas/connector_template.py +5 -0
  21. fides/api/schemas/saas/display_info.py +19 -0
  22. fides/api/schemas/saas/saas_config.py +2 -0
  23. fides/api/schemas/storage/storage.py +2 -0
  24. fides/api/service/connectors/saas/connector_registry_service.py +7 -0
  25. fides/api/service/privacy_request/dsr_package/dsr_report_builder.py +253 -71
  26. fides/api/service/privacy_request/dsr_package/templates/attachments_index.html +4 -2
  27. fides/api/service/privacy_request/dsr_package/templates/collection_index.html +3 -1
  28. fides/api/service/privacy_request/dsr_package/templates/dataset_index.html +1 -1
  29. fides/api/service/privacy_request/request_runner_service.py +52 -6
  30. fides/api/service/privacy_request/request_service.py +18 -1
  31. fides/api/service/storage/storage_uploader_service.py +1 -21
  32. fides/api/service/storage/streaming/dsr_storage.py +1 -5
  33. fides/api/service/storage/streaming/s3/s3_storage_client.py +78 -40
  34. fides/api/service/storage/streaming/s3/streaming_s3.py +9 -21
  35. fides/api/service/storage/streaming/smart_open_client.py +8 -7
  36. fides/api/service/storage/streaming/smart_open_streaming_storage.py +109 -196
  37. fides/api/service/storage/streaming/storage_client_factory.py +7 -3
  38. fides/api/service/storage/util.py +579 -0
  39. fides/api/task/manual/manual_task_graph_task.py +11 -9
  40. fides/api/util/connection_type.py +3 -0
  41. fides/api/util/saas_util.py +12 -1
  42. fides/api/util/text.py +51 -0
  43. fides/common/api/v1/urn_registry.py +3 -0
  44. fides/service/privacy_request/privacy_request_service.py +83 -0
  45. fides/ui-build/static/admin/404.html +1 -1
  46. fides/ui-build/static/admin/_next/static/Pc_eOxj5LbY3XOShbrjSX/_buildManifest.js +1 -0
  47. fides/ui-build/static/admin/_next/static/chunks/1099-79646e64f26d62fa.js +1 -0
  48. fides/ui-build/static/admin/_next/static/chunks/{1345-ab756811e19ff4fc.js → 1345-5e1c5b66e25c566e.js} +1 -1
  49. fides/ui-build/static/admin/_next/static/chunks/{1817-fd21f1f5ef0faffa.js → 1817-3d9e110e007853f0.js} +1 -1
  50. fides/ui-build/static/admin/_next/static/chunks/{1975.16126463309143e3.js → 1975.78e719130cfe3fd6.js} +1 -1
  51. fides/ui-build/static/admin/_next/static/chunks/{2921-0e5cc63a82e31830.js → 2921-52328140bc420d0f.js} +1 -1
  52. fides/ui-build/static/admin/_next/static/chunks/{3620-6cceae71bae5b531.js → 3620-31ebb43dba84cbbd.js} +1 -1
  53. fides/ui-build/static/admin/_next/static/chunks/3729-a1ca1608efc11ac4.js +1 -0
  54. fides/ui-build/static/admin/_next/static/chunks/3847-230bf61b053bc706.js +1 -0
  55. fides/ui-build/static/admin/_next/static/chunks/{3855-64541570e2f838fb.js → 3855-ef5194cdb228beb6.js} +1 -1
  56. fides/ui-build/static/admin/_next/static/chunks/{3872-7a18d18a5e287e4e.js → 3872-a91143aa35fa8ef8.js} +1 -1
  57. fides/ui-build/static/admin/_next/static/chunks/{3923-5c87b3d7f1626678.js → 3923-bb2417b8dcade7a4.js} +1 -1
  58. fides/ui-build/static/admin/_next/static/chunks/{401-3902e3e98790d401.js → 401-4af0a912e249d30f.js} +1 -1
  59. fides/ui-build/static/admin/_next/static/chunks/{4121-64ef70ef906bbdd0.js → 4121-c8d5d717e31899e1.js} +1 -1
  60. fides/ui-build/static/admin/_next/static/chunks/4164-355644b916ae0094.js +1 -0
  61. fides/ui-build/static/admin/_next/static/chunks/{4608-70521532195124de.js → 4608-23bbd4c3c4a59f42.js} +1 -1
  62. fides/ui-build/static/admin/_next/static/chunks/4786-0827aae7aceadd22.js +1 -0
  63. fides/ui-build/static/admin/_next/static/chunks/4808-78ca630f2d2503cd.js +1 -0
  64. fides/ui-build/static/admin/_next/static/chunks/{4844-351f99b6644b654e.js → 4844-46324c3d848b8b6a.js} +1 -1
  65. fides/ui-build/static/admin/_next/static/chunks/{5258-c6f96dc740eb5fb1.js → 5258-b0de22a8521686ab.js} +1 -1
  66. fides/ui-build/static/admin/_next/static/chunks/{5487-338800277d36b8d7.js → 5487-8c635883dcaa9c2a.js} +1 -1
  67. fides/ui-build/static/admin/_next/static/chunks/549-38ea1d91ee2addaa.js +1 -0
  68. fides/ui-build/static/admin/_next/static/chunks/{6084-da63f20d9416a982.js → 6084-0096d7de64ef8015.js} +1 -1
  69. fides/ui-build/static/admin/_next/static/chunks/{6853-1d947b75eb07188c.js → 6853-b17673391117c976.js} +1 -1
  70. fides/ui-build/static/admin/_next/static/chunks/{6954-24f9a4f27d67b732.js → 6954-9d46e2276c461c26.js} +1 -1
  71. fides/ui-build/static/admin/_next/static/chunks/{7476-a0dd03bfccf60d0c.js → 7476-d1b0af9ade392e5b.js} +1 -1
  72. fides/ui-build/static/admin/_next/static/chunks/{7630-9fbe06cfb98266fe.js → 7630-da0a7ce4e3a0d62c.js} +1 -1
  73. fides/ui-build/static/admin/_next/static/chunks/{787-3dd31844cf7fec55.js → 787-3499983fa346b380.js} +1 -1
  74. fides/ui-build/static/admin/_next/static/chunks/{79-dcd20e8b09501c17.js → 79-f197fc4db8d530e5.js} +1 -1
  75. fides/ui-build/static/admin/_next/static/chunks/{796-8773e04b64ce2260.js → 796-db1e30119ea973c7.js} +1 -1
  76. fides/ui-build/static/admin/_next/static/chunks/8002-971e29181f72edd1.js +1 -0
  77. fides/ui-build/static/admin/_next/static/chunks/{9046-57eab238570b8bf4.js → 9046-712156d461165f56.js} +1 -1
  78. fides/ui-build/static/admin/_next/static/chunks/{9676.bf0a8a6ff6dfd2af.js → 9676.9fd9552ef744c717.js} +1 -1
  79. fides/ui-build/static/admin/_next/static/chunks/{9826-756c958aecab59a2.js → 9826-b0b3d3cfb13bfbc1.js} +1 -1
  80. fides/ui-build/static/admin/_next/static/chunks/{9951-cdf73904a3adb27b.js → 9951-a88367a129b724ba.js} +1 -1
  81. fides/ui-build/static/admin/_next/static/chunks/pages/{404-dd625a559ada46ca.js → 404-471a6b18e712f050.js} +1 -1
  82. fides/ui-build/static/admin/_next/static/chunks/pages/{_app-b6b09b2878b77b21.js → _app-ef8e1c986bc5b795.js} +12 -12
  83. fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-92cf5e313be1f9e2.js → manual-9dc7e70ab5b05723.js} +1 -1
  84. fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-d6c525ee731a2993.js → multiple-4b79a1652297ed9a.js} +1 -1
  85. fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-5664a3ea796e5ffb.js → add-systems-1632a59203fe8eab.js} +1 -1
  86. fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-78f13de90111fd80.js → add-vendors-1ca9df7ca91bd101.js} +1 -1
  87. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-0fc678f3d6d2fcec.js → configure-07bdbc9ae4137db4.js} +1 -1
  88. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-126db59dc25ca326.js → [id]-f80cf2d3966816fd.js} +1 -1
  89. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-289605267d6cce7e.js → privacy-experience-2795cd4115a77c94.js} +1 -1
  90. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-e9fd9b28ac9705af.js → [id]-e02921dc82dccbb1.js} +1 -1
  91. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-28c003b6043bd16c.js → new-98f9e4ba3610628a.js} +1 -1
  92. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-c643eff04525298e.js → privacy-notices-17ed82777810d1c6.js} +1 -1
  93. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-3ef5d01779a26455.js → properties-226efa1dcd41437f.js} +1 -1
  94. fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-28b192e2c074b0f3.js +1 -0
  95. fides/ui-build/static/admin/_next/static/chunks/pages/{consent-8d4be9e7ec7d2a35.js → consent-09610b10923d9268.js} +1 -1
  96. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-f27ec4578c674181.js → [resourceUrn]-da1a48336daff6f8.js} +1 -1
  97. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-27b6c255bd9e73b6.js → [projectUrn]-d8e776f1e64e4ba8.js} +1 -1
  98. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-0f66dac32040519c.js → projects-75b9629b0d9cdf96.js} +1 -1
  99. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-3b938562df81c4b0.js → [resourceUrn]-470da05db63767cd.js} +1 -1
  100. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-6984c033b8fe3a13.js +1 -0
  101. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-2f0a33ef9ba1f1da.js +1 -0
  102. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-e9d4f25b20ff6781.js +1 -0
  103. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-9c428d3ef0985915.js +1 -0
  104. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-21c141279e66237a.js → activity-b6ae7adb8ef0b525.js} +1 -1
  105. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-3bc6a207693fd175.js → [resourceUrn]-c3a97e6721ca0abe.js} +1 -1
  106. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-da16e73df395ad1d.js → detection-a0a7de552ef71f5b.js} +1 -1
  107. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-04b242632a114405.js → [resourceUrn]-109754fec0755339.js} +1 -1
  108. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-900fe50183a40d72.js → discovery-88654783b06b3b21.js} +1 -1
  109. fides/ui-build/static/admin/_next/static/chunks/pages/{datamap-4f1f7c3a9531a8f4.js → datamap-89136e6800dc9369.js} +1 -1
  110. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-343294dcb10d9532.js → [...subfieldNames]-8f58192dcb54883d.js} +1 -1
  111. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-1c097a0809fa5b6f.js → [collectionName]-dcb4ab380a77aa1e.js} +1 -1
  112. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-b47fa2498b534719.js → [datasetId]-6f16d43071fb9c11.js} +1 -1
  113. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-a31f881cab25704a.js → new-97f06e21580f1f6a.js} +1 -1
  114. fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-858c59c9e67e318d.js → dataset-674bb3940f088ecc.js} +1 -1
  115. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-16c28d272225afb6.js → [id]-6f77d8647fca71e0.js} +1 -1
  116. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/new-821dd1269834cfa2.js +1 -0
  117. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-23e4caf79faa8106.js +1 -0
  118. fides/ui-build/static/admin/_next/static/chunks/pages/{index-fec557d99211f577.js → index-23eb64eed81dcb69.js} +1 -1
  119. fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-3a4cd3fe9094fba3.js +1 -0
  120. fides/ui-build/static/admin/_next/static/chunks/pages/integrations-57e618d7b16ac69a.js +1 -0
  121. fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-4a08ca7762a19700.js → [id]-c9a323eb6a929476.js} +1 -1
  122. fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-343a965dcdb3d11e.js → add-template-b9bb09e46921a590.js} +1 -1
  123. fides/ui-build/static/admin/_next/static/chunks/pages/{messaging-3ade4c54b1c8a11e.js → messaging-82c631a12b5a008c.js} +1 -1
  124. fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-9103bfb854f71410.js → ant-components-bc0e2adf6e0d3ac7.js} +1 -1
  125. fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-3b97029bd4d3c3ea.js → AntForm-86ffcc1ad3fa912e.js} +1 -1
  126. fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-9d9beb8f0d8a278c.js → FormikAntFormItem-ec04f595465bdf69.js} +1 -1
  127. fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-84a4d8fc60f839ed.js → FormikControlled-41d309754ff0c1de.js} +1 -1
  128. fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-1fccf542ab2e33bf.js → FormikField-cab1f78cec7808f9.js} +1 -1
  129. fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-aa75263ae1ba67bb.js → forms-eb6058221403b156.js} +1 -1
  130. fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-db334a1cbb102255.js → table-migration-38360083348c3d6c.js} +1 -1
  131. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{[id]-11c1e4545c8f528c.js → [id]-0d0bb9eb004a3336.js} +1 -1
  132. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{messaging-192a986f61c23268.js → messaging-f9320a58f489f5b7.js} +1 -1
  133. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{storage-9216ac993d71387e.js → storage-d0cfa8aeddd43a40.js} +1 -1
  134. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-e55ec84d5380401d.js → configure-72ca94ec5ed85733.js} +1 -1
  135. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-5a5edc8a4aa7c30a.js +1 -0
  136. fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-a74b51b704b80cb2.js → [id]-5ec775c4904fdbfe.js} +1 -1
  137. fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-8d23f0c55ff6510a.js → add-property-a6812c0916f2949e.js} +1 -1
  138. fides/ui-build/static/admin/_next/static/chunks/pages/{properties-77acceac4f99e7af.js → properties-da734840e4f9d04b.js} +1 -1
  139. fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-6903f42a0412bfa6.js +1 -0
  140. fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-6aad3f563ed03b3f.js → alpha-3e72e9f91991c119.js} +1 -1
  141. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-c1b8f3606d160bb1.js → about-6aab092f4871cecb.js} +1 -1
  142. fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-d9f7f78810d58d08.js → [purpose_id]-9495e2eb506606c7.js} +1 -1
  143. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-ee2c7dde99b1dafb.js → consent-be47008304106395.js} +1 -1
  144. fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-ae1b57589da7b175.js +1 -0
  145. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-31c270d228e00581.js → domain-records-23a6d7a921150188.js} +1 -1
  146. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-996b3f250dd3ea1f.js → domains-2a9e8859ab4d9de6.js} +1 -1
  147. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-ee94981326ddcbf4.js → email-templates-4f9f0fdf9925ae90.js} +1 -1
  148. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-0b831c58966782b8.js → locations-46f7af35cee4a8bb.js} +1 -1
  149. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-94271ba4a224a353.js → organization-a596a96cb8d0aa8e.js} +1 -1
  150. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-41b8136e50320fd3.js → regulations-6ed5fc2410e00857.js} +1 -1
  151. fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-52b45569cbc82e60.js → test-datasets-86811e3cda277e77.js} +1 -1
  152. fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-36d74e93e54aabaf.js → [id]-5a43f108d8047d5b.js} +1 -1
  153. fides/ui-build/static/admin/_next/static/chunks/pages/{systems-24dfc8e2279ced2e.js → systems-045a841e22e85ea8.js} +1 -1
  154. fides/ui-build/static/admin/_next/static/chunks/pages/{taxonomy-d9675cf5e6083b27.js → taxonomy-1b3f2d4bcb0e164d.js} +1 -1
  155. fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/{[id]-866826d7959df487.js → [id]-05d61c80a556b2d5.js} +1 -1
  156. fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-e63b61a8f99ccd57.js → user-management-2cab41659f1ee7da.js} +1 -1
  157. fides/ui-build/static/admin/_next/static/chunks/{webpack-6d0a487039bcf30c.js → webpack-678e89d68dbcd94f.js} +1 -1
  158. fides/ui-build/static/admin/_next/static/css/650df9c348000a26.css +1 -0
  159. fides/ui-build/static/admin/_next/static/css/{dbcf63488933a4d5.css → 98fab0b3e6aa43ed.css} +1 -1
  160. fides/ui-build/static/admin/add-systems/manual.html +1 -1
  161. fides/ui-build/static/admin/add-systems/multiple.html +1 -1
  162. fides/ui-build/static/admin/add-systems.html +1 -1
  163. fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
  164. fides/ui-build/static/admin/consent/configure.html +1 -1
  165. fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
  166. fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
  167. fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
  168. fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
  169. fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
  170. fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
  171. fides/ui-build/static/admin/consent/properties.html +1 -1
  172. fides/ui-build/static/admin/consent/reporting.html +1 -1
  173. fides/ui-build/static/admin/consent.html +1 -1
  174. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
  175. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
  176. fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
  177. fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
  178. fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
  179. fides/ui-build/static/admin/data-catalog.html +1 -1
  180. fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
  181. fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
  182. fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
  183. fides/ui-build/static/admin/data-discovery/activity.html +1 -1
  184. fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
  185. fides/ui-build/static/admin/data-discovery/detection.html +1 -1
  186. fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
  187. fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
  188. fides/ui-build/static/admin/datamap.html +1 -1
  189. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
  190. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
  191. fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
  192. fides/ui-build/static/admin/dataset/new.html +1 -1
  193. fides/ui-build/static/admin/dataset.html +1 -1
  194. fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
  195. fides/ui-build/static/admin/datastore-connection/new.html +1 -1
  196. fides/ui-build/static/admin/datastore-connection.html +1 -1
  197. fides/ui-build/static/admin/index.html +1 -1
  198. fides/ui-build/static/admin/integrations/[id].html +1 -1
  199. fides/ui-build/static/admin/integrations.html +1 -1
  200. fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
  201. fides/ui-build/static/admin/lib/fides-headless.js +1 -1
  202. fides/ui-build/static/admin/lib/fides-preview.js +1 -1
  203. fides/ui-build/static/admin/lib/fides-tcf.js +2 -2
  204. fides/ui-build/static/admin/lib/fides.js +2 -2
  205. fides/ui-build/static/admin/login/[provider].html +1 -1
  206. fides/ui-build/static/admin/login.html +1 -1
  207. fides/ui-build/static/admin/messaging/[id].html +1 -1
  208. fides/ui-build/static/admin/messaging/add-template.html +1 -1
  209. fides/ui-build/static/admin/messaging.html +1 -1
  210. fides/ui-build/static/admin/poc/ant-components.html +1 -1
  211. fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
  212. fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
  213. fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
  214. fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
  215. fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
  216. fides/ui-build/static/admin/poc/forms.html +1 -1
  217. fides/ui-build/static/admin/poc/table-migration.html +1 -1
  218. fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
  219. fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
  220. fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
  221. fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
  222. fides/ui-build/static/admin/privacy-requests.html +1 -1
  223. fides/ui-build/static/admin/properties/[id].html +1 -1
  224. fides/ui-build/static/admin/properties/add-property.html +1 -1
  225. fides/ui-build/static/admin/properties.html +1 -1
  226. fides/ui-build/static/admin/reporting/datamap.html +1 -1
  227. fides/ui-build/static/admin/settings/about/alpha.html +1 -1
  228. fides/ui-build/static/admin/settings/about.html +1 -1
  229. fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
  230. fides/ui-build/static/admin/settings/consent.html +1 -1
  231. fides/ui-build/static/admin/settings/custom-fields.html +1 -1
  232. fides/ui-build/static/admin/settings/domain-records.html +1 -1
  233. fides/ui-build/static/admin/settings/domains.html +1 -1
  234. fides/ui-build/static/admin/settings/email-templates.html +1 -1
  235. fides/ui-build/static/admin/settings/locations.html +1 -1
  236. fides/ui-build/static/admin/settings/organization.html +1 -1
  237. fides/ui-build/static/admin/settings/regulations.html +1 -1
  238. fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
  239. fides/ui-build/static/admin/systems/configure/[id].html +1 -1
  240. fides/ui-build/static/admin/systems.html +1 -1
  241. fides/ui-build/static/admin/taxonomy.html +1 -1
  242. fides/ui-build/static/admin/user-management/new.html +1 -1
  243. fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
  244. fides/ui-build/static/admin/user-management.html +1 -1
  245. fides/ui-build/static/admin/_next/static/_BLI2ArqQzY5XnXbrcxa2/_buildManifest.js +0 -1
  246. fides/ui-build/static/admin/_next/static/chunks/1099-7b2085a3931da9e4.js +0 -1
  247. fides/ui-build/static/admin/_next/static/chunks/1138-0d846ffef62c580f.js +0 -1
  248. fides/ui-build/static/admin/_next/static/chunks/3729-7d2d52400f1f7413.js +0 -1
  249. fides/ui-build/static/admin/_next/static/chunks/4786-53ef1662f2d0d98c.js +0 -1
  250. fides/ui-build/static/admin/_next/static/chunks/4808-8713433c84a62efe.js +0 -1
  251. fides/ui-build/static/admin/_next/static/chunks/549-e6453a3526023e85.js +0 -1
  252. fides/ui-build/static/admin/_next/static/chunks/602-80d113e801d7407d.js +0 -1
  253. fides/ui-build/static/admin/_next/static/chunks/8002-dcd02da6e5649a1c.js +0 -1
  254. fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-baa4a2f8f08ac224.js +0 -1
  255. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-ebf5e7fa4e2ffb49.js +0 -1
  256. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-b27c660039d951c9.js +0 -1
  257. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-8ce5d24af470888e.js +0 -1
  258. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-8e35e33928abbcdc.js +0 -1
  259. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/new-68f502d8b0b5792c.js +0 -1
  260. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-1eb9acb17b133fd1.js +0 -1
  261. fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-e613543818d6cbd2.js +0 -1
  262. fides/ui-build/static/admin/_next/static/chunks/pages/integrations-8069f7c33695fd45.js +0 -1
  263. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-48f447b31c786b80.js +0 -1
  264. fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-e60d398e255f4e00.js +0 -1
  265. fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-a4dad8ca9de2d07b.js +0 -1
  266. fides/ui-build/static/admin/_next/static/css/92441453b27e9c34.css +0 -1
  267. {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/WHEEL +0 -0
  268. {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/entry_points.txt +0 -0
  269. {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/licenses/LICENSE +0 -0
  270. {ethyca_fides-2.68.1b3.dist-info → ethyca_fides-2.69.0.dist-info}/top_level.txt +0 -0
  271. /fides/ui-build/static/admin/_next/static/{_BLI2ArqQzY5XnXbrcxa2 → Pc_eOxj5LbY3XOShbrjSX}/_ssgManifest.js +0 -0
@@ -0,0 +1,26 @@
1
+ from enum import Enum
2
+
3
+
4
+ class IntegrationFeature(str, Enum):
5
+ """
6
+ Features that can be enabled for different integration types.
7
+ These control which tabs and functionality are available in the integration detail view.
8
+ """
9
+
10
+ # Enables data discovery and monitoring functionality - shows "Data discovery" tab
11
+ DATA_DISCOVERY = "DATA_DISCOVERY"
12
+
13
+ # Enables data synchronization to external systems - shows "Data sync" tab
14
+ DATA_SYNC = "DATA_SYNC"
15
+
16
+ # Enables task/workflow management for manual processes - shows "Tasks" tab
17
+ TASKS = "TASKS"
18
+
19
+ # Indicates integration doesn't require connection testing - shows "Details" tab instead of "Connection" tab
20
+ WITHOUT_CONNECTION = "WITHOUT_CONNECTION"
21
+
22
+ # Enables Data Subject Request automation for SAAS integrations
23
+ DSR_AUTOMATION = "DSR_AUTOMATION"
24
+
25
+ # Enables conditions configuration for manual task creation - shows "Conditions" tab
26
+ CONDITIONS = "CONDITIONS"
@@ -39,3 +39,12 @@ class RequestTaskJWE(BaseModel):
39
39
  request_task_id: str
40
40
  scopes: List[str]
41
41
  iat: str
42
+
43
+
44
+ class DownloadTokenJWE(BaseModel):
45
+ """Describes JWE that is given to users to access their privacy request download package"""
46
+
47
+ privacy_request_id: str
48
+ scopes: List[str]
49
+ iat: str
50
+ exp: str
@@ -18,6 +18,7 @@ from fides.api.schemas.user import PrivacyRequestReviewer
18
18
  from fides.api.util.collection_util import Row
19
19
  from fides.api.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key
20
20
  from fides.api.util.enums import ColumnSort
21
+ from fides.api.util.text import normalize_location_code
21
22
  from fides.config import CONFIG
22
23
 
23
24
 
@@ -103,6 +104,7 @@ class PrivacyRequestCreate(FidesSchema):
103
104
  property_id: Optional[str] = None
104
105
  consent_preferences: Optional[List[Consent]] = None # TODO Slated for deprecation
105
106
  source: Optional[PrivacyRequestSource] = None
107
+ location: Optional[str] = None
106
108
 
107
109
  @field_validator("encryption_key")
108
110
  @classmethod
@@ -114,6 +116,18 @@ class PrivacyRequestCreate(FidesSchema):
114
116
  verify_encryption_key(value.encode(CONFIG.security.encoding))
115
117
  return value
116
118
 
119
+ @field_validator("location")
120
+ @classmethod
121
+ def validate_location(
122
+ cls: Type["PrivacyRequestCreate"], value: Optional[str] = None
123
+ ) -> Optional[str]:
124
+ """Validate and normalize location to ISO 3166 format"""
125
+ if value is None:
126
+ return None
127
+
128
+ # nuance here is the validator will coalesce values from the less strict format to ISO 3166 (i.e. "us_ca" -> "US-CA")
129
+ return normalize_location_code(value)
130
+
117
131
 
118
132
  class PrivacyRequestResubmit(PrivacyRequestCreate):
119
133
  """Schema used to copy a privacy request for resubmission"""
@@ -321,6 +335,7 @@ class PrivacyRequestResponse(FidesSchema):
321
335
  custom_privacy_request_fields_approved_by: Optional[str] = None
322
336
  custom_privacy_request_fields_approved_at: Optional[datetime] = None
323
337
  source: Optional[PrivacyRequestSource] = None
338
+ location: Optional[str] = None
324
339
  deleted_at: Optional[datetime] = None
325
340
  deleted_by: Optional[str] = None
326
341
  finalized_at: Optional[datetime] = None
@@ -437,6 +452,7 @@ class PrivacyRequestFilter(FidesSchema):
437
452
  errored_lt: Optional[datetime] = None
438
453
  errored_gt: Optional[datetime] = None
439
454
  external_id: Optional[str] = None
455
+ location: Optional[str] = None
440
456
  action_type: Optional[ActionType] = None
441
457
  verbose: Optional[bool] = False
442
458
  include_identities: Optional[bool] = False
@@ -4,6 +4,8 @@ from fideslang.models import Dataset
4
4
  from pydantic import BaseModel, field_validator
5
5
 
6
6
  from fides.api.models.datasetconfig import validate_masking_strategy_override
7
+ from fides.api.schemas.enums.connection_category import ConnectionCategory
8
+ from fides.api.schemas.enums.integration_feature import IntegrationFeature
7
9
  from fides.api.schemas.policy import ActionType
8
10
  from fides.api.schemas.saas.saas_config import SaaSConfig
9
11
  from fides.api.util.saas_util import load_config_from_string, load_dataset_from_string
@@ -22,6 +24,9 @@ class ConnectorTemplate(BaseModel):
22
24
  authorization_required: bool
23
25
  user_guide: Optional[str] = None
24
26
  supported_actions: List[ActionType]
27
+ category: Optional[ConnectionCategory] = None
28
+ tags: Optional[List[str]] = None
29
+ enabled_features: Optional[List[IntegrationFeature]] = None
25
30
 
26
31
  @field_validator("config")
27
32
  @classmethod
@@ -0,0 +1,19 @@
1
+ from typing import List, Optional
2
+
3
+ from pydantic import BaseModel, ConfigDict
4
+
5
+ from fides.api.schemas.enums.connection_category import ConnectionCategory
6
+ from fides.api.schemas.enums.integration_feature import IntegrationFeature
7
+
8
+
9
+ class SaaSDisplayInfo(BaseModel):
10
+ """
11
+ Optional display information for SAAS integrations to enhance frontend presentation.
12
+ When not provided, smart defaults will be inferred based on the integration type.
13
+ """
14
+
15
+ category: Optional[ConnectionCategory] = None
16
+ tags: Optional[List[str]] = None
17
+ enabled_features: Optional[List[IntegrationFeature]] = None
18
+
19
+ model_config = ConfigDict(use_enum_values=True)
@@ -17,6 +17,7 @@ from fides.api.graph.config import (
17
17
  from fides.api.schemas.base_class import FidesSchema
18
18
  from fides.api.schemas.limiter.rate_limit_config import RateLimitConfig
19
19
  from fides.api.schemas.policy import ActionType
20
+ from fides.api.schemas.saas.display_info import SaaSDisplayInfo
20
21
  from fides.api.schemas.saas.shared_schemas import HTTPMethod
21
22
  from fides.api.service.saas_request.saas_request_override_factory import (
22
23
  SaaSRequestOverrideFactory,
@@ -428,6 +429,7 @@ class SaaSConfig(SaaSConfigBase):
428
429
  rate_limit_config: Optional[RateLimitConfig] = None
429
430
  consent_requests: Optional[ConsentRequestMap] = None
430
431
  user_guide: Optional[str] = None
432
+ display_info: Optional[SaaSDisplayInfo] = None
431
433
 
432
434
  @property
433
435
  def top_level_endpoint_dict(self) -> Dict[str, Endpoint]:
@@ -38,6 +38,7 @@ class StorageDetails(Enum):
38
38
  MAX_RETRIES = "max_retries"
39
39
  AUTH_METHOD = "auth_method"
40
40
  ENABLE_STREAMING = "enable_streaming"
41
+ ENABLE_ACCESS_PACKAGE_REDIRECT = "enable_access_package_redirect"
41
42
  model_config = ConfigDict(extra="forbid")
42
43
 
43
44
 
@@ -60,6 +61,7 @@ class StorageDetailsS3(FileBasedStorageDetails):
60
61
  bucket: str
61
62
  max_retries: Optional[int] = 0
62
63
  enable_streaming: Optional[bool] = False
64
+ enable_access_package_redirect: Optional[bool] = False
63
65
  model_config = ConfigDict(use_enum_values=True)
64
66
 
65
67
 
@@ -26,6 +26,7 @@ from fides.api.service.authentication.authentication_strategy_oauth2_authorizati
26
26
  )
27
27
  from fides.api.util.saas_util import (
28
28
  encode_file_contents,
29
+ extract_display_info_from_config,
29
30
  load_config,
30
31
  load_config_from_string,
31
32
  load_dataset_from_string,
@@ -78,6 +79,8 @@ class FileConnectorTemplateLoader(ConnectorTemplateLoader):
78
79
  == OAuth2AuthorizationCodeAuthenticationStrategy.name
79
80
  )
80
81
 
82
+ display_info = extract_display_info_from_config(config)
83
+
81
84
  try:
82
85
  icon = encode_file_contents(f"data/saas/icon/{connector_type}.svg")
83
86
  except FileNotFoundError:
@@ -100,6 +103,7 @@ class FileConnectorTemplateLoader(ConnectorTemplateLoader):
100
103
  authorization_required=authorization_required,
101
104
  user_guide=config.user_guide,
102
105
  supported_actions=config.supported_actions,
106
+ **display_info,
103
107
  )
104
108
  except Exception:
105
109
  logger.exception("Unable to load {} connector", connector_type)
@@ -165,6 +169,8 @@ class CustomConnectorTemplateLoader(ConnectorTemplateLoader):
165
169
  == OAuth2AuthorizationCodeAuthenticationStrategy.name
166
170
  )
167
171
 
172
+ display_info = extract_display_info_from_config(config)
173
+
168
174
  connector_template = ConnectorTemplate(
169
175
  config=template.config,
170
176
  dataset=template.dataset,
@@ -173,6 +179,7 @@ class CustomConnectorTemplateLoader(ConnectorTemplateLoader):
173
179
  authorization_required=authorization_required,
174
180
  user_guide=config.user_guide,
175
181
  supported_actions=config.supported_actions,
182
+ **display_info,
176
183
  )
177
184
 
178
185
  # register the template in the loader's template dictionary
@@ -1,8 +1,8 @@
1
+ import copy
1
2
  import json
2
3
  import os
3
4
  import time as time_module
4
5
  import zipfile
5
- from collections import defaultdict
6
6
  from io import BytesIO
7
7
  from pathlib import Path
8
8
  from typing import TYPE_CHECKING, Any, Optional
@@ -12,7 +12,18 @@ from jinja2 import Environment, FileSystemLoader
12
12
  from loguru import logger
13
13
 
14
14
  from fides.api.schemas.policy import ActionType
15
+ from fides.api.service.storage.util import (
16
+ _get_datasets_from_dsr_data,
17
+ create_attachment_info_dict,
18
+ format_attachment_size,
19
+ generate_attachment_url_from_storage_path,
20
+ is_attachment_field,
21
+ process_attachment_naming,
22
+ process_attachments_contextually,
23
+ resolve_directory_from_context,
24
+ )
15
25
  from fides.api.util.storage_util import StorageJSONEncoder, format_size
26
+ from fides.config import CONFIG
16
27
 
17
28
  DSR_DIRECTORY = Path(__file__).parent.resolve()
18
29
 
@@ -46,6 +57,7 @@ class DsrReportBuilder:
46
57
  self,
47
58
  privacy_request: "PrivacyRequest",
48
59
  dsr_data: dict[str, Any],
60
+ enable_streaming: bool = False,
49
61
  ):
50
62
  """
51
63
  Initializes the DSR report builder.
@@ -54,7 +66,6 @@ class DsrReportBuilder:
54
66
  jinja2.filters.FILTERS["pretty_print"] = lambda value, indent=4: json.dumps(
55
67
  value, indent=indent, cls=StorageJSONEncoder
56
68
  )
57
-
58
69
  # Initialize instance zip file variables
59
70
  self.baos = BytesIO()
60
71
 
@@ -70,15 +81,29 @@ class DsrReportBuilder:
70
81
  "text_color": TEXT_COLOR,
71
82
  "header_color": HEADER_COLOR,
72
83
  "border_color": BORDER_COLOR,
84
+ "download_link_ttl_days": self._get_download_link_ttl_days(),
85
+ "enable_streaming": enable_streaming,
73
86
  }
74
87
  self.main_links: dict[str, Any] = {} # used to track the generated pages
75
88
 
76
89
  # report data to populate the templates
77
90
  self.request_data = _map_privacy_request(privacy_request)
78
91
  self.dsr_data = dsr_data
92
+ self.enable_streaming = enable_streaming
93
+
94
+ # Track used filenames per dataset to prevent conflicts within the same dataset
95
+ # Maps dataset_name -> set of used filenames
96
+ self.used_filenames_per_dataset: dict[str, set[str]] = {}
97
+
98
+ # Track attachments by their unique identifier to prevent duplicate processing
99
+ # Maps (download_url, file_name) -> unique_filename
100
+ self.processed_attachments: dict[tuple[str, str], str] = {}
101
+ # Track which attachments were processed as dataset attachments (not top-level)
102
+ self.dataset_processed_attachments: set[tuple[str, str]] = set()
79
103
 
80
- # Track used filenames across all attachments
81
- self.used_filenames: set[str] = set()
104
+ def _get_download_link_ttl_days(self) -> int:
105
+ """Get the download link TTL in days from the security configuration."""
106
+ return int(CONFIG.security.subject_request_download_link_ttl_seconds / 86400)
82
107
 
83
108
  def _populate_template(
84
109
  self,
@@ -148,34 +173,12 @@ class DsrReportBuilder:
148
173
  ),
149
174
  )
150
175
 
151
- def _get_unique_filename(self, filename: str) -> str:
152
- """
153
- Generates a unique filename by appending a counter if the file already exists.
154
- Now tracks filenames across all directories to ensure global uniqueness.
155
-
156
- Args:
157
- filename: The original filename
158
-
159
- Returns:
160
- A unique filename that won't conflict with existing files
161
- """
162
- base_name, extension = os.path.splitext(filename)
163
- counter = 1
164
- unique_filename = filename
165
-
166
- # Check if file exists in used_filenames set
167
- while unique_filename in self.used_filenames:
168
- unique_filename = f"{base_name}_{counter}{extension}"
169
- counter += 1
170
-
171
- # Add the new filename to the set
172
- self.used_filenames.add(unique_filename)
173
- return unique_filename
174
-
176
+ # pylint: disable=too-many-branches
175
177
  def _write_attachment_content(
176
178
  self,
177
179
  attachments: list[dict[str, Any]],
178
180
  directory: str,
181
+ dataset_name: str = "attachments",
179
182
  ) -> dict[str, dict[str, str]]:
180
183
  """
181
184
  Processes attachments and returns a dictionary mapping filenames to their download URLs and sizes.
@@ -194,33 +197,133 @@ class DsrReportBuilder:
194
197
  if not isinstance(attachment, dict):
195
198
  continue
196
199
 
197
- file_name = attachment.get("file_name")
198
- if not file_name:
199
- logger.warning("Skipping attachment with no file name")
200
+ # Get or create the used_filenames set for this dataset
201
+ if dataset_name not in self.used_filenames_per_dataset:
202
+ self.used_filenames_per_dataset[dataset_name] = set()
203
+ used_filenames = self.used_filenames_per_dataset[dataset_name]
204
+
205
+ # Process attachment naming using shared utility
206
+ result = process_attachment_naming(
207
+ attachment, used_filenames, self.processed_attachments, dataset_name
208
+ )
209
+
210
+ if result is None: # Skip if processing failed
200
211
  continue
201
212
 
213
+ unique_filename, attachment_key = result
214
+ # Track that this attachment was processed as a dataset attachment
215
+ self.dataset_processed_attachments.add(attachment_key)
216
+
217
+ # Format file size using shared utility
218
+ file_size = format_attachment_size(attachment.get("file_size"))
219
+
220
+ # Determine the actual directory for this attachment based on its context
221
+ actual_directory = resolve_directory_from_context(attachment, directory)
222
+
223
+ # Generate attachment URL using shared utility with actual storage path
202
224
  download_url = attachment.get("download_url")
203
225
  if not download_url:
204
- logger.warning("Skipping attachment with no download URL")
205
226
  continue
206
227
 
207
- file_size = attachment.get("file_size")
208
- if isinstance(file_size, (int, float)):
209
- file_size = format_size(float(file_size))
210
- else:
211
- file_size = "Unknown"
228
+ attachment_url = generate_attachment_url_from_storage_path(
229
+ download_url,
230
+ unique_filename,
231
+ actual_directory, # This is the base_path where the file will be stored
232
+ actual_directory, # This is the HTML template directory
233
+ self.enable_streaming,
234
+ )
212
235
 
213
- # Get a unique filename to prevent duplicates
214
- unique_filename = self._get_unique_filename(file_name)
236
+ # Create attachment info dictionary using shared utility
237
+ file_name = attachment.get("file_name")
238
+ if not file_name:
239
+ continue
215
240
 
216
- # Add to processed attachments
217
- processed_attachments.append(
218
- (unique_filename, {"url": download_url, "size": file_size})
241
+ attachment_info = create_attachment_info_dict(
242
+ attachment_url, file_size, file_name
219
243
  )
220
244
 
245
+ processed_attachments.append((unique_filename, attachment_info))
246
+
221
247
  # Convert list of tuples to dictionary
222
248
  return dict(processed_attachments)
223
249
 
250
+ def _get_processed_attachments_list(
251
+ self, data: dict[str, Any]
252
+ ) -> list[dict[str, Any]]:
253
+ """Get all processed attachments using shared contextual logic.
254
+
255
+ Args:
256
+ data: The DSR data dictionary
257
+
258
+ Returns:
259
+ List of processed attachment dictionaries
260
+ """
261
+ # Create temporary sets for compatibility with the shared function
262
+ used_filenames_data = set()
263
+ used_filenames_attachments = set()
264
+
265
+ # Populate the temporary sets from our per-dataset tracking
266
+ for dataset_name, filenames in self.used_filenames_per_dataset.items():
267
+ if dataset_name == "attachments":
268
+ used_filenames_attachments.update(filenames)
269
+ else:
270
+ used_filenames_data.update(filenames)
271
+
272
+ processed_attachments_list = process_attachments_contextually(
273
+ data,
274
+ used_filenames_data,
275
+ used_filenames_attachments,
276
+ self.processed_attachments,
277
+ enable_streaming=self.enable_streaming,
278
+ )
279
+
280
+ # Trust the contextual processing completely - it already correctly determines
281
+ # context based on the attachment's location in the DSR data structure
282
+ filtered_list = processed_attachments_list
283
+
284
+ return filtered_list
285
+
286
+ def _generate_attachment_url_from_index(
287
+ self, context: dict[str, Any], unique_filename: str
288
+ ) -> str:
289
+ """Generate the correct URL from attachments/index.html to an attachment file.
290
+
291
+ Args:
292
+ context: The attachment context information
293
+ unique_filename: The unique filename of the attachment
294
+
295
+ Returns:
296
+ The relative URL from attachments/index.html to the attachment file
297
+ """
298
+ if context.get("type") == "top_level":
299
+ # Top-level attachments are in the same directory as the index
300
+ return unique_filename
301
+ if context.get("type") in ["direct", "nested"]:
302
+ # Dataset attachments are in data/dataset/collection/attachments/
303
+ # From attachments/index.html, we need to go to ../data/dataset/collection/attachments/filename
304
+ dataset = context.get("dataset", "unknown")
305
+ collection = context.get("collection", "unknown")
306
+ return f"../data/{dataset}/{collection}/attachments/{unique_filename}"
307
+ # Fallback for other cases - return just the filename
308
+ return unique_filename
309
+
310
+ def _create_attachment_info_with_corrected_url(
311
+ self, attachment_info: dict[str, str], correct_url: str
312
+ ) -> dict[str, str]:
313
+ """Create attachment info with corrected URL.
314
+
315
+ Args:
316
+ attachment_info: The original attachment info
317
+ correct_url: The corrected URL
318
+
319
+ Returns:
320
+ New attachment info with corrected URL and safe_url
321
+ """
322
+ corrected_attachment_info = attachment_info.copy()
323
+ corrected_attachment_info["url"] = correct_url
324
+ corrected_attachment_info["safe_url"] = correct_url
325
+ return corrected_attachment_info
326
+
224
327
  def _add_collection(
225
328
  self, rows: list[dict[str, Any]], dataset_name: str, collection_name: str
226
329
  ) -> None:
@@ -235,10 +338,11 @@ class DsrReportBuilder:
235
338
  items_content = []
236
339
 
237
340
  for index, collection_item in enumerate(rows, 1):
238
- # Create a copy of the item data to avoid modifying the original
239
- item_data = collection_item.copy()
341
+ # Create a deep copy of the item data to avoid modifying the original DSR data
342
+ # This ensures the comprehensive attachments index can access unmodified attachments
343
+ item_data = copy.deepcopy(collection_item)
240
344
 
241
- # Process any attachments in the item
345
+ # Process any attachments in the item - First check for direct attachments key
242
346
  if "attachments" in item_data and isinstance(
243
347
  item_data["attachments"], list
244
348
  ):
@@ -246,9 +350,32 @@ class DsrReportBuilder:
246
350
  attachment_links = self._write_attachment_content(
247
351
  item_data["attachments"],
248
352
  f"data/{dataset_name}/{collection_name}",
353
+ dataset_name,
249
354
  )
250
355
  # Add the attachment URLs to the item data
251
356
  item_data["attachments"] = attachment_links
357
+ else:
358
+ # Check for nested attachment fields (ManualTask format)
359
+ attachment_fields_found = []
360
+ for field_name, field_value in item_data.items():
361
+ if isinstance(field_value, list) and field_value:
362
+ # Check if this field contains attachment-like data
363
+ first_item = field_value[0]
364
+ if isinstance(first_item, dict) and all(
365
+ key in first_item
366
+ for key in ["file_name", "download_url", "file_size"]
367
+ ):
368
+ attachment_fields_found.append(field_name)
369
+
370
+ # Process attachments and get their URLs
371
+ attachment_links = self._write_attachment_content(
372
+ field_value,
373
+ f"data/{dataset_name}/{collection_name}",
374
+ dataset_name,
375
+ )
376
+
377
+ # Replace the field value with processed attachment links
378
+ item_data[field_name] = attachment_links
252
379
 
253
380
  # Add item content to the list
254
381
  items_content.append(
@@ -294,34 +421,67 @@ class DsrReportBuilder:
294
421
  ),
295
422
  )
296
423
 
297
- def _get_datasets_from_dsr_data(self) -> dict[str, Any]:
424
+ def _add_comprehensive_attachments_index(self) -> None:
298
425
  """
299
- Returns the datasets from the DSR data.
426
+ Creates a comprehensive attachments index that includes ALL attachments
427
+ from all datasets and top-level attachments, with links pointing to their
428
+ actual storage locations.
300
429
  """
301
- # pre-process data to split the dataset:collection keys
302
- datasets: dict[str, Any] = defaultdict(lambda: defaultdict(list))
303
- for key, rows in self.dsr_data.items():
304
-
305
- # we handle attachments separately
306
- if key == "attachments":
430
+ # Get all processed attachments using shared logic on original DSR data
431
+ processed_attachments_list = self._get_processed_attachments_list(self.dsr_data)
432
+
433
+ # Create a comprehensive attachment links dictionary with deduplication
434
+ all_attachment_links = {}
435
+ seen_attachment_keys = set()
436
+
437
+ for processed_attachment in processed_attachments_list:
438
+ unique_filename = processed_attachment["unique_filename"]
439
+ attachment_info = processed_attachment["attachment_info"]
440
+ context = processed_attachment["context"]
441
+ attachment = processed_attachment["attachment"]
442
+
443
+ # Create a unique key based on download_url to avoid duplicates
444
+ attachment_key = attachment.get("download_url")
445
+ if attachment_key in seen_attachment_keys:
307
446
  continue
447
+ seen_attachment_keys.add(attachment_key)
308
448
 
309
- parts = key.split(":", 1)
310
- if len(parts) > 1:
311
- dataset_name, collection_name = parts
449
+ # Generate the correct URL based on streaming settings
450
+ if self.enable_streaming:
451
+ # For streaming mode, use local attachment references
452
+ correct_url = self._generate_attachment_url_from_index(
453
+ context, unique_filename
454
+ )
312
455
  else:
313
- for row in rows:
314
- if "system_name" in row:
315
- dataset_name = row["system_name"]
316
- collection_name = parts[0]
317
- break
318
- else:
319
- dataset_name = "manual"
320
- collection_name = parts[0]
456
+ # For non-streaming mode, use original download URLs
457
+ correct_url = attachment.get("download_url", unique_filename)
458
+
459
+ # Create a descriptive key that includes the source location
460
+ if context.get("type") == "top_level":
461
+ key = f"Top-level: {unique_filename}"
462
+ elif context.get("type") in ["direct", "nested"]:
463
+ dataset = context.get("dataset", "unknown")
464
+ collection = context.get("collection", "unknown")
465
+ key = f"{dataset}/{collection}: {unique_filename}"
466
+ else:
467
+ key = unique_filename
321
468
 
322
- datasets[dataset_name][collection_name].extend(rows)
469
+ # Create new attachment info with the correct URL
470
+ corrected_attachment_info = self._create_attachment_info_with_corrected_url(
471
+ attachment_info, correct_url
472
+ )
473
+ all_attachment_links[key] = corrected_attachment_info
323
474
 
324
- return datasets
475
+ # Generate comprehensive attachments index page
476
+ self._add_file(
477
+ "attachments/index.html",
478
+ self._populate_template(
479
+ "templates/attachments_index.html",
480
+ "All Attachments",
481
+ "All files attached to this privacy request",
482
+ all_attachment_links,
483
+ ),
484
+ )
325
485
 
326
486
  def generate(self) -> BytesIO:
327
487
  """
@@ -343,7 +503,7 @@ class DsrReportBuilder:
343
503
  )
344
504
 
345
505
  # pre-process data to split the dataset:collection keys
346
- datasets: dict[str, Any] = self._get_datasets_from_dsr_data()
506
+ datasets: dict[str, Any] = _get_datasets_from_dsr_data(self.dsr_data)
347
507
 
348
508
  # Sort datasets alphabetically, excluding special cases
349
509
  regular_datasets = [
@@ -360,10 +520,32 @@ class DsrReportBuilder:
360
520
  self._add_dataset("dataset", datasets["dataset"])
361
521
  self.main_links["Additional Data"] = "data/dataset/index.html"
362
522
 
363
- # Add Additional Attachments last if it exists
364
- if "attachments" in self.dsr_data:
365
- self._add_attachments(self.dsr_data["attachments"])
366
- self.main_links["Additional Attachments"] = "attachments/index.html"
523
+ # Add comprehensive attachments index that includes ALL attachments
524
+ # Check if there are any attachments at all (top-level or in datasets)
525
+ has_top_level_attachments = (
526
+ "attachments" in self.dsr_data and self.dsr_data["attachments"]
527
+ )
528
+ has_dataset_attachments = any(
529
+ any(
530
+ "attachments" in item
531
+ or any(
532
+ is_attachment_field(field_value)
533
+ for field_value in item.values()
534
+ if isinstance(field_value, list)
535
+ )
536
+ for item in collection_items
537
+ if isinstance(item, dict)
538
+ )
539
+ for collection in datasets.values()
540
+ if isinstance(collection, dict)
541
+ for collection_items in collection.values()
542
+ if isinstance(collection_items, list)
543
+ )
544
+ has_attachments = has_top_level_attachments or has_dataset_attachments
545
+
546
+ if has_attachments:
547
+ self._add_comprehensive_attachments_index()
548
+ self.main_links["All Attachments"] = "attachments/index.html"
367
549
 
368
550
  # create the main index once all the datasets have been added
369
551
  self._add_file(
@@ -14,7 +14,9 @@
14
14
  </a>
15
15
  </div>
16
16
  <h1>Attachments</h1>
17
- <p class="expiration-notice">Note: All download links will expire in 7 days.</p>
17
+ {% if not enable_streaming %}
18
+ <p class="expiration-notice">Note: All download links will expire in {{ download_link_ttl_days }} days.</p>
19
+ {% endif %}
18
20
  <div class="table table-hover">
19
21
  <div class="table-row">
20
22
  <div class="table-cell" style="text-align: left;">File Name</div>
@@ -22,7 +24,7 @@
22
24
  </div>
23
25
  {% for name, info in data.items() %}
24
26
  <a href="{{ info.url }}" class="table-row" target="_blank">
25
- <div class="table-cell" style="text-align: left;">{{ name }}</div>
27
+ <div class="table-cell" style="text-align: left;">{% if enable_streaming %}{{ name }}{% else %}{{ info.original_name }}{% endif %}</div>
26
28
  <div class="table-cell" style="text-align: left;">{{ info.size }}</div>
27
29
  </a>
28
30
  {% endfor %}