ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0rc0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ethyca-fides might be problematic. Click here for more details.

Files changed (326) hide show
  1. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/METADATA +1 -1
  2. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/RECORD +292 -279
  3. fides/_version.py +3 -3
  4. fides/api/alembic/migrations/versions/7e9a2b52f498_adding_masking_secrets.py +60 -0
  5. fides/api/alembic/migrations/versions/a7065df4dcf1_add_finalized_fields_to_privacy_request.py +65 -0
  6. fides/api/alembic/migrations/versions/d0031087eacb_create_manualtaskconditionaldependency_.py +106 -0
  7. fides/api/api/v1/endpoints/dataset_config_endpoints.py +13 -5
  8. fides/api/api/v1/endpoints/drp_endpoints.py +7 -1
  9. fides/api/api/v1/endpoints/privacy_request_endpoints.py +44 -1
  10. fides/api/api/v1/endpoints/user_endpoints.py +83 -7
  11. fides/api/app_setup.py +3 -2
  12. fides/api/common_exceptions.py +4 -0
  13. fides/api/db/base.py +1 -0
  14. fides/api/db/database.py +1 -1
  15. fides/api/graph/execution.py +30 -0
  16. fides/api/graph/graph.py +13 -2
  17. fides/api/graph/traversal.py +126 -39
  18. fides/api/models/manual_task/__init__.py +2 -0
  19. fides/api/models/manual_task/conditional_dependency.py +144 -0
  20. fides/api/models/{manual_task.py → manual_task/manual_task.py} +10 -0
  21. fides/api/models/masking_secret.py +72 -0
  22. fides/api/models/policy.py +23 -0
  23. fides/api/models/privacy_request/execution_log.py +1 -0
  24. fides/api/models/privacy_request/privacy_request.py +31 -13
  25. fides/api/oauth/roles.py +2 -0
  26. fides/api/schemas/application_config.py +11 -1
  27. fides/api/schemas/masking/masking_secrets.py +1 -1
  28. fides/api/schemas/policy.py +1 -0
  29. fides/api/schemas/privacy_request.py +5 -0
  30. fides/api/service/connectors/base_connector.py +1 -0
  31. fides/api/service/connectors/bigquery_connector.py +67 -19
  32. fides/api/service/connectors/dynamodb_connector.py +2 -1
  33. fides/api/service/connectors/fides_connector.py +1 -0
  34. fides/api/service/connectors/http_connector.py +1 -0
  35. fides/api/service/connectors/manual_task_connector.py +1 -0
  36. fides/api/service/connectors/manual_webhook_connector.py +2 -1
  37. fides/api/service/connectors/mongodb_connector.py +1 -0
  38. fides/api/service/connectors/okta_connector.py +1 -0
  39. fides/api/service/connectors/query_configs/bigquery_query_config.py +91 -32
  40. fides/api/service/connectors/rds_mysql_connector.py +1 -0
  41. fides/api/service/connectors/rds_postgres_connector.py +1 -0
  42. fides/api/service/connectors/s3_connector.py +1 -0
  43. fides/api/service/connectors/saas_connector.py +1 -0
  44. fides/api/service/connectors/scylla_connector.py +1 -0
  45. fides/api/service/connectors/sql_connector.py +36 -4
  46. fides/api/service/connectors/website_connector.py +1 -0
  47. fides/api/service/privacy_request/dsr_package/templates/welcome.html +2 -2
  48. fides/api/service/privacy_request/request_runner_service.py +142 -53
  49. fides/api/service/privacy_request/request_service.py +1 -22
  50. fides/api/task/conditional_dependencies/__init__.py +0 -0
  51. fides/api/task/conditional_dependencies/evaluator.py +109 -0
  52. fides/api/task/conditional_dependencies/schemas.py +54 -0
  53. fides/api/task/create_request_tasks.py +1 -1
  54. fides/api/task/deprecated_graph_task.py +24 -6
  55. fides/api/task/execute_request_tasks.py +89 -12
  56. fides/api/task/filter_results.py +1 -1
  57. fides/api/task/graph_task.py +38 -3
  58. fides/api/task/manual/manual_task_address.py +46 -0
  59. fides/api/task/manual/manual_task_graph_task.py +118 -126
  60. fides/api/task/manual/manual_task_utils.py +52 -105
  61. fides/api/util/aws_util.py +5 -1
  62. fides/api/util/cache.py +5 -0
  63. fides/api/util/encryption/secrets_util.py +48 -18
  64. fides/common/api/scope_registry.py +3 -0
  65. fides/common/api/v1/urn_registry.py +1 -1
  66. fides/config/execution_settings.py +4 -0
  67. fides/config/utils.py +1 -0
  68. fides/service/privacy_request/privacy_request_service.py +6 -1
  69. fides/ui-build/static/admin/404.html +1 -1
  70. fides/ui-build/static/admin/_next/static/GKmhMPa_1gMto8JZO8ENy/_buildManifest.js +1 -0
  71. fides/ui-build/static/admin/_next/static/chunks/1316-2606e19807c08aa5.js +1 -0
  72. fides/ui-build/static/admin/_next/static/chunks/1467-8808ec8836e033f9.js +1 -0
  73. fides/ui-build/static/admin/_next/static/chunks/1817-b78b58ae3b75d75a.js +1 -0
  74. fides/ui-build/static/admin/_next/static/chunks/1975.7d4634a0e823a02b.js +1 -0
  75. fides/ui-build/static/admin/_next/static/chunks/{203-5a663f465ba26bb4.js → 203-cd78ea279cecba60.js} +1 -1
  76. fides/ui-build/static/admin/_next/static/chunks/2150-930ffaf2c4718edc.js +1 -0
  77. fides/ui-build/static/admin/_next/static/chunks/255-1bc0cbef7a59cdc6.js +1 -0
  78. fides/ui-build/static/admin/_next/static/chunks/{3450-0ba194991d0cca88.js → 3450-ca4ba70da999f264.js} +1 -1
  79. fides/ui-build/static/admin/_next/static/chunks/346-aa3b88efb85f2e28.js +1 -0
  80. fides/ui-build/static/admin/_next/static/chunks/3550-d04125c828d591a1.js +1 -0
  81. fides/ui-build/static/admin/_next/static/chunks/{3855-e172870d3e21b0dd.js → 3855-509ca7ac99b5eada.js} +1 -1
  82. fides/ui-build/static/admin/_next/static/chunks/{3872-46cebf7ec1b31a2b.js → 3872-0a0f0032ca39a93f.js} +1 -1
  83. fides/ui-build/static/admin/_next/static/chunks/409-ea70638a59296659.js +1 -0
  84. fides/ui-build/static/admin/_next/static/chunks/{4121-2bc09fc4ddbfe5cb.js → 4121-877e19d3fa078c7b.js} +1 -1
  85. fides/ui-build/static/admin/_next/static/chunks/{4230-60100f7ef3ddcde1.js → 4230-114e31621c19ea69.js} +1 -1
  86. fides/ui-build/static/admin/_next/static/chunks/4259.d1507e0db19cbed7.js +1 -0
  87. fides/ui-build/static/admin/_next/static/chunks/431-1db919f6569a4021.js +1 -0
  88. fides/ui-build/static/admin/_next/static/chunks/{4608-bbb7bf511a05c3c2.js → 4608-43acf39319177bee.js} +1 -1
  89. fides/ui-build/static/admin/_next/static/chunks/5163-e682273cd76a7d07.js +1 -0
  90. fides/ui-build/static/admin/_next/static/chunks/{5309-b2c4803370634ff8.js → 5309-ce5702b9faeaff55.js} +1 -1
  91. fides/ui-build/static/admin/_next/static/chunks/{905-ffdbd0b14167e8bd.js → 5596-4378b2927dae65b2.js} +3 -3
  92. fides/ui-build/static/admin/_next/static/chunks/5619-9b50cec521203989.js +1 -0
  93. fides/ui-build/static/admin/_next/static/chunks/{6084-7178ff6ea6822475.js → 6084-ddbad3149364725d.js} +1 -1
  94. fides/ui-build/static/admin/_next/static/chunks/6148-59a59d5c5925344f.js +1 -0
  95. fides/ui-build/static/admin/_next/static/chunks/6419-d0c00d661b01f8fa.js +1 -0
  96. fides/ui-build/static/admin/_next/static/chunks/{6662-507be5d46e5b719b.js → 6662-4392ba1e4c254ef7.js} +1 -1
  97. fides/ui-build/static/admin/_next/static/chunks/6780-fc7d9ddb1a03e7b3.js +1 -0
  98. fides/ui-build/static/admin/_next/static/chunks/{6853-2ad3e08fe6f9f5f2.js → 6853-d0190d2cca9dbde2.js} +1 -1
  99. fides/ui-build/static/admin/_next/static/chunks/{6882-6af16fef26c21e06.js → 6882-59ea739e3616ce83.js} +1 -1
  100. fides/ui-build/static/admin/_next/static/chunks/6954-9c4912fbce87c4df.js +1 -0
  101. fides/ui-build/static/admin/_next/static/chunks/{7476-281ee9a8286556f3.js → 7476-cc0d9a94ed7aad53.js} +1 -1
  102. fides/ui-build/static/admin/_next/static/chunks/{787-fb41002f797eb2df.js → 787-c57185ad89c4e288.js} +1 -1
  103. fides/ui-build/static/admin/_next/static/chunks/{79-7e87aff851423d4a.js → 79-2aab56be75e16187.js} +1 -1
  104. fides/ui-build/static/admin/_next/static/chunks/{796-329a5f823ec258a5.js → 796-3bdda2a7868464af.js} +1 -1
  105. fides/ui-build/static/admin/_next/static/chunks/8237-841439bef6682177.js +1 -0
  106. fides/ui-build/static/admin/_next/static/chunks/8765-f622a35b40a7ec63.js +1 -0
  107. fides/ui-build/static/admin/_next/static/chunks/{9046-5c4c22c375de25b1.js → 9046-04a8c092fef1cd83.js} +1 -1
  108. fides/ui-build/static/admin/_next/static/chunks/9187-7438242f0d380bb0.js +1 -0
  109. fides/ui-build/static/admin/_next/static/chunks/{9226-746771d47dff6266.js → 9226-2dcac54ab3fb94be.js} +1 -1
  110. fides/ui-build/static/admin/_next/static/chunks/9278-08cc704317fe535e.js +1 -0
  111. fides/ui-build/static/admin/_next/static/chunks/{9951-9b753ad7c3f51bdf.js → 9951-7c6639e5d062779e.js} +1 -1
  112. fides/ui-build/static/admin/_next/static/chunks/pages/{_app-39ccb07327c2c5d5.js → _app-4b5bff46158a19a3.js} +56 -56
  113. fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-98777246bec9dc2a.js → manual-2a655ff3a97f2492.js} +1 -1
  114. fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-a71c0aff4e0e6535.js → add-systems-0902f0bb4080643e.js} +1 -1
  115. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-1edf582ba3cd3bbb.js → [id]-f22ddd9b48a5c418.js} +1 -1
  116. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-06bb3b0bf097fcdb.js → new-e74cb5ea87f15b40.js} +1 -1
  117. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-685771e5f7196d87.js → privacy-experience-21f997c69fc3b4c2.js} +1 -1
  118. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-6ccedc70dc447089.js → [id]-da4124b7600a2a1d.js} +1 -1
  119. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-944bca1cc57985b5.js → new-a57d251c88ce68ae.js} +1 -1
  120. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-84f4bd14ce8673bc.js → privacy-notices-ad105181bc91209b.js} +1 -1
  121. fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-b0c4235fe6d0b0c8.js +1 -0
  122. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-11d52f1570759c4d.js → [resourceUrn]-aad6047a4604b945.js} +1 -1
  123. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-32eac8bbd217615a.js → projects-29784a11fe0fbd0a.js} +1 -1
  124. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-b83afa5565d0c84e.js → [resourceUrn]-b6b98cea25dd94fa.js} +1 -1
  125. fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-6f630d42ac9fb6b4.js → data-catalog-d5b01abcb76792ce.js} +1 -1
  126. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-7caea7bb58c1f153.js +1 -0
  127. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-a9a70856f7be1542.js +1 -0
  128. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-9ddb52ebb7ac4c71.js → action-center-1f0ea5c92ae9a2b4.js} +1 -1
  129. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-9aa744d56cdacb0d.js → activity-ad6a84a6276f914c.js} +1 -1
  130. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-14bd7500362ff224.js → [resourceUrn]-f98dd251babb7e28.js} +1 -1
  131. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9e7dfd5a6acc2e8f.js → discovery-56eb4c014f0d96a3.js} +1 -1
  132. fides/ui-build/static/admin/_next/static/chunks/pages/datamap-d23b3ae139f0428b.js +1 -0
  133. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-c0d2bfd465df20e0.js → [...subfieldNames]-15301bd6bf7cf718.js} +1 -1
  134. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-28280a8a39a6e37c.js → [collectionName]-0fa72873e464f581.js} +1 -1
  135. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-82fb246d87e58ebd.js → new-0d50084fbdf9b84c.js} +1 -1
  136. fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-20165c31ab1bc7cf.js → dataset-f3348d0a92543bab.js} +1 -1
  137. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-b4a6bcc87d126840.js → [id]-7d6027570d05c57f.js} +1 -1
  138. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-f95d7b0bbfc58f5a.js → new-8446418c7ad28f77.js} +1 -1
  139. fides/ui-build/static/admin/_next/static/chunks/pages/{fides-js-docs-5d8fd1af75f19e2f.js → fides-js-docs-1f4335dca5c09860.js} +1 -1
  140. fides/ui-build/static/admin/_next/static/chunks/pages/{index-1919aab9e5834b51.js → index-fbf9b845bb901238.js} +1 -1
  141. fides/ui-build/static/admin/_next/static/chunks/pages/integrations/{[id]-0a58aee2d1e7fa01.js → [id]-f53fe1f2cbebda7c.js} +1 -1
  142. fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-e2d5d7e2a5265e68.js → integrations-142abe3e3e3e5bf7.js} +1 -1
  143. fides/ui-build/static/admin/_next/static/chunks/pages/messaging-1bae386d8c190348.js +1 -0
  144. fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-69ad86b7a8a9a115.js → table-migration-05616e2ae20ff4f8.js} +1 -1
  145. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-1b6b0d703cf59389.js +1 -0
  146. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-f1d818242d8550f8.js +1 -0
  147. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-d40a26bddb126c5c.js +1 -0
  148. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-ccd8d9e06cf2d278.js +1 -0
  149. fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-fd1a67892056830a.js +1 -0
  150. fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-fc201657f4a782c7.js → [purpose_id]-e891d01ece59669e.js} +1 -1
  151. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-c2d39cba8396ef3a.js → consent-f61b87e79367865b.js} +1 -1
  152. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-d992103cc55901ae.js → custom-fields-f8eea5d508c60c64.js} +1 -1
  153. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-023e1895552817de.js → locations-ed6a140b362c5baa.js} +1 -1
  154. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-ac403c0886b20e20.js → organization-ff9a34264d48c35f.js} +1 -1
  155. fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-7a3396ac819c7904.js → test-datasets-a4b6d41ca679298b.js} +1 -1
  156. fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-8314a819837f5b2a.js → [id]-c8f5fbaa83dd9945.js} +1 -1
  157. fides/ui-build/static/admin/_next/static/chunks/pages/{systems-21f423a7c417aa9d.js → systems-c05b49ddec1a1b4f.js} +1 -1
  158. fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-df0d88716578e295.js +1 -0
  159. fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-da68efc31998dc66.js +1 -0
  160. fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-173ac3a1ed2b05a6.js → user-management-e98dfc7d4f2a4e16.js} +1 -1
  161. fides/ui-build/static/admin/_next/static/chunks/webpack-90e8ec1fc5c6455b.js +1 -0
  162. fides/ui-build/static/admin/_next/static/css/{5bfb2473e5701527.css → 23cf870196941c9a.css} +1 -1
  163. fides/ui-build/static/admin/_next/static/css/{94965f224bc991e9.css → 8bc1833f1fa53ff0.css} +1 -1
  164. fides/ui-build/static/admin/_next/static/css/d9924caa849931b3.css +1 -0
  165. fides/ui-build/static/admin/_next/static/css/dbcf63488933a4d5.css +29 -0
  166. fides/ui-build/static/admin/add-systems/manual.html +1 -1
  167. fides/ui-build/static/admin/add-systems/multiple.html +1 -1
  168. fides/ui-build/static/admin/add-systems.html +1 -1
  169. fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
  170. fides/ui-build/static/admin/consent/configure.html +1 -1
  171. fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
  172. fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
  173. fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
  174. fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
  175. fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
  176. fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
  177. fides/ui-build/static/admin/consent/properties.html +1 -1
  178. fides/ui-build/static/admin/consent/reporting.html +1 -1
  179. fides/ui-build/static/admin/consent.html +1 -1
  180. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
  181. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
  182. fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
  183. fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
  184. fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
  185. fides/ui-build/static/admin/data-catalog.html +1 -1
  186. fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
  187. fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
  188. fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
  189. fides/ui-build/static/admin/data-discovery/activity.html +1 -1
  190. fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
  191. fides/ui-build/static/admin/data-discovery/detection.html +1 -1
  192. fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
  193. fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
  194. fides/ui-build/static/admin/datamap.html +1 -1
  195. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
  196. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
  197. fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
  198. fides/ui-build/static/admin/dataset/new.html +1 -1
  199. fides/ui-build/static/admin/dataset.html +1 -1
  200. fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
  201. fides/ui-build/static/admin/datastore-connection/new.html +1 -1
  202. fides/ui-build/static/admin/datastore-connection.html +1 -1
  203. fides/ui-build/static/admin/index.html +1 -1
  204. fides/ui-build/static/admin/integrations/[id].html +1 -1
  205. fides/ui-build/static/admin/integrations.html +1 -1
  206. fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
  207. fides/ui-build/static/admin/lib/fides-headless.js +1 -1
  208. fides/ui-build/static/admin/lib/fides-preview.js +1 -1
  209. fides/ui-build/static/admin/lib/fides-tcf.js +3 -3
  210. fides/ui-build/static/admin/lib/fides.js +3 -3
  211. fides/ui-build/static/admin/login/[provider].html +1 -1
  212. fides/ui-build/static/admin/login.html +1 -1
  213. fides/ui-build/static/admin/messaging/[id].html +1 -1
  214. fides/ui-build/static/admin/messaging/add-template.html +1 -1
  215. fides/ui-build/static/admin/messaging.html +1 -1
  216. fides/ui-build/static/admin/poc/ant-components.html +1 -1
  217. fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
  218. fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
  219. fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
  220. fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
  221. fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
  222. fides/ui-build/static/admin/poc/forms.html +1 -1
  223. fides/ui-build/static/admin/poc/table-migration.html +1 -1
  224. fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
  225. fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
  226. fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
  227. fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
  228. fides/ui-build/static/admin/privacy-requests.html +1 -1
  229. fides/ui-build/static/admin/properties/[id].html +1 -1
  230. fides/ui-build/static/admin/properties/add-property.html +1 -1
  231. fides/ui-build/static/admin/properties.html +1 -1
  232. fides/ui-build/static/admin/reporting/datamap.html +1 -1
  233. fides/ui-build/static/admin/settings/about/alpha.html +1 -1
  234. fides/ui-build/static/admin/settings/about.html +1 -1
  235. fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
  236. fides/ui-build/static/admin/settings/consent.html +1 -1
  237. fides/ui-build/static/admin/settings/custom-fields.html +1 -1
  238. fides/ui-build/static/admin/settings/domain-records.html +1 -1
  239. fides/ui-build/static/admin/settings/domains.html +1 -1
  240. fides/ui-build/static/admin/settings/email-templates.html +1 -1
  241. fides/ui-build/static/admin/settings/locations.html +1 -1
  242. fides/ui-build/static/admin/settings/organization.html +1 -1
  243. fides/ui-build/static/admin/settings/regulations.html +1 -1
  244. fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
  245. fides/ui-build/static/admin/systems/configure/[id].html +1 -1
  246. fides/ui-build/static/admin/systems.html +1 -1
  247. fides/ui-build/static/admin/taxonomy.html +1 -1
  248. fides/ui-build/static/admin/user-management/new.html +1 -1
  249. fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
  250. fides/ui-build/static/admin/user-management.html +1 -1
  251. fides/ui-build/static/admin/_next/static/8108ANFxs99VY7KZ_Xev2/_buildManifest.js +0 -1
  252. fides/ui-build/static/admin/_next/static/chunks/1316-6cc72a45ebf7ff81.js +0 -1
  253. fides/ui-build/static/admin/_next/static/chunks/1807-3beab149351d5ded.js +0 -1
  254. fides/ui-build/static/admin/_next/static/chunks/1817-e601e737e3cc7a0e.js +0 -1
  255. fides/ui-build/static/admin/_next/static/chunks/255-7db55b0e3a0f9dea.js +0 -1
  256. fides/ui-build/static/admin/_next/static/chunks/2599-6c4d22e75028d8b6.js +0 -1
  257. fides/ui-build/static/admin/_next/static/chunks/2858-0b44609b6be7850b.js +0 -1
  258. fides/ui-build/static/admin/_next/static/chunks/2866-a73888c17a195cbe.js +0 -1
  259. fides/ui-build/static/admin/_next/static/chunks/3615-5e2d062d684b8fa1.js +0 -1
  260. fides/ui-build/static/admin/_next/static/chunks/409-a257e14acebcd73b.js +0 -1
  261. fides/ui-build/static/admin/_next/static/chunks/431-34f0b91c26f8d9ab.js +0 -1
  262. fides/ui-build/static/admin/_next/static/chunks/570-c99f07161bd339cd.js +0 -1
  263. fides/ui-build/static/admin/_next/static/chunks/6780-e3d40aa17a4bf2e9.js +0 -1
  264. fides/ui-build/static/admin/_next/static/chunks/6954-bb875d9ac89f6030.js +0 -1
  265. fides/ui-build/static/admin/_next/static/chunks/7062.fda15dcb7df85675.js +0 -1
  266. fides/ui-build/static/admin/_next/static/chunks/7c79804f.7a7112aece470725.js +0 -1
  267. fides/ui-build/static/admin/_next/static/chunks/9014-eeae6f581158e645.js +0 -1
  268. fides/ui-build/static/admin/_next/static/chunks/9278-9b1b5970f0702668.js +0 -1
  269. fides/ui-build/static/admin/_next/static/chunks/9392.f4520f66206d347d.js +0 -1
  270. fides/ui-build/static/admin/_next/static/chunks/ea076c0a.84423f606aef37cd.js +0 -1
  271. fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-afdbd4665657cfa1.js +0 -1
  272. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-2265ecb899d45fbc.js +0 -1
  273. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-5d522637871ac6c8.js +0 -1
  274. fides/ui-build/static/admin/_next/static/chunks/pages/datamap-7674b97d655c193b.js +0 -1
  275. fides/ui-build/static/admin/_next/static/chunks/pages/messaging-5094ffea13f32ed9.js +0 -1
  276. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-32600543eb7b584f.js +0 -1
  277. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-10ce53ea356f8bad.js +0 -1
  278. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-5501bbb129fee9c4.js +0 -1
  279. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-cbe4c8f9096b6543.js +0 -1
  280. fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-e130c0197362e8f3.js +0 -1
  281. fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-6387fcc8cce872eb.js +0 -1
  282. fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-ff5738706da07801.js +0 -1
  283. fides/ui-build/static/admin/_next/static/chunks/webpack-ff0cd6bff75588da.js +0 -1
  284. fides/ui-build/static/admin/_next/static/css/2cadb5f62dcd7c2b.css +0 -1
  285. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/WHEEL +0 -0
  286. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/entry_points.txt +0 -0
  287. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/licenses/LICENSE +0 -0
  288. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/top_level.txt +0 -0
  289. /fides/ui-build/static/admin/_next/static/{8108ANFxs99VY7KZ_Xev2 → GKmhMPa_1gMto8JZO8ENy}/_ssgManifest.js +0 -0
  290. /fides/ui-build/static/admin/_next/static/chunks/{2921-455e6357b74d2f76.js → 2921-86f1547ac40a5cdf.js} +0 -0
  291. /fides/ui-build/static/admin/_next/static/chunks/{3923-6cc911dafccc5f63.js → 3923-13a6b4da2d51bf8f.js} +0 -0
  292. /fides/ui-build/static/admin/_next/static/chunks/{401-1b529d5800aa1f3a.js → 401-3cc1fee61494e3bd.js} +0 -0
  293. /fides/ui-build/static/admin/_next/static/chunks/{5574-b13021775a15bfd2.js → 5574-9312f97b637d9ee2.js} +0 -0
  294. /fides/ui-build/static/admin/_next/static/chunks/{7630-9aac73191ed5ed13.js → 7630-b1c93688013ef013.js} +0 -0
  295. /fides/ui-build/static/admin/_next/static/chunks/{9826-111aaee8bd8dbd09.js → 9826-3c578665c6d3b21d.js} +0 -0
  296. /fides/ui-build/static/admin/_next/static/chunks/pages/{404-aece2c920ea14514.js → 404-2d803dab6a00f353.js} +0 -0
  297. /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-dc75dc6e37e52f05.js → multiple-8ff7f37913ad736a.js} +0 -0
  298. /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-24d226b5a8de5c74.js → add-vendors-d00c9034cdeb0236.js} +0 -0
  299. /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-6a8ef51138ac926a.js → configure-0e1ca0f4c8e7f4da.js} +0 -0
  300. /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-6f86ab63a08a6528.js → properties-057cad65e7414a44.js} +0 -0
  301. /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-73d3cbf68f7c3a31.js → consent-e17c56eec8d91371.js} +0 -0
  302. /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-6ba9e160dae64695.js → [projectUrn]-80a6cc8e8573514a.js} +0 -0
  303. /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{resources-7648bbd4f6711e4d.js → resources-6c3714ee97a718c1.js} +0 -0
  304. /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-393e20924c83373e.js → [resourceUrn]-31e6c54794a9883e.js} +0 -0
  305. /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-8733807dad4bc96e.js → detection-2822a423a7ad0550.js} +0 -0
  306. /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-006b695e5af5ef24.js → [datasetId]-a8e8b5f4ee7af86c.js} +0 -0
  307. /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-c391c6fad56eec48.js → datastore-connection-0f29b47402292070.js} +0 -0
  308. /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-53fecfb9dd6a1e0c.js → [id]-5627d0d0668077f9.js} +0 -0
  309. /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-76b01cec5fde10a9.js → add-template-feca66ad5c5fe54a.js} +0 -0
  310. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-5c08e8447c45ce44.js → ant-components-64a322d01aae5ca7.js} +0 -0
  311. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-06ad5f34585480aa.js → AntForm-8bca16a7726e7eb2.js} +0 -0
  312. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-6f071c2bc9446cb0.js → FormikAntFormItem-b0f246fc3b67ebf7.js} +0 -0
  313. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-efcc38c58991ac9e.js → FormikControlled-1a0852b090bfc392.js} +0 -0
  314. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-430ba5c979abfb7c.js → FormikField-11f3de1b45e36583.js} +0 -0
  315. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-5c561880bf131afb.js → forms-1b73a1c2b6c6285f.js} +0 -0
  316. /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-d888a69a3bbe040e.js → configure-e551a860ec727802.js} +0 -0
  317. /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-d3d8e3d7583ec635.js → [id]-dd99183f93763ae4.js} +0 -0
  318. /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-1af10ed303815d46.js → add-property-0bdbc1fcbf553b8f.js} +0 -0
  319. /fides/ui-build/static/admin/_next/static/chunks/pages/{properties-cebc0dc186be499a.js → properties-e959378bb32b6b73.js} +0 -0
  320. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-5e1322de868d615e.js → alpha-8f98a4895e74725e.js} +0 -0
  321. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-241f95e372b65d0f.js → about-8155a35a62fdb5ae.js} +0 -0
  322. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-41242f805599feda.js → domain-records-51333dbd21cb37c8.js} +0 -0
  323. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-2e885f74c92f669c.js → domains-bde86e5f6c09da5a.js} +0 -0
  324. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-ff112655ad5f41e5.js → email-templates-4f9a5cc8bea7725b.js} +0 -0
  325. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-86062a18e081a52a.js → regulations-102efd9199e87124.js} +0 -0
  326. /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-a2524414e968f862.js → new-bc4eb541906781e6.js} +0 -0
@@ -14,6 +14,7 @@ from fides.api.common_exceptions import (
14
14
  ClientUnsuccessfulException,
15
15
  IdentityNotFoundException,
16
16
  ManualWebhookFieldsUnset,
17
+ MaskingSecretsExpired,
17
18
  MessageDispatchException,
18
19
  NoCachedManualWebhookEntry,
19
20
  PrivacyRequestExit,
@@ -75,6 +76,7 @@ from fides.api.task.graph_task import (
75
76
  from fides.api.task.manual.manual_task_utils import create_manual_task_artificial_graphs
76
77
  from fides.api.tasks import DatabaseTask, celery_app
77
78
  from fides.api.tasks.scheduled.scheduler import scheduler
79
+ from fides.api.util.cache import get_all_masking_secret_keys
78
80
  from fides.api.util.collection_util import Row
79
81
  from fides.api.util.logger import Pii, _log_exception, _log_warning
80
82
  from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
@@ -478,12 +480,21 @@ def run_privacy_request(
478
480
  connection_configs
479
481
  )
480
482
 
483
+ # If the privacy request requires manual finalization and has not yet been finalized, we exit here
484
+ if (
485
+ privacy_request.status
486
+ == PrivacyRequestStatus.requires_manual_finalization
487
+ and privacy_request.finalized_at is None
488
+ ):
489
+ return
490
+
481
491
  # Access CHECKPOINT
482
492
  if (
483
493
  policy.get_rules_for_action(action_type=ActionType.access)
484
494
  or policy.get_rules_for_action(action_type=ActionType.erasure)
485
495
  ) and can_run_checkpoint(
486
- request_checkpoint=CurrentStep.access, from_checkpoint=resume_step
496
+ request_checkpoint=CurrentStep.access,
497
+ from_checkpoint=resume_step,
487
498
  ):
488
499
  privacy_request.cache_failed_checkpoint_details(CurrentStep.access)
489
500
  access_runner(
@@ -528,9 +539,12 @@ def run_privacy_request(
528
539
  if policy.get_rules_for_action(
529
540
  action_type=ActionType.erasure
530
541
  ) and can_run_checkpoint(
531
- request_checkpoint=CurrentStep.erasure, from_checkpoint=resume_step
542
+ request_checkpoint=CurrentStep.erasure,
543
+ from_checkpoint=resume_step,
532
544
  ):
533
545
  privacy_request.cache_failed_checkpoint_details(CurrentStep.erasure)
546
+ _verify_masking_secrets(policy, privacy_request, resume_step)
547
+
534
548
  # We only need to run the erasure once until masking strategies are handled
535
549
  erasure_runner(
536
550
  privacy_request=privacy_request,
@@ -652,58 +666,111 @@ def run_privacy_request(
652
666
  if not proceed:
653
667
  return
654
668
 
655
- legacy_request_completion_enabled = ConfigProxy(
656
- session
657
- ).notifications.send_request_completion_notification
658
-
659
- action_type = (
660
- MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS
661
- if policy.get_rules_for_action(action_type=ActionType.access)
662
- else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
663
- )
664
-
665
- if message_send_enabled(
666
- session,
667
- privacy_request.property_id,
668
- action_type,
669
- legacy_request_completion_enabled,
670
- ) and not policy.get_rules_for_action(action_type=ActionType.consent):
671
- try:
672
- if not access_result_urls:
673
- # For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
674
- # from the database because the Privacy Request would have exited
675
- # processing and lost access to the access_result_urls in memory
676
- access_result_urls = (
677
- privacy_request.access_result_urls or {}
678
- ).get("access_result_urls", [])
679
- initiate_privacy_request_completion_email(
680
- session,
681
- policy,
682
- access_result_urls,
683
- identity_data,
684
- privacy_request.property_id,
685
- )
686
- except (IdentityNotFoundException, MessageDispatchException) as e:
687
- privacy_request.error_processing(db=session)
688
- # If dev mode, log traceback
689
- _log_exception(e, CONFIG.dev_mode)
690
- return
691
-
692
- # Only mark as complete if not in error state
693
- if privacy_request.status != PrivacyRequestStatus.error:
694
- privacy_request.finished_processing_at = datetime.utcnow()
695
- AuditLog.create(
696
- db=session,
697
- data={
698
- "user_id": "system",
699
- "privacy_request_id": privacy_request.id,
700
- "action": AuditLogAction.finished,
701
- "message": "",
702
- },
669
+ # Request finalization CHECKPOINT
670
+ if can_run_checkpoint(
671
+ request_checkpoint=CurrentStep.finalization,
672
+ from_checkpoint=resume_step,
673
+ ):
674
+ privacy_request.cache_failed_checkpoint_details(
675
+ CurrentStep.finalization,
703
676
  )
704
- privacy_request.status = PrivacyRequestStatus.complete
705
- logger.info("Privacy request run completed.")
706
- privacy_request.save(db=session)
677
+ if privacy_request.status != PrivacyRequestStatus.error:
678
+ erasure_rules = policy.get_rules_for_action(
679
+ action_type=ActionType.erasure
680
+ )
681
+ if (
682
+ privacy_request.finalized_at is None
683
+ and erasure_rules
684
+ and CONFIG.execution.erasure_request_finalization_required
685
+ ):
686
+ logger.info(
687
+ "Marking privacy request '{}' as requires manual finalization.",
688
+ privacy_request.id,
689
+ )
690
+ privacy_request.status = (
691
+ PrivacyRequestStatus.requires_manual_finalization
692
+ )
693
+ privacy_request.save(db=session)
694
+ return
695
+
696
+ # Finally, mark the request as complete
697
+ if privacy_request.finalized_at:
698
+ logger.info(
699
+ "Marking privacy request '{}' as finalized.",
700
+ privacy_request.id,
701
+ )
702
+ privacy_request.add_success_execution_log(
703
+ session,
704
+ connection_key=None,
705
+ dataset_name="Request finalized",
706
+ collection_name=None,
707
+ message=f"Request finalized for privacy request: {privacy_request.id}",
708
+ action_type=privacy_request.policy.get_action_type(), # type: ignore
709
+ )
710
+
711
+ logger.info(
712
+ "Marking privacy request '{}' as complete.",
713
+ privacy_request.id,
714
+ )
715
+ AuditLog.create(
716
+ db=session,
717
+ data={
718
+ "user_id": "system",
719
+ "privacy_request_id": privacy_request.id,
720
+ "action": AuditLogAction.finished,
721
+ "message": "",
722
+ },
723
+ )
724
+ privacy_request.status = PrivacyRequestStatus.complete
725
+ privacy_request.finished_processing_at = datetime.utcnow()
726
+ privacy_request.save(db=session)
727
+
728
+ # Send a final email to the user confirming request completion
729
+ if privacy_request.status == PrivacyRequestStatus.complete:
730
+ legacy_request_completion_enabled = ConfigProxy(
731
+ session
732
+ ).notifications.send_request_completion_notification
733
+
734
+ action_type = (
735
+ MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS
736
+ if policy.get_rules_for_action(
737
+ action_type=ActionType.access
738
+ )
739
+ else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
740
+ )
741
+
742
+ if message_send_enabled(
743
+ session,
744
+ privacy_request.property_id,
745
+ action_type,
746
+ legacy_request_completion_enabled,
747
+ ) and not policy.get_rules_for_action(
748
+ action_type=ActionType.consent
749
+ ):
750
+ if not access_result_urls:
751
+ # For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
752
+ # from the database because the Privacy Request would have exited
753
+ # processing and lost access to the access_result_urls in memory
754
+ access_result_urls = (
755
+ privacy_request.access_result_urls or {}
756
+ ).get("access_result_urls", [])
757
+
758
+ try:
759
+ initiate_privacy_request_completion_email(
760
+ session,
761
+ policy,
762
+ access_result_urls,
763
+ identity_data,
764
+ privacy_request.property_id,
765
+ )
766
+ except (
767
+ IdentityNotFoundException,
768
+ MessageDispatchException,
769
+ ) as e:
770
+ privacy_request.error_processing(db=session)
771
+ # If dev mode, log traceback
772
+ _log_exception(e, CONFIG.dev_mode)
773
+ return
707
774
 
708
775
 
709
776
  def initiate_privacy_request_completion_email(
@@ -1000,3 +1067,25 @@ def run_webhooks_and_report_status(
1000
1067
  return False
1001
1068
 
1002
1069
  return True
1070
+
1071
+
1072
+ def _verify_masking_secrets(
1073
+ policy: Policy, privacy_request: PrivacyRequest, resume_step: Optional[CurrentStep]
1074
+ ) -> None:
1075
+ """
1076
+ Checks that the required masking secrets are still cached for the given request.
1077
+ Raises an exception if masking secrets are needed for the given policy but they don't exist.
1078
+ """
1079
+
1080
+ if resume_step is None:
1081
+ return
1082
+
1083
+ # if masking can be performed without any masking secrets, we skip the cache check
1084
+ if (
1085
+ policy.generate_masking_secrets()
1086
+ and not get_all_masking_secret_keys(privacy_request.id)
1087
+ and not privacy_request.masking_secrets
1088
+ ):
1089
+ raise MaskingSecretsExpired(
1090
+ f"The masking secrets for privacy request ID '{privacy_request.id}' have expired. Please submit a new erasure request."
1091
+ )
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  import json
4
4
  from asyncio import sleep
5
5
  from datetime import datetime, timedelta
6
- from typing import Any, Dict, List, Optional, Set
6
+ from typing import Any, Dict, Optional, Set
7
7
 
8
8
  from httpx import AsyncClient
9
9
  from loguru import logger
@@ -11,7 +11,6 @@ from sqlalchemy import text
11
11
  from sqlalchemy.sql.elements import TextClause
12
12
 
13
13
  from fides.api.common_exceptions import PrivacyRequestNotFound
14
- from fides.api.models.policy import Policy
15
14
  from fides.api.models.privacy_request import (
16
15
  EXITED_EXECUTION_LOG_STATUSES,
17
16
  PrivacyRequest,
@@ -19,14 +18,12 @@ from fides.api.models.privacy_request import (
19
18
  )
20
19
  from fides.api.models.worker_task import ExecutionLogStatus
21
20
  from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
22
- from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
23
21
  from fides.api.schemas.policy import ActionType
24
22
  from fides.api.schemas.privacy_request import (
25
23
  PrivacyRequestResponse,
26
24
  PrivacyRequestStatus,
27
25
  )
28
26
  from fides.api.schemas.redis_cache import Identity
29
- from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
30
27
  from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
31
28
  from fides.api.tasks.scheduled.scheduler import scheduler
32
29
  from fides.api.util.cache import (
@@ -69,7 +66,6 @@ def build_required_privacy_request_kwargs(
69
66
 
70
67
  def cache_data(
71
68
  privacy_request: PrivacyRequest,
72
- policy: Policy,
73
69
  identity: Identity,
74
70
  encryption_key: Optional[str],
75
71
  drp_request_body: Optional[DrpPrivacyRequestCreate],
@@ -82,23 +78,6 @@ def cache_data(
82
78
  privacy_request.cache_custom_privacy_request_fields(custom_privacy_request_fields)
83
79
  privacy_request.cache_encryption(encryption_key) # handles None already
84
80
 
85
- # Store masking secrets in the cache
86
- logger.info("Caching masking secrets for privacy request {}", privacy_request.id)
87
- erasure_rules = policy.get_rules_for_action(action_type=ActionType.erasure)
88
- unique_masking_strategies_by_name: Set[str] = set()
89
- for rule in erasure_rules:
90
- strategy_name: str = rule.masking_strategy["strategy"] # type: ignore
91
- configuration = rule.masking_strategy["configuration"] # type: ignore
92
- if strategy_name in unique_masking_strategies_by_name:
93
- continue
94
- unique_masking_strategies_by_name.add(strategy_name)
95
- masking_strategy = MaskingStrategy.get_strategy(strategy_name, configuration)
96
- if masking_strategy.secrets_required():
97
- masking_secrets: List[MaskingSecretCache] = (
98
- masking_strategy.generate_secrets_for_cache()
99
- )
100
- for masking_secret in masking_secrets:
101
- privacy_request.cache_masking_secret(masking_secret)
102
81
  if drp_request_body:
103
82
  privacy_request.cache_drp_request_body(drp_request_body)
104
83
 
File without changes
@@ -0,0 +1,109 @@
1
+ import operator as py_operator
2
+ from typing import Any, Union
3
+
4
+ from loguru import logger
5
+ from sqlalchemy.orm import Session
6
+
7
+ from fides.api.graph.config import FieldPath
8
+ from fides.api.task.conditional_dependencies.schemas import (
9
+ Condition,
10
+ ConditionGroup,
11
+ ConditionLeaf,
12
+ GroupOperator,
13
+ Operator,
14
+ )
15
+
16
+ operator_methods = {
17
+ Operator.exists: lambda a, _: a is not None,
18
+ Operator.not_exists: lambda a, _: a is None,
19
+ Operator.eq: py_operator.eq,
20
+ Operator.neq: py_operator.ne,
21
+ Operator.lt: lambda a, b: a < b if a is not None else False,
22
+ Operator.lte: lambda a, b: a <= b if a is not None else False,
23
+ Operator.gt: lambda a, b: a > b if a is not None else False,
24
+ Operator.gte: lambda a, b: a >= b if a is not None else False,
25
+ Operator.list_contains: lambda a, b: b in a if isinstance(a, list) else False,
26
+ Operator.not_in_list: lambda a, b: a not in b if isinstance(b, list) else True,
27
+ }
28
+
29
+
30
+ class ConditionEvaluator:
31
+ """Evaluates nested conditions for manual task creation"""
32
+
33
+ def __init__(self, db: Session):
34
+ self.db = db
35
+
36
+ def evaluate_rule(self, rule: Condition, data: Union[dict, Any]) -> bool:
37
+ """Evaluate a nested condition rule against input data"""
38
+ if isinstance(rule, ConditionLeaf):
39
+ return self._evaluate_leaf_condition(rule, data)
40
+ # ConditionGroup
41
+ return self._evaluate_group_condition(rule, data)
42
+
43
+ def _evaluate_leaf_condition(
44
+ self, condition: ConditionLeaf, data: Union[dict, Any]
45
+ ) -> bool:
46
+ """Evaluate a leaf condition against input data"""
47
+ actual_value = self._get_nested_value(data, condition.field_address.split("."))
48
+ # Apply operator and return result
49
+ return self._apply_operator(actual_value, condition.operator, condition.value)
50
+
51
+ def _evaluate_group_condition(
52
+ self, group: ConditionGroup, data: Union[dict, Any]
53
+ ) -> bool:
54
+ """Evaluate a group condition against input data"""
55
+ results = [
56
+ self.evaluate_rule(condition, data) for condition in group.conditions
57
+ ]
58
+
59
+ logical_operators = {GroupOperator.and_: all, GroupOperator.or_: any}
60
+ operator_func = logical_operators.get(group.logical_operator)
61
+
62
+ if operator_func is None:
63
+ logger.warning(f"Unknown logical operator: {group.logical_operator}")
64
+ return False
65
+
66
+ return operator_func(results)
67
+
68
+ def _get_nested_value(self, data: Union[dict, Any], keys: list[str]) -> Any:
69
+ """Get nested value from data using dot notation
70
+
71
+ Supports both simple dictionary access and Fides reference structures:
72
+ - Simple dict: data["user"]["name"]
73
+ - Fides FieldAddress: data.get_field_value(FieldAddress("dataset", "collection", "field_address"))
74
+ - Fides Collection: data.get_field_value(FieldPath("field_address", "subfield"))
75
+ """
76
+ if not keys:
77
+ return data
78
+
79
+ current = data
80
+
81
+ # Try Fides reference structures first
82
+ if hasattr(current, "get_field_value"):
83
+ try:
84
+ field_path = FieldPath(*keys) if len(keys) > 1 else FieldPath(keys[0])
85
+ return current.get_field_value(field_path)
86
+ except (AttributeError, ValueError):
87
+ pass
88
+
89
+ # Fall back to dictionary access
90
+ for key in keys:
91
+ if not isinstance(current, dict):
92
+ current = current.get(key, {}) if hasattr(current, "get") else None
93
+ else:
94
+ current = current.get(key, {})
95
+
96
+ return current if current != {} else None
97
+
98
+ def _apply_operator(
99
+ self, actual_value: Any, operator: Operator, expected_value: Any
100
+ ) -> bool:
101
+ """Apply operator to actual and expected values"""
102
+
103
+ # Get the method for the operator and execute it
104
+ operator_method = operator_methods.get(operator)
105
+ if operator_method is None:
106
+ logger.warning(f"Unknown operator: {operator}")
107
+ return False
108
+
109
+ return operator_method(actual_value, expected_value)
@@ -0,0 +1,54 @@
1
+ from enum import Enum
2
+ from typing import List, Optional, Union
3
+
4
+ from pydantic import BaseModel, Field, model_validator
5
+
6
+
7
+ class Operator(str, Enum):
8
+ eq = "eq"
9
+ neq = "neq"
10
+ lt = "lt"
11
+ lte = "lte"
12
+ gt = "gt"
13
+ gte = "gte"
14
+ exists = "exists"
15
+ not_exists = "not_exists"
16
+ list_contains = "list_contains"
17
+ not_in_list = "not_in_list"
18
+
19
+
20
+ class GroupOperator(str, Enum):
21
+ and_ = "and"
22
+ or_ = "or"
23
+
24
+
25
+ # Leaf condition (e.g., "user.name exists", "user.created_at >= 2024-01-01")
26
+ class ConditionLeaf(BaseModel):
27
+ field_address: str = Field(
28
+ description="Field path to check (e.g., 'user.name', 'billing.subscription.status')"
29
+ )
30
+ operator: Operator = Field(description="Operator to apply")
31
+ value: Optional[
32
+ Union[str, int, float, bool, List[Union[str, int, float, bool]]]
33
+ ] = Field(default=None, description="Expected value for comparison")
34
+
35
+
36
+ # Nested logical condition (AND, OR)
37
+ class ConditionGroup(BaseModel):
38
+ logical_operator: GroupOperator = Field(
39
+ description="Logical operator: 'and' or 'or'"
40
+ )
41
+ conditions: list["Condition"] = Field(
42
+ description="List of conditions or nested groups"
43
+ )
44
+
45
+ @model_validator(mode="after")
46
+ def validate_conditions(self) -> "ConditionGroup":
47
+ if not self.conditions:
48
+ raise ValueError("conditions list cannot be empty")
49
+ return self
50
+
51
+
52
+ # Use model_rebuild to allow recursive nesting
53
+ Condition = Union[ConditionLeaf, ConditionGroup]
54
+ ConditionGroup.model_rebuild()
@@ -33,8 +33,8 @@ from fides.api.models.worker_task import ExecutionLogStatus
33
33
  from fides.api.schemas.policy import ActionType
34
34
  from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
35
35
  from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
36
+ from fides.api.task.manual.manual_task_address import ManualTaskAddress
36
37
  from fides.api.task.manual.manual_task_utils import (
37
- ManualTaskAddress,
38
38
  create_manual_task_instances_for_privacy_request,
39
39
  )
40
40
  from fides.api.util.logger_context_utils import log_context
@@ -1,4 +1,5 @@
1
1
  # pylint: disable=too-many-lines
2
+ from functools import partial
2
3
  from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
3
4
 
4
5
  import dask
@@ -252,16 +253,33 @@ def run_erasure_request_deprecated( # pylint: disable = too-many-arguments
252
253
 
253
254
  access_request_data[ROOT_COLLECTION_ADDRESS.value] = [identity]
254
255
 
255
- dsk: Dict[CollectionAddress, Any] = {
256
- k: (
257
- t.erasure_request,
256
+ # Build the dask task graph for erasure, ensuring we also pass along the
257
+ # upstream access data that may be required by the connector (e.g. BigQuery
258
+ # delete statements). We accomplish this by partially applying the
259
+ # `inputs` kwarg on each task's `erasure_request` method. The resulting
260
+ # callable accepts the original positional arguments expected by Dask.
261
+
262
+ dsk: Dict[CollectionAddress, Any] = {}
263
+ for k, t in env.items():
264
+ # Collect upstream access data in the same order as the input keys
265
+ upstream_access_data: List[List[Row]] = [
266
+ access_request_data.get(str(input_key), [])
267
+ for input_key in t.execution_node.input_keys
268
+ ]
269
+
270
+ # Bind the `inputs` keyword argument so it is supplied when the task executes
271
+ erasure_fn_with_inputs = partial(
272
+ t.erasure_request, inputs=upstream_access_data
273
+ )
274
+
275
+ # Build the task tuple: (callable, retrieved_data, *prereqs)
276
+ dsk[k] = (
277
+ erasure_fn_with_inputs,
258
278
  access_request_data.get(
259
279
  str(k), []
260
- ), # Pass in the results of the access request for this collection
280
+ ), # Data retrieved for this collection
261
281
  *_evaluate_erasure_dependencies(t, erasure_end_nodes),
262
282
  )
263
- for k, t in env.items()
264
- }
265
283
 
266
284
  # root node returns 0 to be consistent with the output of the other erasure tasks
267
285
  dsk[ROOT_COLLECTION_ADDRESS] = 0
@@ -29,8 +29,8 @@ from fides.api.task.graph_task import (
29
29
  GraphTask,
30
30
  mark_current_and_downstream_nodes_as_failed,
31
31
  )
32
+ from fides.api.task.manual.manual_task_address import ManualTaskAddress
32
33
  from fides.api.task.manual.manual_task_graph_task import ManualTaskGraphTask
33
- from fides.api.task.manual.manual_task_utils import ManualTaskAddress
34
34
  from fides.api.task.task_resources import TaskResources
35
35
  from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
36
36
  from fides.api.util.cache import cache_task_tracking_key
@@ -298,17 +298,11 @@ def run_access_node(
298
298
  )
299
299
  # Currently, upstream tasks and "input keys" (which are built by data dependencies)
300
300
  # are the same, but they may not be the same in the future.
301
- ordered_upstream_tasks: List[Optional[RequestTask]] = (
302
- _order_tasks_by_input_key(
301
+ upstream_access_data: List[List[Row]] = (
302
+ _build_upstream_access_data(
303
303
  graph_task.execution_node.input_keys, upstream_results
304
304
  )
305
305
  )
306
- # Pass in access data dependencies in the same order as the input keys.
307
- # If we don't have access data for an upstream node, pass in an empty list
308
- upstream_access_data: List[List[Row]] = [
309
- upstream.get_access_data() if upstream else []
310
- for upstream in ordered_upstream_tasks
311
- ]
312
306
  # Run the main access function
313
307
  graph_task.access_request(*upstream_access_data)
314
308
 
@@ -359,7 +353,7 @@ def run_erasure_node(
359
353
  session,
360
354
  ) as resources:
361
355
  # Build GraphTask resource to facilitate execution
362
- graph_task: GraphTask = create_graph_task(
356
+ erasure_graph_task: GraphTask = create_graph_task(
363
357
  session, request_task, resources
364
358
  )
365
359
  # Get access data that was saved in the erasure format that was collected from the
@@ -368,8 +362,24 @@ def run_erasure_node(
368
362
  request_task.get_data_for_erasures() or []
369
363
  )
370
364
 
371
- # Run the main erasure function!
372
- graph_task.erasure_request(retrieved_data)
365
+ upstream_access_data: List[List[Row]] = []
366
+
367
+ try:
368
+ upstream_access_data = (
369
+ get_upstream_access_data_for_erasure_task(
370
+ request_task, session, resources
371
+ )
372
+ )
373
+ except Exception as e:
374
+ logger.error(
375
+ f"Unable to get upstream access data for erasure task {request_task.collection_address}: {e}"
376
+ )
377
+
378
+ # Run the main erasure function, passing along the upstream access data.
379
+ # The extra data is currently only needed for generating BigQuery delete statements.
380
+ erasure_graph_task.erasure_request(
381
+ retrieved_data, inputs=upstream_access_data
382
+ )
373
383
 
374
384
  queue_downstream_tasks_with_retries(
375
385
  self,
@@ -487,6 +497,73 @@ def _order_tasks_by_input_key(
487
497
  return tasks
488
498
 
489
499
 
500
+ def get_upstream_access_data_for_erasure_task(
501
+ erasure_request_task: RequestTask,
502
+ session: Session,
503
+ resources: TaskResources,
504
+ ) -> List[List[Row]]:
505
+ """
506
+ Retrieves upstream access data for a given erasure request task.
507
+
508
+ This function finds the corresponding access task for the erasure task,
509
+ creates a GraphTask to extract input keys, and builds the upstream access data
510
+ needed for erasure operations (particularly for BigQuery delete statements).
511
+
512
+ Args:
513
+ erasure_request_task: The erasure task that needs upstream access data
514
+ session: Database session for querying
515
+ resources: Task resources for creating GraphTask
516
+
517
+ Returns:
518
+ List[List[Row]]: Upstream access data ordered by input keys
519
+
520
+ Raises:
521
+ Exception: If the corresponding access task cannot be found
522
+ """
523
+ # Get the corresponding access task for the current erasure task
524
+ access_request_task = (
525
+ session.query(RequestTask)
526
+ .filter(
527
+ RequestTask.privacy_request_id == erasure_request_task.privacy_request_id,
528
+ RequestTask.collection_address == erasure_request_task.collection_address,
529
+ RequestTask.action_type == ActionType.access,
530
+ )
531
+ .first()
532
+ )
533
+
534
+ if not access_request_task:
535
+ raise Exception(
536
+ f"Unable to find access request task for erasure task {erasure_request_task.collection_address}"
537
+ )
538
+
539
+ # Convert the request task to a GraphTask to get the input_keys
540
+ access_graph_task: GraphTask = create_graph_task(
541
+ session, access_request_task, resources
542
+ )
543
+
544
+ # Build and return the upstream access data
545
+ return _build_upstream_access_data(
546
+ access_graph_task.execution_node.input_keys,
547
+ access_request_task.upstream_tasks_objects(session),
548
+ )
549
+
550
+
551
+ def _build_upstream_access_data(
552
+ input_keys: List[CollectionAddress],
553
+ upstream_tasks: Query,
554
+ ) -> List[List[Row]]:
555
+ """
556
+ Helper function to build the access data for the current node.
557
+ The access data is passed in the same order as the input keys.
558
+ If we don't have access data for an upstream node, return an empty list.
559
+ """
560
+
561
+ ordered_upstream: List[Optional[RequestTask]] = _order_tasks_by_input_key(
562
+ input_keys, upstream_tasks
563
+ )
564
+ return [task.get_access_data() if task else [] for task in ordered_upstream]
565
+
566
+
490
567
  mapping = {
491
568
  ActionType.access.value: run_access_node,
492
569
  ActionType.erasure.value: run_erasure_node,
@@ -6,7 +6,7 @@ from loguru import logger
6
6
 
7
7
  from fides.api.graph.config import CollectionAddress, FieldPath
8
8
  from fides.api.graph.graph import DatasetGraph
9
- from fides.api.task.manual.manual_task_utils import ManualTaskAddress
9
+ from fides.api.task.manual.manual_task_address import ManualTaskAddress
10
10
  from fides.api.util.collection_util import Row
11
11
 
12
12