ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0rc0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ethyca-fides might be problematic. Click here for more details.

Files changed (326) hide show
  1. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/METADATA +1 -1
  2. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/RECORD +292 -279
  3. fides/_version.py +3 -3
  4. fides/api/alembic/migrations/versions/7e9a2b52f498_adding_masking_secrets.py +60 -0
  5. fides/api/alembic/migrations/versions/a7065df4dcf1_add_finalized_fields_to_privacy_request.py +65 -0
  6. fides/api/alembic/migrations/versions/d0031087eacb_create_manualtaskconditionaldependency_.py +106 -0
  7. fides/api/api/v1/endpoints/dataset_config_endpoints.py +13 -5
  8. fides/api/api/v1/endpoints/drp_endpoints.py +7 -1
  9. fides/api/api/v1/endpoints/privacy_request_endpoints.py +44 -1
  10. fides/api/api/v1/endpoints/user_endpoints.py +83 -7
  11. fides/api/app_setup.py +3 -2
  12. fides/api/common_exceptions.py +4 -0
  13. fides/api/db/base.py +1 -0
  14. fides/api/db/database.py +1 -1
  15. fides/api/graph/execution.py +30 -0
  16. fides/api/graph/graph.py +13 -2
  17. fides/api/graph/traversal.py +126 -39
  18. fides/api/models/manual_task/__init__.py +2 -0
  19. fides/api/models/manual_task/conditional_dependency.py +144 -0
  20. fides/api/models/{manual_task.py → manual_task/manual_task.py} +10 -0
  21. fides/api/models/masking_secret.py +72 -0
  22. fides/api/models/policy.py +23 -0
  23. fides/api/models/privacy_request/execution_log.py +1 -0
  24. fides/api/models/privacy_request/privacy_request.py +31 -13
  25. fides/api/oauth/roles.py +2 -0
  26. fides/api/schemas/application_config.py +11 -1
  27. fides/api/schemas/masking/masking_secrets.py +1 -1
  28. fides/api/schemas/policy.py +1 -0
  29. fides/api/schemas/privacy_request.py +5 -0
  30. fides/api/service/connectors/base_connector.py +1 -0
  31. fides/api/service/connectors/bigquery_connector.py +67 -19
  32. fides/api/service/connectors/dynamodb_connector.py +2 -1
  33. fides/api/service/connectors/fides_connector.py +1 -0
  34. fides/api/service/connectors/http_connector.py +1 -0
  35. fides/api/service/connectors/manual_task_connector.py +1 -0
  36. fides/api/service/connectors/manual_webhook_connector.py +2 -1
  37. fides/api/service/connectors/mongodb_connector.py +1 -0
  38. fides/api/service/connectors/okta_connector.py +1 -0
  39. fides/api/service/connectors/query_configs/bigquery_query_config.py +91 -32
  40. fides/api/service/connectors/rds_mysql_connector.py +1 -0
  41. fides/api/service/connectors/rds_postgres_connector.py +1 -0
  42. fides/api/service/connectors/s3_connector.py +1 -0
  43. fides/api/service/connectors/saas_connector.py +1 -0
  44. fides/api/service/connectors/scylla_connector.py +1 -0
  45. fides/api/service/connectors/sql_connector.py +36 -4
  46. fides/api/service/connectors/website_connector.py +1 -0
  47. fides/api/service/privacy_request/dsr_package/templates/welcome.html +2 -2
  48. fides/api/service/privacy_request/request_runner_service.py +142 -53
  49. fides/api/service/privacy_request/request_service.py +1 -22
  50. fides/api/task/conditional_dependencies/__init__.py +0 -0
  51. fides/api/task/conditional_dependencies/evaluator.py +109 -0
  52. fides/api/task/conditional_dependencies/schemas.py +54 -0
  53. fides/api/task/create_request_tasks.py +1 -1
  54. fides/api/task/deprecated_graph_task.py +24 -6
  55. fides/api/task/execute_request_tasks.py +89 -12
  56. fides/api/task/filter_results.py +1 -1
  57. fides/api/task/graph_task.py +38 -3
  58. fides/api/task/manual/manual_task_address.py +46 -0
  59. fides/api/task/manual/manual_task_graph_task.py +118 -126
  60. fides/api/task/manual/manual_task_utils.py +52 -105
  61. fides/api/util/aws_util.py +5 -1
  62. fides/api/util/cache.py +5 -0
  63. fides/api/util/encryption/secrets_util.py +48 -18
  64. fides/common/api/scope_registry.py +3 -0
  65. fides/common/api/v1/urn_registry.py +1 -1
  66. fides/config/execution_settings.py +4 -0
  67. fides/config/utils.py +1 -0
  68. fides/service/privacy_request/privacy_request_service.py +6 -1
  69. fides/ui-build/static/admin/404.html +1 -1
  70. fides/ui-build/static/admin/_next/static/GKmhMPa_1gMto8JZO8ENy/_buildManifest.js +1 -0
  71. fides/ui-build/static/admin/_next/static/chunks/1316-2606e19807c08aa5.js +1 -0
  72. fides/ui-build/static/admin/_next/static/chunks/1467-8808ec8836e033f9.js +1 -0
  73. fides/ui-build/static/admin/_next/static/chunks/1817-b78b58ae3b75d75a.js +1 -0
  74. fides/ui-build/static/admin/_next/static/chunks/1975.7d4634a0e823a02b.js +1 -0
  75. fides/ui-build/static/admin/_next/static/chunks/{203-5a663f465ba26bb4.js → 203-cd78ea279cecba60.js} +1 -1
  76. fides/ui-build/static/admin/_next/static/chunks/2150-930ffaf2c4718edc.js +1 -0
  77. fides/ui-build/static/admin/_next/static/chunks/255-1bc0cbef7a59cdc6.js +1 -0
  78. fides/ui-build/static/admin/_next/static/chunks/{3450-0ba194991d0cca88.js → 3450-ca4ba70da999f264.js} +1 -1
  79. fides/ui-build/static/admin/_next/static/chunks/346-aa3b88efb85f2e28.js +1 -0
  80. fides/ui-build/static/admin/_next/static/chunks/3550-d04125c828d591a1.js +1 -0
  81. fides/ui-build/static/admin/_next/static/chunks/{3855-e172870d3e21b0dd.js → 3855-509ca7ac99b5eada.js} +1 -1
  82. fides/ui-build/static/admin/_next/static/chunks/{3872-46cebf7ec1b31a2b.js → 3872-0a0f0032ca39a93f.js} +1 -1
  83. fides/ui-build/static/admin/_next/static/chunks/409-ea70638a59296659.js +1 -0
  84. fides/ui-build/static/admin/_next/static/chunks/{4121-2bc09fc4ddbfe5cb.js → 4121-877e19d3fa078c7b.js} +1 -1
  85. fides/ui-build/static/admin/_next/static/chunks/{4230-60100f7ef3ddcde1.js → 4230-114e31621c19ea69.js} +1 -1
  86. fides/ui-build/static/admin/_next/static/chunks/4259.d1507e0db19cbed7.js +1 -0
  87. fides/ui-build/static/admin/_next/static/chunks/431-1db919f6569a4021.js +1 -0
  88. fides/ui-build/static/admin/_next/static/chunks/{4608-bbb7bf511a05c3c2.js → 4608-43acf39319177bee.js} +1 -1
  89. fides/ui-build/static/admin/_next/static/chunks/5163-e682273cd76a7d07.js +1 -0
  90. fides/ui-build/static/admin/_next/static/chunks/{5309-b2c4803370634ff8.js → 5309-ce5702b9faeaff55.js} +1 -1
  91. fides/ui-build/static/admin/_next/static/chunks/{905-ffdbd0b14167e8bd.js → 5596-4378b2927dae65b2.js} +3 -3
  92. fides/ui-build/static/admin/_next/static/chunks/5619-9b50cec521203989.js +1 -0
  93. fides/ui-build/static/admin/_next/static/chunks/{6084-7178ff6ea6822475.js → 6084-ddbad3149364725d.js} +1 -1
  94. fides/ui-build/static/admin/_next/static/chunks/6148-59a59d5c5925344f.js +1 -0
  95. fides/ui-build/static/admin/_next/static/chunks/6419-d0c00d661b01f8fa.js +1 -0
  96. fides/ui-build/static/admin/_next/static/chunks/{6662-507be5d46e5b719b.js → 6662-4392ba1e4c254ef7.js} +1 -1
  97. fides/ui-build/static/admin/_next/static/chunks/6780-fc7d9ddb1a03e7b3.js +1 -0
  98. fides/ui-build/static/admin/_next/static/chunks/{6853-2ad3e08fe6f9f5f2.js → 6853-d0190d2cca9dbde2.js} +1 -1
  99. fides/ui-build/static/admin/_next/static/chunks/{6882-6af16fef26c21e06.js → 6882-59ea739e3616ce83.js} +1 -1
  100. fides/ui-build/static/admin/_next/static/chunks/6954-9c4912fbce87c4df.js +1 -0
  101. fides/ui-build/static/admin/_next/static/chunks/{7476-281ee9a8286556f3.js → 7476-cc0d9a94ed7aad53.js} +1 -1
  102. fides/ui-build/static/admin/_next/static/chunks/{787-fb41002f797eb2df.js → 787-c57185ad89c4e288.js} +1 -1
  103. fides/ui-build/static/admin/_next/static/chunks/{79-7e87aff851423d4a.js → 79-2aab56be75e16187.js} +1 -1
  104. fides/ui-build/static/admin/_next/static/chunks/{796-329a5f823ec258a5.js → 796-3bdda2a7868464af.js} +1 -1
  105. fides/ui-build/static/admin/_next/static/chunks/8237-841439bef6682177.js +1 -0
  106. fides/ui-build/static/admin/_next/static/chunks/8765-f622a35b40a7ec63.js +1 -0
  107. fides/ui-build/static/admin/_next/static/chunks/{9046-5c4c22c375de25b1.js → 9046-04a8c092fef1cd83.js} +1 -1
  108. fides/ui-build/static/admin/_next/static/chunks/9187-7438242f0d380bb0.js +1 -0
  109. fides/ui-build/static/admin/_next/static/chunks/{9226-746771d47dff6266.js → 9226-2dcac54ab3fb94be.js} +1 -1
  110. fides/ui-build/static/admin/_next/static/chunks/9278-08cc704317fe535e.js +1 -0
  111. fides/ui-build/static/admin/_next/static/chunks/{9951-9b753ad7c3f51bdf.js → 9951-7c6639e5d062779e.js} +1 -1
  112. fides/ui-build/static/admin/_next/static/chunks/pages/{_app-39ccb07327c2c5d5.js → _app-4b5bff46158a19a3.js} +56 -56
  113. fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-98777246bec9dc2a.js → manual-2a655ff3a97f2492.js} +1 -1
  114. fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-a71c0aff4e0e6535.js → add-systems-0902f0bb4080643e.js} +1 -1
  115. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-1edf582ba3cd3bbb.js → [id]-f22ddd9b48a5c418.js} +1 -1
  116. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-06bb3b0bf097fcdb.js → new-e74cb5ea87f15b40.js} +1 -1
  117. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-685771e5f7196d87.js → privacy-experience-21f997c69fc3b4c2.js} +1 -1
  118. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-6ccedc70dc447089.js → [id]-da4124b7600a2a1d.js} +1 -1
  119. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-944bca1cc57985b5.js → new-a57d251c88ce68ae.js} +1 -1
  120. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-84f4bd14ce8673bc.js → privacy-notices-ad105181bc91209b.js} +1 -1
  121. fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-b0c4235fe6d0b0c8.js +1 -0
  122. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-11d52f1570759c4d.js → [resourceUrn]-aad6047a4604b945.js} +1 -1
  123. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-32eac8bbd217615a.js → projects-29784a11fe0fbd0a.js} +1 -1
  124. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-b83afa5565d0c84e.js → [resourceUrn]-b6b98cea25dd94fa.js} +1 -1
  125. fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-6f630d42ac9fb6b4.js → data-catalog-d5b01abcb76792ce.js} +1 -1
  126. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-7caea7bb58c1f153.js +1 -0
  127. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-a9a70856f7be1542.js +1 -0
  128. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-9ddb52ebb7ac4c71.js → action-center-1f0ea5c92ae9a2b4.js} +1 -1
  129. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-9aa744d56cdacb0d.js → activity-ad6a84a6276f914c.js} +1 -1
  130. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-14bd7500362ff224.js → [resourceUrn]-f98dd251babb7e28.js} +1 -1
  131. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9e7dfd5a6acc2e8f.js → discovery-56eb4c014f0d96a3.js} +1 -1
  132. fides/ui-build/static/admin/_next/static/chunks/pages/datamap-d23b3ae139f0428b.js +1 -0
  133. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-c0d2bfd465df20e0.js → [...subfieldNames]-15301bd6bf7cf718.js} +1 -1
  134. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-28280a8a39a6e37c.js → [collectionName]-0fa72873e464f581.js} +1 -1
  135. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-82fb246d87e58ebd.js → new-0d50084fbdf9b84c.js} +1 -1
  136. fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-20165c31ab1bc7cf.js → dataset-f3348d0a92543bab.js} +1 -1
  137. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-b4a6bcc87d126840.js → [id]-7d6027570d05c57f.js} +1 -1
  138. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-f95d7b0bbfc58f5a.js → new-8446418c7ad28f77.js} +1 -1
  139. fides/ui-build/static/admin/_next/static/chunks/pages/{fides-js-docs-5d8fd1af75f19e2f.js → fides-js-docs-1f4335dca5c09860.js} +1 -1
  140. fides/ui-build/static/admin/_next/static/chunks/pages/{index-1919aab9e5834b51.js → index-fbf9b845bb901238.js} +1 -1
  141. fides/ui-build/static/admin/_next/static/chunks/pages/integrations/{[id]-0a58aee2d1e7fa01.js → [id]-f53fe1f2cbebda7c.js} +1 -1
  142. fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-e2d5d7e2a5265e68.js → integrations-142abe3e3e3e5bf7.js} +1 -1
  143. fides/ui-build/static/admin/_next/static/chunks/pages/messaging-1bae386d8c190348.js +1 -0
  144. fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-69ad86b7a8a9a115.js → table-migration-05616e2ae20ff4f8.js} +1 -1
  145. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-1b6b0d703cf59389.js +1 -0
  146. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-f1d818242d8550f8.js +1 -0
  147. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-d40a26bddb126c5c.js +1 -0
  148. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-ccd8d9e06cf2d278.js +1 -0
  149. fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-fd1a67892056830a.js +1 -0
  150. fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-fc201657f4a782c7.js → [purpose_id]-e891d01ece59669e.js} +1 -1
  151. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-c2d39cba8396ef3a.js → consent-f61b87e79367865b.js} +1 -1
  152. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-d992103cc55901ae.js → custom-fields-f8eea5d508c60c64.js} +1 -1
  153. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-023e1895552817de.js → locations-ed6a140b362c5baa.js} +1 -1
  154. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-ac403c0886b20e20.js → organization-ff9a34264d48c35f.js} +1 -1
  155. fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-7a3396ac819c7904.js → test-datasets-a4b6d41ca679298b.js} +1 -1
  156. fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-8314a819837f5b2a.js → [id]-c8f5fbaa83dd9945.js} +1 -1
  157. fides/ui-build/static/admin/_next/static/chunks/pages/{systems-21f423a7c417aa9d.js → systems-c05b49ddec1a1b4f.js} +1 -1
  158. fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-df0d88716578e295.js +1 -0
  159. fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-da68efc31998dc66.js +1 -0
  160. fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-173ac3a1ed2b05a6.js → user-management-e98dfc7d4f2a4e16.js} +1 -1
  161. fides/ui-build/static/admin/_next/static/chunks/webpack-90e8ec1fc5c6455b.js +1 -0
  162. fides/ui-build/static/admin/_next/static/css/{5bfb2473e5701527.css → 23cf870196941c9a.css} +1 -1
  163. fides/ui-build/static/admin/_next/static/css/{94965f224bc991e9.css → 8bc1833f1fa53ff0.css} +1 -1
  164. fides/ui-build/static/admin/_next/static/css/d9924caa849931b3.css +1 -0
  165. fides/ui-build/static/admin/_next/static/css/dbcf63488933a4d5.css +29 -0
  166. fides/ui-build/static/admin/add-systems/manual.html +1 -1
  167. fides/ui-build/static/admin/add-systems/multiple.html +1 -1
  168. fides/ui-build/static/admin/add-systems.html +1 -1
  169. fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
  170. fides/ui-build/static/admin/consent/configure.html +1 -1
  171. fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
  172. fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
  173. fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
  174. fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
  175. fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
  176. fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
  177. fides/ui-build/static/admin/consent/properties.html +1 -1
  178. fides/ui-build/static/admin/consent/reporting.html +1 -1
  179. fides/ui-build/static/admin/consent.html +1 -1
  180. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
  181. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
  182. fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
  183. fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
  184. fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
  185. fides/ui-build/static/admin/data-catalog.html +1 -1
  186. fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
  187. fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
  188. fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
  189. fides/ui-build/static/admin/data-discovery/activity.html +1 -1
  190. fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
  191. fides/ui-build/static/admin/data-discovery/detection.html +1 -1
  192. fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
  193. fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
  194. fides/ui-build/static/admin/datamap.html +1 -1
  195. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
  196. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
  197. fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
  198. fides/ui-build/static/admin/dataset/new.html +1 -1
  199. fides/ui-build/static/admin/dataset.html +1 -1
  200. fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
  201. fides/ui-build/static/admin/datastore-connection/new.html +1 -1
  202. fides/ui-build/static/admin/datastore-connection.html +1 -1
  203. fides/ui-build/static/admin/index.html +1 -1
  204. fides/ui-build/static/admin/integrations/[id].html +1 -1
  205. fides/ui-build/static/admin/integrations.html +1 -1
  206. fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
  207. fides/ui-build/static/admin/lib/fides-headless.js +1 -1
  208. fides/ui-build/static/admin/lib/fides-preview.js +1 -1
  209. fides/ui-build/static/admin/lib/fides-tcf.js +3 -3
  210. fides/ui-build/static/admin/lib/fides.js +3 -3
  211. fides/ui-build/static/admin/login/[provider].html +1 -1
  212. fides/ui-build/static/admin/login.html +1 -1
  213. fides/ui-build/static/admin/messaging/[id].html +1 -1
  214. fides/ui-build/static/admin/messaging/add-template.html +1 -1
  215. fides/ui-build/static/admin/messaging.html +1 -1
  216. fides/ui-build/static/admin/poc/ant-components.html +1 -1
  217. fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
  218. fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
  219. fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
  220. fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
  221. fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
  222. fides/ui-build/static/admin/poc/forms.html +1 -1
  223. fides/ui-build/static/admin/poc/table-migration.html +1 -1
  224. fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
  225. fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
  226. fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
  227. fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
  228. fides/ui-build/static/admin/privacy-requests.html +1 -1
  229. fides/ui-build/static/admin/properties/[id].html +1 -1
  230. fides/ui-build/static/admin/properties/add-property.html +1 -1
  231. fides/ui-build/static/admin/properties.html +1 -1
  232. fides/ui-build/static/admin/reporting/datamap.html +1 -1
  233. fides/ui-build/static/admin/settings/about/alpha.html +1 -1
  234. fides/ui-build/static/admin/settings/about.html +1 -1
  235. fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
  236. fides/ui-build/static/admin/settings/consent.html +1 -1
  237. fides/ui-build/static/admin/settings/custom-fields.html +1 -1
  238. fides/ui-build/static/admin/settings/domain-records.html +1 -1
  239. fides/ui-build/static/admin/settings/domains.html +1 -1
  240. fides/ui-build/static/admin/settings/email-templates.html +1 -1
  241. fides/ui-build/static/admin/settings/locations.html +1 -1
  242. fides/ui-build/static/admin/settings/organization.html +1 -1
  243. fides/ui-build/static/admin/settings/regulations.html +1 -1
  244. fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
  245. fides/ui-build/static/admin/systems/configure/[id].html +1 -1
  246. fides/ui-build/static/admin/systems.html +1 -1
  247. fides/ui-build/static/admin/taxonomy.html +1 -1
  248. fides/ui-build/static/admin/user-management/new.html +1 -1
  249. fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
  250. fides/ui-build/static/admin/user-management.html +1 -1
  251. fides/ui-build/static/admin/_next/static/8108ANFxs99VY7KZ_Xev2/_buildManifest.js +0 -1
  252. fides/ui-build/static/admin/_next/static/chunks/1316-6cc72a45ebf7ff81.js +0 -1
  253. fides/ui-build/static/admin/_next/static/chunks/1807-3beab149351d5ded.js +0 -1
  254. fides/ui-build/static/admin/_next/static/chunks/1817-e601e737e3cc7a0e.js +0 -1
  255. fides/ui-build/static/admin/_next/static/chunks/255-7db55b0e3a0f9dea.js +0 -1
  256. fides/ui-build/static/admin/_next/static/chunks/2599-6c4d22e75028d8b6.js +0 -1
  257. fides/ui-build/static/admin/_next/static/chunks/2858-0b44609b6be7850b.js +0 -1
  258. fides/ui-build/static/admin/_next/static/chunks/2866-a73888c17a195cbe.js +0 -1
  259. fides/ui-build/static/admin/_next/static/chunks/3615-5e2d062d684b8fa1.js +0 -1
  260. fides/ui-build/static/admin/_next/static/chunks/409-a257e14acebcd73b.js +0 -1
  261. fides/ui-build/static/admin/_next/static/chunks/431-34f0b91c26f8d9ab.js +0 -1
  262. fides/ui-build/static/admin/_next/static/chunks/570-c99f07161bd339cd.js +0 -1
  263. fides/ui-build/static/admin/_next/static/chunks/6780-e3d40aa17a4bf2e9.js +0 -1
  264. fides/ui-build/static/admin/_next/static/chunks/6954-bb875d9ac89f6030.js +0 -1
  265. fides/ui-build/static/admin/_next/static/chunks/7062.fda15dcb7df85675.js +0 -1
  266. fides/ui-build/static/admin/_next/static/chunks/7c79804f.7a7112aece470725.js +0 -1
  267. fides/ui-build/static/admin/_next/static/chunks/9014-eeae6f581158e645.js +0 -1
  268. fides/ui-build/static/admin/_next/static/chunks/9278-9b1b5970f0702668.js +0 -1
  269. fides/ui-build/static/admin/_next/static/chunks/9392.f4520f66206d347d.js +0 -1
  270. fides/ui-build/static/admin/_next/static/chunks/ea076c0a.84423f606aef37cd.js +0 -1
  271. fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-afdbd4665657cfa1.js +0 -1
  272. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-2265ecb899d45fbc.js +0 -1
  273. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-5d522637871ac6c8.js +0 -1
  274. fides/ui-build/static/admin/_next/static/chunks/pages/datamap-7674b97d655c193b.js +0 -1
  275. fides/ui-build/static/admin/_next/static/chunks/pages/messaging-5094ffea13f32ed9.js +0 -1
  276. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-32600543eb7b584f.js +0 -1
  277. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-10ce53ea356f8bad.js +0 -1
  278. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-5501bbb129fee9c4.js +0 -1
  279. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-cbe4c8f9096b6543.js +0 -1
  280. fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-e130c0197362e8f3.js +0 -1
  281. fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-6387fcc8cce872eb.js +0 -1
  282. fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-ff5738706da07801.js +0 -1
  283. fides/ui-build/static/admin/_next/static/chunks/webpack-ff0cd6bff75588da.js +0 -1
  284. fides/ui-build/static/admin/_next/static/css/2cadb5f62dcd7c2b.css +0 -1
  285. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/WHEEL +0 -0
  286. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/entry_points.txt +0 -0
  287. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/licenses/LICENSE +0 -0
  288. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/top_level.txt +0 -0
  289. /fides/ui-build/static/admin/_next/static/{8108ANFxs99VY7KZ_Xev2 → GKmhMPa_1gMto8JZO8ENy}/_ssgManifest.js +0 -0
  290. /fides/ui-build/static/admin/_next/static/chunks/{2921-455e6357b74d2f76.js → 2921-86f1547ac40a5cdf.js} +0 -0
  291. /fides/ui-build/static/admin/_next/static/chunks/{3923-6cc911dafccc5f63.js → 3923-13a6b4da2d51bf8f.js} +0 -0
  292. /fides/ui-build/static/admin/_next/static/chunks/{401-1b529d5800aa1f3a.js → 401-3cc1fee61494e3bd.js} +0 -0
  293. /fides/ui-build/static/admin/_next/static/chunks/{5574-b13021775a15bfd2.js → 5574-9312f97b637d9ee2.js} +0 -0
  294. /fides/ui-build/static/admin/_next/static/chunks/{7630-9aac73191ed5ed13.js → 7630-b1c93688013ef013.js} +0 -0
  295. /fides/ui-build/static/admin/_next/static/chunks/{9826-111aaee8bd8dbd09.js → 9826-3c578665c6d3b21d.js} +0 -0
  296. /fides/ui-build/static/admin/_next/static/chunks/pages/{404-aece2c920ea14514.js → 404-2d803dab6a00f353.js} +0 -0
  297. /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-dc75dc6e37e52f05.js → multiple-8ff7f37913ad736a.js} +0 -0
  298. /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-24d226b5a8de5c74.js → add-vendors-d00c9034cdeb0236.js} +0 -0
  299. /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-6a8ef51138ac926a.js → configure-0e1ca0f4c8e7f4da.js} +0 -0
  300. /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-6f86ab63a08a6528.js → properties-057cad65e7414a44.js} +0 -0
  301. /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-73d3cbf68f7c3a31.js → consent-e17c56eec8d91371.js} +0 -0
  302. /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-6ba9e160dae64695.js → [projectUrn]-80a6cc8e8573514a.js} +0 -0
  303. /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{resources-7648bbd4f6711e4d.js → resources-6c3714ee97a718c1.js} +0 -0
  304. /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-393e20924c83373e.js → [resourceUrn]-31e6c54794a9883e.js} +0 -0
  305. /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-8733807dad4bc96e.js → detection-2822a423a7ad0550.js} +0 -0
  306. /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-006b695e5af5ef24.js → [datasetId]-a8e8b5f4ee7af86c.js} +0 -0
  307. /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-c391c6fad56eec48.js → datastore-connection-0f29b47402292070.js} +0 -0
  308. /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-53fecfb9dd6a1e0c.js → [id]-5627d0d0668077f9.js} +0 -0
  309. /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-76b01cec5fde10a9.js → add-template-feca66ad5c5fe54a.js} +0 -0
  310. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-5c08e8447c45ce44.js → ant-components-64a322d01aae5ca7.js} +0 -0
  311. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-06ad5f34585480aa.js → AntForm-8bca16a7726e7eb2.js} +0 -0
  312. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-6f071c2bc9446cb0.js → FormikAntFormItem-b0f246fc3b67ebf7.js} +0 -0
  313. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-efcc38c58991ac9e.js → FormikControlled-1a0852b090bfc392.js} +0 -0
  314. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-430ba5c979abfb7c.js → FormikField-11f3de1b45e36583.js} +0 -0
  315. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-5c561880bf131afb.js → forms-1b73a1c2b6c6285f.js} +0 -0
  316. /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-d888a69a3bbe040e.js → configure-e551a860ec727802.js} +0 -0
  317. /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-d3d8e3d7583ec635.js → [id]-dd99183f93763ae4.js} +0 -0
  318. /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-1af10ed303815d46.js → add-property-0bdbc1fcbf553b8f.js} +0 -0
  319. /fides/ui-build/static/admin/_next/static/chunks/pages/{properties-cebc0dc186be499a.js → properties-e959378bb32b6b73.js} +0 -0
  320. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-5e1322de868d615e.js → alpha-8f98a4895e74725e.js} +0 -0
  321. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-241f95e372b65d0f.js → about-8155a35a62fdb5ae.js} +0 -0
  322. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-41242f805599feda.js → domain-records-51333dbd21cb37c8.js} +0 -0
  323. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-2e885f74c92f669c.js → domains-bde86e5f6c09da5a.js} +0 -0
  324. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-ff112655ad5f41e5.js → email-templates-4f9a5cc8bea7725b.js} +0 -0
  325. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-86062a18e081a52a.js → regulations-102efd9199e87124.js} +0 -0
  326. /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-a2524414e968f862.js → new-bc4eb541906781e6.js} +0 -0
@@ -1,6 +1,8 @@
1
+ # pylint: disable=too-many-instance-attributes, too-many-branches, too-many-statements
1
2
  from __future__ import annotations
2
3
 
3
4
  import json
5
+ from collections import defaultdict
4
6
  from itertools import chain
5
7
  from typing import Any, Callable, Dict, List, Optional, Set, Tuple, cast
6
8
 
@@ -99,37 +101,70 @@ class BaseTraversal:
99
101
  self.traversal_node_dict = {k: TraversalNode(v) for k, v in graph.nodes.items()}
100
102
  self.edges: Set[Edge] = graph.edges.copy()
101
103
  self.root_node = artificial_traversal_node(ROOT_COLLECTION_ADDRESS)
104
+
105
+ # Pre-index edges by node address for O(1) lookup
106
+ self.edges_by_node: Dict[CollectionAddress, List[Edge]] = defaultdict(list)
107
+ for edge in self.edges:
108
+ self.edges_by_node[edge.f1.collection_address()].append(edge)
109
+ self.edges_by_node[edge.f2.collection_address()].append(edge)
110
+
111
+ # Pre-compute string versions of node dependencies
112
+ # This avoids expensive hash operations during traversal
113
+ self.node_after_str: Dict[str, Set[str]] = {}
114
+ self.dataset_after_str: Dict[str, Set[str]] = {}
115
+
116
+ for addr, traversal_node in self.traversal_node_dict.items():
117
+ # Collection-level after dependencies
118
+ self.node_after_str[addr.value] = {
119
+ dep.value for dep in traversal_node.node.collection.after
120
+ }
121
+ # Dataset-level after dependencies (need to find all collections in those datasets)
122
+ dataset_deps = set()
123
+ for dataset_name in traversal_node.node.dataset.after:
124
+ for other_addr in self.traversal_node_dict.keys():
125
+ if other_addr.dataset == dataset_name:
126
+ dataset_deps.add(other_addr.value)
127
+ self.dataset_after_str[addr.value] = dataset_deps
128
+
129
+ # Add root node to the pre-computed dependencies (it has no dependencies)
130
+ self.node_after_str[ROOT_COLLECTION_ADDRESS.value] = set()
131
+ self.dataset_after_str[ROOT_COLLECTION_ADDRESS.value] = set()
132
+
102
133
  for (
103
134
  start_field_address,
104
135
  seed_key,
105
136
  ) in self.extract_seed_field_addresses().items():
106
- self.edges.add(
107
- Edge(
108
- FieldAddress(
109
- ROOT_COLLECTION_ADDRESS.dataset,
110
- ROOT_COLLECTION_ADDRESS.collection,
111
- seed_key,
112
- ),
113
- start_field_address,
114
- )
137
+ edge = Edge(
138
+ FieldAddress(
139
+ ROOT_COLLECTION_ADDRESS.dataset,
140
+ ROOT_COLLECTION_ADDRESS.collection,
141
+ seed_key,
142
+ ),
143
+ start_field_address,
115
144
  )
145
+ self.edges.add(edge)
146
+ # Add to edge index
147
+ self.edges_by_node[ROOT_COLLECTION_ADDRESS].append(edge)
148
+ self.edges_by_node[start_field_address.collection_address()].append(edge)
116
149
 
117
150
  # Ensure manual_task collections execute right after ROOT
118
- from fides.api.task.manual.manual_task_utils import ManualTaskAddress
151
+ from fides.api.task.manual.manual_task_address import ManualTaskAddress
119
152
 
120
153
  for addr in self.traversal_node_dict.keys():
121
154
  if ManualTaskAddress.is_manual_task_address(addr):
122
155
  # Add a simple synthetic edge ROOT.id -> manual_data.id
123
- self.edges.add(
124
- Edge(
125
- FieldAddress(
126
- ROOT_COLLECTION_ADDRESS.dataset,
127
- ROOT_COLLECTION_ADDRESS.collection,
128
- "id",
129
- ),
130
- addr.field_address(FieldPath("id")),
131
- )
156
+ edge = Edge(
157
+ FieldAddress(
158
+ ROOT_COLLECTION_ADDRESS.dataset,
159
+ ROOT_COLLECTION_ADDRESS.collection,
160
+ "id",
161
+ ),
162
+ addr.field_address(FieldPath("id")),
132
163
  )
164
+ self.edges.add(edge)
165
+ # Add to edge index
166
+ self.edges_by_node[ROOT_COLLECTION_ADDRESS].append(edge)
167
+ self.edges_by_node[addr].append(edge)
133
168
 
134
169
  self._verify_traversal()
135
170
 
@@ -204,28 +239,52 @@ class BaseTraversal:
204
239
  logger.info(
205
240
  "Starting traversal",
206
241
  )
207
- remaining_node_keys: Set[CollectionAddress] = set(
208
- self.traversal_node_dict.keys()
209
- )
242
+
243
+ # Use string sets instead of CollectionAddress sets
244
+ # This avoids expensive hash operations
245
+ remaining_node_keys_str: Set[str] = {
246
+ addr.value for addr in self.traversal_node_dict.keys()
247
+ }
210
248
  finished_nodes: dict[CollectionAddress, TraversalNode] = {}
211
249
  running_node_queue: MatchingQueue[TraversalNode] = MatchingQueue(self.root_node)
212
250
 
213
- remaining_edges: Set[Edge] = self.edges.copy()
251
+ # Instead of copying entire edge set, use a more efficient approach
252
+ # We'll simulate Edge.delete_edges behavior without the expensive set operations
253
+ deleted_edges_tracker: Dict[Edge, bool] = {}
254
+
214
255
  while not running_node_queue.is_empty():
215
256
  # this is to support the "run traversal_node A AFTER traversal_node B functionality:"
216
257
  n = running_node_queue.pop_first_match(
217
- lambda x: x.can_run_given(remaining_node_keys)
258
+ lambda x: x.can_run_given_str(
259
+ remaining_node_keys_str, self.node_after_str, self.dataset_after_str
260
+ )
218
261
  )
219
262
 
220
263
  if n:
221
264
  node_run_fn(n, environment)
222
265
  # delete all edges between the traversal_node that's just run and any completed nodes
223
266
  for finished_node_address, finished_node in finished_nodes.items():
224
- completed_edges: Set[Edge] = Edge.delete_edges(
225
- remaining_edges,
226
- finished_node_address,
227
- cast(TraversalNode, n).address, # type: ignore[redundant-cast]
267
+ # Find edges to delete manually instead of using Edge.delete_edges
268
+ completed_edges: Set[Edge] = set()
269
+
270
+ # Only check edges connected to the relevant nodes
271
+ relevant_edges = set()
272
+ relevant_edges.update(
273
+ self.edges_by_node.get(finished_node_address, [])
228
274
  )
275
+ relevant_edges.update(self.edges_by_node.get(n.address, []))
276
+
277
+ for edge in relevant_edges:
278
+ # Skip if already deleted
279
+ if deleted_edges_tracker.get(edge, False):
280
+ continue
281
+
282
+ # Check if this edge spans between the two nodes (bidirectional check)
283
+ if edge.spans(
284
+ finished_node_address, cast(TraversalNode, n).address # type: ignore[redundant-cast]
285
+ ):
286
+ completed_edges.add(edge)
287
+ deleted_edges_tracker[edge] = True
229
288
 
230
289
  def edge_ends_with_collection(_edge: Edge) -> bool:
231
290
  # append edges that end in this traversal_node
@@ -236,14 +295,16 @@ class BaseTraversal:
236
295
  for edge in filter(edge_ends_with_collection, completed_edges):
237
296
  # note, this will not work for self-reference
238
297
  finished_node.add_child(n, edge)
298
+
239
299
  # next edges = take all edges including n that are _not_ in edges_from_completed_nodes
240
300
  # in the form (field_address_this, field_address_foreign)
241
301
 
302
+ # Use pre-indexed edges instead of iterating through all edges
242
303
  edges_to_children = pydash.collections.filter_(
243
304
  [
244
305
  e.split_by_address(cast(TraversalNode, n).address) # type: ignore[redundant-cast]
245
- for e in remaining_edges
246
- if e.contains(n.address)
306
+ for e in self.edges_by_node[n.address]
307
+ if not deleted_edges_tracker.get(e, False)
247
308
  ]
248
309
  )
249
310
  if not edges_to_children:
@@ -259,7 +320,7 @@ class BaseTraversal:
259
320
  self.traversal_node_dict[nxt_address]
260
321
  )
261
322
  finished_nodes[n.address] = n
262
- remaining_node_keys.difference_update({n.address})
323
+ remaining_node_keys_str.discard(n.address.value) # Use string value
263
324
  else:
264
325
  # traversal traversal_node dict diff finished nodes
265
326
  logger.error(
@@ -271,12 +332,17 @@ class BaseTraversal:
271
332
  [{', '.join([str(tn.address) for tn in running_node_queue.data])}]""",
272
333
  )
273
334
 
335
+ # Convert back to CollectionAddress set for filtering
274
336
  remaining_node_keys = {
275
337
  key
276
- for key in remaining_node_keys
277
- if not self.should_exclude_node(self.traversal_node_dict[key])
338
+ for key in self.traversal_node_dict.keys()
339
+ if key.value in remaining_node_keys_str
340
+ and not self.should_exclude_node(self.traversal_node_dict[key])
278
341
  }
279
342
 
343
+ # Update string set after filtering
344
+ remaining_node_keys_str = {key.value for key in remaining_node_keys}
345
+
280
346
  # error if there are nodes that have not been visited
281
347
  if remaining_node_keys:
282
348
  logger.error(
@@ -289,12 +355,16 @@ class BaseTraversal:
289
355
  )
290
356
 
291
357
  # filter out remaining_edges if the nodes they link are allowed to remain unreachable
292
- remaining_edges = {
293
- edge
294
- for edge in remaining_edges
295
- if edge.f1.collection_address() in remaining_node_keys
296
- and edge.f2.collection_address() in remaining_node_keys
297
- }
358
+ remaining_edges = set()
359
+ for node_key in remaining_node_keys:
360
+ for edge in self.edges_by_node.get(node_key, []):
361
+ if not deleted_edges_tracker.get(edge, False):
362
+ # Check if both ends of the edge are in remaining nodes
363
+ if (
364
+ edge.f1.collection_address() in remaining_node_keys
365
+ and edge.f2.collection_address() in remaining_node_keys
366
+ ):
367
+ remaining_edges.add(edge)
298
368
 
299
369
  # error if there are edges that have not been visited
300
370
  if remaining_edges:
@@ -478,6 +548,23 @@ class TraversalNode(Contextualizable):
478
548
  return False
479
549
  return True
480
550
 
551
+ def can_run_given_str(
552
+ self,
553
+ remaining_node_keys_str: Set[str],
554
+ node_after_str: Dict[str, Set[str]],
555
+ dataset_after_str: Dict[str, Set[str]],
556
+ ) -> bool:
557
+ """Optimized version using pre-computed string sets to avoid expensive hash operations."""
558
+ # Check collection-level dependencies
559
+ node_deps = node_after_str.get(self.address.value, set())
560
+ if node_deps & remaining_node_keys_str:
561
+ return False
562
+ # Check dataset-level dependencies
563
+ dataset_deps = dataset_after_str.get(self.address.value, set())
564
+ if dataset_deps & remaining_node_keys_str:
565
+ return False
566
+ return True
567
+
481
568
  def is_root_node(self) -> bool:
482
569
  """This traversal_node is the defined traversal start"""
483
570
  return self.address == ROOT_COLLECTION_ADDRESS
@@ -0,0 +1,2 @@
1
+ from .conditional_dependency import *
2
+ from .manual_task import *
@@ -0,0 +1,144 @@
1
+ from enum import Enum
2
+ from typing import TYPE_CHECKING, Optional, Union
3
+
4
+ from sqlalchemy import Column, ForeignKey, Index, Integer, String
5
+ from sqlalchemy.dialects.postgresql import JSONB
6
+ from sqlalchemy.ext.declarative import declared_attr
7
+ from sqlalchemy.orm import Session, relationship
8
+
9
+ from fides.api.db.base_class import Base, FidesBase
10
+ from fides.api.db.util import EnumColumn
11
+ from fides.api.task.conditional_dependencies.schemas import (
12
+ ConditionGroup,
13
+ ConditionLeaf,
14
+ )
15
+
16
+ if TYPE_CHECKING:
17
+ from fides.api.models.manual_task.manual_task import ManualTask
18
+
19
+
20
+ class ManualTaskConditionalDependencyType(str, Enum):
21
+ """Enum for manual task conditional dependency types.
22
+
23
+ This enum defines the two types of nodes in a conditional dependency tree:
24
+
25
+ - leaf: A terminal node that represents a single condition (e.g., "user.age >= 18")
26
+ - group: A non-terminal node that groups multiple conditions with logical operators (AND/OR)
27
+
28
+ Examples:
29
+ leaf: Used for simple field comparisons like:
30
+ - "user.name exists"
31
+ - "user.age >= 18"
32
+ - "billing.subscription.status == 'active'"
33
+
34
+ group: Used to combine multiple conditions with logical operators:
35
+ - AND group: "user.age >= 18 AND user.active == true"
36
+ - OR group: "user.role == 'admin' OR user.verified == true"
37
+ - Nested groups: "(user.age >= 18 AND (user.role == 'admin' OR user.verified == true))"
38
+ """
39
+
40
+ leaf = "leaf"
41
+ group = "group"
42
+
43
+
44
+ class ManualTaskConditionalDependency(Base):
45
+ """Model for storing conditional dependencies."""
46
+
47
+ @declared_attr
48
+ def __tablename__(cls) -> str:
49
+ """Overriding base class method to set the table name."""
50
+ return "manual_task_conditional_dependency"
51
+
52
+ # We need to redefine it here so that self-referential relationships
53
+ # can properly reference the `id` column instead of the built-in Python function.
54
+ id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
55
+
56
+ # Foreign key relationships
57
+ manual_task_id = Column(
58
+ String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
59
+ )
60
+ parent_id = Column(
61
+ String,
62
+ ForeignKey("manual_task_conditional_dependency.id", ondelete="CASCADE"),
63
+ nullable=True,
64
+ )
65
+
66
+ # Condition metadata
67
+ condition_type = Column(
68
+ EnumColumn(ManualTaskConditionalDependencyType), nullable=False
69
+ ) # leaf or group
70
+ field_address = Column(String, nullable=True) # For leaf conditions
71
+ operator = Column(String, nullable=True) # For leaf conditions
72
+ value = Column(JSONB, nullable=True) # For leaf conditions
73
+ logical_operator = Column(String, nullable=True) # 'and' or 'or' for groups
74
+
75
+ # Ordering
76
+ sort_order = Column(Integer, nullable=False, default=0)
77
+
78
+ __table_args__ = (
79
+ Index("ix_manual_task_conditional_dependency_manual_task_id", "manual_task_id"),
80
+ Index("ix_manual_task_conditional_dependency_parent_id", "parent_id"),
81
+ Index("ix_manual_task_conditional_dependency_condition_type", "condition_type"),
82
+ Index("ix_manual_task_conditional_dependency_sort_order", "sort_order"),
83
+ )
84
+
85
+ # Relationships
86
+ task = relationship("ManualTask", back_populates="conditional_dependencies")
87
+ parent = relationship(
88
+ "ManualTaskConditionalDependency",
89
+ remote_side=[id],
90
+ back_populates="children",
91
+ foreign_keys=[parent_id],
92
+ )
93
+ children = relationship(
94
+ "ManualTaskConditionalDependency",
95
+ back_populates="parent",
96
+ cascade="all, delete-orphan",
97
+ foreign_keys=[parent_id],
98
+ )
99
+
100
+ def to_condition_leaf(self) -> ConditionLeaf:
101
+ """Convert to ConditionLeaf if this is a leaf condition"""
102
+ if self.condition_type != "leaf":
103
+ raise ValueError("Cannot convert group condition to leaf")
104
+
105
+ return ConditionLeaf(
106
+ field_address=self.field_address, operator=self.operator, value=self.value
107
+ )
108
+
109
+ def to_condition_group(self) -> ConditionGroup:
110
+ """Convert to ConditionGroup if this is a group condition"""
111
+ if self.condition_type != "group":
112
+ raise ValueError("Cannot convert leaf condition to group")
113
+
114
+ # Recursively build children
115
+ child_conditions = []
116
+ children_list = [child for child in self.children] # type: ignore[attr-defined]
117
+ for child in sorted(children_list, key=lambda x: x.sort_order):
118
+ if child.condition_type == "leaf":
119
+ child_conditions.append(child.to_condition_leaf())
120
+ else:
121
+ child_conditions.append(child.to_condition_group())
122
+
123
+ return ConditionGroup(
124
+ logical_operator=self.logical_operator, conditions=child_conditions
125
+ )
126
+
127
+ @classmethod
128
+ def get_root_condition(
129
+ cls, db: Session, manual_task_id: str
130
+ ) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
131
+ """Get the root condition for a config"""
132
+ root = (
133
+ db.query(cls)
134
+ .filter(cls.manual_task_id == manual_task_id, cls.parent_id.is_(None))
135
+ .first()
136
+ )
137
+
138
+ if not root:
139
+ return None
140
+
141
+ if root.condition_type == "leaf":
142
+ return root.to_condition_leaf()
143
+
144
+ return root.to_condition_group()
@@ -26,6 +26,9 @@ from fides.api.schemas.base_class import FidesSchema
26
26
  if TYPE_CHECKING:
27
27
  from fides.api.models.attachment import Attachment
28
28
  from fides.api.models.fides_user import FidesUser
29
+ from fides.api.models.manual_task.conditional_dependency import (
30
+ ManualTaskConditionalDependency,
31
+ )
29
32
 
30
33
  # ------------------------------------------------------------
31
34
  # Enums
@@ -242,6 +245,13 @@ class ManualTask(Base):
242
245
  viewonly=True, # No cascade delete - submissions are historical data
243
246
  )
244
247
 
248
+ conditional_dependencies = relationship(
249
+ "ManualTaskConditionalDependency",
250
+ back_populates="task",
251
+ uselist=True,
252
+ cascade="all, delete-orphan",
253
+ )
254
+
245
255
  # Properties
246
256
  @property
247
257
  def assigned_users(self) -> list[str]:
@@ -0,0 +1,72 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, Any, Union
4
+ from urllib.parse import unquote_to_bytes
5
+
6
+ from sqlalchemy import Column, ForeignKey, String
7
+ from sqlalchemy.ext.declarative import declared_attr
8
+ from sqlalchemy.orm import Session, relationship
9
+ from sqlalchemy_utils.types.encrypted.encrypted_type import (
10
+ AesGcmEngine,
11
+ StringEncryptedType,
12
+ )
13
+
14
+ from fides.api.db.base_class import Base, JSONTypeOverride
15
+ from fides.api.util.custom_json_encoder import ENCODED_BYTES_PREFIX
16
+ from fides.config import CONFIG
17
+
18
+ if TYPE_CHECKING:
19
+ from fides.api.models.privacy_request import PrivacyRequest
20
+
21
+
22
+ class MaskingSecret(Base):
23
+ """SQLAlchemy model for storing secret caching information"""
24
+
25
+ @declared_attr
26
+ def __tablename__(self) -> str:
27
+ return "masking_secret"
28
+
29
+ privacy_request_id = Column(
30
+ String, ForeignKey("privacyrequest.id", ondelete="CASCADE"), nullable=False
31
+ )
32
+ secret = Column(
33
+ StringEncryptedType(
34
+ JSONTypeOverride,
35
+ CONFIG.security.app_encryption_key,
36
+ AesGcmEngine,
37
+ "pkcs5",
38
+ ),
39
+ nullable=False,
40
+ )
41
+ masking_strategy = Column(String, nullable=False)
42
+ secret_type = Column(String, nullable=False)
43
+ privacy_request = relationship("PrivacyRequest", back_populates="masking_secrets")
44
+
45
+ def set_secret(self, secret: Union[str, bytes]) -> None:
46
+ """Set the secret value, handling both string and bytes types"""
47
+ if isinstance(secret, str):
48
+ self.secret = secret.encode("utf-8")
49
+ elif isinstance(secret, bytes):
50
+ self.secret = secret
51
+ else:
52
+ raise ValueError("Secret must be either string or bytes")
53
+
54
+ def get_secret(self) -> Union[str, bytes]:
55
+ """Retrieve the secret in its original type"""
56
+ secret = self.secret
57
+ if isinstance(secret, str) and secret.startswith(ENCODED_BYTES_PREFIX):
58
+ return unquote_to_bytes(secret)[len(ENCODED_BYTES_PREFIX) :]
59
+ return secret
60
+
61
+ @classmethod
62
+ def create(
63
+ cls,
64
+ db: Session,
65
+ *,
66
+ data: dict[str, Any],
67
+ check_name: bool = True,
68
+ ) -> "MaskingSecret":
69
+ """
70
+ Create a new masking secret. Handles both string and bytes secrets automatically, encrypting them for storage.
71
+ """
72
+ return super().create(db=db, data=data, check_name=check_name)
@@ -25,7 +25,9 @@ from fides.api.models.client import ClientDetail
25
25
  from fides.api.models.connectionconfig import ConnectionConfig
26
26
  from fides.api.models.sql_models import DataCategory # type: ignore
27
27
  from fides.api.models.storage import StorageConfig, get_active_default_storage_config
28
+ from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
28
29
  from fides.api.schemas.policy import ActionType, DrpAction
30
+ from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
29
31
  from fides.api.util.data_category import _validate_data_category
30
32
  from fides.config import CONFIG
31
33
 
@@ -140,6 +142,27 @@ class Policy(Base):
140
142
  except IndexError:
141
143
  return None
142
144
 
145
+ def generate_masking_secrets(self) -> Optional[List[MaskingSecretCache]]:
146
+ """
147
+ Returns a list of masking secrets for the masking strategies in the policy.
148
+ """
149
+
150
+ masking_secrets: List[MaskingSecretCache] = []
151
+ erasure_rules = self.get_rules_for_action(action_type=ActionType.erasure)
152
+ unique_masking_strategies_by_name: Set[str] = set()
153
+ for rule in erasure_rules:
154
+ strategy_name: str = rule.masking_strategy["strategy"] # type: ignore
155
+ configuration = rule.masking_strategy["configuration"] # type: ignore
156
+ if strategy_name in unique_masking_strategies_by_name:
157
+ continue
158
+ unique_masking_strategies_by_name.add(strategy_name)
159
+ masking_strategy = MaskingStrategy.get_strategy(
160
+ strategy_name, configuration
161
+ )
162
+ if masking_strategy.secrets_required():
163
+ masking_secrets.extend(masking_strategy.generate_secrets_for_cache())
164
+ return masking_secrets
165
+
143
166
  def applies_to(self, node: "TraversalNode") -> bool:
144
167
  """
145
168
  Returns True if any data category in the traversal node starts with any of the policy's target categories.
@@ -24,6 +24,7 @@ EXECUTION_CHECKPOINTS = [
24
24
  CurrentStep.finalize_consent,
25
25
  CurrentStep.email_post_send,
26
26
  CurrentStep.post_webhooks,
27
+ CurrentStep.finalization,
27
28
  ]
28
29
 
29
30
  COMPLETED_EXECUTION_LOG_STATUSES = [
@@ -50,6 +50,7 @@ from fides.api.models.comment import Comment, CommentReference, CommentReference
50
50
  from fides.api.models.fides_user import FidesUser
51
51
  from fides.api.models.field_types import EncryptedLargeDataDescriptor
52
52
  from fides.api.models.manual_webhook import AccessManualWebhook
53
+ from fides.api.models.masking_secret import MaskingSecret
53
54
  from fides.api.models.policy import (
54
55
  Policy,
55
56
  PolicyPreWebhook,
@@ -98,7 +99,6 @@ from fides.api.util.cache import (
98
99
  get_drp_request_body_cache_key,
99
100
  get_encryption_cache_key,
100
101
  get_identity_cache_key,
101
- get_masking_secret_cache_key,
102
102
  )
103
103
  from fides.api.util.collection_util import Row, extract_key_for_address
104
104
  from fides.api.util.constants import API_DATE_FORMAT
@@ -145,6 +145,12 @@ class PrivacyRequest(
145
145
  ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
146
146
  nullable=True,
147
147
  )
148
+ finalized_at = Column(DateTime(timezone=True), nullable=True)
149
+ finalized_by = Column(
150
+ String,
151
+ ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
152
+ nullable=True,
153
+ )
148
154
  submitted_by = Column(
149
155
  String,
150
156
  ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
@@ -289,6 +295,13 @@ class PrivacyRequest(
289
295
  order_by="RequestTask.created_at",
290
296
  )
291
297
 
298
+ masking_secrets: "RelationshipProperty[List[MaskingSecret]]" = relationship(
299
+ "MaskingSecret",
300
+ back_populates="privacy_request",
301
+ uselist=True,
302
+ passive_deletes="all",
303
+ )
304
+
292
305
  @property
293
306
  def days_left(self: PrivacyRequest) -> Union[int, None]:
294
307
  if self.due_date is None:
@@ -566,19 +579,24 @@ class PrivacyRequest(
566
579
  encryption_key,
567
580
  )
568
581
 
569
- def cache_masking_secret(self, masking_secret: MaskingSecretCache) -> None:
570
- """Sets masking encryption secrets in the Fides app cache if provided"""
571
- if not masking_secret:
582
+ def persist_masking_secrets(
583
+ self, masking_secrets: List[MaskingSecretCache]
584
+ ) -> None:
585
+ """Persists masking encryption secrets to database."""
586
+ if not masking_secrets:
572
587
  return
573
- cache: FidesopsRedis = get_cache()
574
- cache.set_with_autoexpire(
575
- get_masking_secret_cache_key(
576
- self.id,
577
- masking_strategy=masking_secret.masking_strategy,
578
- secret_type=masking_secret.secret_type,
579
- ),
580
- FidesopsRedis.encode_obj(masking_secret.secret),
581
- )
588
+
589
+ session = Session.object_session(self)
590
+ for masking_secret in masking_secrets:
591
+ MaskingSecret.create(
592
+ db=session,
593
+ data={
594
+ "privacy_request_id": self.id,
595
+ "secret": masking_secret.secret,
596
+ "masking_strategy": masking_secret.masking_strategy,
597
+ "secret_type": masking_secret.secret_type,
598
+ },
599
+ )
582
600
 
583
601
  def get_cached_identity_data(self) -> Dict[str, Any]:
584
602
  """Retrieves any identity data pertaining to this request from the cache"""
fides/api/oauth/roles.py CHANGED
@@ -47,6 +47,7 @@ from fides.common.api.scope_registry import (
47
47
  SYSTEM_READ,
48
48
  USER_PERMISSION_ASSIGN_OWNERS,
49
49
  USER_READ,
50
+ USER_READ_OWN,
50
51
  WEBHOOK_READ,
51
52
  )
52
53
 
@@ -126,6 +127,7 @@ viewer_scopes = [ # Intentionally omitted USER_PERMISSION_READ and PRIVACY_REQU
126
127
 
127
128
  respondent_scopes = [
128
129
  PRIVACY_REQUEST_MANUAL_STEPS_RESPOND, # allows respondents to respond to assigned manual steps
130
+ USER_READ_OWN,
129
131
  ]
130
132
 
131
133
  external_respondent_scopes = [
@@ -11,6 +11,14 @@ from fides.api.schemas.messaging.messaging import MessagingServiceType
11
11
  from fides.config.admin_ui_settings import ErrorNotificationMode
12
12
 
13
13
 
14
+ class SqlDryRunMode(str, Enum):
15
+ """SQL dry run mode for controlling execution of SQL statements in privacy requests"""
16
+
17
+ none = "none"
18
+ access = "access"
19
+ erasure = "erasure"
20
+
21
+
14
22
  class StorageTypeApiAccepted(Enum):
15
23
  """Enum for storage destination types accepted in API updates"""
16
24
 
@@ -62,7 +70,9 @@ class ExecutionApplicationConfig(FidesSchema):
62
70
  subject_identity_verification_required: Optional[bool] = None
63
71
  disable_consent_identity_verification: Optional[bool] = None
64
72
  require_manual_request_approval: Optional[bool] = None
65
- model_config = ConfigDict(extra="forbid")
73
+ sql_dry_run: Optional[SqlDryRunMode] = None
74
+
75
+ model_config = ConfigDict(use_enum_values=True, extra="forbid")
66
76
 
67
77
 
68
78
  class AdminUIConfig(FidesSchema):
@@ -5,7 +5,7 @@ from typing import Callable, Generic, TypeVar
5
5
  T = TypeVar("T")
6
6
 
7
7
 
8
- class SecretType(Enum):
8
+ class SecretType(str, Enum):
9
9
  """Enum that holds all possible types of secrets across all masking strategies"""
10
10
 
11
11
  key = "key"