ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0rc0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ethyca-fides might be problematic. Click here for more details.

Files changed (326) hide show
  1. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/METADATA +1 -1
  2. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/RECORD +292 -279
  3. fides/_version.py +3 -3
  4. fides/api/alembic/migrations/versions/7e9a2b52f498_adding_masking_secrets.py +60 -0
  5. fides/api/alembic/migrations/versions/a7065df4dcf1_add_finalized_fields_to_privacy_request.py +65 -0
  6. fides/api/alembic/migrations/versions/d0031087eacb_create_manualtaskconditionaldependency_.py +106 -0
  7. fides/api/api/v1/endpoints/dataset_config_endpoints.py +13 -5
  8. fides/api/api/v1/endpoints/drp_endpoints.py +7 -1
  9. fides/api/api/v1/endpoints/privacy_request_endpoints.py +44 -1
  10. fides/api/api/v1/endpoints/user_endpoints.py +83 -7
  11. fides/api/app_setup.py +3 -2
  12. fides/api/common_exceptions.py +4 -0
  13. fides/api/db/base.py +1 -0
  14. fides/api/db/database.py +1 -1
  15. fides/api/graph/execution.py +30 -0
  16. fides/api/graph/graph.py +13 -2
  17. fides/api/graph/traversal.py +126 -39
  18. fides/api/models/manual_task/__init__.py +2 -0
  19. fides/api/models/manual_task/conditional_dependency.py +144 -0
  20. fides/api/models/{manual_task.py → manual_task/manual_task.py} +10 -0
  21. fides/api/models/masking_secret.py +72 -0
  22. fides/api/models/policy.py +23 -0
  23. fides/api/models/privacy_request/execution_log.py +1 -0
  24. fides/api/models/privacy_request/privacy_request.py +31 -13
  25. fides/api/oauth/roles.py +2 -0
  26. fides/api/schemas/application_config.py +11 -1
  27. fides/api/schemas/masking/masking_secrets.py +1 -1
  28. fides/api/schemas/policy.py +1 -0
  29. fides/api/schemas/privacy_request.py +5 -0
  30. fides/api/service/connectors/base_connector.py +1 -0
  31. fides/api/service/connectors/bigquery_connector.py +67 -19
  32. fides/api/service/connectors/dynamodb_connector.py +2 -1
  33. fides/api/service/connectors/fides_connector.py +1 -0
  34. fides/api/service/connectors/http_connector.py +1 -0
  35. fides/api/service/connectors/manual_task_connector.py +1 -0
  36. fides/api/service/connectors/manual_webhook_connector.py +2 -1
  37. fides/api/service/connectors/mongodb_connector.py +1 -0
  38. fides/api/service/connectors/okta_connector.py +1 -0
  39. fides/api/service/connectors/query_configs/bigquery_query_config.py +91 -32
  40. fides/api/service/connectors/rds_mysql_connector.py +1 -0
  41. fides/api/service/connectors/rds_postgres_connector.py +1 -0
  42. fides/api/service/connectors/s3_connector.py +1 -0
  43. fides/api/service/connectors/saas_connector.py +1 -0
  44. fides/api/service/connectors/scylla_connector.py +1 -0
  45. fides/api/service/connectors/sql_connector.py +36 -4
  46. fides/api/service/connectors/website_connector.py +1 -0
  47. fides/api/service/privacy_request/dsr_package/templates/welcome.html +2 -2
  48. fides/api/service/privacy_request/request_runner_service.py +142 -53
  49. fides/api/service/privacy_request/request_service.py +1 -22
  50. fides/api/task/conditional_dependencies/__init__.py +0 -0
  51. fides/api/task/conditional_dependencies/evaluator.py +109 -0
  52. fides/api/task/conditional_dependencies/schemas.py +54 -0
  53. fides/api/task/create_request_tasks.py +1 -1
  54. fides/api/task/deprecated_graph_task.py +24 -6
  55. fides/api/task/execute_request_tasks.py +89 -12
  56. fides/api/task/filter_results.py +1 -1
  57. fides/api/task/graph_task.py +38 -3
  58. fides/api/task/manual/manual_task_address.py +46 -0
  59. fides/api/task/manual/manual_task_graph_task.py +118 -126
  60. fides/api/task/manual/manual_task_utils.py +52 -105
  61. fides/api/util/aws_util.py +5 -1
  62. fides/api/util/cache.py +5 -0
  63. fides/api/util/encryption/secrets_util.py +48 -18
  64. fides/common/api/scope_registry.py +3 -0
  65. fides/common/api/v1/urn_registry.py +1 -1
  66. fides/config/execution_settings.py +4 -0
  67. fides/config/utils.py +1 -0
  68. fides/service/privacy_request/privacy_request_service.py +6 -1
  69. fides/ui-build/static/admin/404.html +1 -1
  70. fides/ui-build/static/admin/_next/static/GKmhMPa_1gMto8JZO8ENy/_buildManifest.js +1 -0
  71. fides/ui-build/static/admin/_next/static/chunks/1316-2606e19807c08aa5.js +1 -0
  72. fides/ui-build/static/admin/_next/static/chunks/1467-8808ec8836e033f9.js +1 -0
  73. fides/ui-build/static/admin/_next/static/chunks/1817-b78b58ae3b75d75a.js +1 -0
  74. fides/ui-build/static/admin/_next/static/chunks/1975.7d4634a0e823a02b.js +1 -0
  75. fides/ui-build/static/admin/_next/static/chunks/{203-5a663f465ba26bb4.js → 203-cd78ea279cecba60.js} +1 -1
  76. fides/ui-build/static/admin/_next/static/chunks/2150-930ffaf2c4718edc.js +1 -0
  77. fides/ui-build/static/admin/_next/static/chunks/255-1bc0cbef7a59cdc6.js +1 -0
  78. fides/ui-build/static/admin/_next/static/chunks/{3450-0ba194991d0cca88.js → 3450-ca4ba70da999f264.js} +1 -1
  79. fides/ui-build/static/admin/_next/static/chunks/346-aa3b88efb85f2e28.js +1 -0
  80. fides/ui-build/static/admin/_next/static/chunks/3550-d04125c828d591a1.js +1 -0
  81. fides/ui-build/static/admin/_next/static/chunks/{3855-e172870d3e21b0dd.js → 3855-509ca7ac99b5eada.js} +1 -1
  82. fides/ui-build/static/admin/_next/static/chunks/{3872-46cebf7ec1b31a2b.js → 3872-0a0f0032ca39a93f.js} +1 -1
  83. fides/ui-build/static/admin/_next/static/chunks/409-ea70638a59296659.js +1 -0
  84. fides/ui-build/static/admin/_next/static/chunks/{4121-2bc09fc4ddbfe5cb.js → 4121-877e19d3fa078c7b.js} +1 -1
  85. fides/ui-build/static/admin/_next/static/chunks/{4230-60100f7ef3ddcde1.js → 4230-114e31621c19ea69.js} +1 -1
  86. fides/ui-build/static/admin/_next/static/chunks/4259.d1507e0db19cbed7.js +1 -0
  87. fides/ui-build/static/admin/_next/static/chunks/431-1db919f6569a4021.js +1 -0
  88. fides/ui-build/static/admin/_next/static/chunks/{4608-bbb7bf511a05c3c2.js → 4608-43acf39319177bee.js} +1 -1
  89. fides/ui-build/static/admin/_next/static/chunks/5163-e682273cd76a7d07.js +1 -0
  90. fides/ui-build/static/admin/_next/static/chunks/{5309-b2c4803370634ff8.js → 5309-ce5702b9faeaff55.js} +1 -1
  91. fides/ui-build/static/admin/_next/static/chunks/{905-ffdbd0b14167e8bd.js → 5596-4378b2927dae65b2.js} +3 -3
  92. fides/ui-build/static/admin/_next/static/chunks/5619-9b50cec521203989.js +1 -0
  93. fides/ui-build/static/admin/_next/static/chunks/{6084-7178ff6ea6822475.js → 6084-ddbad3149364725d.js} +1 -1
  94. fides/ui-build/static/admin/_next/static/chunks/6148-59a59d5c5925344f.js +1 -0
  95. fides/ui-build/static/admin/_next/static/chunks/6419-d0c00d661b01f8fa.js +1 -0
  96. fides/ui-build/static/admin/_next/static/chunks/{6662-507be5d46e5b719b.js → 6662-4392ba1e4c254ef7.js} +1 -1
  97. fides/ui-build/static/admin/_next/static/chunks/6780-fc7d9ddb1a03e7b3.js +1 -0
  98. fides/ui-build/static/admin/_next/static/chunks/{6853-2ad3e08fe6f9f5f2.js → 6853-d0190d2cca9dbde2.js} +1 -1
  99. fides/ui-build/static/admin/_next/static/chunks/{6882-6af16fef26c21e06.js → 6882-59ea739e3616ce83.js} +1 -1
  100. fides/ui-build/static/admin/_next/static/chunks/6954-9c4912fbce87c4df.js +1 -0
  101. fides/ui-build/static/admin/_next/static/chunks/{7476-281ee9a8286556f3.js → 7476-cc0d9a94ed7aad53.js} +1 -1
  102. fides/ui-build/static/admin/_next/static/chunks/{787-fb41002f797eb2df.js → 787-c57185ad89c4e288.js} +1 -1
  103. fides/ui-build/static/admin/_next/static/chunks/{79-7e87aff851423d4a.js → 79-2aab56be75e16187.js} +1 -1
  104. fides/ui-build/static/admin/_next/static/chunks/{796-329a5f823ec258a5.js → 796-3bdda2a7868464af.js} +1 -1
  105. fides/ui-build/static/admin/_next/static/chunks/8237-841439bef6682177.js +1 -0
  106. fides/ui-build/static/admin/_next/static/chunks/8765-f622a35b40a7ec63.js +1 -0
  107. fides/ui-build/static/admin/_next/static/chunks/{9046-5c4c22c375de25b1.js → 9046-04a8c092fef1cd83.js} +1 -1
  108. fides/ui-build/static/admin/_next/static/chunks/9187-7438242f0d380bb0.js +1 -0
  109. fides/ui-build/static/admin/_next/static/chunks/{9226-746771d47dff6266.js → 9226-2dcac54ab3fb94be.js} +1 -1
  110. fides/ui-build/static/admin/_next/static/chunks/9278-08cc704317fe535e.js +1 -0
  111. fides/ui-build/static/admin/_next/static/chunks/{9951-9b753ad7c3f51bdf.js → 9951-7c6639e5d062779e.js} +1 -1
  112. fides/ui-build/static/admin/_next/static/chunks/pages/{_app-39ccb07327c2c5d5.js → _app-4b5bff46158a19a3.js} +56 -56
  113. fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-98777246bec9dc2a.js → manual-2a655ff3a97f2492.js} +1 -1
  114. fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-a71c0aff4e0e6535.js → add-systems-0902f0bb4080643e.js} +1 -1
  115. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-1edf582ba3cd3bbb.js → [id]-f22ddd9b48a5c418.js} +1 -1
  116. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-06bb3b0bf097fcdb.js → new-e74cb5ea87f15b40.js} +1 -1
  117. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-685771e5f7196d87.js → privacy-experience-21f997c69fc3b4c2.js} +1 -1
  118. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-6ccedc70dc447089.js → [id]-da4124b7600a2a1d.js} +1 -1
  119. fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-944bca1cc57985b5.js → new-a57d251c88ce68ae.js} +1 -1
  120. fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-84f4bd14ce8673bc.js → privacy-notices-ad105181bc91209b.js} +1 -1
  121. fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-b0c4235fe6d0b0c8.js +1 -0
  122. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-11d52f1570759c4d.js → [resourceUrn]-aad6047a4604b945.js} +1 -1
  123. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-32eac8bbd217615a.js → projects-29784a11fe0fbd0a.js} +1 -1
  124. fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-b83afa5565d0c84e.js → [resourceUrn]-b6b98cea25dd94fa.js} +1 -1
  125. fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-6f630d42ac9fb6b4.js → data-catalog-d5b01abcb76792ce.js} +1 -1
  126. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-7caea7bb58c1f153.js +1 -0
  127. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-a9a70856f7be1542.js +1 -0
  128. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-9ddb52ebb7ac4c71.js → action-center-1f0ea5c92ae9a2b4.js} +1 -1
  129. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-9aa744d56cdacb0d.js → activity-ad6a84a6276f914c.js} +1 -1
  130. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-14bd7500362ff224.js → [resourceUrn]-f98dd251babb7e28.js} +1 -1
  131. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9e7dfd5a6acc2e8f.js → discovery-56eb4c014f0d96a3.js} +1 -1
  132. fides/ui-build/static/admin/_next/static/chunks/pages/datamap-d23b3ae139f0428b.js +1 -0
  133. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-c0d2bfd465df20e0.js → [...subfieldNames]-15301bd6bf7cf718.js} +1 -1
  134. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-28280a8a39a6e37c.js → [collectionName]-0fa72873e464f581.js} +1 -1
  135. fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-82fb246d87e58ebd.js → new-0d50084fbdf9b84c.js} +1 -1
  136. fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-20165c31ab1bc7cf.js → dataset-f3348d0a92543bab.js} +1 -1
  137. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-b4a6bcc87d126840.js → [id]-7d6027570d05c57f.js} +1 -1
  138. fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-f95d7b0bbfc58f5a.js → new-8446418c7ad28f77.js} +1 -1
  139. fides/ui-build/static/admin/_next/static/chunks/pages/{fides-js-docs-5d8fd1af75f19e2f.js → fides-js-docs-1f4335dca5c09860.js} +1 -1
  140. fides/ui-build/static/admin/_next/static/chunks/pages/{index-1919aab9e5834b51.js → index-fbf9b845bb901238.js} +1 -1
  141. fides/ui-build/static/admin/_next/static/chunks/pages/integrations/{[id]-0a58aee2d1e7fa01.js → [id]-f53fe1f2cbebda7c.js} +1 -1
  142. fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-e2d5d7e2a5265e68.js → integrations-142abe3e3e3e5bf7.js} +1 -1
  143. fides/ui-build/static/admin/_next/static/chunks/pages/messaging-1bae386d8c190348.js +1 -0
  144. fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-69ad86b7a8a9a115.js → table-migration-05616e2ae20ff4f8.js} +1 -1
  145. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-1b6b0d703cf59389.js +1 -0
  146. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-f1d818242d8550f8.js +1 -0
  147. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-d40a26bddb126c5c.js +1 -0
  148. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-ccd8d9e06cf2d278.js +1 -0
  149. fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-fd1a67892056830a.js +1 -0
  150. fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-fc201657f4a782c7.js → [purpose_id]-e891d01ece59669e.js} +1 -1
  151. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-c2d39cba8396ef3a.js → consent-f61b87e79367865b.js} +1 -1
  152. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-d992103cc55901ae.js → custom-fields-f8eea5d508c60c64.js} +1 -1
  153. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-023e1895552817de.js → locations-ed6a140b362c5baa.js} +1 -1
  154. fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-ac403c0886b20e20.js → organization-ff9a34264d48c35f.js} +1 -1
  155. fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-7a3396ac819c7904.js → test-datasets-a4b6d41ca679298b.js} +1 -1
  156. fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-8314a819837f5b2a.js → [id]-c8f5fbaa83dd9945.js} +1 -1
  157. fides/ui-build/static/admin/_next/static/chunks/pages/{systems-21f423a7c417aa9d.js → systems-c05b49ddec1a1b4f.js} +1 -1
  158. fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-df0d88716578e295.js +1 -0
  159. fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-da68efc31998dc66.js +1 -0
  160. fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-173ac3a1ed2b05a6.js → user-management-e98dfc7d4f2a4e16.js} +1 -1
  161. fides/ui-build/static/admin/_next/static/chunks/webpack-90e8ec1fc5c6455b.js +1 -0
  162. fides/ui-build/static/admin/_next/static/css/{5bfb2473e5701527.css → 23cf870196941c9a.css} +1 -1
  163. fides/ui-build/static/admin/_next/static/css/{94965f224bc991e9.css → 8bc1833f1fa53ff0.css} +1 -1
  164. fides/ui-build/static/admin/_next/static/css/d9924caa849931b3.css +1 -0
  165. fides/ui-build/static/admin/_next/static/css/dbcf63488933a4d5.css +29 -0
  166. fides/ui-build/static/admin/add-systems/manual.html +1 -1
  167. fides/ui-build/static/admin/add-systems/multiple.html +1 -1
  168. fides/ui-build/static/admin/add-systems.html +1 -1
  169. fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
  170. fides/ui-build/static/admin/consent/configure.html +1 -1
  171. fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
  172. fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
  173. fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
  174. fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
  175. fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
  176. fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
  177. fides/ui-build/static/admin/consent/properties.html +1 -1
  178. fides/ui-build/static/admin/consent/reporting.html +1 -1
  179. fides/ui-build/static/admin/consent.html +1 -1
  180. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
  181. fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
  182. fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
  183. fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
  184. fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
  185. fides/ui-build/static/admin/data-catalog.html +1 -1
  186. fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
  187. fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
  188. fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
  189. fides/ui-build/static/admin/data-discovery/activity.html +1 -1
  190. fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
  191. fides/ui-build/static/admin/data-discovery/detection.html +1 -1
  192. fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
  193. fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
  194. fides/ui-build/static/admin/datamap.html +1 -1
  195. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
  196. fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
  197. fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
  198. fides/ui-build/static/admin/dataset/new.html +1 -1
  199. fides/ui-build/static/admin/dataset.html +1 -1
  200. fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
  201. fides/ui-build/static/admin/datastore-connection/new.html +1 -1
  202. fides/ui-build/static/admin/datastore-connection.html +1 -1
  203. fides/ui-build/static/admin/index.html +1 -1
  204. fides/ui-build/static/admin/integrations/[id].html +1 -1
  205. fides/ui-build/static/admin/integrations.html +1 -1
  206. fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
  207. fides/ui-build/static/admin/lib/fides-headless.js +1 -1
  208. fides/ui-build/static/admin/lib/fides-preview.js +1 -1
  209. fides/ui-build/static/admin/lib/fides-tcf.js +3 -3
  210. fides/ui-build/static/admin/lib/fides.js +3 -3
  211. fides/ui-build/static/admin/login/[provider].html +1 -1
  212. fides/ui-build/static/admin/login.html +1 -1
  213. fides/ui-build/static/admin/messaging/[id].html +1 -1
  214. fides/ui-build/static/admin/messaging/add-template.html +1 -1
  215. fides/ui-build/static/admin/messaging.html +1 -1
  216. fides/ui-build/static/admin/poc/ant-components.html +1 -1
  217. fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
  218. fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
  219. fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
  220. fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
  221. fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
  222. fides/ui-build/static/admin/poc/forms.html +1 -1
  223. fides/ui-build/static/admin/poc/table-migration.html +1 -1
  224. fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
  225. fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
  226. fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
  227. fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
  228. fides/ui-build/static/admin/privacy-requests.html +1 -1
  229. fides/ui-build/static/admin/properties/[id].html +1 -1
  230. fides/ui-build/static/admin/properties/add-property.html +1 -1
  231. fides/ui-build/static/admin/properties.html +1 -1
  232. fides/ui-build/static/admin/reporting/datamap.html +1 -1
  233. fides/ui-build/static/admin/settings/about/alpha.html +1 -1
  234. fides/ui-build/static/admin/settings/about.html +1 -1
  235. fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
  236. fides/ui-build/static/admin/settings/consent.html +1 -1
  237. fides/ui-build/static/admin/settings/custom-fields.html +1 -1
  238. fides/ui-build/static/admin/settings/domain-records.html +1 -1
  239. fides/ui-build/static/admin/settings/domains.html +1 -1
  240. fides/ui-build/static/admin/settings/email-templates.html +1 -1
  241. fides/ui-build/static/admin/settings/locations.html +1 -1
  242. fides/ui-build/static/admin/settings/organization.html +1 -1
  243. fides/ui-build/static/admin/settings/regulations.html +1 -1
  244. fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
  245. fides/ui-build/static/admin/systems/configure/[id].html +1 -1
  246. fides/ui-build/static/admin/systems.html +1 -1
  247. fides/ui-build/static/admin/taxonomy.html +1 -1
  248. fides/ui-build/static/admin/user-management/new.html +1 -1
  249. fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
  250. fides/ui-build/static/admin/user-management.html +1 -1
  251. fides/ui-build/static/admin/_next/static/8108ANFxs99VY7KZ_Xev2/_buildManifest.js +0 -1
  252. fides/ui-build/static/admin/_next/static/chunks/1316-6cc72a45ebf7ff81.js +0 -1
  253. fides/ui-build/static/admin/_next/static/chunks/1807-3beab149351d5ded.js +0 -1
  254. fides/ui-build/static/admin/_next/static/chunks/1817-e601e737e3cc7a0e.js +0 -1
  255. fides/ui-build/static/admin/_next/static/chunks/255-7db55b0e3a0f9dea.js +0 -1
  256. fides/ui-build/static/admin/_next/static/chunks/2599-6c4d22e75028d8b6.js +0 -1
  257. fides/ui-build/static/admin/_next/static/chunks/2858-0b44609b6be7850b.js +0 -1
  258. fides/ui-build/static/admin/_next/static/chunks/2866-a73888c17a195cbe.js +0 -1
  259. fides/ui-build/static/admin/_next/static/chunks/3615-5e2d062d684b8fa1.js +0 -1
  260. fides/ui-build/static/admin/_next/static/chunks/409-a257e14acebcd73b.js +0 -1
  261. fides/ui-build/static/admin/_next/static/chunks/431-34f0b91c26f8d9ab.js +0 -1
  262. fides/ui-build/static/admin/_next/static/chunks/570-c99f07161bd339cd.js +0 -1
  263. fides/ui-build/static/admin/_next/static/chunks/6780-e3d40aa17a4bf2e9.js +0 -1
  264. fides/ui-build/static/admin/_next/static/chunks/6954-bb875d9ac89f6030.js +0 -1
  265. fides/ui-build/static/admin/_next/static/chunks/7062.fda15dcb7df85675.js +0 -1
  266. fides/ui-build/static/admin/_next/static/chunks/7c79804f.7a7112aece470725.js +0 -1
  267. fides/ui-build/static/admin/_next/static/chunks/9014-eeae6f581158e645.js +0 -1
  268. fides/ui-build/static/admin/_next/static/chunks/9278-9b1b5970f0702668.js +0 -1
  269. fides/ui-build/static/admin/_next/static/chunks/9392.f4520f66206d347d.js +0 -1
  270. fides/ui-build/static/admin/_next/static/chunks/ea076c0a.84423f606aef37cd.js +0 -1
  271. fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-afdbd4665657cfa1.js +0 -1
  272. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-2265ecb899d45fbc.js +0 -1
  273. fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-5d522637871ac6c8.js +0 -1
  274. fides/ui-build/static/admin/_next/static/chunks/pages/datamap-7674b97d655c193b.js +0 -1
  275. fides/ui-build/static/admin/_next/static/chunks/pages/messaging-5094ffea13f32ed9.js +0 -1
  276. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-32600543eb7b584f.js +0 -1
  277. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-10ce53ea356f8bad.js +0 -1
  278. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-5501bbb129fee9c4.js +0 -1
  279. fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-cbe4c8f9096b6543.js +0 -1
  280. fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-e130c0197362e8f3.js +0 -1
  281. fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-6387fcc8cce872eb.js +0 -1
  282. fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-ff5738706da07801.js +0 -1
  283. fides/ui-build/static/admin/_next/static/chunks/webpack-ff0cd6bff75588da.js +0 -1
  284. fides/ui-build/static/admin/_next/static/css/2cadb5f62dcd7c2b.css +0 -1
  285. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/WHEEL +0 -0
  286. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/entry_points.txt +0 -0
  287. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/licenses/LICENSE +0 -0
  288. {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/top_level.txt +0 -0
  289. /fides/ui-build/static/admin/_next/static/{8108ANFxs99VY7KZ_Xev2 → GKmhMPa_1gMto8JZO8ENy}/_ssgManifest.js +0 -0
  290. /fides/ui-build/static/admin/_next/static/chunks/{2921-455e6357b74d2f76.js → 2921-86f1547ac40a5cdf.js} +0 -0
  291. /fides/ui-build/static/admin/_next/static/chunks/{3923-6cc911dafccc5f63.js → 3923-13a6b4da2d51bf8f.js} +0 -0
  292. /fides/ui-build/static/admin/_next/static/chunks/{401-1b529d5800aa1f3a.js → 401-3cc1fee61494e3bd.js} +0 -0
  293. /fides/ui-build/static/admin/_next/static/chunks/{5574-b13021775a15bfd2.js → 5574-9312f97b637d9ee2.js} +0 -0
  294. /fides/ui-build/static/admin/_next/static/chunks/{7630-9aac73191ed5ed13.js → 7630-b1c93688013ef013.js} +0 -0
  295. /fides/ui-build/static/admin/_next/static/chunks/{9826-111aaee8bd8dbd09.js → 9826-3c578665c6d3b21d.js} +0 -0
  296. /fides/ui-build/static/admin/_next/static/chunks/pages/{404-aece2c920ea14514.js → 404-2d803dab6a00f353.js} +0 -0
  297. /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-dc75dc6e37e52f05.js → multiple-8ff7f37913ad736a.js} +0 -0
  298. /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-24d226b5a8de5c74.js → add-vendors-d00c9034cdeb0236.js} +0 -0
  299. /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-6a8ef51138ac926a.js → configure-0e1ca0f4c8e7f4da.js} +0 -0
  300. /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-6f86ab63a08a6528.js → properties-057cad65e7414a44.js} +0 -0
  301. /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-73d3cbf68f7c3a31.js → consent-e17c56eec8d91371.js} +0 -0
  302. /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-6ba9e160dae64695.js → [projectUrn]-80a6cc8e8573514a.js} +0 -0
  303. /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{resources-7648bbd4f6711e4d.js → resources-6c3714ee97a718c1.js} +0 -0
  304. /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-393e20924c83373e.js → [resourceUrn]-31e6c54794a9883e.js} +0 -0
  305. /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-8733807dad4bc96e.js → detection-2822a423a7ad0550.js} +0 -0
  306. /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-006b695e5af5ef24.js → [datasetId]-a8e8b5f4ee7af86c.js} +0 -0
  307. /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-c391c6fad56eec48.js → datastore-connection-0f29b47402292070.js} +0 -0
  308. /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-53fecfb9dd6a1e0c.js → [id]-5627d0d0668077f9.js} +0 -0
  309. /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-76b01cec5fde10a9.js → add-template-feca66ad5c5fe54a.js} +0 -0
  310. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-5c08e8447c45ce44.js → ant-components-64a322d01aae5ca7.js} +0 -0
  311. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-06ad5f34585480aa.js → AntForm-8bca16a7726e7eb2.js} +0 -0
  312. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-6f071c2bc9446cb0.js → FormikAntFormItem-b0f246fc3b67ebf7.js} +0 -0
  313. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-efcc38c58991ac9e.js → FormikControlled-1a0852b090bfc392.js} +0 -0
  314. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-430ba5c979abfb7c.js → FormikField-11f3de1b45e36583.js} +0 -0
  315. /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-5c561880bf131afb.js → forms-1b73a1c2b6c6285f.js} +0 -0
  316. /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-d888a69a3bbe040e.js → configure-e551a860ec727802.js} +0 -0
  317. /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-d3d8e3d7583ec635.js → [id]-dd99183f93763ae4.js} +0 -0
  318. /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-1af10ed303815d46.js → add-property-0bdbc1fcbf553b8f.js} +0 -0
  319. /fides/ui-build/static/admin/_next/static/chunks/pages/{properties-cebc0dc186be499a.js → properties-e959378bb32b6b73.js} +0 -0
  320. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-5e1322de868d615e.js → alpha-8f98a4895e74725e.js} +0 -0
  321. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-241f95e372b65d0f.js → about-8155a35a62fdb5ae.js} +0 -0
  322. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-41242f805599feda.js → domain-records-51333dbd21cb37c8.js} +0 -0
  323. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-2e885f74c92f669c.js → domains-bde86e5f6c09da5a.js} +0 -0
  324. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-ff112655ad5f41e5.js → email-templates-4f9a5cc8bea7725b.js} +0 -0
  325. /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-86062a18e081a52a.js → regulations-102efd9199e87124.js} +0 -0
  326. /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-a2524414e968f862.js → new-bc4eb541906781e6.js} +0 -0
@@ -28,6 +28,7 @@ class CurrentStep(EnumType):
28
28
  finalize_consent = "finalize_consent"
29
29
  email_post_send = "email_post_send"
30
30
  post_webhooks = "post_webhooks"
31
+ finalization = "finalization"
31
32
 
32
33
 
33
34
  # action types we actively support in policies/requests
@@ -124,6 +124,8 @@ class PrivacyRequestResubmit(PrivacyRequestCreate):
124
124
  identity_verified_at: Optional[datetime] = None
125
125
  custom_privacy_request_fields_approved_at: Optional[datetime] = None
126
126
  custom_privacy_request_fields_approved_by: Optional[str] = None
127
+ finalized_at: Optional[datetime] = None
128
+ finalized_by: Optional[str] = None
127
129
 
128
130
 
129
131
  class ConsentRequestCreate(FidesSchema):
@@ -287,6 +289,7 @@ class PrivacyRequestStatus(str, EnumType):
287
289
  complete = "complete"
288
290
  paused = "paused"
289
291
  awaiting_email_send = "awaiting_email_send"
292
+ requires_manual_finalization = "requires_manual_finalization"
290
293
  canceled = "canceled"
291
294
  error = "error"
292
295
 
@@ -320,6 +323,8 @@ class PrivacyRequestResponse(FidesSchema):
320
323
  source: Optional[PrivacyRequestSource] = None
321
324
  deleted_at: Optional[datetime] = None
322
325
  deleted_by: Optional[str] = None
326
+ finalized_at: Optional[datetime] = None
327
+ finalized_by: Optional[str] = None
323
328
  model_config = ConfigDict(from_attributes=True, use_enum_values=True)
324
329
 
325
330
 
@@ -82,6 +82,7 @@ class BaseConnector(Generic[DB_CONNECTOR_TYPE], ABC):
82
82
  privacy_request: PrivacyRequest,
83
83
  request_task: RequestTask,
84
84
  rows: List[Row],
85
+ input_data: Optional[Dict[str, List[Any]]] = None,
85
86
  ) -> int:
86
87
  """Execute a masking request. Return the number of rows that have been updated
87
88
 
@@ -1,13 +1,8 @@
1
- from typing import List, Optional
1
+ from typing import Any, Dict, List, Optional
2
2
 
3
3
  from loguru import logger
4
4
  from sqlalchemy import text
5
- from sqlalchemy.engine import ( # type: ignore
6
- Connection,
7
- Engine,
8
- LegacyCursorResult,
9
- create_engine,
10
- )
5
+ from sqlalchemy.engine import Connection, Engine, create_engine # type: ignore
11
6
  from sqlalchemy.orm import Session
12
7
  from sqlalchemy.sql import Executable # type: ignore
13
8
  from sqlalchemy.sql.elements import TextClause
@@ -17,6 +12,7 @@ from fides.api.graph.execution import ExecutionNode
17
12
  from fides.api.models.connectionconfig import ConnectionTestStatus
18
13
  from fides.api.models.policy import Policy
19
14
  from fides.api.models.privacy_request import PrivacyRequest, RequestTask
15
+ from fides.api.schemas.application_config import SqlDryRunMode
20
16
  from fides.api.schemas.connection_configuration.connection_secrets_bigquery import (
21
17
  BigQuerySchema,
22
18
  )
@@ -99,6 +95,16 @@ class BigQueryConnector(SQLConnector):
99
95
  logger.info(
100
96
  f"Executing {len(partition_clauses)} partition queries for node '{query_config.node.address}' in DSR execution"
101
97
  )
98
+
99
+ if self.should_dry_run(SqlDryRunMode.access):
100
+ for partition_clause in partition_clauses:
101
+ existing_bind_params = stmt.compile().params
102
+ partitioned_stmt = text(
103
+ f"{stmt} AND ({text(partition_clause)})"
104
+ ).params(existing_bind_params)
105
+ logger.warning(f"SQL DRY RUN - Would execute SQL: {partitioned_stmt}")
106
+ return []
107
+
102
108
  rows = []
103
109
  for partition_clause in partition_clauses:
104
110
  logger.debug(
@@ -135,6 +141,39 @@ class BigQueryConnector(SQLConnector):
135
141
  logger.exception(f"Error testing connection to remote BigQuery {str(e)}")
136
142
  raise ConnectionException(f"Connection error: {e}")
137
143
 
144
+ def _execute_statements_with_sql_dry_run(
145
+ self,
146
+ statements: List[Executable],
147
+ sql_dry_run_enabled: bool,
148
+ client: Engine,
149
+ ) -> int:
150
+ """
151
+ Execute SQL statements with sql_dry_run support.
152
+
153
+ Args:
154
+ statements: List of SQL statements to execute
155
+ sql_dry_run_enabled: Whether sql_dry_run mode is enabled
156
+ client: Database engine
157
+
158
+ Returns:
159
+ int: Number of affected rows (0 in sql_dry_run mode)
160
+ """
161
+ if not statements:
162
+ return 0
163
+
164
+ if sql_dry_run_enabled:
165
+ for stmt in statements:
166
+ logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
167
+ return 0
168
+
169
+ row_count = 0
170
+ with client.connect() as connection:
171
+ for stmt in statements:
172
+ results = connection.execute(stmt)
173
+ logger.debug(f"Affected {results.rowcount} rows")
174
+ row_count += results.rowcount
175
+ return row_count
176
+
138
177
  def mask_data(
139
178
  self,
140
179
  node: ExecutionNode,
@@ -142,22 +181,31 @@ class BigQueryConnector(SQLConnector):
142
181
  privacy_request: PrivacyRequest,
143
182
  request_task: RequestTask,
144
183
  rows: List[Row],
184
+ input_data: Optional[Dict[str, List[Any]]] = None,
145
185
  ) -> int:
146
186
  """Execute a masking request. Returns the number of records updated or deleted"""
187
+
147
188
  query_config = self.query_config(node)
148
189
  update_or_delete_ct = 0
149
190
  client = self.client()
150
- for row in rows:
151
- update_or_delete_stmts: List[Executable] = (
152
- query_config.generate_masking_stmt(
153
- node, row, policy, privacy_request, client
154
- )
191
+
192
+ if query_config.uses_delete_masking_strategy():
193
+ delete_stmts = query_config.generate_delete(client, input_data or {})
194
+ logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
195
+ update_or_delete_ct += self._execute_statements_with_sql_dry_run(
196
+ delete_stmts, self.should_dry_run(SqlDryRunMode.erasure), client
155
197
  )
156
- if update_or_delete_stmts:
157
- with client.connect() as connection:
158
- for update_or_delete_stmt in update_or_delete_stmts:
159
- results: LegacyCursorResult = connection.execute(
160
- update_or_delete_stmt
161
- )
162
- update_or_delete_ct = update_or_delete_ct + results.rowcount
198
+ else:
199
+ for row in rows:
200
+ update_or_delete_stmts: List[Executable] = query_config.generate_update(
201
+ row, policy, privacy_request, client
202
+ )
203
+ logger.debug(
204
+ f"Generated {len(update_or_delete_stmts)} UPDATE statements"
205
+ )
206
+ update_or_delete_ct += self._execute_statements_with_sql_dry_run(
207
+ update_or_delete_stmts,
208
+ self.should_dry_run(SqlDryRunMode.erasure),
209
+ client,
210
+ )
163
211
  return update_or_delete_ct
@@ -140,8 +140,9 @@ class DynamoDBConnector(BaseConnector[Any]): # type: ignore
140
140
  privacy_request: PrivacyRequest,
141
141
  request_task: RequestTask,
142
142
  rows: List[Row],
143
+ input_data: Optional[Dict[str, List[Any]]] = None,
143
144
  ) -> int:
144
- """Execute a masking requestfor DynamoDB"""
145
+ """Execute a masking request for DynamoDB"""
145
146
 
146
147
  query_config = self.query_config(node)
147
148
  collection_name = node.address.collection
@@ -142,6 +142,7 @@ class FidesConnector(BaseConnector[FidesClient]):
142
142
  privacy_request: PrivacyRequest,
143
143
  request_task: RequestTask,
144
144
  rows: List[Row],
145
+ input_data: Optional[Dict[str, List[Any]]] = None,
145
146
  ) -> int:
146
147
  """Execute an erasure request on remote fides"""
147
148
  identity_data = {
@@ -90,6 +90,7 @@ class HTTPSConnector(BaseConnector[None]):
90
90
  privacy_request: PrivacyRequest,
91
91
  request_task: RequestTask,
92
92
  rows: List[Row],
93
+ input_data: Optional[Dict[str, List[Any]]] = None,
93
94
  ) -> int:
94
95
  """Currently not supported as webhooks are not called at the collection level"""
95
96
  raise NotImplementedError(
@@ -80,6 +80,7 @@ class ManualTaskConnector(BaseConnector):
80
80
  privacy_request: PrivacyRequest,
81
81
  request_task: RequestTask,
82
82
  rows: List[Row],
83
+ input_data: Optional[Dict[str, List[Any]]] = None,
83
84
  ) -> int:
84
85
  """
85
86
  Manual tasks don't support erasure operations.
@@ -1,4 +1,4 @@
1
- from typing import Any, Dict, List
1
+ from typing import Any, Dict, List, Optional
2
2
 
3
3
  from fides.api.graph.execution import ExecutionNode
4
4
  from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
@@ -53,6 +53,7 @@ class ManualWebhookConnector(BaseConnector[None]):
53
53
  privacy_request: PrivacyRequest,
54
54
  request_task: RequestTask,
55
55
  rows: List[Row],
56
+ input_data: Optional[List[List[Row]]] = None,
56
57
  ) -> None:
57
58
  """
58
59
  Not applicable for a manual webhook. Manual webhooks are not called as part of the traversal.
@@ -126,6 +126,7 @@ class MongoDBConnector(BaseConnector[MongoClient]):
126
126
  privacy_request: PrivacyRequest,
127
127
  request_task: RequestTask,
128
128
  rows: List[Row],
129
+ input_data: Optional[Dict[str, List[Any]]] = None,
129
130
  ) -> int:
130
131
  """Execute a masking request"""
131
132
  query_config = self.query_config(node)
@@ -82,6 +82,7 @@ class OktaConnector(BaseConnector):
82
82
  privacy_request: PrivacyRequest,
83
83
  request_task: RequestTask,
84
84
  rows: List[Row],
85
+ input_data: Optional[Dict[str, List[Any]]] = None,
85
86
  ) -> int:
86
87
  """DSR execution not supported for Okta connector"""
87
88
  return 0
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Union, cast
3
3
  import pydash
4
4
  from fideslang.models import MaskingStrategies
5
5
  from loguru import logger
6
- from sqlalchemy import MetaData, Table, text
6
+ from sqlalchemy import MetaData, Table, or_, text
7
7
  from sqlalchemy.engine import Engine
8
8
  from sqlalchemy.sql import Delete, Update
9
9
  from sqlalchemy.sql.elements import ColumnElement, TextClause
@@ -125,6 +125,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
125
125
  policy: Policy,
126
126
  request: PrivacyRequest,
127
127
  client: Engine,
128
+ input_data: Optional[Dict[str, List[Any]]] = None,
128
129
  ) -> Union[List[Update], List[Delete]]:
129
130
  """
130
131
  Generate a masking statement for BigQuery.
@@ -137,7 +138,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
137
138
  logger.info(
138
139
  f"Masking override detected for collection {node.address.value}: {masking_override.strategy.value}"
139
140
  )
140
- return self.generate_delete(row, client)
141
+ return self.generate_delete(client, input_data or {})
141
142
  return self.generate_update(row, policy, request, client)
142
143
 
143
144
  def generate_update(
@@ -198,9 +199,16 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
198
199
 
199
200
  table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
200
201
  where_clauses: List[ColumnElement] = [
201
- getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items()
202
+ table.c[k] == v for k, v in non_empty_reference_field_keys.items()
202
203
  ]
203
204
 
205
+ # Create update values using Column objects as keys to handle column names with spaces
206
+ update_values = {}
207
+ for column_name, value in final_update_map.items():
208
+ # Use bracket notation to access columns with spaces in their names
209
+ column = table.c[column_name]
210
+ update_values[column] = value
211
+
204
212
  if self.partitioning:
205
213
  partition_clauses = self.get_partition_clauses()
206
214
  partitioned_queries = []
@@ -211,34 +219,37 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
211
219
  partitioned_queries.append(
212
220
  table.update()
213
221
  .where(*(where_clauses + [text(partition_clause)]))
214
- .values(**final_update_map)
222
+ .values(update_values)
215
223
  )
216
224
 
217
225
  return partitioned_queries
218
226
 
219
- return [table.update().where(*where_clauses).values(**final_update_map)]
227
+ return [table.update().where(*where_clauses).values(update_values)]
220
228
 
221
- def generate_delete(self, row: Row, client: Engine) -> List[Delete]:
222
- """Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
229
+ def generate_delete(
230
+ self,
231
+ client: Engine,
232
+ input_data: Optional[Dict[str, List[Any]]] = None,
233
+ ) -> List[Delete]:
234
+ """
235
+ Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
223
236
 
224
237
  Used when a collection-level masking override is present and the masking strategy is DELETE.
225
238
 
226
239
  A List of multiple DELETE statements are returned for partitioned tables; for a non-partitioned table,
227
240
  a single DELETE statement is returned in a List for consistent typing.
228
-
229
- TODO: DRY up this method and `generate_update` a bit
230
241
  """
231
242
 
232
- non_empty_reference_field_keys: Dict[str, Field] = filter_nonempty_values(
233
- {
234
- fpath.string_path: fld.cast(row[fpath.string_path])
235
- for fpath, fld in self.reference_field_paths.items()
236
- if fpath.string_path in row
237
- }
238
- )
243
+ if not input_data:
244
+ logger.warning(
245
+ "No input data provided for node '{}', skipping DELETE statement generation",
246
+ self.node.address,
247
+ )
248
+ return []
239
249
 
240
- valid = len(non_empty_reference_field_keys) > 0
241
- if not valid:
250
+ filtered_data = self.node.typed_filtered_values(input_data)
251
+
252
+ if not filtered_data:
242
253
  logger.warning(
243
254
  "There is not enough data to generate a valid DELETE statement for {}",
244
255
  self.node.address,
@@ -246,9 +257,17 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
246
257
  return []
247
258
 
248
259
  table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
249
- where_clauses: List[ColumnElement] = [
250
- getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items()
251
- ]
260
+
261
+ # Build individual reference clauses
262
+ where_clauses: List[ColumnElement] = []
263
+ for column_name, values in filtered_data.items():
264
+ if len(values) == 1:
265
+ where_clauses.append(table.c[column_name] == values[0])
266
+ else:
267
+ where_clauses.append(table.c[column_name].in_(values))
268
+
269
+ # Combine reference clauses with OR instead of AND
270
+ combined_reference_clause = or_(*where_clauses)
252
271
 
253
272
  if self.partitioning:
254
273
  partition_clauses = self.get_partition_clauses()
@@ -259,12 +278,25 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
259
278
 
260
279
  for partition_clause in partition_clauses:
261
280
  partitioned_queries.append(
262
- table.delete().where(*(where_clauses + [text(partition_clause)]))
281
+ table.delete()
282
+ .where(combined_reference_clause)
283
+ .where(text(partition_clause))
263
284
  )
264
285
 
265
286
  return partitioned_queries
266
287
 
267
- return [table.delete().where(*where_clauses)]
288
+ return [table.delete().where(combined_reference_clause)]
289
+
290
+ def uses_delete_masking_strategy(self) -> bool:
291
+ """Check if this collection uses DELETE masking strategy.
292
+
293
+ Returns True if masking override is present and strategy is DELETE.
294
+ """
295
+ masking_override = self.node.collection.masking_strategy_override
296
+ return (
297
+ masking_override is not None
298
+ and masking_override.strategy == MaskingStrategies.DELETE
299
+ )
268
300
 
269
301
  def format_fields_for_query(
270
302
  self,
@@ -281,6 +313,25 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
281
313
  formatted_fields.append(field_path.levels[0])
282
314
  return formatted_fields
283
315
 
316
+ def format_clause_for_query(
317
+ self, string_path: str, operator: str, operand: str
318
+ ) -> str:
319
+ """
320
+ Returns clauses with proper BigQuery backtick escaping for column names.
321
+ Handles column names with spaces and nested fields (dot-separated) by escaping each part individually.
322
+ """
323
+ # For nested fields (containing dots), escape each part individually
324
+ if "." in string_path:
325
+ parts = string_path.split(".")
326
+ escaped_field = ".".join(f"`{part}`" for part in parts)
327
+ else:
328
+ # For simple fields, wrap the entire name in backticks
329
+ escaped_field = f"`{string_path}`"
330
+
331
+ if operator == "IN":
332
+ return f"{escaped_field} IN ({operand})"
333
+ return f"{escaped_field} {operator} :{operand}"
334
+
284
335
  def generate_raw_query_without_tuples(
285
336
  self, field_list: List[str], filters: Dict[str, List[Any]]
286
337
  ) -> Optional[TextClause]:
@@ -290,27 +341,35 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
290
341
 
291
342
  This is an override of the base class method that supports nested fields for BigQuery.
292
343
 
293
- Examples with dot-delimited field names, notice the periods are replaced with underscores in the parameter bindings:
344
+ Examples with field names containing dots and spaces, notice these are replaced with underscores in the parameter bindings:
294
345
 
295
346
  1. Single value filter:
296
347
  field_list = ["id", "name", "email"]
297
348
  filters = {"user.id": [123]}
298
349
 
299
- Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (user.id = :user_id)
350
+ Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`user`.`id` = :user_id)
300
351
  With parameter binding: user_id = 123
301
352
 
302
- 2. Multiple value filter:
353
+ 2. Field with spaces:
354
+ field_list = ["id", "custom id", "email"]
355
+ filters = {"custom id": ["abc123"]}
356
+
357
+ Generates: SELECT id, `custom id`, email FROM `project_id.dataset_id.table_name` WHERE (`custom id` = :custom_id)
358
+ With parameter binding: custom_id = "abc123"
359
+
360
+ 3. Multiple value filter with nested field:
303
361
  field_list = ["id", "name", "email"]
304
- filters = {"user.status": ["active", "pending"]}
362
+ filters = {"contact_info.primary_email": ["active", "pending"]}
305
363
 
306
- Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (user.status IN (:user_status_in_stmt_generated_0, :user_status_in_stmt_generated_1))
307
- With parameter bindings: user_status_in_stmt_generated_0 = "active", user_status_in_stmt_generated_1 = "pending"
364
+ Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`contact_info`.`primary_email` IN (:contact_info_primary_email_in_stmt_generated_0, :contact_info_primary_email_in_stmt_generated_1))
365
+ With parameter bindings: contact_info_primary_email_in_stmt_generated_0 = "active", contact_info_primary_email_in_stmt_generated_1 = "pending"
308
366
  """
309
367
  clauses = []
310
368
  query_data = {}
311
369
  for field_name, field_value in filters.items():
312
- # Replace dots with underscores in field names for parameter binding
313
- field_binding_name = field_name.replace(".", "_")
370
+ # Replace dots and spaces with underscores in field names for parameter binding
371
+ # SQLAlchemy parameter names cannot contain spaces or special characters
372
+ field_binding_name = field_name.replace(".", "_").replace(" ", "_")
314
373
  data = set(field_value)
315
374
  if len(data) == 1:
316
375
  clauses.append(
@@ -333,7 +392,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
333
392
  clauses.append(self.format_clause_for_query(field_name, "IN", operand))
334
393
 
335
394
  if len(clauses) > 0:
336
- formatted_fields = ", ".join(field_list)
395
+ formatted_fields = ", ".join([f"`{field}`" for field in field_list])
337
396
  query_str = self.get_formatted_query_string(formatted_fields, clauses)
338
397
  return text(query_str).params(query_data)
339
398
 
@@ -158,6 +158,7 @@ class RDSMySQLConnector(RDSConnectorMixin, SQLConnector):
158
158
  privacy_request: PrivacyRequest,
159
159
  request_task: RequestTask,
160
160
  rows: List[Row],
161
+ input_data: Optional[Dict[str, List[Any]]] = None,
161
162
  ) -> int:
162
163
  """DSR execution not yet supported for RDS MySQL"""
163
164
  return 0
@@ -147,6 +147,7 @@ class RDSPostgresConnector(RDSConnectorMixin, SQLConnector):
147
147
  privacy_request: PrivacyRequest,
148
148
  request_task: RequestTask,
149
149
  rows: List[Row],
150
+ input_data: Optional[Dict[str, List[Any]]] = None,
150
151
  ) -> int:
151
152
  """DSR execution not yet supported for RDS Postgres"""
152
153
  return 0
@@ -66,6 +66,7 @@ class S3Connector(BaseConnector):
66
66
  privacy_request: PrivacyRequest,
67
67
  request_task: RequestTask,
68
68
  rows: List[Row],
69
+ input_data: Optional[Dict[str, List[Any]]] = None,
69
70
  ) -> int:
70
71
  """DSR execution not yet supported for S3"""
71
72
  return 0
@@ -517,6 +517,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
517
517
  privacy_request: PrivacyRequest,
518
518
  request_task: RequestTask,
519
519
  rows: List[Row],
520
+ input_data: Optional[Dict[str, List[Any]]] = None,
520
521
  ) -> int:
521
522
  """Execute a masking request. Return the number of rows that have been updated."""
522
523
  self.set_privacy_request_state(privacy_request, node, request_task)
@@ -150,6 +150,7 @@ class ScyllaConnector(BaseConnector[Cluster]):
150
150
  privacy_request: PrivacyRequest,
151
151
  request_task: RequestTask,
152
152
  rows: List[Row],
153
+ input_data: Optional[Dict[str, List[Any]]] = None,
153
154
  ) -> int:
154
155
  """Execute a masking request"""
155
156
  query_config = self.query_config(node)
@@ -27,11 +27,13 @@ from fides.api.graph.execution import ExecutionNode
27
27
  from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
28
28
  from fides.api.models.policy import Policy
29
29
  from fides.api.models.privacy_request import PrivacyRequest, RequestTask
30
+ from fides.api.schemas.application_config import SqlDryRunMode
30
31
  from fides.api.schemas.connection_configuration import ConnectionConfigSecretsSchema
31
32
  from fides.api.service.connectors.base_connector import BaseConnector
32
33
  from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
33
34
  from fides.api.util.collection_util import Row
34
35
  from fides.config import get_config
36
+ from fides.config.config_proxy import ConfigProxy
35
37
 
36
38
  from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip
37
39
  Dataset as CtlDataset,
@@ -58,6 +60,23 @@ class SQLConnector(BaseConnector[Engine]):
58
60
  )
59
61
  self.ssh_server: sshtunnel._ForwardServer = None
60
62
 
63
+ def should_dry_run(self, mode_to_check: SqlDryRunMode) -> bool:
64
+ """
65
+ Check if SQL dry run is enabled for the specified mode.
66
+
67
+ Args:
68
+ mode_to_check: The SqlDryRunMode to check for
69
+
70
+ Returns:
71
+ bool: True if the current mode matches the mode to check
72
+ """
73
+ from fides.api.api.deps import get_autoclose_db_session as get_db
74
+
75
+ with get_db() as db:
76
+ config_proxy = ConfigProxy(db)
77
+ current_mode = getattr(config_proxy.execution, "sql_dry_run", None)
78
+ return current_mode == mode_to_check
79
+
61
80
  @staticmethod
62
81
  def cursor_result_to_rows(results: CursorResult) -> List[Row]:
63
82
  """Convert SQLAlchemy results to a list of dictionaries"""
@@ -140,6 +159,10 @@ class SQLConnector(BaseConnector[Engine]):
140
159
  if query is None:
141
160
  return []
142
161
 
162
+ if self.should_dry_run(SqlDryRunMode.access):
163
+ logger.warning(f"SQL DRY RUN - Would execute SQL: {query}")
164
+ return []
165
+
143
166
  with client.connect() as connection:
144
167
  self.set_schema(connection)
145
168
  results = connection.execute(query)
@@ -160,6 +183,10 @@ class SQLConnector(BaseConnector[Engine]):
160
183
  if stmt is None:
161
184
  return []
162
185
 
186
+ if self.should_dry_run(SqlDryRunMode.access):
187
+ logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
188
+ return []
189
+
163
190
  logger.info("Starting data retrieval for {}", node.address)
164
191
  with client.connect() as connection:
165
192
  self.set_schema(connection)
@@ -178,20 +205,25 @@ class SQLConnector(BaseConnector[Engine]):
178
205
  privacy_request: PrivacyRequest,
179
206
  request_task: RequestTask,
180
207
  rows: List[Row],
208
+ input_data: Optional[Dict[str, List[Any]]] = None,
181
209
  ) -> int:
182
210
  """Execute a masking request. Returns the number of records masked"""
183
211
  query_config = self.query_config(node)
184
212
  update_ct = 0
185
213
  client = self.client()
214
+
186
215
  for row in rows:
187
216
  update_stmt: Optional[TextClause] = query_config.generate_update_stmt(
188
217
  row, policy, privacy_request
189
218
  )
190
219
  if update_stmt is not None:
191
- with client.connect() as connection:
192
- self.set_schema(connection)
193
- results: LegacyCursorResult = connection.execute(update_stmt)
194
- update_ct = update_ct + results.rowcount
220
+ if self.should_dry_run(SqlDryRunMode.erasure):
221
+ logger.warning(f"SQL DRY RUN - Would execute SQL: {update_stmt}")
222
+ else:
223
+ with client.connect() as connection:
224
+ self.set_schema(connection)
225
+ results: LegacyCursorResult = connection.execute(update_stmt)
226
+ update_ct = update_ct + results.rowcount
195
227
  return update_ct
196
228
 
197
229
  def close(self) -> None:
@@ -75,6 +75,7 @@ class WebsiteConnector(BaseConnector):
75
75
  privacy_request: PrivacyRequest,
76
76
  request_task: RequestTask,
77
77
  rows: List[Row],
78
+ input_data: Optional[Dict[str, List[Any]]] = None,
78
79
  ) -> int:
79
80
  """DSR execution not supported for website connector"""
80
81
  return 0
@@ -48,8 +48,8 @@
48
48
  <div class="dsr-information-text">
49
49
  <p>
50
50
  This web link contains all data requested as part of your Data Subject Request (DSR). Your
51
- information has been compiled from the following areas. Click on each section to open a file and view
52
- your data in PDF format.
51
+ information has been compiled from the following areas. Click on each section to open and view
52
+ your data.
53
53
  </p>
54
54
  </div>
55
55
  <div class="dataset-list">