ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0rc0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/METADATA +1 -1
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/RECORD +292 -279
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/7e9a2b52f498_adding_masking_secrets.py +60 -0
- fides/api/alembic/migrations/versions/a7065df4dcf1_add_finalized_fields_to_privacy_request.py +65 -0
- fides/api/alembic/migrations/versions/d0031087eacb_create_manualtaskconditionaldependency_.py +106 -0
- fides/api/api/v1/endpoints/dataset_config_endpoints.py +13 -5
- fides/api/api/v1/endpoints/drp_endpoints.py +7 -1
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +44 -1
- fides/api/api/v1/endpoints/user_endpoints.py +83 -7
- fides/api/app_setup.py +3 -2
- fides/api/common_exceptions.py +4 -0
- fides/api/db/base.py +1 -0
- fides/api/db/database.py +1 -1
- fides/api/graph/execution.py +30 -0
- fides/api/graph/graph.py +13 -2
- fides/api/graph/traversal.py +126 -39
- fides/api/models/manual_task/__init__.py +2 -0
- fides/api/models/manual_task/conditional_dependency.py +144 -0
- fides/api/models/{manual_task.py → manual_task/manual_task.py} +10 -0
- fides/api/models/masking_secret.py +72 -0
- fides/api/models/policy.py +23 -0
- fides/api/models/privacy_request/execution_log.py +1 -0
- fides/api/models/privacy_request/privacy_request.py +31 -13
- fides/api/oauth/roles.py +2 -0
- fides/api/schemas/application_config.py +11 -1
- fides/api/schemas/masking/masking_secrets.py +1 -1
- fides/api/schemas/policy.py +1 -0
- fides/api/schemas/privacy_request.py +5 -0
- fides/api/service/connectors/base_connector.py +1 -0
- fides/api/service/connectors/bigquery_connector.py +67 -19
- fides/api/service/connectors/dynamodb_connector.py +2 -1
- fides/api/service/connectors/fides_connector.py +1 -0
- fides/api/service/connectors/http_connector.py +1 -0
- fides/api/service/connectors/manual_task_connector.py +1 -0
- fides/api/service/connectors/manual_webhook_connector.py +2 -1
- fides/api/service/connectors/mongodb_connector.py +1 -0
- fides/api/service/connectors/okta_connector.py +1 -0
- fides/api/service/connectors/query_configs/bigquery_query_config.py +91 -32
- fides/api/service/connectors/rds_mysql_connector.py +1 -0
- fides/api/service/connectors/rds_postgres_connector.py +1 -0
- fides/api/service/connectors/s3_connector.py +1 -0
- fides/api/service/connectors/saas_connector.py +1 -0
- fides/api/service/connectors/scylla_connector.py +1 -0
- fides/api/service/connectors/sql_connector.py +36 -4
- fides/api/service/connectors/website_connector.py +1 -0
- fides/api/service/privacy_request/dsr_package/templates/welcome.html +2 -2
- fides/api/service/privacy_request/request_runner_service.py +142 -53
- fides/api/service/privacy_request/request_service.py +1 -22
- fides/api/task/conditional_dependencies/__init__.py +0 -0
- fides/api/task/conditional_dependencies/evaluator.py +109 -0
- fides/api/task/conditional_dependencies/schemas.py +54 -0
- fides/api/task/create_request_tasks.py +1 -1
- fides/api/task/deprecated_graph_task.py +24 -6
- fides/api/task/execute_request_tasks.py +89 -12
- fides/api/task/filter_results.py +1 -1
- fides/api/task/graph_task.py +38 -3
- fides/api/task/manual/manual_task_address.py +46 -0
- fides/api/task/manual/manual_task_graph_task.py +118 -126
- fides/api/task/manual/manual_task_utils.py +52 -105
- fides/api/util/aws_util.py +5 -1
- fides/api/util/cache.py +5 -0
- fides/api/util/encryption/secrets_util.py +48 -18
- fides/common/api/scope_registry.py +3 -0
- fides/common/api/v1/urn_registry.py +1 -1
- fides/config/execution_settings.py +4 -0
- fides/config/utils.py +1 -0
- fides/service/privacy_request/privacy_request_service.py +6 -1
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/GKmhMPa_1gMto8JZO8ENy/_buildManifest.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1316-2606e19807c08aa5.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1467-8808ec8836e033f9.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1817-b78b58ae3b75d75a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1975.7d4634a0e823a02b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{203-5a663f465ba26bb4.js → 203-cd78ea279cecba60.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/2150-930ffaf2c4718edc.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/255-1bc0cbef7a59cdc6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3450-0ba194991d0cca88.js → 3450-ca4ba70da999f264.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/346-aa3b88efb85f2e28.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3550-d04125c828d591a1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3855-e172870d3e21b0dd.js → 3855-509ca7ac99b5eada.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3872-46cebf7ec1b31a2b.js → 3872-0a0f0032ca39a93f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/409-ea70638a59296659.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4121-2bc09fc4ddbfe5cb.js → 4121-877e19d3fa078c7b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4230-60100f7ef3ddcde1.js → 4230-114e31621c19ea69.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4259.d1507e0db19cbed7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/431-1db919f6569a4021.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4608-bbb7bf511a05c3c2.js → 4608-43acf39319177bee.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/5163-e682273cd76a7d07.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{5309-b2c4803370634ff8.js → 5309-ce5702b9faeaff55.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{905-ffdbd0b14167e8bd.js → 5596-4378b2927dae65b2.js} +3 -3
- fides/ui-build/static/admin/_next/static/chunks/5619-9b50cec521203989.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6084-7178ff6ea6822475.js → 6084-ddbad3149364725d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6148-59a59d5c5925344f.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6419-d0c00d661b01f8fa.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6662-507be5d46e5b719b.js → 6662-4392ba1e4c254ef7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6780-fc7d9ddb1a03e7b3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6853-2ad3e08fe6f9f5f2.js → 6853-d0190d2cca9dbde2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{6882-6af16fef26c21e06.js → 6882-59ea739e3616ce83.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-9c4912fbce87c4df.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{7476-281ee9a8286556f3.js → 7476-cc0d9a94ed7aad53.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{787-fb41002f797eb2df.js → 787-c57185ad89c4e288.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{79-7e87aff851423d4a.js → 79-2aab56be75e16187.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{796-329a5f823ec258a5.js → 796-3bdda2a7868464af.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/8237-841439bef6682177.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/8765-f622a35b40a7ec63.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9046-5c4c22c375de25b1.js → 9046-04a8c092fef1cd83.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9187-7438242f0d380bb0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9226-746771d47dff6266.js → 9226-2dcac54ab3fb94be.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9278-08cc704317fe535e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9951-9b753ad7c3f51bdf.js → 9951-7c6639e5d062779e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-39ccb07327c2c5d5.js → _app-4b5bff46158a19a3.js} +56 -56
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-98777246bec9dc2a.js → manual-2a655ff3a97f2492.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-a71c0aff4e0e6535.js → add-systems-0902f0bb4080643e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-1edf582ba3cd3bbb.js → [id]-f22ddd9b48a5c418.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-06bb3b0bf097fcdb.js → new-e74cb5ea87f15b40.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-685771e5f7196d87.js → privacy-experience-21f997c69fc3b4c2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-6ccedc70dc447089.js → [id]-da4124b7600a2a1d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-944bca1cc57985b5.js → new-a57d251c88ce68ae.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-84f4bd14ce8673bc.js → privacy-notices-ad105181bc91209b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-b0c4235fe6d0b0c8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-11d52f1570759c4d.js → [resourceUrn]-aad6047a4604b945.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-32eac8bbd217615a.js → projects-29784a11fe0fbd0a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-b83afa5565d0c84e.js → [resourceUrn]-b6b98cea25dd94fa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-6f630d42ac9fb6b4.js → data-catalog-d5b01abcb76792ce.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-7caea7bb58c1f153.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-a9a70856f7be1542.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-9ddb52ebb7ac4c71.js → action-center-1f0ea5c92ae9a2b4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-9aa744d56cdacb0d.js → activity-ad6a84a6276f914c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-14bd7500362ff224.js → [resourceUrn]-f98dd251babb7e28.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9e7dfd5a6acc2e8f.js → discovery-56eb4c014f0d96a3.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-d23b3ae139f0428b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-c0d2bfd465df20e0.js → [...subfieldNames]-15301bd6bf7cf718.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-28280a8a39a6e37c.js → [collectionName]-0fa72873e464f581.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-82fb246d87e58ebd.js → new-0d50084fbdf9b84c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-20165c31ab1bc7cf.js → dataset-f3348d0a92543bab.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-b4a6bcc87d126840.js → [id]-7d6027570d05c57f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-f95d7b0bbfc58f5a.js → new-8446418c7ad28f77.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{fides-js-docs-5d8fd1af75f19e2f.js → fides-js-docs-1f4335dca5c09860.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{index-1919aab9e5834b51.js → index-fbf9b845bb901238.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/{[id]-0a58aee2d1e7fa01.js → [id]-f53fe1f2cbebda7c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-e2d5d7e2a5265e68.js → integrations-142abe3e3e3e5bf7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-1bae386d8c190348.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-69ad86b7a8a9a115.js → table-migration-05616e2ae20ff4f8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-1b6b0d703cf59389.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-f1d818242d8550f8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-d40a26bddb126c5c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-ccd8d9e06cf2d278.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-fd1a67892056830a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-fc201657f4a782c7.js → [purpose_id]-e891d01ece59669e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-c2d39cba8396ef3a.js → consent-f61b87e79367865b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-d992103cc55901ae.js → custom-fields-f8eea5d508c60c64.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-023e1895552817de.js → locations-ed6a140b362c5baa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-ac403c0886b20e20.js → organization-ff9a34264d48c35f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-7a3396ac819c7904.js → test-datasets-a4b6d41ca679298b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-8314a819837f5b2a.js → [id]-c8f5fbaa83dd9945.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{systems-21f423a7c417aa9d.js → systems-c05b49ddec1a1b4f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-df0d88716578e295.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-da68efc31998dc66.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-173ac3a1ed2b05a6.js → user-management-e98dfc7d4f2a4e16.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/webpack-90e8ec1fc5c6455b.js +1 -0
- fides/ui-build/static/admin/_next/static/css/{5bfb2473e5701527.css → 23cf870196941c9a.css} +1 -1
- fides/ui-build/static/admin/_next/static/css/{94965f224bc991e9.css → 8bc1833f1fa53ff0.css} +1 -1
- fides/ui-build/static/admin/_next/static/css/d9924caa849931b3.css +1 -0
- fides/ui-build/static/admin/_next/static/css/dbcf63488933a4d5.css +29 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +3 -3
- fides/ui-build/static/admin/lib/fides.js +3 -3
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/8108ANFxs99VY7KZ_Xev2/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1316-6cc72a45ebf7ff81.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1807-3beab149351d5ded.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1817-e601e737e3cc7a0e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/255-7db55b0e3a0f9dea.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2599-6c4d22e75028d8b6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2858-0b44609b6be7850b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2866-a73888c17a195cbe.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3615-5e2d062d684b8fa1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/409-a257e14acebcd73b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/431-34f0b91c26f8d9ab.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/570-c99f07161bd339cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6780-e3d40aa17a4bf2e9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-bb875d9ac89f6030.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7062.fda15dcb7df85675.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7c79804f.7a7112aece470725.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9014-eeae6f581158e645.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9278-9b1b5970f0702668.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9392.f4520f66206d347d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/ea076c0a.84423f606aef37cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-afdbd4665657cfa1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-2265ecb899d45fbc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-5d522637871ac6c8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-7674b97d655c193b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-5094ffea13f32ed9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-32600543eb7b584f.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-10ce53ea356f8bad.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-5501bbb129fee9c4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-cbe4c8f9096b6543.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-e130c0197362e8f3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-6387fcc8cce872eb.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-ff5738706da07801.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/webpack-ff0cd6bff75588da.js +0 -1
- fides/ui-build/static/admin/_next/static/css/2cadb5f62dcd7c2b.css +0 -1
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{8108ANFxs99VY7KZ_Xev2 → GKmhMPa_1gMto8JZO8ENy}/_ssgManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{2921-455e6357b74d2f76.js → 2921-86f1547ac40a5cdf.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3923-6cc911dafccc5f63.js → 3923-13a6b4da2d51bf8f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{401-1b529d5800aa1f3a.js → 401-3cc1fee61494e3bd.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{5574-b13021775a15bfd2.js → 5574-9312f97b637d9ee2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7630-9aac73191ed5ed13.js → 7630-b1c93688013ef013.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{9826-111aaee8bd8dbd09.js → 9826-3c578665c6d3b21d.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{404-aece2c920ea14514.js → 404-2d803dab6a00f353.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-dc75dc6e37e52f05.js → multiple-8ff7f37913ad736a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-24d226b5a8de5c74.js → add-vendors-d00c9034cdeb0236.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-6a8ef51138ac926a.js → configure-0e1ca0f4c8e7f4da.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-6f86ab63a08a6528.js → properties-057cad65e7414a44.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-73d3cbf68f7c3a31.js → consent-e17c56eec8d91371.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-6ba9e160dae64695.js → [projectUrn]-80a6cc8e8573514a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{resources-7648bbd4f6711e4d.js → resources-6c3714ee97a718c1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-393e20924c83373e.js → [resourceUrn]-31e6c54794a9883e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-8733807dad4bc96e.js → detection-2822a423a7ad0550.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-006b695e5af5ef24.js → [datasetId]-a8e8b5f4ee7af86c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-c391c6fad56eec48.js → datastore-connection-0f29b47402292070.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-53fecfb9dd6a1e0c.js → [id]-5627d0d0668077f9.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-76b01cec5fde10a9.js → add-template-feca66ad5c5fe54a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-5c08e8447c45ce44.js → ant-components-64a322d01aae5ca7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-06ad5f34585480aa.js → AntForm-8bca16a7726e7eb2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-6f071c2bc9446cb0.js → FormikAntFormItem-b0f246fc3b67ebf7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-efcc38c58991ac9e.js → FormikControlled-1a0852b090bfc392.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-430ba5c979abfb7c.js → FormikField-11f3de1b45e36583.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-5c561880bf131afb.js → forms-1b73a1c2b6c6285f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-d888a69a3bbe040e.js → configure-e551a860ec727802.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-d3d8e3d7583ec635.js → [id]-dd99183f93763ae4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-1af10ed303815d46.js → add-property-0bdbc1fcbf553b8f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{properties-cebc0dc186be499a.js → properties-e959378bb32b6b73.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-5e1322de868d615e.js → alpha-8f98a4895e74725e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-241f95e372b65d0f.js → about-8155a35a62fdb5ae.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-41242f805599feda.js → domain-records-51333dbd21cb37c8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-2e885f74c92f669c.js → domains-bde86e5f6c09da5a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-ff112655ad5f41e5.js → email-templates-4f9a5cc8bea7725b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-86062a18e081a52a.js → regulations-102efd9199e87124.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-a2524414e968f862.js → new-bc4eb541906781e6.js} +0 -0
fides/api/schemas/policy.py
CHANGED
|
@@ -124,6 +124,8 @@ class PrivacyRequestResubmit(PrivacyRequestCreate):
|
|
|
124
124
|
identity_verified_at: Optional[datetime] = None
|
|
125
125
|
custom_privacy_request_fields_approved_at: Optional[datetime] = None
|
|
126
126
|
custom_privacy_request_fields_approved_by: Optional[str] = None
|
|
127
|
+
finalized_at: Optional[datetime] = None
|
|
128
|
+
finalized_by: Optional[str] = None
|
|
127
129
|
|
|
128
130
|
|
|
129
131
|
class ConsentRequestCreate(FidesSchema):
|
|
@@ -287,6 +289,7 @@ class PrivacyRequestStatus(str, EnumType):
|
|
|
287
289
|
complete = "complete"
|
|
288
290
|
paused = "paused"
|
|
289
291
|
awaiting_email_send = "awaiting_email_send"
|
|
292
|
+
requires_manual_finalization = "requires_manual_finalization"
|
|
290
293
|
canceled = "canceled"
|
|
291
294
|
error = "error"
|
|
292
295
|
|
|
@@ -320,6 +323,8 @@ class PrivacyRequestResponse(FidesSchema):
|
|
|
320
323
|
source: Optional[PrivacyRequestSource] = None
|
|
321
324
|
deleted_at: Optional[datetime] = None
|
|
322
325
|
deleted_by: Optional[str] = None
|
|
326
|
+
finalized_at: Optional[datetime] = None
|
|
327
|
+
finalized_by: Optional[str] = None
|
|
323
328
|
model_config = ConfigDict(from_attributes=True, use_enum_values=True)
|
|
324
329
|
|
|
325
330
|
|
|
@@ -82,6 +82,7 @@ class BaseConnector(Generic[DB_CONNECTOR_TYPE], ABC):
|
|
|
82
82
|
privacy_request: PrivacyRequest,
|
|
83
83
|
request_task: RequestTask,
|
|
84
84
|
rows: List[Row],
|
|
85
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
85
86
|
) -> int:
|
|
86
87
|
"""Execute a masking request. Return the number of rows that have been updated
|
|
87
88
|
|
|
@@ -1,13 +1,8 @@
|
|
|
1
|
-
from typing import List, Optional
|
|
1
|
+
from typing import Any, Dict, List, Optional
|
|
2
2
|
|
|
3
3
|
from loguru import logger
|
|
4
4
|
from sqlalchemy import text
|
|
5
|
-
from sqlalchemy.engine import
|
|
6
|
-
Connection,
|
|
7
|
-
Engine,
|
|
8
|
-
LegacyCursorResult,
|
|
9
|
-
create_engine,
|
|
10
|
-
)
|
|
5
|
+
from sqlalchemy.engine import Connection, Engine, create_engine # type: ignore
|
|
11
6
|
from sqlalchemy.orm import Session
|
|
12
7
|
from sqlalchemy.sql import Executable # type: ignore
|
|
13
8
|
from sqlalchemy.sql.elements import TextClause
|
|
@@ -17,6 +12,7 @@ from fides.api.graph.execution import ExecutionNode
|
|
|
17
12
|
from fides.api.models.connectionconfig import ConnectionTestStatus
|
|
18
13
|
from fides.api.models.policy import Policy
|
|
19
14
|
from fides.api.models.privacy_request import PrivacyRequest, RequestTask
|
|
15
|
+
from fides.api.schemas.application_config import SqlDryRunMode
|
|
20
16
|
from fides.api.schemas.connection_configuration.connection_secrets_bigquery import (
|
|
21
17
|
BigQuerySchema,
|
|
22
18
|
)
|
|
@@ -99,6 +95,16 @@ class BigQueryConnector(SQLConnector):
|
|
|
99
95
|
logger.info(
|
|
100
96
|
f"Executing {len(partition_clauses)} partition queries for node '{query_config.node.address}' in DSR execution"
|
|
101
97
|
)
|
|
98
|
+
|
|
99
|
+
if self.should_dry_run(SqlDryRunMode.access):
|
|
100
|
+
for partition_clause in partition_clauses:
|
|
101
|
+
existing_bind_params = stmt.compile().params
|
|
102
|
+
partitioned_stmt = text(
|
|
103
|
+
f"{stmt} AND ({text(partition_clause)})"
|
|
104
|
+
).params(existing_bind_params)
|
|
105
|
+
logger.warning(f"SQL DRY RUN - Would execute SQL: {partitioned_stmt}")
|
|
106
|
+
return []
|
|
107
|
+
|
|
102
108
|
rows = []
|
|
103
109
|
for partition_clause in partition_clauses:
|
|
104
110
|
logger.debug(
|
|
@@ -135,6 +141,39 @@ class BigQueryConnector(SQLConnector):
|
|
|
135
141
|
logger.exception(f"Error testing connection to remote BigQuery {str(e)}")
|
|
136
142
|
raise ConnectionException(f"Connection error: {e}")
|
|
137
143
|
|
|
144
|
+
def _execute_statements_with_sql_dry_run(
|
|
145
|
+
self,
|
|
146
|
+
statements: List[Executable],
|
|
147
|
+
sql_dry_run_enabled: bool,
|
|
148
|
+
client: Engine,
|
|
149
|
+
) -> int:
|
|
150
|
+
"""
|
|
151
|
+
Execute SQL statements with sql_dry_run support.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
statements: List of SQL statements to execute
|
|
155
|
+
sql_dry_run_enabled: Whether sql_dry_run mode is enabled
|
|
156
|
+
client: Database engine
|
|
157
|
+
|
|
158
|
+
Returns:
|
|
159
|
+
int: Number of affected rows (0 in sql_dry_run mode)
|
|
160
|
+
"""
|
|
161
|
+
if not statements:
|
|
162
|
+
return 0
|
|
163
|
+
|
|
164
|
+
if sql_dry_run_enabled:
|
|
165
|
+
for stmt in statements:
|
|
166
|
+
logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
|
|
167
|
+
return 0
|
|
168
|
+
|
|
169
|
+
row_count = 0
|
|
170
|
+
with client.connect() as connection:
|
|
171
|
+
for stmt in statements:
|
|
172
|
+
results = connection.execute(stmt)
|
|
173
|
+
logger.debug(f"Affected {results.rowcount} rows")
|
|
174
|
+
row_count += results.rowcount
|
|
175
|
+
return row_count
|
|
176
|
+
|
|
138
177
|
def mask_data(
|
|
139
178
|
self,
|
|
140
179
|
node: ExecutionNode,
|
|
@@ -142,22 +181,31 @@ class BigQueryConnector(SQLConnector):
|
|
|
142
181
|
privacy_request: PrivacyRequest,
|
|
143
182
|
request_task: RequestTask,
|
|
144
183
|
rows: List[Row],
|
|
184
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
145
185
|
) -> int:
|
|
146
186
|
"""Execute a masking request. Returns the number of records updated or deleted"""
|
|
187
|
+
|
|
147
188
|
query_config = self.query_config(node)
|
|
148
189
|
update_or_delete_ct = 0
|
|
149
190
|
client = self.client()
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
191
|
+
|
|
192
|
+
if query_config.uses_delete_masking_strategy():
|
|
193
|
+
delete_stmts = query_config.generate_delete(client, input_data or {})
|
|
194
|
+
logger.debug(f"Generated {len(delete_stmts)} DELETE statements")
|
|
195
|
+
update_or_delete_ct += self._execute_statements_with_sql_dry_run(
|
|
196
|
+
delete_stmts, self.should_dry_run(SqlDryRunMode.erasure), client
|
|
155
197
|
)
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
198
|
+
else:
|
|
199
|
+
for row in rows:
|
|
200
|
+
update_or_delete_stmts: List[Executable] = query_config.generate_update(
|
|
201
|
+
row, policy, privacy_request, client
|
|
202
|
+
)
|
|
203
|
+
logger.debug(
|
|
204
|
+
f"Generated {len(update_or_delete_stmts)} UPDATE statements"
|
|
205
|
+
)
|
|
206
|
+
update_or_delete_ct += self._execute_statements_with_sql_dry_run(
|
|
207
|
+
update_or_delete_stmts,
|
|
208
|
+
self.should_dry_run(SqlDryRunMode.erasure),
|
|
209
|
+
client,
|
|
210
|
+
)
|
|
163
211
|
return update_or_delete_ct
|
|
@@ -140,8 +140,9 @@ class DynamoDBConnector(BaseConnector[Any]): # type: ignore
|
|
|
140
140
|
privacy_request: PrivacyRequest,
|
|
141
141
|
request_task: RequestTask,
|
|
142
142
|
rows: List[Row],
|
|
143
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
143
144
|
) -> int:
|
|
144
|
-
"""Execute a masking
|
|
145
|
+
"""Execute a masking request for DynamoDB"""
|
|
145
146
|
|
|
146
147
|
query_config = self.query_config(node)
|
|
147
148
|
collection_name = node.address.collection
|
|
@@ -142,6 +142,7 @@ class FidesConnector(BaseConnector[FidesClient]):
|
|
|
142
142
|
privacy_request: PrivacyRequest,
|
|
143
143
|
request_task: RequestTask,
|
|
144
144
|
rows: List[Row],
|
|
145
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
145
146
|
) -> int:
|
|
146
147
|
"""Execute an erasure request on remote fides"""
|
|
147
148
|
identity_data = {
|
|
@@ -90,6 +90,7 @@ class HTTPSConnector(BaseConnector[None]):
|
|
|
90
90
|
privacy_request: PrivacyRequest,
|
|
91
91
|
request_task: RequestTask,
|
|
92
92
|
rows: List[Row],
|
|
93
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
93
94
|
) -> int:
|
|
94
95
|
"""Currently not supported as webhooks are not called at the collection level"""
|
|
95
96
|
raise NotImplementedError(
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any, Dict, List
|
|
1
|
+
from typing import Any, Dict, List, Optional
|
|
2
2
|
|
|
3
3
|
from fides.api.graph.execution import ExecutionNode
|
|
4
4
|
from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
|
|
@@ -53,6 +53,7 @@ class ManualWebhookConnector(BaseConnector[None]):
|
|
|
53
53
|
privacy_request: PrivacyRequest,
|
|
54
54
|
request_task: RequestTask,
|
|
55
55
|
rows: List[Row],
|
|
56
|
+
input_data: Optional[List[List[Row]]] = None,
|
|
56
57
|
) -> None:
|
|
57
58
|
"""
|
|
58
59
|
Not applicable for a manual webhook. Manual webhooks are not called as part of the traversal.
|
|
@@ -126,6 +126,7 @@ class MongoDBConnector(BaseConnector[MongoClient]):
|
|
|
126
126
|
privacy_request: PrivacyRequest,
|
|
127
127
|
request_task: RequestTask,
|
|
128
128
|
rows: List[Row],
|
|
129
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
129
130
|
) -> int:
|
|
130
131
|
"""Execute a masking request"""
|
|
131
132
|
query_config = self.query_config(node)
|
|
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Union, cast
|
|
|
3
3
|
import pydash
|
|
4
4
|
from fideslang.models import MaskingStrategies
|
|
5
5
|
from loguru import logger
|
|
6
|
-
from sqlalchemy import MetaData, Table, text
|
|
6
|
+
from sqlalchemy import MetaData, Table, or_, text
|
|
7
7
|
from sqlalchemy.engine import Engine
|
|
8
8
|
from sqlalchemy.sql import Delete, Update
|
|
9
9
|
from sqlalchemy.sql.elements import ColumnElement, TextClause
|
|
@@ -125,6 +125,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
125
125
|
policy: Policy,
|
|
126
126
|
request: PrivacyRequest,
|
|
127
127
|
client: Engine,
|
|
128
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
128
129
|
) -> Union[List[Update], List[Delete]]:
|
|
129
130
|
"""
|
|
130
131
|
Generate a masking statement for BigQuery.
|
|
@@ -137,7 +138,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
137
138
|
logger.info(
|
|
138
139
|
f"Masking override detected for collection {node.address.value}: {masking_override.strategy.value}"
|
|
139
140
|
)
|
|
140
|
-
return self.generate_delete(
|
|
141
|
+
return self.generate_delete(client, input_data or {})
|
|
141
142
|
return self.generate_update(row, policy, request, client)
|
|
142
143
|
|
|
143
144
|
def generate_update(
|
|
@@ -198,9 +199,16 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
198
199
|
|
|
199
200
|
table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
|
|
200
201
|
where_clauses: List[ColumnElement] = [
|
|
201
|
-
|
|
202
|
+
table.c[k] == v for k, v in non_empty_reference_field_keys.items()
|
|
202
203
|
]
|
|
203
204
|
|
|
205
|
+
# Create update values using Column objects as keys to handle column names with spaces
|
|
206
|
+
update_values = {}
|
|
207
|
+
for column_name, value in final_update_map.items():
|
|
208
|
+
# Use bracket notation to access columns with spaces in their names
|
|
209
|
+
column = table.c[column_name]
|
|
210
|
+
update_values[column] = value
|
|
211
|
+
|
|
204
212
|
if self.partitioning:
|
|
205
213
|
partition_clauses = self.get_partition_clauses()
|
|
206
214
|
partitioned_queries = []
|
|
@@ -211,34 +219,37 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
211
219
|
partitioned_queries.append(
|
|
212
220
|
table.update()
|
|
213
221
|
.where(*(where_clauses + [text(partition_clause)]))
|
|
214
|
-
.values(
|
|
222
|
+
.values(update_values)
|
|
215
223
|
)
|
|
216
224
|
|
|
217
225
|
return partitioned_queries
|
|
218
226
|
|
|
219
|
-
return [table.update().where(*where_clauses).values(
|
|
227
|
+
return [table.update().where(*where_clauses).values(update_values)]
|
|
220
228
|
|
|
221
|
-
def generate_delete(
|
|
222
|
-
|
|
229
|
+
def generate_delete(
|
|
230
|
+
self,
|
|
231
|
+
client: Engine,
|
|
232
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
233
|
+
) -> List[Delete]:
|
|
234
|
+
"""
|
|
235
|
+
Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
|
|
223
236
|
|
|
224
237
|
Used when a collection-level masking override is present and the masking strategy is DELETE.
|
|
225
238
|
|
|
226
239
|
A List of multiple DELETE statements are returned for partitioned tables; for a non-partitioned table,
|
|
227
240
|
a single DELETE statement is returned in a List for consistent typing.
|
|
228
|
-
|
|
229
|
-
TODO: DRY up this method and `generate_update` a bit
|
|
230
241
|
"""
|
|
231
242
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
)
|
|
243
|
+
if not input_data:
|
|
244
|
+
logger.warning(
|
|
245
|
+
"No input data provided for node '{}', skipping DELETE statement generation",
|
|
246
|
+
self.node.address,
|
|
247
|
+
)
|
|
248
|
+
return []
|
|
239
249
|
|
|
240
|
-
|
|
241
|
-
|
|
250
|
+
filtered_data = self.node.typed_filtered_values(input_data)
|
|
251
|
+
|
|
252
|
+
if not filtered_data:
|
|
242
253
|
logger.warning(
|
|
243
254
|
"There is not enough data to generate a valid DELETE statement for {}",
|
|
244
255
|
self.node.address,
|
|
@@ -246,9 +257,17 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
246
257
|
return []
|
|
247
258
|
|
|
248
259
|
table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
]
|
|
260
|
+
|
|
261
|
+
# Build individual reference clauses
|
|
262
|
+
where_clauses: List[ColumnElement] = []
|
|
263
|
+
for column_name, values in filtered_data.items():
|
|
264
|
+
if len(values) == 1:
|
|
265
|
+
where_clauses.append(table.c[column_name] == values[0])
|
|
266
|
+
else:
|
|
267
|
+
where_clauses.append(table.c[column_name].in_(values))
|
|
268
|
+
|
|
269
|
+
# Combine reference clauses with OR instead of AND
|
|
270
|
+
combined_reference_clause = or_(*where_clauses)
|
|
252
271
|
|
|
253
272
|
if self.partitioning:
|
|
254
273
|
partition_clauses = self.get_partition_clauses()
|
|
@@ -259,12 +278,25 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
259
278
|
|
|
260
279
|
for partition_clause in partition_clauses:
|
|
261
280
|
partitioned_queries.append(
|
|
262
|
-
table.delete()
|
|
281
|
+
table.delete()
|
|
282
|
+
.where(combined_reference_clause)
|
|
283
|
+
.where(text(partition_clause))
|
|
263
284
|
)
|
|
264
285
|
|
|
265
286
|
return partitioned_queries
|
|
266
287
|
|
|
267
|
-
return [table.delete().where(
|
|
288
|
+
return [table.delete().where(combined_reference_clause)]
|
|
289
|
+
|
|
290
|
+
def uses_delete_masking_strategy(self) -> bool:
|
|
291
|
+
"""Check if this collection uses DELETE masking strategy.
|
|
292
|
+
|
|
293
|
+
Returns True if masking override is present and strategy is DELETE.
|
|
294
|
+
"""
|
|
295
|
+
masking_override = self.node.collection.masking_strategy_override
|
|
296
|
+
return (
|
|
297
|
+
masking_override is not None
|
|
298
|
+
and masking_override.strategy == MaskingStrategies.DELETE
|
|
299
|
+
)
|
|
268
300
|
|
|
269
301
|
def format_fields_for_query(
|
|
270
302
|
self,
|
|
@@ -281,6 +313,25 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
281
313
|
formatted_fields.append(field_path.levels[0])
|
|
282
314
|
return formatted_fields
|
|
283
315
|
|
|
316
|
+
def format_clause_for_query(
|
|
317
|
+
self, string_path: str, operator: str, operand: str
|
|
318
|
+
) -> str:
|
|
319
|
+
"""
|
|
320
|
+
Returns clauses with proper BigQuery backtick escaping for column names.
|
|
321
|
+
Handles column names with spaces and nested fields (dot-separated) by escaping each part individually.
|
|
322
|
+
"""
|
|
323
|
+
# For nested fields (containing dots), escape each part individually
|
|
324
|
+
if "." in string_path:
|
|
325
|
+
parts = string_path.split(".")
|
|
326
|
+
escaped_field = ".".join(f"`{part}`" for part in parts)
|
|
327
|
+
else:
|
|
328
|
+
# For simple fields, wrap the entire name in backticks
|
|
329
|
+
escaped_field = f"`{string_path}`"
|
|
330
|
+
|
|
331
|
+
if operator == "IN":
|
|
332
|
+
return f"{escaped_field} IN ({operand})"
|
|
333
|
+
return f"{escaped_field} {operator} :{operand}"
|
|
334
|
+
|
|
284
335
|
def generate_raw_query_without_tuples(
|
|
285
336
|
self, field_list: List[str], filters: Dict[str, List[Any]]
|
|
286
337
|
) -> Optional[TextClause]:
|
|
@@ -290,27 +341,35 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
290
341
|
|
|
291
342
|
This is an override of the base class method that supports nested fields for BigQuery.
|
|
292
343
|
|
|
293
|
-
Examples with
|
|
344
|
+
Examples with field names containing dots and spaces, notice these are replaced with underscores in the parameter bindings:
|
|
294
345
|
|
|
295
346
|
1. Single value filter:
|
|
296
347
|
field_list = ["id", "name", "email"]
|
|
297
348
|
filters = {"user.id": [123]}
|
|
298
349
|
|
|
299
|
-
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (user
|
|
350
|
+
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`user`.`id` = :user_id)
|
|
300
351
|
With parameter binding: user_id = 123
|
|
301
352
|
|
|
302
|
-
2.
|
|
353
|
+
2. Field with spaces:
|
|
354
|
+
field_list = ["id", "custom id", "email"]
|
|
355
|
+
filters = {"custom id": ["abc123"]}
|
|
356
|
+
|
|
357
|
+
Generates: SELECT id, `custom id`, email FROM `project_id.dataset_id.table_name` WHERE (`custom id` = :custom_id)
|
|
358
|
+
With parameter binding: custom_id = "abc123"
|
|
359
|
+
|
|
360
|
+
3. Multiple value filter with nested field:
|
|
303
361
|
field_list = ["id", "name", "email"]
|
|
304
|
-
filters = {"
|
|
362
|
+
filters = {"contact_info.primary_email": ["active", "pending"]}
|
|
305
363
|
|
|
306
|
-
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (
|
|
307
|
-
With parameter bindings:
|
|
364
|
+
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`contact_info`.`primary_email` IN (:contact_info_primary_email_in_stmt_generated_0, :contact_info_primary_email_in_stmt_generated_1))
|
|
365
|
+
With parameter bindings: contact_info_primary_email_in_stmt_generated_0 = "active", contact_info_primary_email_in_stmt_generated_1 = "pending"
|
|
308
366
|
"""
|
|
309
367
|
clauses = []
|
|
310
368
|
query_data = {}
|
|
311
369
|
for field_name, field_value in filters.items():
|
|
312
|
-
# Replace dots with underscores in field names for parameter binding
|
|
313
|
-
|
|
370
|
+
# Replace dots and spaces with underscores in field names for parameter binding
|
|
371
|
+
# SQLAlchemy parameter names cannot contain spaces or special characters
|
|
372
|
+
field_binding_name = field_name.replace(".", "_").replace(" ", "_")
|
|
314
373
|
data = set(field_value)
|
|
315
374
|
if len(data) == 1:
|
|
316
375
|
clauses.append(
|
|
@@ -333,7 +392,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
333
392
|
clauses.append(self.format_clause_for_query(field_name, "IN", operand))
|
|
334
393
|
|
|
335
394
|
if len(clauses) > 0:
|
|
336
|
-
formatted_fields = ", ".join(field_list)
|
|
395
|
+
formatted_fields = ", ".join([f"`{field}`" for field in field_list])
|
|
337
396
|
query_str = self.get_formatted_query_string(formatted_fields, clauses)
|
|
338
397
|
return text(query_str).params(query_data)
|
|
339
398
|
|
|
@@ -158,6 +158,7 @@ class RDSMySQLConnector(RDSConnectorMixin, SQLConnector):
|
|
|
158
158
|
privacy_request: PrivacyRequest,
|
|
159
159
|
request_task: RequestTask,
|
|
160
160
|
rows: List[Row],
|
|
161
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
161
162
|
) -> int:
|
|
162
163
|
"""DSR execution not yet supported for RDS MySQL"""
|
|
163
164
|
return 0
|
|
@@ -147,6 +147,7 @@ class RDSPostgresConnector(RDSConnectorMixin, SQLConnector):
|
|
|
147
147
|
privacy_request: PrivacyRequest,
|
|
148
148
|
request_task: RequestTask,
|
|
149
149
|
rows: List[Row],
|
|
150
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
150
151
|
) -> int:
|
|
151
152
|
"""DSR execution not yet supported for RDS Postgres"""
|
|
152
153
|
return 0
|
|
@@ -517,6 +517,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
|
|
|
517
517
|
privacy_request: PrivacyRequest,
|
|
518
518
|
request_task: RequestTask,
|
|
519
519
|
rows: List[Row],
|
|
520
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
520
521
|
) -> int:
|
|
521
522
|
"""Execute a masking request. Return the number of rows that have been updated."""
|
|
522
523
|
self.set_privacy_request_state(privacy_request, node, request_task)
|
|
@@ -150,6 +150,7 @@ class ScyllaConnector(BaseConnector[Cluster]):
|
|
|
150
150
|
privacy_request: PrivacyRequest,
|
|
151
151
|
request_task: RequestTask,
|
|
152
152
|
rows: List[Row],
|
|
153
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
153
154
|
) -> int:
|
|
154
155
|
"""Execute a masking request"""
|
|
155
156
|
query_config = self.query_config(node)
|
|
@@ -27,11 +27,13 @@ from fides.api.graph.execution import ExecutionNode
|
|
|
27
27
|
from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
|
|
28
28
|
from fides.api.models.policy import Policy
|
|
29
29
|
from fides.api.models.privacy_request import PrivacyRequest, RequestTask
|
|
30
|
+
from fides.api.schemas.application_config import SqlDryRunMode
|
|
30
31
|
from fides.api.schemas.connection_configuration import ConnectionConfigSecretsSchema
|
|
31
32
|
from fides.api.service.connectors.base_connector import BaseConnector
|
|
32
33
|
from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
|
|
33
34
|
from fides.api.util.collection_util import Row
|
|
34
35
|
from fides.config import get_config
|
|
36
|
+
from fides.config.config_proxy import ConfigProxy
|
|
35
37
|
|
|
36
38
|
from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip
|
|
37
39
|
Dataset as CtlDataset,
|
|
@@ -58,6 +60,23 @@ class SQLConnector(BaseConnector[Engine]):
|
|
|
58
60
|
)
|
|
59
61
|
self.ssh_server: sshtunnel._ForwardServer = None
|
|
60
62
|
|
|
63
|
+
def should_dry_run(self, mode_to_check: SqlDryRunMode) -> bool:
|
|
64
|
+
"""
|
|
65
|
+
Check if SQL dry run is enabled for the specified mode.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
mode_to_check: The SqlDryRunMode to check for
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
bool: True if the current mode matches the mode to check
|
|
72
|
+
"""
|
|
73
|
+
from fides.api.api.deps import get_autoclose_db_session as get_db
|
|
74
|
+
|
|
75
|
+
with get_db() as db:
|
|
76
|
+
config_proxy = ConfigProxy(db)
|
|
77
|
+
current_mode = getattr(config_proxy.execution, "sql_dry_run", None)
|
|
78
|
+
return current_mode == mode_to_check
|
|
79
|
+
|
|
61
80
|
@staticmethod
|
|
62
81
|
def cursor_result_to_rows(results: CursorResult) -> List[Row]:
|
|
63
82
|
"""Convert SQLAlchemy results to a list of dictionaries"""
|
|
@@ -140,6 +159,10 @@ class SQLConnector(BaseConnector[Engine]):
|
|
|
140
159
|
if query is None:
|
|
141
160
|
return []
|
|
142
161
|
|
|
162
|
+
if self.should_dry_run(SqlDryRunMode.access):
|
|
163
|
+
logger.warning(f"SQL DRY RUN - Would execute SQL: {query}")
|
|
164
|
+
return []
|
|
165
|
+
|
|
143
166
|
with client.connect() as connection:
|
|
144
167
|
self.set_schema(connection)
|
|
145
168
|
results = connection.execute(query)
|
|
@@ -160,6 +183,10 @@ class SQLConnector(BaseConnector[Engine]):
|
|
|
160
183
|
if stmt is None:
|
|
161
184
|
return []
|
|
162
185
|
|
|
186
|
+
if self.should_dry_run(SqlDryRunMode.access):
|
|
187
|
+
logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
|
|
188
|
+
return []
|
|
189
|
+
|
|
163
190
|
logger.info("Starting data retrieval for {}", node.address)
|
|
164
191
|
with client.connect() as connection:
|
|
165
192
|
self.set_schema(connection)
|
|
@@ -178,20 +205,25 @@ class SQLConnector(BaseConnector[Engine]):
|
|
|
178
205
|
privacy_request: PrivacyRequest,
|
|
179
206
|
request_task: RequestTask,
|
|
180
207
|
rows: List[Row],
|
|
208
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
181
209
|
) -> int:
|
|
182
210
|
"""Execute a masking request. Returns the number of records masked"""
|
|
183
211
|
query_config = self.query_config(node)
|
|
184
212
|
update_ct = 0
|
|
185
213
|
client = self.client()
|
|
214
|
+
|
|
186
215
|
for row in rows:
|
|
187
216
|
update_stmt: Optional[TextClause] = query_config.generate_update_stmt(
|
|
188
217
|
row, policy, privacy_request
|
|
189
218
|
)
|
|
190
219
|
if update_stmt is not None:
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
220
|
+
if self.should_dry_run(SqlDryRunMode.erasure):
|
|
221
|
+
logger.warning(f"SQL DRY RUN - Would execute SQL: {update_stmt}")
|
|
222
|
+
else:
|
|
223
|
+
with client.connect() as connection:
|
|
224
|
+
self.set_schema(connection)
|
|
225
|
+
results: LegacyCursorResult = connection.execute(update_stmt)
|
|
226
|
+
update_ct = update_ct + results.rowcount
|
|
195
227
|
return update_ct
|
|
196
228
|
|
|
197
229
|
def close(self) -> None:
|
|
@@ -75,6 +75,7 @@ class WebsiteConnector(BaseConnector):
|
|
|
75
75
|
privacy_request: PrivacyRequest,
|
|
76
76
|
request_task: RequestTask,
|
|
77
77
|
rows: List[Row],
|
|
78
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
78
79
|
) -> int:
|
|
79
80
|
"""DSR execution not supported for website connector"""
|
|
80
81
|
return 0
|
|
@@ -48,8 +48,8 @@
|
|
|
48
48
|
<div class="dsr-information-text">
|
|
49
49
|
<p>
|
|
50
50
|
This web link contains all data requested as part of your Data Subject Request (DSR). Your
|
|
51
|
-
information has been compiled from the following areas. Click on each section to open
|
|
52
|
-
your data
|
|
51
|
+
information has been compiled from the following areas. Click on each section to open and view
|
|
52
|
+
your data.
|
|
53
53
|
</p>
|
|
54
54
|
</div>
|
|
55
55
|
<div class="dataset-list">
|