ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0.dist-info}/METADATA +1 -1
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0.dist-info}/RECORD +297 -282
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/7e9a2b52f498_adding_masking_secrets.py +60 -0
- fides/api/alembic/migrations/versions/a7065df4dcf1_add_finalized_fields_to_privacy_request.py +65 -0
- fides/api/alembic/migrations/versions/d0031087eacb_create_manualtaskconditionaldependency_.py +106 -0
- fides/api/api/v1/endpoints/dataset_config_endpoints.py +13 -5
- fides/api/api/v1/endpoints/drp_endpoints.py +7 -1
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +44 -1
- fides/api/api/v1/endpoints/user_endpoints.py +83 -7
- fides/api/app_setup.py +3 -2
- fides/api/common_exceptions.py +8 -0
- fides/api/db/base.py +1 -0
- fides/api/db/database.py +1 -1
- fides/api/graph/execution.py +46 -0
- fides/api/graph/graph.py +13 -2
- fides/api/graph/traversal.py +126 -39
- fides/api/models/manual_task/__init__.py +2 -0
- fides/api/models/manual_task/conditional_dependency.py +144 -0
- fides/api/models/{manual_task.py → manual_task/manual_task.py} +10 -0
- fides/api/models/masking_secret.py +72 -0
- fides/api/models/policy.py +23 -0
- fides/api/models/privacy_request/execution_log.py +1 -0
- fides/api/models/privacy_request/privacy_request.py +64 -26
- fides/api/oauth/roles.py +2 -0
- fides/api/schemas/application_config.py +12 -1
- fides/api/schemas/connection_configuration/connection_secrets_datahub.py +10 -1
- fides/api/schemas/masking/masking_secrets.py +1 -1
- fides/api/schemas/policy.py +1 -0
- fides/api/schemas/privacy_request.py +5 -0
- fides/api/service/connectors/base_connector.py +15 -0
- fides/api/service/connectors/bigquery_connector.py +72 -19
- fides/api/service/connectors/dynamodb_connector.py +2 -1
- fides/api/service/connectors/fides_connector.py +1 -0
- fides/api/service/connectors/http_connector.py +1 -0
- fides/api/service/connectors/manual_task_connector.py +1 -0
- fides/api/service/connectors/manual_webhook_connector.py +2 -1
- fides/api/service/connectors/mongodb_connector.py +1 -0
- fides/api/service/connectors/okta_connector.py +1 -0
- fides/api/service/connectors/query_configs/bigquery_query_config.py +95 -36
- fides/api/service/connectors/query_configs/snowflake_query_config.py +3 -3
- fides/api/service/connectors/rds_mysql_connector.py +1 -0
- fides/api/service/connectors/rds_postgres_connector.py +1 -0
- fides/api/service/connectors/s3_connector.py +1 -0
- fides/api/service/connectors/saas_connector.py +1 -0
- fides/api/service/connectors/scylla_connector.py +1 -0
- fides/api/service/connectors/snowflake_connector.py +55 -2
- fides/api/service/connectors/sql_connector.py +159 -13
- fides/api/service/connectors/website_connector.py +1 -0
- fides/api/service/privacy_request/dsr_package/templates/welcome.html +2 -2
- fides/api/service/privacy_request/request_runner_service.py +145 -55
- fides/api/service/privacy_request/request_service.py +172 -52
- fides/api/task/conditional_dependencies/__init__.py +0 -0
- fides/api/task/conditional_dependencies/evaluator.py +109 -0
- fides/api/task/conditional_dependencies/schemas.py +54 -0
- fides/api/task/create_request_tasks.py +1 -1
- fides/api/task/deprecated_graph_task.py +24 -6
- fides/api/task/execute_request_tasks.py +93 -12
- fides/api/task/filter_results.py +1 -1
- fides/api/task/graph_task.py +86 -5
- fides/api/task/manual/manual_task_address.py +46 -0
- fides/api/task/manual/manual_task_graph_task.py +118 -126
- fides/api/task/manual/manual_task_utils.py +52 -105
- fides/api/util/aws_util.py +5 -1
- fides/api/util/cache.py +61 -0
- fides/api/util/encryption/secrets_util.py +48 -18
- fides/api/util/memory_watchdog.py +286 -0
- fides/common/api/scope_registry.py +3 -0
- fides/common/api/v1/urn_registry.py +1 -1
- fides/config/execution_settings.py +12 -0
- fides/config/utils.py +2 -0
- fides/service/privacy_request/privacy_request_service.py +6 -1
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/MNlh9olWjgbqAHKyQY3LF/_buildManifest.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1316-2606e19807c08aa5.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1467-8808ec8836e033f9.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1817-b78b58ae3b75d75a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1975.7d4634a0e823a02b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{203-5a663f465ba26bb4.js → 203-cd78ea279cecba60.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/2150-930ffaf2c4718edc.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/255-1bc0cbef7a59cdc6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3450-0ba194991d0cca88.js → 3450-69f4e16978971bb8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/346-aa3b88efb85f2e28.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3550-d04125c828d591a1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3855-e172870d3e21b0dd.js → 3855-509ca7ac99b5eada.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3872-46cebf7ec1b31a2b.js → 3872-0a0f0032ca39a93f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/409-ea70638a59296659.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4121-2bc09fc4ddbfe5cb.js → 4121-877e19d3fa078c7b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4230-60100f7ef3ddcde1.js → 4230-114e31621c19ea69.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4259.d1507e0db19cbed7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/431-1db919f6569a4021.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4608-bbb7bf511a05c3c2.js → 4608-f16f281f2d05d963.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/5163-e682273cd76a7d07.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/5309-d9a488457898263b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{905-ffdbd0b14167e8bd.js → 5596-4378b2927dae65b2.js} +3 -3
- fides/ui-build/static/admin/_next/static/chunks/5619-9b50cec521203989.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6084-7178ff6ea6822475.js → 6084-ddbad3149364725d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6148-59a59d5c5925344f.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6419-d0c00d661b01f8fa.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6662-507be5d46e5b719b.js → 6662-a9e54ead3dc53644.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6780-b42a27e72707936d.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6853-2ad3e08fe6f9f5f2.js → 6853-d0190d2cca9dbde2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{6882-6af16fef26c21e06.js → 6882-59ea739e3616ce83.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-ba98e778a5b45ebf.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{7476-281ee9a8286556f3.js → 7476-cc0d9a94ed7aad53.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/7725-539d3a906f627531.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{787-fb41002f797eb2df.js → 787-c57185ad89c4e288.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{79-7e87aff851423d4a.js → 79-2aab56be75e16187.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{796-329a5f823ec258a5.js → 796-3bdda2a7868464af.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/8735-40caf91800a3610c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/8765-f622a35b40a7ec63.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/9046-7085a401297c5520.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/9187-7438242f0d380bb0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9226-746771d47dff6266.js → 9226-2dcac54ab3fb94be.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9278-08cc704317fe535e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9951-9b753ad7c3f51bdf.js → 9951-7c6639e5d062779e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-39ccb07327c2c5d5.js → _app-750d6bd16c971bb9.js} +56 -56
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-98777246bec9dc2a.js → manual-2a655ff3a97f2492.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-a71c0aff4e0e6535.js → add-systems-0902f0bb4080643e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-1edf582ba3cd3bbb.js → [id]-f22ddd9b48a5c418.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-06bb3b0bf097fcdb.js → new-e74cb5ea87f15b40.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-685771e5f7196d87.js → privacy-experience-21f997c69fc3b4c2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-6ccedc70dc447089.js → [id]-da4124b7600a2a1d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-944bca1cc57985b5.js → new-a57d251c88ce68ae.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-84f4bd14ce8673bc.js → privacy-notices-ad105181bc91209b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-b0c4235fe6d0b0c8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-11d52f1570759c4d.js → [resourceUrn]-aad6047a4604b945.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-32eac8bbd217615a.js → projects-29784a11fe0fbd0a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-b83afa5565d0c84e.js → [resourceUrn]-b6b98cea25dd94fa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-6f630d42ac9fb6b4.js → data-catalog-7770a8dc34bd0fc0.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-3e5725cd06d7fe6c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-92b0bd97d8e79340.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-ee3c0a103346fc06.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-9aa744d56cdacb0d.js → activity-ad6a84a6276f914c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-14bd7500362ff224.js → [resourceUrn]-f98dd251babb7e28.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9e7dfd5a6acc2e8f.js → discovery-56eb4c014f0d96a3.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-d23b3ae139f0428b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-c0d2bfd465df20e0.js → [...subfieldNames]-15301bd6bf7cf718.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-28280a8a39a6e37c.js → [collectionName]-0fa72873e464f581.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-82fb246d87e58ebd.js → new-0d50084fbdf9b84c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-20165c31ab1bc7cf.js → dataset-f3348d0a92543bab.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-b4a6bcc87d126840.js → [id]-7d6027570d05c57f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-f95d7b0bbfc58f5a.js → new-c6614583b14dc9f2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{fides-js-docs-5d8fd1af75f19e2f.js → fides-js-docs-1f4335dca5c09860.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{index-1919aab9e5834b51.js → index-fbf9b845bb901238.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-8e346fb36e8034d2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-e2d5d7e2a5265e68.js → integrations-7f15cd8538cdc24d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-1bae386d8c190348.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-69ad86b7a8a9a115.js → table-migration-05616e2ae20ff4f8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-79f1576b1126975c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-f1d818242d8550f8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-d40a26bddb126c5c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-96a08c4431b5462c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-9d1840f8309b706e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-fc201657f4a782c7.js → [purpose_id]-e891d01ece59669e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-c2d39cba8396ef3a.js → consent-f61b87e79367865b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-d992103cc55901ae.js → custom-fields-f8eea5d508c60c64.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-023e1895552817de.js → locations-ed6a140b362c5baa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-ac403c0886b20e20.js → organization-ff9a34264d48c35f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-7a3396ac819c7904.js → test-datasets-a4b6d41ca679298b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-8314a819837f5b2a.js → [id]-c8f5fbaa83dd9945.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{systems-21f423a7c417aa9d.js → systems-c05b49ddec1a1b4f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-df0d88716578e295.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-da68efc31998dc66.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-173ac3a1ed2b05a6.js → user-management-e98dfc7d4f2a4e16.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/webpack-90e8ec1fc5c6455b.js +1 -0
- fides/ui-build/static/admin/_next/static/css/{5bfb2473e5701527.css → 23cf870196941c9a.css} +1 -1
- fides/ui-build/static/admin/_next/static/css/{94965f224bc991e9.css → 8bc1833f1fa53ff0.css} +1 -1
- fides/ui-build/static/admin/_next/static/css/d9924caa849931b3.css +1 -0
- fides/ui-build/static/admin/_next/static/css/dbcf63488933a4d5.css +29 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +4 -4
- fides/ui-build/static/admin/lib/fides.js +4 -4
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/8108ANFxs99VY7KZ_Xev2/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1316-6cc72a45ebf7ff81.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1807-3beab149351d5ded.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1817-e601e737e3cc7a0e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/255-7db55b0e3a0f9dea.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2599-6c4d22e75028d8b6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2858-0b44609b6be7850b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2866-a73888c17a195cbe.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3615-5e2d062d684b8fa1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/409-a257e14acebcd73b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/431-34f0b91c26f8d9ab.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5309-b2c4803370634ff8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/570-c99f07161bd339cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6780-e3d40aa17a4bf2e9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-bb875d9ac89f6030.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7062.fda15dcb7df85675.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7c79804f.7a7112aece470725.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9014-eeae6f581158e645.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9046-5c4c22c375de25b1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9278-9b1b5970f0702668.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9392.f4520f66206d347d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/ea076c0a.84423f606aef37cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-afdbd4665657cfa1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-2265ecb899d45fbc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-5d522637871ac6c8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-9ddb52ebb7ac4c71.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-7674b97d655c193b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-0a58aee2d1e7fa01.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-5094ffea13f32ed9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-32600543eb7b584f.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-10ce53ea356f8bad.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-5501bbb129fee9c4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-cbe4c8f9096b6543.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-e130c0197362e8f3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-6387fcc8cce872eb.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-ff5738706da07801.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/webpack-ff0cd6bff75588da.js +0 -1
- fides/ui-build/static/admin/_next/static/css/2cadb5f62dcd7c2b.css +0 -1
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{8108ANFxs99VY7KZ_Xev2 → MNlh9olWjgbqAHKyQY3LF}/_ssgManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{2921-455e6357b74d2f76.js → 2921-86f1547ac40a5cdf.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3923-6cc911dafccc5f63.js → 3923-13a6b4da2d51bf8f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{401-1b529d5800aa1f3a.js → 401-3cc1fee61494e3bd.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{5574-b13021775a15bfd2.js → 5574-9312f97b637d9ee2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7630-9aac73191ed5ed13.js → 7630-b1c93688013ef013.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{9826-111aaee8bd8dbd09.js → 9826-3c578665c6d3b21d.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{404-aece2c920ea14514.js → 404-2d803dab6a00f353.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-dc75dc6e37e52f05.js → multiple-8ff7f37913ad736a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-24d226b5a8de5c74.js → add-vendors-d00c9034cdeb0236.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-6a8ef51138ac926a.js → configure-0e1ca0f4c8e7f4da.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-6f86ab63a08a6528.js → properties-057cad65e7414a44.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-73d3cbf68f7c3a31.js → consent-e17c56eec8d91371.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-6ba9e160dae64695.js → [projectUrn]-80a6cc8e8573514a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{resources-7648bbd4f6711e4d.js → resources-6c3714ee97a718c1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-393e20924c83373e.js → [resourceUrn]-31e6c54794a9883e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-8733807dad4bc96e.js → detection-2822a423a7ad0550.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-006b695e5af5ef24.js → [datasetId]-a8e8b5f4ee7af86c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-c391c6fad56eec48.js → datastore-connection-3bd77864da523d41.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-53fecfb9dd6a1e0c.js → [id]-5627d0d0668077f9.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-76b01cec5fde10a9.js → add-template-feca66ad5c5fe54a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-5c08e8447c45ce44.js → ant-components-64a322d01aae5ca7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-06ad5f34585480aa.js → AntForm-8bca16a7726e7eb2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-6f071c2bc9446cb0.js → FormikAntFormItem-b0f246fc3b67ebf7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-efcc38c58991ac9e.js → FormikControlled-1a0852b090bfc392.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-430ba5c979abfb7c.js → FormikField-11f3de1b45e36583.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-5c561880bf131afb.js → forms-1b73a1c2b6c6285f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-d888a69a3bbe040e.js → configure-e551a860ec727802.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-d3d8e3d7583ec635.js → [id]-dd99183f93763ae4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-1af10ed303815d46.js → add-property-0bdbc1fcbf553b8f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{properties-cebc0dc186be499a.js → properties-e959378bb32b6b73.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-5e1322de868d615e.js → alpha-1066f0c202ef744c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-241f95e372b65d0f.js → about-37ba24a72a06862e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-41242f805599feda.js → domain-records-51333dbd21cb37c8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-2e885f74c92f669c.js → domains-bde86e5f6c09da5a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-ff112655ad5f41e5.js → email-templates-4f9a5cc8bea7725b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-86062a18e081a52a.js → regulations-102efd9199e87124.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-a2524414e968f862.js → new-de8cb3739ab99c09.js} +0 -0
|
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Union, cast
|
|
|
3
3
|
import pydash
|
|
4
4
|
from fideslang.models import MaskingStrategies
|
|
5
5
|
from loguru import logger
|
|
6
|
-
from sqlalchemy import MetaData, Table, text
|
|
6
|
+
from sqlalchemy import MetaData, Table, or_, text
|
|
7
7
|
from sqlalchemy.engine import Engine
|
|
8
8
|
from sqlalchemy.sql import Delete, Update
|
|
9
9
|
from sqlalchemy.sql.elements import ColumnElement, TextClause
|
|
@@ -93,7 +93,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
93
93
|
|
|
94
94
|
return where_clauses
|
|
95
95
|
|
|
96
|
-
def
|
|
96
|
+
def generate_table_name(self) -> str:
|
|
97
97
|
"""
|
|
98
98
|
Prepends the dataset ID and project ID to the base table name
|
|
99
99
|
if the BigQuery namespace meta is provided.
|
|
@@ -116,7 +116,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
116
116
|
Returns a query string with backtick formatting for tables that have the same names as
|
|
117
117
|
BigQuery reserved words.
|
|
118
118
|
"""
|
|
119
|
-
return f'SELECT {field_list} FROM `{self.
|
|
119
|
+
return f'SELECT {field_list} FROM `{self.generate_table_name()}` WHERE ({" OR ".join(clauses)})'
|
|
120
120
|
|
|
121
121
|
def generate_masking_stmt(
|
|
122
122
|
self,
|
|
@@ -125,6 +125,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
125
125
|
policy: Policy,
|
|
126
126
|
request: PrivacyRequest,
|
|
127
127
|
client: Engine,
|
|
128
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
128
129
|
) -> Union[List[Update], List[Delete]]:
|
|
129
130
|
"""
|
|
130
131
|
Generate a masking statement for BigQuery.
|
|
@@ -137,7 +138,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
137
138
|
logger.info(
|
|
138
139
|
f"Masking override detected for collection {node.address.value}: {masking_override.strategy.value}"
|
|
139
140
|
)
|
|
140
|
-
return self.generate_delete(
|
|
141
|
+
return self.generate_delete(client, input_data or {})
|
|
141
142
|
return self.generate_update(row, policy, request, client)
|
|
142
143
|
|
|
143
144
|
def generate_update(
|
|
@@ -196,11 +197,18 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
196
197
|
)
|
|
197
198
|
return []
|
|
198
199
|
|
|
199
|
-
table = Table(self.
|
|
200
|
+
table = Table(self.generate_table_name(), MetaData(bind=client), autoload=True)
|
|
200
201
|
where_clauses: List[ColumnElement] = [
|
|
201
|
-
|
|
202
|
+
table.c[k] == v for k, v in non_empty_reference_field_keys.items()
|
|
202
203
|
]
|
|
203
204
|
|
|
205
|
+
# Create update values using Column objects as keys to handle column names with spaces
|
|
206
|
+
update_values = {}
|
|
207
|
+
for column_name, value in final_update_map.items():
|
|
208
|
+
# Use bracket notation to access columns with spaces in their names
|
|
209
|
+
column = table.c[column_name]
|
|
210
|
+
update_values[column] = value
|
|
211
|
+
|
|
204
212
|
if self.partitioning:
|
|
205
213
|
partition_clauses = self.get_partition_clauses()
|
|
206
214
|
partitioned_queries = []
|
|
@@ -211,44 +219,55 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
211
219
|
partitioned_queries.append(
|
|
212
220
|
table.update()
|
|
213
221
|
.where(*(where_clauses + [text(partition_clause)]))
|
|
214
|
-
.values(
|
|
222
|
+
.values(update_values)
|
|
215
223
|
)
|
|
216
224
|
|
|
217
225
|
return partitioned_queries
|
|
218
226
|
|
|
219
|
-
return [table.update().where(*where_clauses).values(
|
|
227
|
+
return [table.update().where(*where_clauses).values(update_values)]
|
|
220
228
|
|
|
221
|
-
def generate_delete(
|
|
222
|
-
|
|
229
|
+
def generate_delete(
|
|
230
|
+
self,
|
|
231
|
+
client: Engine,
|
|
232
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
233
|
+
) -> List[Delete]:
|
|
234
|
+
"""
|
|
235
|
+
Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement.
|
|
223
236
|
|
|
224
237
|
Used when a collection-level masking override is present and the masking strategy is DELETE.
|
|
225
238
|
|
|
226
239
|
A List of multiple DELETE statements are returned for partitioned tables; for a non-partitioned table,
|
|
227
240
|
a single DELETE statement is returned in a List for consistent typing.
|
|
228
|
-
|
|
229
|
-
TODO: DRY up this method and `generate_update` a bit
|
|
230
241
|
"""
|
|
231
242
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
)
|
|
243
|
+
if not input_data:
|
|
244
|
+
logger.warning(
|
|
245
|
+
"No input data provided for node '{}', skipping DELETE statement generation",
|
|
246
|
+
self.node.address,
|
|
247
|
+
)
|
|
248
|
+
return []
|
|
239
249
|
|
|
240
|
-
|
|
241
|
-
|
|
250
|
+
filtered_data = self.node.typed_filtered_values(input_data)
|
|
251
|
+
|
|
252
|
+
if not filtered_data:
|
|
242
253
|
logger.warning(
|
|
243
254
|
"There is not enough data to generate a valid DELETE statement for {}",
|
|
244
255
|
self.node.address,
|
|
245
256
|
)
|
|
246
257
|
return []
|
|
247
258
|
|
|
248
|
-
table = Table(self.
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
]
|
|
259
|
+
table = Table(self.generate_table_name(), MetaData(bind=client), autoload=True)
|
|
260
|
+
|
|
261
|
+
# Build individual reference clauses
|
|
262
|
+
where_clauses: List[ColumnElement] = []
|
|
263
|
+
for column_name, values in filtered_data.items():
|
|
264
|
+
if len(values) == 1:
|
|
265
|
+
where_clauses.append(table.c[column_name] == values[0])
|
|
266
|
+
else:
|
|
267
|
+
where_clauses.append(table.c[column_name].in_(values))
|
|
268
|
+
|
|
269
|
+
# Combine reference clauses with OR instead of AND
|
|
270
|
+
combined_reference_clause = or_(*where_clauses)
|
|
252
271
|
|
|
253
272
|
if self.partitioning:
|
|
254
273
|
partition_clauses = self.get_partition_clauses()
|
|
@@ -259,12 +278,25 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
259
278
|
|
|
260
279
|
for partition_clause in partition_clauses:
|
|
261
280
|
partitioned_queries.append(
|
|
262
|
-
table.delete()
|
|
281
|
+
table.delete()
|
|
282
|
+
.where(combined_reference_clause)
|
|
283
|
+
.where(text(partition_clause))
|
|
263
284
|
)
|
|
264
285
|
|
|
265
286
|
return partitioned_queries
|
|
266
287
|
|
|
267
|
-
return [table.delete().where(
|
|
288
|
+
return [table.delete().where(combined_reference_clause)]
|
|
289
|
+
|
|
290
|
+
def uses_delete_masking_strategy(self) -> bool:
|
|
291
|
+
"""Check if this collection uses DELETE masking strategy.
|
|
292
|
+
|
|
293
|
+
Returns True if masking override is present and strategy is DELETE.
|
|
294
|
+
"""
|
|
295
|
+
masking_override = self.node.collection.masking_strategy_override
|
|
296
|
+
return (
|
|
297
|
+
masking_override is not None
|
|
298
|
+
and masking_override.strategy == MaskingStrategies.DELETE
|
|
299
|
+
)
|
|
268
300
|
|
|
269
301
|
def format_fields_for_query(
|
|
270
302
|
self,
|
|
@@ -281,6 +313,25 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
281
313
|
formatted_fields.append(field_path.levels[0])
|
|
282
314
|
return formatted_fields
|
|
283
315
|
|
|
316
|
+
def format_clause_for_query(
|
|
317
|
+
self, string_path: str, operator: str, operand: str
|
|
318
|
+
) -> str:
|
|
319
|
+
"""
|
|
320
|
+
Returns clauses with proper BigQuery backtick escaping for column names.
|
|
321
|
+
Handles column names with spaces and nested fields (dot-separated) by escaping each part individually.
|
|
322
|
+
"""
|
|
323
|
+
# For nested fields (containing dots), escape each part individually
|
|
324
|
+
if "." in string_path:
|
|
325
|
+
parts = string_path.split(".")
|
|
326
|
+
escaped_field = ".".join(f"`{part}`" for part in parts)
|
|
327
|
+
else:
|
|
328
|
+
# For simple fields, wrap the entire name in backticks
|
|
329
|
+
escaped_field = f"`{string_path}`"
|
|
330
|
+
|
|
331
|
+
if operator == "IN":
|
|
332
|
+
return f"{escaped_field} IN ({operand})"
|
|
333
|
+
return f"{escaped_field} {operator} :{operand}"
|
|
334
|
+
|
|
284
335
|
def generate_raw_query_without_tuples(
|
|
285
336
|
self, field_list: List[str], filters: Dict[str, List[Any]]
|
|
286
337
|
) -> Optional[TextClause]:
|
|
@@ -290,27 +341,35 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
290
341
|
|
|
291
342
|
This is an override of the base class method that supports nested fields for BigQuery.
|
|
292
343
|
|
|
293
|
-
Examples with
|
|
344
|
+
Examples with field names containing dots and spaces, notice these are replaced with underscores in the parameter bindings:
|
|
294
345
|
|
|
295
346
|
1. Single value filter:
|
|
296
347
|
field_list = ["id", "name", "email"]
|
|
297
348
|
filters = {"user.id": [123]}
|
|
298
349
|
|
|
299
|
-
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (user
|
|
350
|
+
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`user`.`id` = :user_id)
|
|
300
351
|
With parameter binding: user_id = 123
|
|
301
352
|
|
|
302
|
-
2.
|
|
353
|
+
2. Field with spaces:
|
|
354
|
+
field_list = ["id", "custom id", "email"]
|
|
355
|
+
filters = {"custom id": ["abc123"]}
|
|
356
|
+
|
|
357
|
+
Generates: SELECT id, `custom id`, email FROM `project_id.dataset_id.table_name` WHERE (`custom id` = :custom_id)
|
|
358
|
+
With parameter binding: custom_id = "abc123"
|
|
359
|
+
|
|
360
|
+
3. Multiple value filter with nested field:
|
|
303
361
|
field_list = ["id", "name", "email"]
|
|
304
|
-
filters = {"
|
|
362
|
+
filters = {"contact_info.primary_email": ["active", "pending"]}
|
|
305
363
|
|
|
306
|
-
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (
|
|
307
|
-
With parameter bindings:
|
|
364
|
+
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`contact_info`.`primary_email` IN (:contact_info_primary_email_in_stmt_generated_0, :contact_info_primary_email_in_stmt_generated_1))
|
|
365
|
+
With parameter bindings: contact_info_primary_email_in_stmt_generated_0 = "active", contact_info_primary_email_in_stmt_generated_1 = "pending"
|
|
308
366
|
"""
|
|
309
367
|
clauses = []
|
|
310
368
|
query_data = {}
|
|
311
369
|
for field_name, field_value in filters.items():
|
|
312
|
-
# Replace dots with underscores in field names for parameter binding
|
|
313
|
-
|
|
370
|
+
# Replace dots and spaces with underscores in field names for parameter binding
|
|
371
|
+
# SQLAlchemy parameter names cannot contain spaces or special characters
|
|
372
|
+
field_binding_name = field_name.replace(".", "_").replace(" ", "_")
|
|
314
373
|
data = set(field_value)
|
|
315
374
|
if len(data) == 1:
|
|
316
375
|
clauses.append(
|
|
@@ -333,7 +392,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
333
392
|
clauses.append(self.format_clause_for_query(field_name, "IN", operand))
|
|
334
393
|
|
|
335
394
|
if len(clauses) > 0:
|
|
336
|
-
formatted_fields = ", ".join(field_list)
|
|
395
|
+
formatted_fields = ", ".join([f"`{field}`" for field in field_list])
|
|
337
396
|
query_str = self.get_formatted_query_string(formatted_fields, clauses)
|
|
338
397
|
return text(query_str).params(query_data)
|
|
339
398
|
|
|
@@ -30,7 +30,7 @@ class SnowflakeQueryConfig(SQLQueryConfig):
|
|
|
30
30
|
"""Returns field names in clauses surrounded by quotation marks as required by Snowflake syntax."""
|
|
31
31
|
return f'"{string_path}" {operator} (:{operand})'
|
|
32
32
|
|
|
33
|
-
def
|
|
33
|
+
def generate_table_name(self) -> str:
|
|
34
34
|
"""
|
|
35
35
|
Prepends the dataset name and schema to the base table name
|
|
36
36
|
if the Snowflake namespace meta is provided.
|
|
@@ -57,7 +57,7 @@ class SnowflakeQueryConfig(SQLQueryConfig):
|
|
|
57
57
|
clauses: List[str],
|
|
58
58
|
) -> str:
|
|
59
59
|
"""Returns a query string with double quotation mark formatting as required by Snowflake syntax."""
|
|
60
|
-
return f'SELECT {field_list} FROM {self.
|
|
60
|
+
return f'SELECT {field_list} FROM {self.generate_table_name()} WHERE ({" OR ".join(clauses)})'
|
|
61
61
|
|
|
62
62
|
def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]:
|
|
63
63
|
"""Adds the appropriate formatting for update statements in this datastore."""
|
|
@@ -69,4 +69,4 @@ class SnowflakeQueryConfig(SQLQueryConfig):
|
|
|
69
69
|
where_clauses: List[str],
|
|
70
70
|
) -> str:
|
|
71
71
|
"""Returns a parameterized update statement in Snowflake dialect."""
|
|
72
|
-
return f'UPDATE {self.
|
|
72
|
+
return f'UPDATE {self.generate_table_name()} SET {", ".join(update_clauses)} WHERE {" AND ".join(where_clauses)}'
|
|
@@ -158,6 +158,7 @@ class RDSMySQLConnector(RDSConnectorMixin, SQLConnector):
|
|
|
158
158
|
privacy_request: PrivacyRequest,
|
|
159
159
|
request_task: RequestTask,
|
|
160
160
|
rows: List[Row],
|
|
161
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
161
162
|
) -> int:
|
|
162
163
|
"""DSR execution not yet supported for RDS MySQL"""
|
|
163
164
|
return 0
|
|
@@ -147,6 +147,7 @@ class RDSPostgresConnector(RDSConnectorMixin, SQLConnector):
|
|
|
147
147
|
privacy_request: PrivacyRequest,
|
|
148
148
|
request_task: RequestTask,
|
|
149
149
|
rows: List[Row],
|
|
150
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
150
151
|
) -> int:
|
|
151
152
|
"""DSR execution not yet supported for RDS Postgres"""
|
|
152
153
|
return 0
|
|
@@ -517,6 +517,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
|
|
|
517
517
|
privacy_request: PrivacyRequest,
|
|
518
518
|
request_task: RequestTask,
|
|
519
519
|
rows: List[Row],
|
|
520
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
520
521
|
) -> int:
|
|
521
522
|
"""Execute a masking request. Return the number of rows that have been updated."""
|
|
522
523
|
self.set_privacy_request_state(privacy_request, node, request_task)
|
|
@@ -150,6 +150,7 @@ class ScyllaConnector(BaseConnector[Cluster]):
|
|
|
150
150
|
privacy_request: PrivacyRequest,
|
|
151
151
|
request_task: RequestTask,
|
|
152
152
|
rows: List[Row],
|
|
153
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
153
154
|
) -> int:
|
|
154
155
|
"""Execute a masking request"""
|
|
155
156
|
query_config = self.query_config(node)
|
|
@@ -3,11 +3,11 @@ from typing import Any, Dict, Union
|
|
|
3
3
|
from cryptography.hazmat.backends import default_backend
|
|
4
4
|
from cryptography.hazmat.primitives import serialization
|
|
5
5
|
from snowflake.sqlalchemy import URL as Snowflake_URL
|
|
6
|
+
from sqlalchemy import text
|
|
6
7
|
from sqlalchemy.orm import Session
|
|
7
8
|
|
|
8
9
|
from fides.api.graph.execution import ExecutionNode
|
|
9
10
|
from fides.api.schemas.connection_configuration import SnowflakeSchema
|
|
10
|
-
from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
|
|
11
11
|
from fides.api.service.connectors.query_configs.snowflake_query_config import (
|
|
12
12
|
SnowflakeQueryConfig,
|
|
13
13
|
)
|
|
@@ -69,10 +69,63 @@ class SnowflakeConnector(SQLConnector):
|
|
|
69
69
|
connect_args["private_key"] = private_key
|
|
70
70
|
return connect_args
|
|
71
71
|
|
|
72
|
-
def query_config(self, node: ExecutionNode) ->
|
|
72
|
+
def query_config(self, node: ExecutionNode) -> SnowflakeQueryConfig:
|
|
73
73
|
"""Query wrapper corresponding to the input execution_node."""
|
|
74
74
|
|
|
75
75
|
db: Session = Session.object_session(self.configuration)
|
|
76
76
|
return SnowflakeQueryConfig(
|
|
77
77
|
node, SQLConnector.get_namespace_meta(db, node.address.dataset)
|
|
78
78
|
)
|
|
79
|
+
|
|
80
|
+
def get_qualified_table_name(self, node: ExecutionNode) -> str:
|
|
81
|
+
"""Get fully qualified Snowflake table name using existing query config logic"""
|
|
82
|
+
query_config = self.query_config(node)
|
|
83
|
+
return query_config.generate_table_name()
|
|
84
|
+
|
|
85
|
+
def table_exists(self, qualified_table_name: str) -> bool:
|
|
86
|
+
"""
|
|
87
|
+
Check if table exists in Snowflake using the proper three-part naming convention.
|
|
88
|
+
|
|
89
|
+
Snowflake supports database.schema.table naming, and the generic SQLConnector
|
|
90
|
+
table_exists method doesn't handle quoted identifiers properly.
|
|
91
|
+
"""
|
|
92
|
+
try:
|
|
93
|
+
client = self.create_client()
|
|
94
|
+
with client.connect() as connection:
|
|
95
|
+
# Remove quotes and split the parts
|
|
96
|
+
clean_name = qualified_table_name.replace('"', "")
|
|
97
|
+
parts = clean_name.split(".")
|
|
98
|
+
|
|
99
|
+
if len(parts) == 1:
|
|
100
|
+
# Simple table name - use current schema
|
|
101
|
+
table_name = parts[0]
|
|
102
|
+
result = connection.execute(text(f'DESC TABLE "{table_name}"'))
|
|
103
|
+
elif len(parts) == 2:
|
|
104
|
+
# schema.table format
|
|
105
|
+
schema_name, table_name = parts
|
|
106
|
+
result = connection.execute(
|
|
107
|
+
text(f'DESC TABLE "{schema_name}"."{table_name}"')
|
|
108
|
+
)
|
|
109
|
+
elif len(parts) >= 3:
|
|
110
|
+
# database.schema.table format
|
|
111
|
+
database_name, schema_name, table_name = (
|
|
112
|
+
parts[-3],
|
|
113
|
+
parts[-2],
|
|
114
|
+
parts[-1],
|
|
115
|
+
)
|
|
116
|
+
# Use the database.schema.table format
|
|
117
|
+
result = connection.execute(
|
|
118
|
+
text(
|
|
119
|
+
f'DESC TABLE "{database_name}"."{schema_name}"."{table_name}"'
|
|
120
|
+
)
|
|
121
|
+
)
|
|
122
|
+
else:
|
|
123
|
+
return False
|
|
124
|
+
|
|
125
|
+
# If we get here without an exception, the table exists
|
|
126
|
+
result.close()
|
|
127
|
+
return True
|
|
128
|
+
|
|
129
|
+
except Exception:
|
|
130
|
+
# Table doesn't exist or other error
|
|
131
|
+
return False
|
|
@@ -6,7 +6,7 @@ import paramiko
|
|
|
6
6
|
import sshtunnel # type: ignore
|
|
7
7
|
from aiohttp.client_exceptions import ClientResponseError
|
|
8
8
|
from loguru import logger
|
|
9
|
-
from sqlalchemy import Column, select
|
|
9
|
+
from sqlalchemy import Column, inspect, select
|
|
10
10
|
from sqlalchemy.dialects.postgresql import JSONB
|
|
11
11
|
from sqlalchemy.engine import ( # type: ignore
|
|
12
12
|
Connection,
|
|
@@ -22,16 +22,19 @@ from sqlalchemy.sql.elements import TextClause
|
|
|
22
22
|
from fides.api.common_exceptions import (
|
|
23
23
|
ConnectionException,
|
|
24
24
|
SSHTunnelConfigNotFoundException,
|
|
25
|
+
TableNotFound,
|
|
25
26
|
)
|
|
26
27
|
from fides.api.graph.execution import ExecutionNode
|
|
27
28
|
from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
|
|
28
29
|
from fides.api.models.policy import Policy
|
|
29
30
|
from fides.api.models.privacy_request import PrivacyRequest, RequestTask
|
|
31
|
+
from fides.api.schemas.application_config import SqlDryRunMode
|
|
30
32
|
from fides.api.schemas.connection_configuration import ConnectionConfigSecretsSchema
|
|
31
33
|
from fides.api.service.connectors.base_connector import BaseConnector
|
|
32
34
|
from fides.api.service.connectors.query_configs.query_config import SQLQueryConfig
|
|
33
35
|
from fides.api.util.collection_util import Row
|
|
34
36
|
from fides.config import get_config
|
|
37
|
+
from fides.config.config_proxy import ConfigProxy
|
|
35
38
|
|
|
36
39
|
from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip
|
|
37
40
|
Dataset as CtlDataset,
|
|
@@ -58,6 +61,23 @@ class SQLConnector(BaseConnector[Engine]):
|
|
|
58
61
|
)
|
|
59
62
|
self.ssh_server: sshtunnel._ForwardServer = None
|
|
60
63
|
|
|
64
|
+
def should_dry_run(self, mode_to_check: SqlDryRunMode) -> bool:
|
|
65
|
+
"""
|
|
66
|
+
Check if SQL dry run is enabled for the specified mode.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
mode_to_check: The SqlDryRunMode to check for
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
bool: True if the current mode matches the mode to check
|
|
73
|
+
"""
|
|
74
|
+
from fides.api.api.deps import get_autoclose_db_session as get_db
|
|
75
|
+
|
|
76
|
+
with get_db() as db:
|
|
77
|
+
config_proxy = ConfigProxy(db)
|
|
78
|
+
current_mode = getattr(config_proxy.execution, "sql_dry_run", None)
|
|
79
|
+
return current_mode == mode_to_check
|
|
80
|
+
|
|
61
81
|
@staticmethod
|
|
62
82
|
def cursor_result_to_rows(results: CursorResult) -> List[Row]:
|
|
63
83
|
"""Convert SQLAlchemy results to a list of dictionaries"""
|
|
@@ -140,6 +160,10 @@ class SQLConnector(BaseConnector[Engine]):
|
|
|
140
160
|
if query is None:
|
|
141
161
|
return []
|
|
142
162
|
|
|
163
|
+
if self.should_dry_run(SqlDryRunMode.access):
|
|
164
|
+
logger.warning(f"SQL DRY RUN - Would execute SQL: {query}")
|
|
165
|
+
return []
|
|
166
|
+
|
|
143
167
|
with client.connect() as connection:
|
|
144
168
|
self.set_schema(connection)
|
|
145
169
|
results = connection.execute(query)
|
|
@@ -160,16 +184,34 @@ class SQLConnector(BaseConnector[Engine]):
|
|
|
160
184
|
if stmt is None:
|
|
161
185
|
return []
|
|
162
186
|
|
|
187
|
+
if self.should_dry_run(SqlDryRunMode.access):
|
|
188
|
+
logger.warning(f"SQL DRY RUN - Would execute SQL: {stmt}")
|
|
189
|
+
return []
|
|
190
|
+
|
|
163
191
|
logger.info("Starting data retrieval for {}", node.address)
|
|
164
192
|
with client.connect() as connection:
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
193
|
+
try:
|
|
194
|
+
self.set_schema(connection)
|
|
195
|
+
if (
|
|
196
|
+
query_config.partitioning
|
|
197
|
+
): # only BigQuery supports partitioning, for now
|
|
198
|
+
return self.partitioned_retrieval(query_config, connection, stmt)
|
|
199
|
+
|
|
200
|
+
results = connection.execute(stmt)
|
|
201
|
+
return self.cursor_result_to_rows(results)
|
|
202
|
+
except Exception as exc:
|
|
203
|
+
# Check if table exists using qualified table name
|
|
204
|
+
qualified_table_name = self.get_qualified_table_name(node)
|
|
205
|
+
if not self.table_exists(qualified_table_name):
|
|
206
|
+
# Central decision point - will raise TableNotFound or ConnectionException
|
|
207
|
+
self.handle_table_not_found(
|
|
208
|
+
node=node,
|
|
209
|
+
table_name=qualified_table_name,
|
|
210
|
+
operation_context="data retrieval",
|
|
211
|
+
original_exception=exc,
|
|
212
|
+
)
|
|
213
|
+
# Table exists or can't check - re-raise original exception
|
|
214
|
+
raise
|
|
173
215
|
|
|
174
216
|
def mask_data(
|
|
175
217
|
self,
|
|
@@ -178,20 +220,41 @@ class SQLConnector(BaseConnector[Engine]):
|
|
|
178
220
|
privacy_request: PrivacyRequest,
|
|
179
221
|
request_task: RequestTask,
|
|
180
222
|
rows: List[Row],
|
|
223
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
181
224
|
) -> int:
|
|
182
225
|
"""Execute a masking request. Returns the number of records masked"""
|
|
183
226
|
query_config = self.query_config(node)
|
|
184
227
|
update_ct = 0
|
|
185
228
|
client = self.client()
|
|
229
|
+
|
|
186
230
|
for row in rows:
|
|
187
231
|
update_stmt: Optional[TextClause] = query_config.generate_update_stmt(
|
|
188
232
|
row, policy, privacy_request
|
|
189
233
|
)
|
|
190
234
|
if update_stmt is not None:
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
235
|
+
if self.should_dry_run(SqlDryRunMode.erasure):
|
|
236
|
+
logger.warning(f"SQL DRY RUN - Would execute SQL: {update_stmt}")
|
|
237
|
+
else:
|
|
238
|
+
with client.connect() as connection:
|
|
239
|
+
try:
|
|
240
|
+
self.set_schema(connection)
|
|
241
|
+
results: LegacyCursorResult = connection.execute(
|
|
242
|
+
update_stmt
|
|
243
|
+
)
|
|
244
|
+
update_ct = update_ct + results.rowcount
|
|
245
|
+
except Exception as exc:
|
|
246
|
+
# Check if table exists using qualified table name
|
|
247
|
+
qualified_table_name = self.get_qualified_table_name(node)
|
|
248
|
+
if not self.table_exists(qualified_table_name):
|
|
249
|
+
# Central decision point - will raise TableNotFound or ConnectionException
|
|
250
|
+
self.handle_table_not_found(
|
|
251
|
+
node=node,
|
|
252
|
+
table_name=qualified_table_name,
|
|
253
|
+
operation_context="data erasure",
|
|
254
|
+
original_exception=exc,
|
|
255
|
+
)
|
|
256
|
+
# Table exists or can't check - re-raise original exception
|
|
257
|
+
raise
|
|
195
258
|
return update_ct
|
|
196
259
|
|
|
197
260
|
def close(self) -> None:
|
|
@@ -258,3 +321,86 @@ class SQLConnector(BaseConnector[Engine]):
|
|
|
258
321
|
raise NotImplementedError(
|
|
259
322
|
"Partitioned retrieval is only supported for BigQuery currently!"
|
|
260
323
|
)
|
|
324
|
+
|
|
325
|
+
def get_qualified_table_name(self, node: ExecutionNode) -> str:
|
|
326
|
+
"""
|
|
327
|
+
Get the fully qualified table name for this database.
|
|
328
|
+
|
|
329
|
+
Default: Returns the simple collection name
|
|
330
|
+
Override: Database-specific connectors can implement namespace resolution
|
|
331
|
+
"""
|
|
332
|
+
return node.collection.name
|
|
333
|
+
|
|
334
|
+
def table_exists(self, qualified_table_name: str) -> bool:
|
|
335
|
+
"""
|
|
336
|
+
Check if table exists using SQLAlchemy introspection.
|
|
337
|
+
|
|
338
|
+
This is a generic implementation that should work for most SQL databases.
|
|
339
|
+
Override: Connectors can implement database-specific table existence checking
|
|
340
|
+
"""
|
|
341
|
+
try:
|
|
342
|
+
client = self.create_client()
|
|
343
|
+
with client.connect() as connection:
|
|
344
|
+
inspector = inspect(connection)
|
|
345
|
+
|
|
346
|
+
# For simple table names
|
|
347
|
+
if "." not in qualified_table_name:
|
|
348
|
+
return inspector.has_table(qualified_table_name)
|
|
349
|
+
|
|
350
|
+
# For qualified names like schema.table or database.schema.table
|
|
351
|
+
parts = qualified_table_name.split(".")
|
|
352
|
+
|
|
353
|
+
if len(parts) == 2:
|
|
354
|
+
# schema.table format
|
|
355
|
+
schema_name, table_name = parts
|
|
356
|
+
return inspector.has_table(table_name, schema=schema_name)
|
|
357
|
+
|
|
358
|
+
if len(parts) >= 3:
|
|
359
|
+
# database.schema.table format (use schema.table)
|
|
360
|
+
schema_name, table_name = parts[-2], parts[-1]
|
|
361
|
+
return inspector.has_table(table_name, schema=schema_name)
|
|
362
|
+
|
|
363
|
+
# Fallback for unexpected format
|
|
364
|
+
return inspector.has_table(qualified_table_name)
|
|
365
|
+
|
|
366
|
+
except Exception as exc:
|
|
367
|
+
# Graceful fallback - if we can't check, assume table exists
|
|
368
|
+
# to preserve existing behavior for connectors that don't implement this
|
|
369
|
+
logger.error("Unable to check if table exists, assuming it does: {}", exc)
|
|
370
|
+
return True
|
|
371
|
+
|
|
372
|
+
def handle_table_not_found(
|
|
373
|
+
self,
|
|
374
|
+
node: ExecutionNode,
|
|
375
|
+
table_name: str,
|
|
376
|
+
operation_context: str,
|
|
377
|
+
original_exception: Optional[Exception] = None,
|
|
378
|
+
) -> None:
|
|
379
|
+
"""
|
|
380
|
+
Central decision point for table-not-found scenarios.
|
|
381
|
+
|
|
382
|
+
Raises TableNotFound (for collection skipping) or ConnectionException (for hard errors).
|
|
383
|
+
The raised exception will be caught by the @retry decorator in graph_task.py.
|
|
384
|
+
|
|
385
|
+
Args:
|
|
386
|
+
node: The ExecutionNode being processed
|
|
387
|
+
table_name: Name of the missing table
|
|
388
|
+
operation_context: Context like "data retrieval" or "data masking"
|
|
389
|
+
original_exception: The original exception that triggered this check
|
|
390
|
+
"""
|
|
391
|
+
if node.has_outgoing_dependencies():
|
|
392
|
+
# Collection has dependencies - cannot skip safely
|
|
393
|
+
error_msg = (
|
|
394
|
+
f"Table '{table_name}' did not exist during {operation_context}. "
|
|
395
|
+
f"Cannot skip collection '{node.address}' because other collections depend on it."
|
|
396
|
+
)
|
|
397
|
+
if original_exception:
|
|
398
|
+
raise ConnectionException(error_msg) from original_exception
|
|
399
|
+
raise ConnectionException(error_msg)
|
|
400
|
+
|
|
401
|
+
# Safe to skip - raise TableNotFound for @retry decorator to catch
|
|
402
|
+
skip_msg = f"Table '{table_name}' did not exist during {operation_context}."
|
|
403
|
+
if original_exception:
|
|
404
|
+
raise TableNotFound(skip_msg) from original_exception
|
|
405
|
+
|
|
406
|
+
raise TableNotFound(skip_msg)
|
|
@@ -75,6 +75,7 @@ class WebsiteConnector(BaseConnector):
|
|
|
75
75
|
privacy_request: PrivacyRequest,
|
|
76
76
|
request_task: RequestTask,
|
|
77
77
|
rows: List[Row],
|
|
78
|
+
input_data: Optional[Dict[str, List[Any]]] = None,
|
|
78
79
|
) -> int:
|
|
79
80
|
"""DSR execution not supported for website connector"""
|
|
80
81
|
return 0
|
|
@@ -48,8 +48,8 @@
|
|
|
48
48
|
<div class="dsr-information-text">
|
|
49
49
|
<p>
|
|
50
50
|
This web link contains all data requested as part of your Data Subject Request (DSR). Your
|
|
51
|
-
information has been compiled from the following areas. Click on each section to open
|
|
52
|
-
your data
|
|
51
|
+
information has been compiled from the following areas. Click on each section to open and view
|
|
52
|
+
your data.
|
|
53
53
|
</p>
|
|
54
54
|
</div>
|
|
55
55
|
<div class="dataset-list">
|