ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0rc0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/METADATA +1 -1
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/RECORD +292 -279
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/7e9a2b52f498_adding_masking_secrets.py +60 -0
- fides/api/alembic/migrations/versions/a7065df4dcf1_add_finalized_fields_to_privacy_request.py +65 -0
- fides/api/alembic/migrations/versions/d0031087eacb_create_manualtaskconditionaldependency_.py +106 -0
- fides/api/api/v1/endpoints/dataset_config_endpoints.py +13 -5
- fides/api/api/v1/endpoints/drp_endpoints.py +7 -1
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +44 -1
- fides/api/api/v1/endpoints/user_endpoints.py +83 -7
- fides/api/app_setup.py +3 -2
- fides/api/common_exceptions.py +4 -0
- fides/api/db/base.py +1 -0
- fides/api/db/database.py +1 -1
- fides/api/graph/execution.py +30 -0
- fides/api/graph/graph.py +13 -2
- fides/api/graph/traversal.py +126 -39
- fides/api/models/manual_task/__init__.py +2 -0
- fides/api/models/manual_task/conditional_dependency.py +144 -0
- fides/api/models/{manual_task.py → manual_task/manual_task.py} +10 -0
- fides/api/models/masking_secret.py +72 -0
- fides/api/models/policy.py +23 -0
- fides/api/models/privacy_request/execution_log.py +1 -0
- fides/api/models/privacy_request/privacy_request.py +31 -13
- fides/api/oauth/roles.py +2 -0
- fides/api/schemas/application_config.py +11 -1
- fides/api/schemas/masking/masking_secrets.py +1 -1
- fides/api/schemas/policy.py +1 -0
- fides/api/schemas/privacy_request.py +5 -0
- fides/api/service/connectors/base_connector.py +1 -0
- fides/api/service/connectors/bigquery_connector.py +67 -19
- fides/api/service/connectors/dynamodb_connector.py +2 -1
- fides/api/service/connectors/fides_connector.py +1 -0
- fides/api/service/connectors/http_connector.py +1 -0
- fides/api/service/connectors/manual_task_connector.py +1 -0
- fides/api/service/connectors/manual_webhook_connector.py +2 -1
- fides/api/service/connectors/mongodb_connector.py +1 -0
- fides/api/service/connectors/okta_connector.py +1 -0
- fides/api/service/connectors/query_configs/bigquery_query_config.py +91 -32
- fides/api/service/connectors/rds_mysql_connector.py +1 -0
- fides/api/service/connectors/rds_postgres_connector.py +1 -0
- fides/api/service/connectors/s3_connector.py +1 -0
- fides/api/service/connectors/saas_connector.py +1 -0
- fides/api/service/connectors/scylla_connector.py +1 -0
- fides/api/service/connectors/sql_connector.py +36 -4
- fides/api/service/connectors/website_connector.py +1 -0
- fides/api/service/privacy_request/dsr_package/templates/welcome.html +2 -2
- fides/api/service/privacy_request/request_runner_service.py +142 -53
- fides/api/service/privacy_request/request_service.py +1 -22
- fides/api/task/conditional_dependencies/__init__.py +0 -0
- fides/api/task/conditional_dependencies/evaluator.py +109 -0
- fides/api/task/conditional_dependencies/schemas.py +54 -0
- fides/api/task/create_request_tasks.py +1 -1
- fides/api/task/deprecated_graph_task.py +24 -6
- fides/api/task/execute_request_tasks.py +89 -12
- fides/api/task/filter_results.py +1 -1
- fides/api/task/graph_task.py +38 -3
- fides/api/task/manual/manual_task_address.py +46 -0
- fides/api/task/manual/manual_task_graph_task.py +118 -126
- fides/api/task/manual/manual_task_utils.py +52 -105
- fides/api/util/aws_util.py +5 -1
- fides/api/util/cache.py +5 -0
- fides/api/util/encryption/secrets_util.py +48 -18
- fides/common/api/scope_registry.py +3 -0
- fides/common/api/v1/urn_registry.py +1 -1
- fides/config/execution_settings.py +4 -0
- fides/config/utils.py +1 -0
- fides/service/privacy_request/privacy_request_service.py +6 -1
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/GKmhMPa_1gMto8JZO8ENy/_buildManifest.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1316-2606e19807c08aa5.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1467-8808ec8836e033f9.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1817-b78b58ae3b75d75a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1975.7d4634a0e823a02b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{203-5a663f465ba26bb4.js → 203-cd78ea279cecba60.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/2150-930ffaf2c4718edc.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/255-1bc0cbef7a59cdc6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3450-0ba194991d0cca88.js → 3450-ca4ba70da999f264.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/346-aa3b88efb85f2e28.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3550-d04125c828d591a1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3855-e172870d3e21b0dd.js → 3855-509ca7ac99b5eada.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3872-46cebf7ec1b31a2b.js → 3872-0a0f0032ca39a93f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/409-ea70638a59296659.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4121-2bc09fc4ddbfe5cb.js → 4121-877e19d3fa078c7b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4230-60100f7ef3ddcde1.js → 4230-114e31621c19ea69.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4259.d1507e0db19cbed7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/431-1db919f6569a4021.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4608-bbb7bf511a05c3c2.js → 4608-43acf39319177bee.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/5163-e682273cd76a7d07.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{5309-b2c4803370634ff8.js → 5309-ce5702b9faeaff55.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{905-ffdbd0b14167e8bd.js → 5596-4378b2927dae65b2.js} +3 -3
- fides/ui-build/static/admin/_next/static/chunks/5619-9b50cec521203989.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6084-7178ff6ea6822475.js → 6084-ddbad3149364725d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6148-59a59d5c5925344f.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6419-d0c00d661b01f8fa.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6662-507be5d46e5b719b.js → 6662-4392ba1e4c254ef7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6780-fc7d9ddb1a03e7b3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6853-2ad3e08fe6f9f5f2.js → 6853-d0190d2cca9dbde2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{6882-6af16fef26c21e06.js → 6882-59ea739e3616ce83.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-9c4912fbce87c4df.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{7476-281ee9a8286556f3.js → 7476-cc0d9a94ed7aad53.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{787-fb41002f797eb2df.js → 787-c57185ad89c4e288.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{79-7e87aff851423d4a.js → 79-2aab56be75e16187.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{796-329a5f823ec258a5.js → 796-3bdda2a7868464af.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/8237-841439bef6682177.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/8765-f622a35b40a7ec63.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9046-5c4c22c375de25b1.js → 9046-04a8c092fef1cd83.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9187-7438242f0d380bb0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9226-746771d47dff6266.js → 9226-2dcac54ab3fb94be.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9278-08cc704317fe535e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9951-9b753ad7c3f51bdf.js → 9951-7c6639e5d062779e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-39ccb07327c2c5d5.js → _app-4b5bff46158a19a3.js} +56 -56
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-98777246bec9dc2a.js → manual-2a655ff3a97f2492.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-a71c0aff4e0e6535.js → add-systems-0902f0bb4080643e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-1edf582ba3cd3bbb.js → [id]-f22ddd9b48a5c418.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-06bb3b0bf097fcdb.js → new-e74cb5ea87f15b40.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-685771e5f7196d87.js → privacy-experience-21f997c69fc3b4c2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-6ccedc70dc447089.js → [id]-da4124b7600a2a1d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-944bca1cc57985b5.js → new-a57d251c88ce68ae.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-84f4bd14ce8673bc.js → privacy-notices-ad105181bc91209b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-b0c4235fe6d0b0c8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-11d52f1570759c4d.js → [resourceUrn]-aad6047a4604b945.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-32eac8bbd217615a.js → projects-29784a11fe0fbd0a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-b83afa5565d0c84e.js → [resourceUrn]-b6b98cea25dd94fa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-6f630d42ac9fb6b4.js → data-catalog-d5b01abcb76792ce.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-7caea7bb58c1f153.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-a9a70856f7be1542.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-9ddb52ebb7ac4c71.js → action-center-1f0ea5c92ae9a2b4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-9aa744d56cdacb0d.js → activity-ad6a84a6276f914c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-14bd7500362ff224.js → [resourceUrn]-f98dd251babb7e28.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9e7dfd5a6acc2e8f.js → discovery-56eb4c014f0d96a3.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-d23b3ae139f0428b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-c0d2bfd465df20e0.js → [...subfieldNames]-15301bd6bf7cf718.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-28280a8a39a6e37c.js → [collectionName]-0fa72873e464f581.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-82fb246d87e58ebd.js → new-0d50084fbdf9b84c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-20165c31ab1bc7cf.js → dataset-f3348d0a92543bab.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-b4a6bcc87d126840.js → [id]-7d6027570d05c57f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-f95d7b0bbfc58f5a.js → new-8446418c7ad28f77.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{fides-js-docs-5d8fd1af75f19e2f.js → fides-js-docs-1f4335dca5c09860.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{index-1919aab9e5834b51.js → index-fbf9b845bb901238.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/{[id]-0a58aee2d1e7fa01.js → [id]-f53fe1f2cbebda7c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-e2d5d7e2a5265e68.js → integrations-142abe3e3e3e5bf7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-1bae386d8c190348.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-69ad86b7a8a9a115.js → table-migration-05616e2ae20ff4f8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-1b6b0d703cf59389.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-f1d818242d8550f8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-d40a26bddb126c5c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-ccd8d9e06cf2d278.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-fd1a67892056830a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-fc201657f4a782c7.js → [purpose_id]-e891d01ece59669e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-c2d39cba8396ef3a.js → consent-f61b87e79367865b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-d992103cc55901ae.js → custom-fields-f8eea5d508c60c64.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-023e1895552817de.js → locations-ed6a140b362c5baa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-ac403c0886b20e20.js → organization-ff9a34264d48c35f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-7a3396ac819c7904.js → test-datasets-a4b6d41ca679298b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-8314a819837f5b2a.js → [id]-c8f5fbaa83dd9945.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{systems-21f423a7c417aa9d.js → systems-c05b49ddec1a1b4f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-df0d88716578e295.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-da68efc31998dc66.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-173ac3a1ed2b05a6.js → user-management-e98dfc7d4f2a4e16.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/webpack-90e8ec1fc5c6455b.js +1 -0
- fides/ui-build/static/admin/_next/static/css/{5bfb2473e5701527.css → 23cf870196941c9a.css} +1 -1
- fides/ui-build/static/admin/_next/static/css/{94965f224bc991e9.css → 8bc1833f1fa53ff0.css} +1 -1
- fides/ui-build/static/admin/_next/static/css/d9924caa849931b3.css +1 -0
- fides/ui-build/static/admin/_next/static/css/dbcf63488933a4d5.css +29 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +3 -3
- fides/ui-build/static/admin/lib/fides.js +3 -3
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/8108ANFxs99VY7KZ_Xev2/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1316-6cc72a45ebf7ff81.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1807-3beab149351d5ded.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1817-e601e737e3cc7a0e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/255-7db55b0e3a0f9dea.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2599-6c4d22e75028d8b6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2858-0b44609b6be7850b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2866-a73888c17a195cbe.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3615-5e2d062d684b8fa1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/409-a257e14acebcd73b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/431-34f0b91c26f8d9ab.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/570-c99f07161bd339cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6780-e3d40aa17a4bf2e9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-bb875d9ac89f6030.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7062.fda15dcb7df85675.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7c79804f.7a7112aece470725.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9014-eeae6f581158e645.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9278-9b1b5970f0702668.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9392.f4520f66206d347d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/ea076c0a.84423f606aef37cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-afdbd4665657cfa1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-2265ecb899d45fbc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-5d522637871ac6c8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-7674b97d655c193b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-5094ffea13f32ed9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-32600543eb7b584f.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-10ce53ea356f8bad.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-5501bbb129fee9c4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-cbe4c8f9096b6543.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-e130c0197362e8f3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-6387fcc8cce872eb.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-ff5738706da07801.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/webpack-ff0cd6bff75588da.js +0 -1
- fides/ui-build/static/admin/_next/static/css/2cadb5f62dcd7c2b.css +0 -1
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{8108ANFxs99VY7KZ_Xev2 → GKmhMPa_1gMto8JZO8ENy}/_ssgManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{2921-455e6357b74d2f76.js → 2921-86f1547ac40a5cdf.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3923-6cc911dafccc5f63.js → 3923-13a6b4da2d51bf8f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{401-1b529d5800aa1f3a.js → 401-3cc1fee61494e3bd.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{5574-b13021775a15bfd2.js → 5574-9312f97b637d9ee2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7630-9aac73191ed5ed13.js → 7630-b1c93688013ef013.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{9826-111aaee8bd8dbd09.js → 9826-3c578665c6d3b21d.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{404-aece2c920ea14514.js → 404-2d803dab6a00f353.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-dc75dc6e37e52f05.js → multiple-8ff7f37913ad736a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-24d226b5a8de5c74.js → add-vendors-d00c9034cdeb0236.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-6a8ef51138ac926a.js → configure-0e1ca0f4c8e7f4da.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-6f86ab63a08a6528.js → properties-057cad65e7414a44.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-73d3cbf68f7c3a31.js → consent-e17c56eec8d91371.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-6ba9e160dae64695.js → [projectUrn]-80a6cc8e8573514a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{resources-7648bbd4f6711e4d.js → resources-6c3714ee97a718c1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-393e20924c83373e.js → [resourceUrn]-31e6c54794a9883e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-8733807dad4bc96e.js → detection-2822a423a7ad0550.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-006b695e5af5ef24.js → [datasetId]-a8e8b5f4ee7af86c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-c391c6fad56eec48.js → datastore-connection-0f29b47402292070.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-53fecfb9dd6a1e0c.js → [id]-5627d0d0668077f9.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-76b01cec5fde10a9.js → add-template-feca66ad5c5fe54a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-5c08e8447c45ce44.js → ant-components-64a322d01aae5ca7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-06ad5f34585480aa.js → AntForm-8bca16a7726e7eb2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-6f071c2bc9446cb0.js → FormikAntFormItem-b0f246fc3b67ebf7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-efcc38c58991ac9e.js → FormikControlled-1a0852b090bfc392.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-430ba5c979abfb7c.js → FormikField-11f3de1b45e36583.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-5c561880bf131afb.js → forms-1b73a1c2b6c6285f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-d888a69a3bbe040e.js → configure-e551a860ec727802.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-d3d8e3d7583ec635.js → [id]-dd99183f93763ae4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-1af10ed303815d46.js → add-property-0bdbc1fcbf553b8f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{properties-cebc0dc186be499a.js → properties-e959378bb32b6b73.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-5e1322de868d615e.js → alpha-8f98a4895e74725e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-241f95e372b65d0f.js → about-8155a35a62fdb5ae.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-41242f805599feda.js → domain-records-51333dbd21cb37c8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-2e885f74c92f669c.js → domains-bde86e5f6c09da5a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-ff112655ad5f41e5.js → email-templates-4f9a5cc8bea7725b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-86062a18e081a52a.js → regulations-102efd9199e87124.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-a2524414e968f862.js → new-bc4eb541906781e6.js} +0 -0
|
@@ -14,6 +14,7 @@ from fides.api.common_exceptions import (
|
|
|
14
14
|
ClientUnsuccessfulException,
|
|
15
15
|
IdentityNotFoundException,
|
|
16
16
|
ManualWebhookFieldsUnset,
|
|
17
|
+
MaskingSecretsExpired,
|
|
17
18
|
MessageDispatchException,
|
|
18
19
|
NoCachedManualWebhookEntry,
|
|
19
20
|
PrivacyRequestExit,
|
|
@@ -75,6 +76,7 @@ from fides.api.task.graph_task import (
|
|
|
75
76
|
from fides.api.task.manual.manual_task_utils import create_manual_task_artificial_graphs
|
|
76
77
|
from fides.api.tasks import DatabaseTask, celery_app
|
|
77
78
|
from fides.api.tasks.scheduled.scheduler import scheduler
|
|
79
|
+
from fides.api.util.cache import get_all_masking_secret_keys
|
|
78
80
|
from fides.api.util.collection_util import Row
|
|
79
81
|
from fides.api.util.logger import Pii, _log_exception, _log_warning
|
|
80
82
|
from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
|
|
@@ -478,12 +480,21 @@ def run_privacy_request(
|
|
|
478
480
|
connection_configs
|
|
479
481
|
)
|
|
480
482
|
|
|
483
|
+
# If the privacy request requires manual finalization and has not yet been finalized, we exit here
|
|
484
|
+
if (
|
|
485
|
+
privacy_request.status
|
|
486
|
+
== PrivacyRequestStatus.requires_manual_finalization
|
|
487
|
+
and privacy_request.finalized_at is None
|
|
488
|
+
):
|
|
489
|
+
return
|
|
490
|
+
|
|
481
491
|
# Access CHECKPOINT
|
|
482
492
|
if (
|
|
483
493
|
policy.get_rules_for_action(action_type=ActionType.access)
|
|
484
494
|
or policy.get_rules_for_action(action_type=ActionType.erasure)
|
|
485
495
|
) and can_run_checkpoint(
|
|
486
|
-
request_checkpoint=CurrentStep.access,
|
|
496
|
+
request_checkpoint=CurrentStep.access,
|
|
497
|
+
from_checkpoint=resume_step,
|
|
487
498
|
):
|
|
488
499
|
privacy_request.cache_failed_checkpoint_details(CurrentStep.access)
|
|
489
500
|
access_runner(
|
|
@@ -528,9 +539,12 @@ def run_privacy_request(
|
|
|
528
539
|
if policy.get_rules_for_action(
|
|
529
540
|
action_type=ActionType.erasure
|
|
530
541
|
) and can_run_checkpoint(
|
|
531
|
-
request_checkpoint=CurrentStep.erasure,
|
|
542
|
+
request_checkpoint=CurrentStep.erasure,
|
|
543
|
+
from_checkpoint=resume_step,
|
|
532
544
|
):
|
|
533
545
|
privacy_request.cache_failed_checkpoint_details(CurrentStep.erasure)
|
|
546
|
+
_verify_masking_secrets(policy, privacy_request, resume_step)
|
|
547
|
+
|
|
534
548
|
# We only need to run the erasure once until masking strategies are handled
|
|
535
549
|
erasure_runner(
|
|
536
550
|
privacy_request=privacy_request,
|
|
@@ -652,58 +666,111 @@ def run_privacy_request(
|
|
|
652
666
|
if not proceed:
|
|
653
667
|
return
|
|
654
668
|
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
|
|
663
|
-
)
|
|
664
|
-
|
|
665
|
-
if message_send_enabled(
|
|
666
|
-
session,
|
|
667
|
-
privacy_request.property_id,
|
|
668
|
-
action_type,
|
|
669
|
-
legacy_request_completion_enabled,
|
|
670
|
-
) and not policy.get_rules_for_action(action_type=ActionType.consent):
|
|
671
|
-
try:
|
|
672
|
-
if not access_result_urls:
|
|
673
|
-
# For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
|
|
674
|
-
# from the database because the Privacy Request would have exited
|
|
675
|
-
# processing and lost access to the access_result_urls in memory
|
|
676
|
-
access_result_urls = (
|
|
677
|
-
privacy_request.access_result_urls or {}
|
|
678
|
-
).get("access_result_urls", [])
|
|
679
|
-
initiate_privacy_request_completion_email(
|
|
680
|
-
session,
|
|
681
|
-
policy,
|
|
682
|
-
access_result_urls,
|
|
683
|
-
identity_data,
|
|
684
|
-
privacy_request.property_id,
|
|
685
|
-
)
|
|
686
|
-
except (IdentityNotFoundException, MessageDispatchException) as e:
|
|
687
|
-
privacy_request.error_processing(db=session)
|
|
688
|
-
# If dev mode, log traceback
|
|
689
|
-
_log_exception(e, CONFIG.dev_mode)
|
|
690
|
-
return
|
|
691
|
-
|
|
692
|
-
# Only mark as complete if not in error state
|
|
693
|
-
if privacy_request.status != PrivacyRequestStatus.error:
|
|
694
|
-
privacy_request.finished_processing_at = datetime.utcnow()
|
|
695
|
-
AuditLog.create(
|
|
696
|
-
db=session,
|
|
697
|
-
data={
|
|
698
|
-
"user_id": "system",
|
|
699
|
-
"privacy_request_id": privacy_request.id,
|
|
700
|
-
"action": AuditLogAction.finished,
|
|
701
|
-
"message": "",
|
|
702
|
-
},
|
|
669
|
+
# Request finalization CHECKPOINT
|
|
670
|
+
if can_run_checkpoint(
|
|
671
|
+
request_checkpoint=CurrentStep.finalization,
|
|
672
|
+
from_checkpoint=resume_step,
|
|
673
|
+
):
|
|
674
|
+
privacy_request.cache_failed_checkpoint_details(
|
|
675
|
+
CurrentStep.finalization,
|
|
703
676
|
)
|
|
704
|
-
privacy_request.status
|
|
705
|
-
|
|
706
|
-
|
|
677
|
+
if privacy_request.status != PrivacyRequestStatus.error:
|
|
678
|
+
erasure_rules = policy.get_rules_for_action(
|
|
679
|
+
action_type=ActionType.erasure
|
|
680
|
+
)
|
|
681
|
+
if (
|
|
682
|
+
privacy_request.finalized_at is None
|
|
683
|
+
and erasure_rules
|
|
684
|
+
and CONFIG.execution.erasure_request_finalization_required
|
|
685
|
+
):
|
|
686
|
+
logger.info(
|
|
687
|
+
"Marking privacy request '{}' as requires manual finalization.",
|
|
688
|
+
privacy_request.id,
|
|
689
|
+
)
|
|
690
|
+
privacy_request.status = (
|
|
691
|
+
PrivacyRequestStatus.requires_manual_finalization
|
|
692
|
+
)
|
|
693
|
+
privacy_request.save(db=session)
|
|
694
|
+
return
|
|
695
|
+
|
|
696
|
+
# Finally, mark the request as complete
|
|
697
|
+
if privacy_request.finalized_at:
|
|
698
|
+
logger.info(
|
|
699
|
+
"Marking privacy request '{}' as finalized.",
|
|
700
|
+
privacy_request.id,
|
|
701
|
+
)
|
|
702
|
+
privacy_request.add_success_execution_log(
|
|
703
|
+
session,
|
|
704
|
+
connection_key=None,
|
|
705
|
+
dataset_name="Request finalized",
|
|
706
|
+
collection_name=None,
|
|
707
|
+
message=f"Request finalized for privacy request: {privacy_request.id}",
|
|
708
|
+
action_type=privacy_request.policy.get_action_type(), # type: ignore
|
|
709
|
+
)
|
|
710
|
+
|
|
711
|
+
logger.info(
|
|
712
|
+
"Marking privacy request '{}' as complete.",
|
|
713
|
+
privacy_request.id,
|
|
714
|
+
)
|
|
715
|
+
AuditLog.create(
|
|
716
|
+
db=session,
|
|
717
|
+
data={
|
|
718
|
+
"user_id": "system",
|
|
719
|
+
"privacy_request_id": privacy_request.id,
|
|
720
|
+
"action": AuditLogAction.finished,
|
|
721
|
+
"message": "",
|
|
722
|
+
},
|
|
723
|
+
)
|
|
724
|
+
privacy_request.status = PrivacyRequestStatus.complete
|
|
725
|
+
privacy_request.finished_processing_at = datetime.utcnow()
|
|
726
|
+
privacy_request.save(db=session)
|
|
727
|
+
|
|
728
|
+
# Send a final email to the user confirming request completion
|
|
729
|
+
if privacy_request.status == PrivacyRequestStatus.complete:
|
|
730
|
+
legacy_request_completion_enabled = ConfigProxy(
|
|
731
|
+
session
|
|
732
|
+
).notifications.send_request_completion_notification
|
|
733
|
+
|
|
734
|
+
action_type = (
|
|
735
|
+
MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS
|
|
736
|
+
if policy.get_rules_for_action(
|
|
737
|
+
action_type=ActionType.access
|
|
738
|
+
)
|
|
739
|
+
else MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION
|
|
740
|
+
)
|
|
741
|
+
|
|
742
|
+
if message_send_enabled(
|
|
743
|
+
session,
|
|
744
|
+
privacy_request.property_id,
|
|
745
|
+
action_type,
|
|
746
|
+
legacy_request_completion_enabled,
|
|
747
|
+
) and not policy.get_rules_for_action(
|
|
748
|
+
action_type=ActionType.consent
|
|
749
|
+
):
|
|
750
|
+
if not access_result_urls:
|
|
751
|
+
# For DSR 3.0, if the request had both access and erasure rules, this needs to be fetched
|
|
752
|
+
# from the database because the Privacy Request would have exited
|
|
753
|
+
# processing and lost access to the access_result_urls in memory
|
|
754
|
+
access_result_urls = (
|
|
755
|
+
privacy_request.access_result_urls or {}
|
|
756
|
+
).get("access_result_urls", [])
|
|
757
|
+
|
|
758
|
+
try:
|
|
759
|
+
initiate_privacy_request_completion_email(
|
|
760
|
+
session,
|
|
761
|
+
policy,
|
|
762
|
+
access_result_urls,
|
|
763
|
+
identity_data,
|
|
764
|
+
privacy_request.property_id,
|
|
765
|
+
)
|
|
766
|
+
except (
|
|
767
|
+
IdentityNotFoundException,
|
|
768
|
+
MessageDispatchException,
|
|
769
|
+
) as e:
|
|
770
|
+
privacy_request.error_processing(db=session)
|
|
771
|
+
# If dev mode, log traceback
|
|
772
|
+
_log_exception(e, CONFIG.dev_mode)
|
|
773
|
+
return
|
|
707
774
|
|
|
708
775
|
|
|
709
776
|
def initiate_privacy_request_completion_email(
|
|
@@ -1000,3 +1067,25 @@ def run_webhooks_and_report_status(
|
|
|
1000
1067
|
return False
|
|
1001
1068
|
|
|
1002
1069
|
return True
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
def _verify_masking_secrets(
|
|
1073
|
+
policy: Policy, privacy_request: PrivacyRequest, resume_step: Optional[CurrentStep]
|
|
1074
|
+
) -> None:
|
|
1075
|
+
"""
|
|
1076
|
+
Checks that the required masking secrets are still cached for the given request.
|
|
1077
|
+
Raises an exception if masking secrets are needed for the given policy but they don't exist.
|
|
1078
|
+
"""
|
|
1079
|
+
|
|
1080
|
+
if resume_step is None:
|
|
1081
|
+
return
|
|
1082
|
+
|
|
1083
|
+
# if masking can be performed without any masking secrets, we skip the cache check
|
|
1084
|
+
if (
|
|
1085
|
+
policy.generate_masking_secrets()
|
|
1086
|
+
and not get_all_masking_secret_keys(privacy_request.id)
|
|
1087
|
+
and not privacy_request.masking_secrets
|
|
1088
|
+
):
|
|
1089
|
+
raise MaskingSecretsExpired(
|
|
1090
|
+
f"The masking secrets for privacy request ID '{privacy_request.id}' have expired. Please submit a new erasure request."
|
|
1091
|
+
)
|
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import json
|
|
4
4
|
from asyncio import sleep
|
|
5
5
|
from datetime import datetime, timedelta
|
|
6
|
-
from typing import Any, Dict,
|
|
6
|
+
from typing import Any, Dict, Optional, Set
|
|
7
7
|
|
|
8
8
|
from httpx import AsyncClient
|
|
9
9
|
from loguru import logger
|
|
@@ -11,7 +11,6 @@ from sqlalchemy import text
|
|
|
11
11
|
from sqlalchemy.sql.elements import TextClause
|
|
12
12
|
|
|
13
13
|
from fides.api.common_exceptions import PrivacyRequestNotFound
|
|
14
|
-
from fides.api.models.policy import Policy
|
|
15
14
|
from fides.api.models.privacy_request import (
|
|
16
15
|
EXITED_EXECUTION_LOG_STATUSES,
|
|
17
16
|
PrivacyRequest,
|
|
@@ -19,14 +18,12 @@ from fides.api.models.privacy_request import (
|
|
|
19
18
|
)
|
|
20
19
|
from fides.api.models.worker_task import ExecutionLogStatus
|
|
21
20
|
from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
|
|
22
|
-
from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
|
|
23
21
|
from fides.api.schemas.policy import ActionType
|
|
24
22
|
from fides.api.schemas.privacy_request import (
|
|
25
23
|
PrivacyRequestResponse,
|
|
26
24
|
PrivacyRequestStatus,
|
|
27
25
|
)
|
|
28
26
|
from fides.api.schemas.redis_cache import Identity
|
|
29
|
-
from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
|
|
30
27
|
from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
|
|
31
28
|
from fides.api.tasks.scheduled.scheduler import scheduler
|
|
32
29
|
from fides.api.util.cache import (
|
|
@@ -69,7 +66,6 @@ def build_required_privacy_request_kwargs(
|
|
|
69
66
|
|
|
70
67
|
def cache_data(
|
|
71
68
|
privacy_request: PrivacyRequest,
|
|
72
|
-
policy: Policy,
|
|
73
69
|
identity: Identity,
|
|
74
70
|
encryption_key: Optional[str],
|
|
75
71
|
drp_request_body: Optional[DrpPrivacyRequestCreate],
|
|
@@ -82,23 +78,6 @@ def cache_data(
|
|
|
82
78
|
privacy_request.cache_custom_privacy_request_fields(custom_privacy_request_fields)
|
|
83
79
|
privacy_request.cache_encryption(encryption_key) # handles None already
|
|
84
80
|
|
|
85
|
-
# Store masking secrets in the cache
|
|
86
|
-
logger.info("Caching masking secrets for privacy request {}", privacy_request.id)
|
|
87
|
-
erasure_rules = policy.get_rules_for_action(action_type=ActionType.erasure)
|
|
88
|
-
unique_masking_strategies_by_name: Set[str] = set()
|
|
89
|
-
for rule in erasure_rules:
|
|
90
|
-
strategy_name: str = rule.masking_strategy["strategy"] # type: ignore
|
|
91
|
-
configuration = rule.masking_strategy["configuration"] # type: ignore
|
|
92
|
-
if strategy_name in unique_masking_strategies_by_name:
|
|
93
|
-
continue
|
|
94
|
-
unique_masking_strategies_by_name.add(strategy_name)
|
|
95
|
-
masking_strategy = MaskingStrategy.get_strategy(strategy_name, configuration)
|
|
96
|
-
if masking_strategy.secrets_required():
|
|
97
|
-
masking_secrets: List[MaskingSecretCache] = (
|
|
98
|
-
masking_strategy.generate_secrets_for_cache()
|
|
99
|
-
)
|
|
100
|
-
for masking_secret in masking_secrets:
|
|
101
|
-
privacy_request.cache_masking_secret(masking_secret)
|
|
102
81
|
if drp_request_body:
|
|
103
82
|
privacy_request.cache_drp_request_body(drp_request_body)
|
|
104
83
|
|
|
File without changes
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import operator as py_operator
|
|
2
|
+
from typing import Any, Union
|
|
3
|
+
|
|
4
|
+
from loguru import logger
|
|
5
|
+
from sqlalchemy.orm import Session
|
|
6
|
+
|
|
7
|
+
from fides.api.graph.config import FieldPath
|
|
8
|
+
from fides.api.task.conditional_dependencies.schemas import (
|
|
9
|
+
Condition,
|
|
10
|
+
ConditionGroup,
|
|
11
|
+
ConditionLeaf,
|
|
12
|
+
GroupOperator,
|
|
13
|
+
Operator,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
operator_methods = {
|
|
17
|
+
Operator.exists: lambda a, _: a is not None,
|
|
18
|
+
Operator.not_exists: lambda a, _: a is None,
|
|
19
|
+
Operator.eq: py_operator.eq,
|
|
20
|
+
Operator.neq: py_operator.ne,
|
|
21
|
+
Operator.lt: lambda a, b: a < b if a is not None else False,
|
|
22
|
+
Operator.lte: lambda a, b: a <= b if a is not None else False,
|
|
23
|
+
Operator.gt: lambda a, b: a > b if a is not None else False,
|
|
24
|
+
Operator.gte: lambda a, b: a >= b if a is not None else False,
|
|
25
|
+
Operator.list_contains: lambda a, b: b in a if isinstance(a, list) else False,
|
|
26
|
+
Operator.not_in_list: lambda a, b: a not in b if isinstance(b, list) else True,
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ConditionEvaluator:
|
|
31
|
+
"""Evaluates nested conditions for manual task creation"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, db: Session):
|
|
34
|
+
self.db = db
|
|
35
|
+
|
|
36
|
+
def evaluate_rule(self, rule: Condition, data: Union[dict, Any]) -> bool:
|
|
37
|
+
"""Evaluate a nested condition rule against input data"""
|
|
38
|
+
if isinstance(rule, ConditionLeaf):
|
|
39
|
+
return self._evaluate_leaf_condition(rule, data)
|
|
40
|
+
# ConditionGroup
|
|
41
|
+
return self._evaluate_group_condition(rule, data)
|
|
42
|
+
|
|
43
|
+
def _evaluate_leaf_condition(
|
|
44
|
+
self, condition: ConditionLeaf, data: Union[dict, Any]
|
|
45
|
+
) -> bool:
|
|
46
|
+
"""Evaluate a leaf condition against input data"""
|
|
47
|
+
actual_value = self._get_nested_value(data, condition.field_address.split("."))
|
|
48
|
+
# Apply operator and return result
|
|
49
|
+
return self._apply_operator(actual_value, condition.operator, condition.value)
|
|
50
|
+
|
|
51
|
+
def _evaluate_group_condition(
|
|
52
|
+
self, group: ConditionGroup, data: Union[dict, Any]
|
|
53
|
+
) -> bool:
|
|
54
|
+
"""Evaluate a group condition against input data"""
|
|
55
|
+
results = [
|
|
56
|
+
self.evaluate_rule(condition, data) for condition in group.conditions
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
logical_operators = {GroupOperator.and_: all, GroupOperator.or_: any}
|
|
60
|
+
operator_func = logical_operators.get(group.logical_operator)
|
|
61
|
+
|
|
62
|
+
if operator_func is None:
|
|
63
|
+
logger.warning(f"Unknown logical operator: {group.logical_operator}")
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
return operator_func(results)
|
|
67
|
+
|
|
68
|
+
def _get_nested_value(self, data: Union[dict, Any], keys: list[str]) -> Any:
|
|
69
|
+
"""Get nested value from data using dot notation
|
|
70
|
+
|
|
71
|
+
Supports both simple dictionary access and Fides reference structures:
|
|
72
|
+
- Simple dict: data["user"]["name"]
|
|
73
|
+
- Fides FieldAddress: data.get_field_value(FieldAddress("dataset", "collection", "field_address"))
|
|
74
|
+
- Fides Collection: data.get_field_value(FieldPath("field_address", "subfield"))
|
|
75
|
+
"""
|
|
76
|
+
if not keys:
|
|
77
|
+
return data
|
|
78
|
+
|
|
79
|
+
current = data
|
|
80
|
+
|
|
81
|
+
# Try Fides reference structures first
|
|
82
|
+
if hasattr(current, "get_field_value"):
|
|
83
|
+
try:
|
|
84
|
+
field_path = FieldPath(*keys) if len(keys) > 1 else FieldPath(keys[0])
|
|
85
|
+
return current.get_field_value(field_path)
|
|
86
|
+
except (AttributeError, ValueError):
|
|
87
|
+
pass
|
|
88
|
+
|
|
89
|
+
# Fall back to dictionary access
|
|
90
|
+
for key in keys:
|
|
91
|
+
if not isinstance(current, dict):
|
|
92
|
+
current = current.get(key, {}) if hasattr(current, "get") else None
|
|
93
|
+
else:
|
|
94
|
+
current = current.get(key, {})
|
|
95
|
+
|
|
96
|
+
return current if current != {} else None
|
|
97
|
+
|
|
98
|
+
def _apply_operator(
|
|
99
|
+
self, actual_value: Any, operator: Operator, expected_value: Any
|
|
100
|
+
) -> bool:
|
|
101
|
+
"""Apply operator to actual and expected values"""
|
|
102
|
+
|
|
103
|
+
# Get the method for the operator and execute it
|
|
104
|
+
operator_method = operator_methods.get(operator)
|
|
105
|
+
if operator_method is None:
|
|
106
|
+
logger.warning(f"Unknown operator: {operator}")
|
|
107
|
+
return False
|
|
108
|
+
|
|
109
|
+
return operator_method(actual_value, expected_value)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from typing import List, Optional, Union
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, Field, model_validator
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Operator(str, Enum):
|
|
8
|
+
eq = "eq"
|
|
9
|
+
neq = "neq"
|
|
10
|
+
lt = "lt"
|
|
11
|
+
lte = "lte"
|
|
12
|
+
gt = "gt"
|
|
13
|
+
gte = "gte"
|
|
14
|
+
exists = "exists"
|
|
15
|
+
not_exists = "not_exists"
|
|
16
|
+
list_contains = "list_contains"
|
|
17
|
+
not_in_list = "not_in_list"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class GroupOperator(str, Enum):
|
|
21
|
+
and_ = "and"
|
|
22
|
+
or_ = "or"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# Leaf condition (e.g., "user.name exists", "user.created_at >= 2024-01-01")
|
|
26
|
+
class ConditionLeaf(BaseModel):
|
|
27
|
+
field_address: str = Field(
|
|
28
|
+
description="Field path to check (e.g., 'user.name', 'billing.subscription.status')"
|
|
29
|
+
)
|
|
30
|
+
operator: Operator = Field(description="Operator to apply")
|
|
31
|
+
value: Optional[
|
|
32
|
+
Union[str, int, float, bool, List[Union[str, int, float, bool]]]
|
|
33
|
+
] = Field(default=None, description="Expected value for comparison")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# Nested logical condition (AND, OR)
|
|
37
|
+
class ConditionGroup(BaseModel):
|
|
38
|
+
logical_operator: GroupOperator = Field(
|
|
39
|
+
description="Logical operator: 'and' or 'or'"
|
|
40
|
+
)
|
|
41
|
+
conditions: list["Condition"] = Field(
|
|
42
|
+
description="List of conditions or nested groups"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
@model_validator(mode="after")
|
|
46
|
+
def validate_conditions(self) -> "ConditionGroup":
|
|
47
|
+
if not self.conditions:
|
|
48
|
+
raise ValueError("conditions list cannot be empty")
|
|
49
|
+
return self
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# Use model_rebuild to allow recursive nesting
|
|
53
|
+
Condition = Union[ConditionLeaf, ConditionGroup]
|
|
54
|
+
ConditionGroup.model_rebuild()
|
|
@@ -33,8 +33,8 @@ from fides.api.models.worker_task import ExecutionLogStatus
|
|
|
33
33
|
from fides.api.schemas.policy import ActionType
|
|
34
34
|
from fides.api.task.deprecated_graph_task import format_data_use_map_for_caching
|
|
35
35
|
from fides.api.task.execute_request_tasks import log_task_queued, queue_request_task
|
|
36
|
+
from fides.api.task.manual.manual_task_address import ManualTaskAddress
|
|
36
37
|
from fides.api.task.manual.manual_task_utils import (
|
|
37
|
-
ManualTaskAddress,
|
|
38
38
|
create_manual_task_instances_for_privacy_request,
|
|
39
39
|
)
|
|
40
40
|
from fides.api.util.logger_context_utils import log_context
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# pylint: disable=too-many-lines
|
|
2
|
+
from functools import partial
|
|
2
3
|
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
|
|
3
4
|
|
|
4
5
|
import dask
|
|
@@ -252,16 +253,33 @@ def run_erasure_request_deprecated( # pylint: disable = too-many-arguments
|
|
|
252
253
|
|
|
253
254
|
access_request_data[ROOT_COLLECTION_ADDRESS.value] = [identity]
|
|
254
255
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
256
|
+
# Build the dask task graph for erasure, ensuring we also pass along the
|
|
257
|
+
# upstream access data that may be required by the connector (e.g. BigQuery
|
|
258
|
+
# delete statements). We accomplish this by partially applying the
|
|
259
|
+
# `inputs` kwarg on each task's `erasure_request` method. The resulting
|
|
260
|
+
# callable accepts the original positional arguments expected by Dask.
|
|
261
|
+
|
|
262
|
+
dsk: Dict[CollectionAddress, Any] = {}
|
|
263
|
+
for k, t in env.items():
|
|
264
|
+
# Collect upstream access data in the same order as the input keys
|
|
265
|
+
upstream_access_data: List[List[Row]] = [
|
|
266
|
+
access_request_data.get(str(input_key), [])
|
|
267
|
+
for input_key in t.execution_node.input_keys
|
|
268
|
+
]
|
|
269
|
+
|
|
270
|
+
# Bind the `inputs` keyword argument so it is supplied when the task executes
|
|
271
|
+
erasure_fn_with_inputs = partial(
|
|
272
|
+
t.erasure_request, inputs=upstream_access_data
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
# Build the task tuple: (callable, retrieved_data, *prereqs)
|
|
276
|
+
dsk[k] = (
|
|
277
|
+
erasure_fn_with_inputs,
|
|
258
278
|
access_request_data.get(
|
|
259
279
|
str(k), []
|
|
260
|
-
), #
|
|
280
|
+
), # Data retrieved for this collection
|
|
261
281
|
*_evaluate_erasure_dependencies(t, erasure_end_nodes),
|
|
262
282
|
)
|
|
263
|
-
for k, t in env.items()
|
|
264
|
-
}
|
|
265
283
|
|
|
266
284
|
# root node returns 0 to be consistent with the output of the other erasure tasks
|
|
267
285
|
dsk[ROOT_COLLECTION_ADDRESS] = 0
|
|
@@ -29,8 +29,8 @@ from fides.api.task.graph_task import (
|
|
|
29
29
|
GraphTask,
|
|
30
30
|
mark_current_and_downstream_nodes_as_failed,
|
|
31
31
|
)
|
|
32
|
+
from fides.api.task.manual.manual_task_address import ManualTaskAddress
|
|
32
33
|
from fides.api.task.manual.manual_task_graph_task import ManualTaskGraphTask
|
|
33
|
-
from fides.api.task.manual.manual_task_utils import ManualTaskAddress
|
|
34
34
|
from fides.api.task.task_resources import TaskResources
|
|
35
35
|
from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
|
|
36
36
|
from fides.api.util.cache import cache_task_tracking_key
|
|
@@ -298,17 +298,11 @@ def run_access_node(
|
|
|
298
298
|
)
|
|
299
299
|
# Currently, upstream tasks and "input keys" (which are built by data dependencies)
|
|
300
300
|
# are the same, but they may not be the same in the future.
|
|
301
|
-
|
|
302
|
-
|
|
301
|
+
upstream_access_data: List[List[Row]] = (
|
|
302
|
+
_build_upstream_access_data(
|
|
303
303
|
graph_task.execution_node.input_keys, upstream_results
|
|
304
304
|
)
|
|
305
305
|
)
|
|
306
|
-
# Pass in access data dependencies in the same order as the input keys.
|
|
307
|
-
# If we don't have access data for an upstream node, pass in an empty list
|
|
308
|
-
upstream_access_data: List[List[Row]] = [
|
|
309
|
-
upstream.get_access_data() if upstream else []
|
|
310
|
-
for upstream in ordered_upstream_tasks
|
|
311
|
-
]
|
|
312
306
|
# Run the main access function
|
|
313
307
|
graph_task.access_request(*upstream_access_data)
|
|
314
308
|
|
|
@@ -359,7 +353,7 @@ def run_erasure_node(
|
|
|
359
353
|
session,
|
|
360
354
|
) as resources:
|
|
361
355
|
# Build GraphTask resource to facilitate execution
|
|
362
|
-
|
|
356
|
+
erasure_graph_task: GraphTask = create_graph_task(
|
|
363
357
|
session, request_task, resources
|
|
364
358
|
)
|
|
365
359
|
# Get access data that was saved in the erasure format that was collected from the
|
|
@@ -368,8 +362,24 @@ def run_erasure_node(
|
|
|
368
362
|
request_task.get_data_for_erasures() or []
|
|
369
363
|
)
|
|
370
364
|
|
|
371
|
-
|
|
372
|
-
|
|
365
|
+
upstream_access_data: List[List[Row]] = []
|
|
366
|
+
|
|
367
|
+
try:
|
|
368
|
+
upstream_access_data = (
|
|
369
|
+
get_upstream_access_data_for_erasure_task(
|
|
370
|
+
request_task, session, resources
|
|
371
|
+
)
|
|
372
|
+
)
|
|
373
|
+
except Exception as e:
|
|
374
|
+
logger.error(
|
|
375
|
+
f"Unable to get upstream access data for erasure task {request_task.collection_address}: {e}"
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
# Run the main erasure function, passing along the upstream access data.
|
|
379
|
+
# The extra data is currently only needed for generating BigQuery delete statements.
|
|
380
|
+
erasure_graph_task.erasure_request(
|
|
381
|
+
retrieved_data, inputs=upstream_access_data
|
|
382
|
+
)
|
|
373
383
|
|
|
374
384
|
queue_downstream_tasks_with_retries(
|
|
375
385
|
self,
|
|
@@ -487,6 +497,73 @@ def _order_tasks_by_input_key(
|
|
|
487
497
|
return tasks
|
|
488
498
|
|
|
489
499
|
|
|
500
|
+
def get_upstream_access_data_for_erasure_task(
|
|
501
|
+
erasure_request_task: RequestTask,
|
|
502
|
+
session: Session,
|
|
503
|
+
resources: TaskResources,
|
|
504
|
+
) -> List[List[Row]]:
|
|
505
|
+
"""
|
|
506
|
+
Retrieves upstream access data for a given erasure request task.
|
|
507
|
+
|
|
508
|
+
This function finds the corresponding access task for the erasure task,
|
|
509
|
+
creates a GraphTask to extract input keys, and builds the upstream access data
|
|
510
|
+
needed for erasure operations (particularly for BigQuery delete statements).
|
|
511
|
+
|
|
512
|
+
Args:
|
|
513
|
+
erasure_request_task: The erasure task that needs upstream access data
|
|
514
|
+
session: Database session for querying
|
|
515
|
+
resources: Task resources for creating GraphTask
|
|
516
|
+
|
|
517
|
+
Returns:
|
|
518
|
+
List[List[Row]]: Upstream access data ordered by input keys
|
|
519
|
+
|
|
520
|
+
Raises:
|
|
521
|
+
Exception: If the corresponding access task cannot be found
|
|
522
|
+
"""
|
|
523
|
+
# Get the corresponding access task for the current erasure task
|
|
524
|
+
access_request_task = (
|
|
525
|
+
session.query(RequestTask)
|
|
526
|
+
.filter(
|
|
527
|
+
RequestTask.privacy_request_id == erasure_request_task.privacy_request_id,
|
|
528
|
+
RequestTask.collection_address == erasure_request_task.collection_address,
|
|
529
|
+
RequestTask.action_type == ActionType.access,
|
|
530
|
+
)
|
|
531
|
+
.first()
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
if not access_request_task:
|
|
535
|
+
raise Exception(
|
|
536
|
+
f"Unable to find access request task for erasure task {erasure_request_task.collection_address}"
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
# Convert the request task to a GraphTask to get the input_keys
|
|
540
|
+
access_graph_task: GraphTask = create_graph_task(
|
|
541
|
+
session, access_request_task, resources
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
# Build and return the upstream access data
|
|
545
|
+
return _build_upstream_access_data(
|
|
546
|
+
access_graph_task.execution_node.input_keys,
|
|
547
|
+
access_request_task.upstream_tasks_objects(session),
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _build_upstream_access_data(
|
|
552
|
+
input_keys: List[CollectionAddress],
|
|
553
|
+
upstream_tasks: Query,
|
|
554
|
+
) -> List[List[Row]]:
|
|
555
|
+
"""
|
|
556
|
+
Helper function to build the access data for the current node.
|
|
557
|
+
The access data is passed in the same order as the input keys.
|
|
558
|
+
If we don't have access data for an upstream node, return an empty list.
|
|
559
|
+
"""
|
|
560
|
+
|
|
561
|
+
ordered_upstream: List[Optional[RequestTask]] = _order_tasks_by_input_key(
|
|
562
|
+
input_keys, upstream_tasks
|
|
563
|
+
)
|
|
564
|
+
return [task.get_access_data() if task else [] for task in ordered_upstream]
|
|
565
|
+
|
|
566
|
+
|
|
490
567
|
mapping = {
|
|
491
568
|
ActionType.access.value: run_access_node,
|
|
492
569
|
ActionType.erasure.value: run_erasure_node,
|
fides/api/task/filter_results.py
CHANGED
|
@@ -6,7 +6,7 @@ from loguru import logger
|
|
|
6
6
|
|
|
7
7
|
from fides.api.graph.config import CollectionAddress, FieldPath
|
|
8
8
|
from fides.api.graph.graph import DatasetGraph
|
|
9
|
-
from fides.api.task.manual.
|
|
9
|
+
from fides.api.task.manual.manual_task_address import ManualTaskAddress
|
|
10
10
|
from fides.api.util.collection_util import Row
|
|
11
11
|
|
|
12
12
|
|