ethyca-fides 2.66.2rc0__py2.py3-none-any.whl → 2.67.0rc0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/METADATA +1 -1
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/RECORD +292 -279
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/7e9a2b52f498_adding_masking_secrets.py +60 -0
- fides/api/alembic/migrations/versions/a7065df4dcf1_add_finalized_fields_to_privacy_request.py +65 -0
- fides/api/alembic/migrations/versions/d0031087eacb_create_manualtaskconditionaldependency_.py +106 -0
- fides/api/api/v1/endpoints/dataset_config_endpoints.py +13 -5
- fides/api/api/v1/endpoints/drp_endpoints.py +7 -1
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +44 -1
- fides/api/api/v1/endpoints/user_endpoints.py +83 -7
- fides/api/app_setup.py +3 -2
- fides/api/common_exceptions.py +4 -0
- fides/api/db/base.py +1 -0
- fides/api/db/database.py +1 -1
- fides/api/graph/execution.py +30 -0
- fides/api/graph/graph.py +13 -2
- fides/api/graph/traversal.py +126 -39
- fides/api/models/manual_task/__init__.py +2 -0
- fides/api/models/manual_task/conditional_dependency.py +144 -0
- fides/api/models/{manual_task.py → manual_task/manual_task.py} +10 -0
- fides/api/models/masking_secret.py +72 -0
- fides/api/models/policy.py +23 -0
- fides/api/models/privacy_request/execution_log.py +1 -0
- fides/api/models/privacy_request/privacy_request.py +31 -13
- fides/api/oauth/roles.py +2 -0
- fides/api/schemas/application_config.py +11 -1
- fides/api/schemas/masking/masking_secrets.py +1 -1
- fides/api/schemas/policy.py +1 -0
- fides/api/schemas/privacy_request.py +5 -0
- fides/api/service/connectors/base_connector.py +1 -0
- fides/api/service/connectors/bigquery_connector.py +67 -19
- fides/api/service/connectors/dynamodb_connector.py +2 -1
- fides/api/service/connectors/fides_connector.py +1 -0
- fides/api/service/connectors/http_connector.py +1 -0
- fides/api/service/connectors/manual_task_connector.py +1 -0
- fides/api/service/connectors/manual_webhook_connector.py +2 -1
- fides/api/service/connectors/mongodb_connector.py +1 -0
- fides/api/service/connectors/okta_connector.py +1 -0
- fides/api/service/connectors/query_configs/bigquery_query_config.py +91 -32
- fides/api/service/connectors/rds_mysql_connector.py +1 -0
- fides/api/service/connectors/rds_postgres_connector.py +1 -0
- fides/api/service/connectors/s3_connector.py +1 -0
- fides/api/service/connectors/saas_connector.py +1 -0
- fides/api/service/connectors/scylla_connector.py +1 -0
- fides/api/service/connectors/sql_connector.py +36 -4
- fides/api/service/connectors/website_connector.py +1 -0
- fides/api/service/privacy_request/dsr_package/templates/welcome.html +2 -2
- fides/api/service/privacy_request/request_runner_service.py +142 -53
- fides/api/service/privacy_request/request_service.py +1 -22
- fides/api/task/conditional_dependencies/__init__.py +0 -0
- fides/api/task/conditional_dependencies/evaluator.py +109 -0
- fides/api/task/conditional_dependencies/schemas.py +54 -0
- fides/api/task/create_request_tasks.py +1 -1
- fides/api/task/deprecated_graph_task.py +24 -6
- fides/api/task/execute_request_tasks.py +89 -12
- fides/api/task/filter_results.py +1 -1
- fides/api/task/graph_task.py +38 -3
- fides/api/task/manual/manual_task_address.py +46 -0
- fides/api/task/manual/manual_task_graph_task.py +118 -126
- fides/api/task/manual/manual_task_utils.py +52 -105
- fides/api/util/aws_util.py +5 -1
- fides/api/util/cache.py +5 -0
- fides/api/util/encryption/secrets_util.py +48 -18
- fides/common/api/scope_registry.py +3 -0
- fides/common/api/v1/urn_registry.py +1 -1
- fides/config/execution_settings.py +4 -0
- fides/config/utils.py +1 -0
- fides/service/privacy_request/privacy_request_service.py +6 -1
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/GKmhMPa_1gMto8JZO8ENy/_buildManifest.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1316-2606e19807c08aa5.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1467-8808ec8836e033f9.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1817-b78b58ae3b75d75a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1975.7d4634a0e823a02b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{203-5a663f465ba26bb4.js → 203-cd78ea279cecba60.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/2150-930ffaf2c4718edc.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/255-1bc0cbef7a59cdc6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3450-0ba194991d0cca88.js → 3450-ca4ba70da999f264.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/346-aa3b88efb85f2e28.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3550-d04125c828d591a1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3855-e172870d3e21b0dd.js → 3855-509ca7ac99b5eada.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3872-46cebf7ec1b31a2b.js → 3872-0a0f0032ca39a93f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/409-ea70638a59296659.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4121-2bc09fc4ddbfe5cb.js → 4121-877e19d3fa078c7b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4230-60100f7ef3ddcde1.js → 4230-114e31621c19ea69.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4259.d1507e0db19cbed7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/431-1db919f6569a4021.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4608-bbb7bf511a05c3c2.js → 4608-43acf39319177bee.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/5163-e682273cd76a7d07.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{5309-b2c4803370634ff8.js → 5309-ce5702b9faeaff55.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{905-ffdbd0b14167e8bd.js → 5596-4378b2927dae65b2.js} +3 -3
- fides/ui-build/static/admin/_next/static/chunks/5619-9b50cec521203989.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6084-7178ff6ea6822475.js → 6084-ddbad3149364725d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6148-59a59d5c5925344f.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6419-d0c00d661b01f8fa.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6662-507be5d46e5b719b.js → 6662-4392ba1e4c254ef7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6780-fc7d9ddb1a03e7b3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6853-2ad3e08fe6f9f5f2.js → 6853-d0190d2cca9dbde2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{6882-6af16fef26c21e06.js → 6882-59ea739e3616ce83.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-9c4912fbce87c4df.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{7476-281ee9a8286556f3.js → 7476-cc0d9a94ed7aad53.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{787-fb41002f797eb2df.js → 787-c57185ad89c4e288.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{79-7e87aff851423d4a.js → 79-2aab56be75e16187.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{796-329a5f823ec258a5.js → 796-3bdda2a7868464af.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/8237-841439bef6682177.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/8765-f622a35b40a7ec63.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9046-5c4c22c375de25b1.js → 9046-04a8c092fef1cd83.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9187-7438242f0d380bb0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9226-746771d47dff6266.js → 9226-2dcac54ab3fb94be.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9278-08cc704317fe535e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9951-9b753ad7c3f51bdf.js → 9951-7c6639e5d062779e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-39ccb07327c2c5d5.js → _app-4b5bff46158a19a3.js} +56 -56
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-98777246bec9dc2a.js → manual-2a655ff3a97f2492.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-a71c0aff4e0e6535.js → add-systems-0902f0bb4080643e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-1edf582ba3cd3bbb.js → [id]-f22ddd9b48a5c418.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-06bb3b0bf097fcdb.js → new-e74cb5ea87f15b40.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-685771e5f7196d87.js → privacy-experience-21f997c69fc3b4c2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-6ccedc70dc447089.js → [id]-da4124b7600a2a1d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-944bca1cc57985b5.js → new-a57d251c88ce68ae.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-84f4bd14ce8673bc.js → privacy-notices-ad105181bc91209b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-b0c4235fe6d0b0c8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-11d52f1570759c4d.js → [resourceUrn]-aad6047a4604b945.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-32eac8bbd217615a.js → projects-29784a11fe0fbd0a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-b83afa5565d0c84e.js → [resourceUrn]-b6b98cea25dd94fa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-6f630d42ac9fb6b4.js → data-catalog-d5b01abcb76792ce.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-7caea7bb58c1f153.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-a9a70856f7be1542.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-9ddb52ebb7ac4c71.js → action-center-1f0ea5c92ae9a2b4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-9aa744d56cdacb0d.js → activity-ad6a84a6276f914c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-14bd7500362ff224.js → [resourceUrn]-f98dd251babb7e28.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9e7dfd5a6acc2e8f.js → discovery-56eb4c014f0d96a3.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-d23b3ae139f0428b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-c0d2bfd465df20e0.js → [...subfieldNames]-15301bd6bf7cf718.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-28280a8a39a6e37c.js → [collectionName]-0fa72873e464f581.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-82fb246d87e58ebd.js → new-0d50084fbdf9b84c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-20165c31ab1bc7cf.js → dataset-f3348d0a92543bab.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-b4a6bcc87d126840.js → [id]-7d6027570d05c57f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-f95d7b0bbfc58f5a.js → new-8446418c7ad28f77.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{fides-js-docs-5d8fd1af75f19e2f.js → fides-js-docs-1f4335dca5c09860.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{index-1919aab9e5834b51.js → index-fbf9b845bb901238.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/{[id]-0a58aee2d1e7fa01.js → [id]-f53fe1f2cbebda7c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-e2d5d7e2a5265e68.js → integrations-142abe3e3e3e5bf7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-1bae386d8c190348.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-69ad86b7a8a9a115.js → table-migration-05616e2ae20ff4f8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-1b6b0d703cf59389.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-f1d818242d8550f8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-d40a26bddb126c5c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-ccd8d9e06cf2d278.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-fd1a67892056830a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-fc201657f4a782c7.js → [purpose_id]-e891d01ece59669e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-c2d39cba8396ef3a.js → consent-f61b87e79367865b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-d992103cc55901ae.js → custom-fields-f8eea5d508c60c64.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-023e1895552817de.js → locations-ed6a140b362c5baa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-ac403c0886b20e20.js → organization-ff9a34264d48c35f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-7a3396ac819c7904.js → test-datasets-a4b6d41ca679298b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-8314a819837f5b2a.js → [id]-c8f5fbaa83dd9945.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{systems-21f423a7c417aa9d.js → systems-c05b49ddec1a1b4f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-df0d88716578e295.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-da68efc31998dc66.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-173ac3a1ed2b05a6.js → user-management-e98dfc7d4f2a4e16.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/webpack-90e8ec1fc5c6455b.js +1 -0
- fides/ui-build/static/admin/_next/static/css/{5bfb2473e5701527.css → 23cf870196941c9a.css} +1 -1
- fides/ui-build/static/admin/_next/static/css/{94965f224bc991e9.css → 8bc1833f1fa53ff0.css} +1 -1
- fides/ui-build/static/admin/_next/static/css/d9924caa849931b3.css +1 -0
- fides/ui-build/static/admin/_next/static/css/dbcf63488933a4d5.css +29 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +3 -3
- fides/ui-build/static/admin/lib/fides.js +3 -3
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/8108ANFxs99VY7KZ_Xev2/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1316-6cc72a45ebf7ff81.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1807-3beab149351d5ded.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1817-e601e737e3cc7a0e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/255-7db55b0e3a0f9dea.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2599-6c4d22e75028d8b6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2858-0b44609b6be7850b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2866-a73888c17a195cbe.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3615-5e2d062d684b8fa1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/409-a257e14acebcd73b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/431-34f0b91c26f8d9ab.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/570-c99f07161bd339cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6780-e3d40aa17a4bf2e9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-bb875d9ac89f6030.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7062.fda15dcb7df85675.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7c79804f.7a7112aece470725.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9014-eeae6f581158e645.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9278-9b1b5970f0702668.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9392.f4520f66206d347d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/ea076c0a.84423f606aef37cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-afdbd4665657cfa1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-2265ecb899d45fbc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-5d522637871ac6c8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-7674b97d655c193b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-5094ffea13f32ed9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-32600543eb7b584f.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-10ce53ea356f8bad.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-5501bbb129fee9c4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-cbe4c8f9096b6543.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-e130c0197362e8f3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-6387fcc8cce872eb.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-ff5738706da07801.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/webpack-ff0cd6bff75588da.js +0 -1
- fides/ui-build/static/admin/_next/static/css/2cadb5f62dcd7c2b.css +0 -1
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.66.2rc0.dist-info → ethyca_fides-2.67.0rc0.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{8108ANFxs99VY7KZ_Xev2 → GKmhMPa_1gMto8JZO8ENy}/_ssgManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{2921-455e6357b74d2f76.js → 2921-86f1547ac40a5cdf.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3923-6cc911dafccc5f63.js → 3923-13a6b4da2d51bf8f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{401-1b529d5800aa1f3a.js → 401-3cc1fee61494e3bd.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{5574-b13021775a15bfd2.js → 5574-9312f97b637d9ee2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7630-9aac73191ed5ed13.js → 7630-b1c93688013ef013.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{9826-111aaee8bd8dbd09.js → 9826-3c578665c6d3b21d.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{404-aece2c920ea14514.js → 404-2d803dab6a00f353.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-dc75dc6e37e52f05.js → multiple-8ff7f37913ad736a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-24d226b5a8de5c74.js → add-vendors-d00c9034cdeb0236.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-6a8ef51138ac926a.js → configure-0e1ca0f4c8e7f4da.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-6f86ab63a08a6528.js → properties-057cad65e7414a44.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-73d3cbf68f7c3a31.js → consent-e17c56eec8d91371.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-6ba9e160dae64695.js → [projectUrn]-80a6cc8e8573514a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{resources-7648bbd4f6711e4d.js → resources-6c3714ee97a718c1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-393e20924c83373e.js → [resourceUrn]-31e6c54794a9883e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-8733807dad4bc96e.js → detection-2822a423a7ad0550.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-006b695e5af5ef24.js → [datasetId]-a8e8b5f4ee7af86c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-c391c6fad56eec48.js → datastore-connection-0f29b47402292070.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-53fecfb9dd6a1e0c.js → [id]-5627d0d0668077f9.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-76b01cec5fde10a9.js → add-template-feca66ad5c5fe54a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-5c08e8447c45ce44.js → ant-components-64a322d01aae5ca7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-06ad5f34585480aa.js → AntForm-8bca16a7726e7eb2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-6f071c2bc9446cb0.js → FormikAntFormItem-b0f246fc3b67ebf7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-efcc38c58991ac9e.js → FormikControlled-1a0852b090bfc392.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-430ba5c979abfb7c.js → FormikField-11f3de1b45e36583.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-5c561880bf131afb.js → forms-1b73a1c2b6c6285f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-d888a69a3bbe040e.js → configure-e551a860ec727802.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-d3d8e3d7583ec635.js → [id]-dd99183f93763ae4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-1af10ed303815d46.js → add-property-0bdbc1fcbf553b8f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{properties-cebc0dc186be499a.js → properties-e959378bb32b6b73.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-5e1322de868d615e.js → alpha-8f98a4895e74725e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-241f95e372b65d0f.js → about-8155a35a62fdb5ae.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-41242f805599feda.js → domain-records-51333dbd21cb37c8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-2e885f74c92f669c.js → domains-bde86e5f6c09da5a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-ff112655ad5f41e5.js → email-templates-4f9a5cc8bea7725b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-86062a18e081a52a.js → regulations-102efd9199e87124.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-a2524414e968f862.js → new-bc4eb541906781e6.js} +0 -0
fides/api/graph/traversal.py
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
# pylint: disable=too-many-instance-attributes, too-many-branches, too-many-statements
|
|
1
2
|
from __future__ import annotations
|
|
2
3
|
|
|
3
4
|
import json
|
|
5
|
+
from collections import defaultdict
|
|
4
6
|
from itertools import chain
|
|
5
7
|
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, cast
|
|
6
8
|
|
|
@@ -99,37 +101,70 @@ class BaseTraversal:
|
|
|
99
101
|
self.traversal_node_dict = {k: TraversalNode(v) for k, v in graph.nodes.items()}
|
|
100
102
|
self.edges: Set[Edge] = graph.edges.copy()
|
|
101
103
|
self.root_node = artificial_traversal_node(ROOT_COLLECTION_ADDRESS)
|
|
104
|
+
|
|
105
|
+
# Pre-index edges by node address for O(1) lookup
|
|
106
|
+
self.edges_by_node: Dict[CollectionAddress, List[Edge]] = defaultdict(list)
|
|
107
|
+
for edge in self.edges:
|
|
108
|
+
self.edges_by_node[edge.f1.collection_address()].append(edge)
|
|
109
|
+
self.edges_by_node[edge.f2.collection_address()].append(edge)
|
|
110
|
+
|
|
111
|
+
# Pre-compute string versions of node dependencies
|
|
112
|
+
# This avoids expensive hash operations during traversal
|
|
113
|
+
self.node_after_str: Dict[str, Set[str]] = {}
|
|
114
|
+
self.dataset_after_str: Dict[str, Set[str]] = {}
|
|
115
|
+
|
|
116
|
+
for addr, traversal_node in self.traversal_node_dict.items():
|
|
117
|
+
# Collection-level after dependencies
|
|
118
|
+
self.node_after_str[addr.value] = {
|
|
119
|
+
dep.value for dep in traversal_node.node.collection.after
|
|
120
|
+
}
|
|
121
|
+
# Dataset-level after dependencies (need to find all collections in those datasets)
|
|
122
|
+
dataset_deps = set()
|
|
123
|
+
for dataset_name in traversal_node.node.dataset.after:
|
|
124
|
+
for other_addr in self.traversal_node_dict.keys():
|
|
125
|
+
if other_addr.dataset == dataset_name:
|
|
126
|
+
dataset_deps.add(other_addr.value)
|
|
127
|
+
self.dataset_after_str[addr.value] = dataset_deps
|
|
128
|
+
|
|
129
|
+
# Add root node to the pre-computed dependencies (it has no dependencies)
|
|
130
|
+
self.node_after_str[ROOT_COLLECTION_ADDRESS.value] = set()
|
|
131
|
+
self.dataset_after_str[ROOT_COLLECTION_ADDRESS.value] = set()
|
|
132
|
+
|
|
102
133
|
for (
|
|
103
134
|
start_field_address,
|
|
104
135
|
seed_key,
|
|
105
136
|
) in self.extract_seed_field_addresses().items():
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
start_field_address,
|
|
114
|
-
)
|
|
137
|
+
edge = Edge(
|
|
138
|
+
FieldAddress(
|
|
139
|
+
ROOT_COLLECTION_ADDRESS.dataset,
|
|
140
|
+
ROOT_COLLECTION_ADDRESS.collection,
|
|
141
|
+
seed_key,
|
|
142
|
+
),
|
|
143
|
+
start_field_address,
|
|
115
144
|
)
|
|
145
|
+
self.edges.add(edge)
|
|
146
|
+
# Add to edge index
|
|
147
|
+
self.edges_by_node[ROOT_COLLECTION_ADDRESS].append(edge)
|
|
148
|
+
self.edges_by_node[start_field_address.collection_address()].append(edge)
|
|
116
149
|
|
|
117
150
|
# Ensure manual_task collections execute right after ROOT
|
|
118
|
-
from fides.api.task.manual.
|
|
151
|
+
from fides.api.task.manual.manual_task_address import ManualTaskAddress
|
|
119
152
|
|
|
120
153
|
for addr in self.traversal_node_dict.keys():
|
|
121
154
|
if ManualTaskAddress.is_manual_task_address(addr):
|
|
122
155
|
# Add a simple synthetic edge ROOT.id -> manual_data.id
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
addr.field_address(FieldPath("id")),
|
|
131
|
-
)
|
|
156
|
+
edge = Edge(
|
|
157
|
+
FieldAddress(
|
|
158
|
+
ROOT_COLLECTION_ADDRESS.dataset,
|
|
159
|
+
ROOT_COLLECTION_ADDRESS.collection,
|
|
160
|
+
"id",
|
|
161
|
+
),
|
|
162
|
+
addr.field_address(FieldPath("id")),
|
|
132
163
|
)
|
|
164
|
+
self.edges.add(edge)
|
|
165
|
+
# Add to edge index
|
|
166
|
+
self.edges_by_node[ROOT_COLLECTION_ADDRESS].append(edge)
|
|
167
|
+
self.edges_by_node[addr].append(edge)
|
|
133
168
|
|
|
134
169
|
self._verify_traversal()
|
|
135
170
|
|
|
@@ -204,28 +239,52 @@ class BaseTraversal:
|
|
|
204
239
|
logger.info(
|
|
205
240
|
"Starting traversal",
|
|
206
241
|
)
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
242
|
+
|
|
243
|
+
# Use string sets instead of CollectionAddress sets
|
|
244
|
+
# This avoids expensive hash operations
|
|
245
|
+
remaining_node_keys_str: Set[str] = {
|
|
246
|
+
addr.value for addr in self.traversal_node_dict.keys()
|
|
247
|
+
}
|
|
210
248
|
finished_nodes: dict[CollectionAddress, TraversalNode] = {}
|
|
211
249
|
running_node_queue: MatchingQueue[TraversalNode] = MatchingQueue(self.root_node)
|
|
212
250
|
|
|
213
|
-
|
|
251
|
+
# Instead of copying entire edge set, use a more efficient approach
|
|
252
|
+
# We'll simulate Edge.delete_edges behavior without the expensive set operations
|
|
253
|
+
deleted_edges_tracker: Dict[Edge, bool] = {}
|
|
254
|
+
|
|
214
255
|
while not running_node_queue.is_empty():
|
|
215
256
|
# this is to support the "run traversal_node A AFTER traversal_node B functionality:"
|
|
216
257
|
n = running_node_queue.pop_first_match(
|
|
217
|
-
lambda x: x.
|
|
258
|
+
lambda x: x.can_run_given_str(
|
|
259
|
+
remaining_node_keys_str, self.node_after_str, self.dataset_after_str
|
|
260
|
+
)
|
|
218
261
|
)
|
|
219
262
|
|
|
220
263
|
if n:
|
|
221
264
|
node_run_fn(n, environment)
|
|
222
265
|
# delete all edges between the traversal_node that's just run and any completed nodes
|
|
223
266
|
for finished_node_address, finished_node in finished_nodes.items():
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
267
|
+
# Find edges to delete manually instead of using Edge.delete_edges
|
|
268
|
+
completed_edges: Set[Edge] = set()
|
|
269
|
+
|
|
270
|
+
# Only check edges connected to the relevant nodes
|
|
271
|
+
relevant_edges = set()
|
|
272
|
+
relevant_edges.update(
|
|
273
|
+
self.edges_by_node.get(finished_node_address, [])
|
|
228
274
|
)
|
|
275
|
+
relevant_edges.update(self.edges_by_node.get(n.address, []))
|
|
276
|
+
|
|
277
|
+
for edge in relevant_edges:
|
|
278
|
+
# Skip if already deleted
|
|
279
|
+
if deleted_edges_tracker.get(edge, False):
|
|
280
|
+
continue
|
|
281
|
+
|
|
282
|
+
# Check if this edge spans between the two nodes (bidirectional check)
|
|
283
|
+
if edge.spans(
|
|
284
|
+
finished_node_address, cast(TraversalNode, n).address # type: ignore[redundant-cast]
|
|
285
|
+
):
|
|
286
|
+
completed_edges.add(edge)
|
|
287
|
+
deleted_edges_tracker[edge] = True
|
|
229
288
|
|
|
230
289
|
def edge_ends_with_collection(_edge: Edge) -> bool:
|
|
231
290
|
# append edges that end in this traversal_node
|
|
@@ -236,14 +295,16 @@ class BaseTraversal:
|
|
|
236
295
|
for edge in filter(edge_ends_with_collection, completed_edges):
|
|
237
296
|
# note, this will not work for self-reference
|
|
238
297
|
finished_node.add_child(n, edge)
|
|
298
|
+
|
|
239
299
|
# next edges = take all edges including n that are _not_ in edges_from_completed_nodes
|
|
240
300
|
# in the form (field_address_this, field_address_foreign)
|
|
241
301
|
|
|
302
|
+
# Use pre-indexed edges instead of iterating through all edges
|
|
242
303
|
edges_to_children = pydash.collections.filter_(
|
|
243
304
|
[
|
|
244
305
|
e.split_by_address(cast(TraversalNode, n).address) # type: ignore[redundant-cast]
|
|
245
|
-
for e in
|
|
246
|
-
if
|
|
306
|
+
for e in self.edges_by_node[n.address]
|
|
307
|
+
if not deleted_edges_tracker.get(e, False)
|
|
247
308
|
]
|
|
248
309
|
)
|
|
249
310
|
if not edges_to_children:
|
|
@@ -259,7 +320,7 @@ class BaseTraversal:
|
|
|
259
320
|
self.traversal_node_dict[nxt_address]
|
|
260
321
|
)
|
|
261
322
|
finished_nodes[n.address] = n
|
|
262
|
-
|
|
323
|
+
remaining_node_keys_str.discard(n.address.value) # Use string value
|
|
263
324
|
else:
|
|
264
325
|
# traversal traversal_node dict diff finished nodes
|
|
265
326
|
logger.error(
|
|
@@ -271,12 +332,17 @@ class BaseTraversal:
|
|
|
271
332
|
[{', '.join([str(tn.address) for tn in running_node_queue.data])}]""",
|
|
272
333
|
)
|
|
273
334
|
|
|
335
|
+
# Convert back to CollectionAddress set for filtering
|
|
274
336
|
remaining_node_keys = {
|
|
275
337
|
key
|
|
276
|
-
for key in
|
|
277
|
-
if
|
|
338
|
+
for key in self.traversal_node_dict.keys()
|
|
339
|
+
if key.value in remaining_node_keys_str
|
|
340
|
+
and not self.should_exclude_node(self.traversal_node_dict[key])
|
|
278
341
|
}
|
|
279
342
|
|
|
343
|
+
# Update string set after filtering
|
|
344
|
+
remaining_node_keys_str = {key.value for key in remaining_node_keys}
|
|
345
|
+
|
|
280
346
|
# error if there are nodes that have not been visited
|
|
281
347
|
if remaining_node_keys:
|
|
282
348
|
logger.error(
|
|
@@ -289,12 +355,16 @@ class BaseTraversal:
|
|
|
289
355
|
)
|
|
290
356
|
|
|
291
357
|
# filter out remaining_edges if the nodes they link are allowed to remain unreachable
|
|
292
|
-
remaining_edges =
|
|
293
|
-
|
|
294
|
-
for edge in
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
358
|
+
remaining_edges = set()
|
|
359
|
+
for node_key in remaining_node_keys:
|
|
360
|
+
for edge in self.edges_by_node.get(node_key, []):
|
|
361
|
+
if not deleted_edges_tracker.get(edge, False):
|
|
362
|
+
# Check if both ends of the edge are in remaining nodes
|
|
363
|
+
if (
|
|
364
|
+
edge.f1.collection_address() in remaining_node_keys
|
|
365
|
+
and edge.f2.collection_address() in remaining_node_keys
|
|
366
|
+
):
|
|
367
|
+
remaining_edges.add(edge)
|
|
298
368
|
|
|
299
369
|
# error if there are edges that have not been visited
|
|
300
370
|
if remaining_edges:
|
|
@@ -478,6 +548,23 @@ class TraversalNode(Contextualizable):
|
|
|
478
548
|
return False
|
|
479
549
|
return True
|
|
480
550
|
|
|
551
|
+
def can_run_given_str(
|
|
552
|
+
self,
|
|
553
|
+
remaining_node_keys_str: Set[str],
|
|
554
|
+
node_after_str: Dict[str, Set[str]],
|
|
555
|
+
dataset_after_str: Dict[str, Set[str]],
|
|
556
|
+
) -> bool:
|
|
557
|
+
"""Optimized version using pre-computed string sets to avoid expensive hash operations."""
|
|
558
|
+
# Check collection-level dependencies
|
|
559
|
+
node_deps = node_after_str.get(self.address.value, set())
|
|
560
|
+
if node_deps & remaining_node_keys_str:
|
|
561
|
+
return False
|
|
562
|
+
# Check dataset-level dependencies
|
|
563
|
+
dataset_deps = dataset_after_str.get(self.address.value, set())
|
|
564
|
+
if dataset_deps & remaining_node_keys_str:
|
|
565
|
+
return False
|
|
566
|
+
return True
|
|
567
|
+
|
|
481
568
|
def is_root_node(self) -> bool:
|
|
482
569
|
"""This traversal_node is the defined traversal start"""
|
|
483
570
|
return self.address == ROOT_COLLECTION_ADDRESS
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from typing import TYPE_CHECKING, Optional, Union
|
|
3
|
+
|
|
4
|
+
from sqlalchemy import Column, ForeignKey, Index, Integer, String
|
|
5
|
+
from sqlalchemy.dialects.postgresql import JSONB
|
|
6
|
+
from sqlalchemy.ext.declarative import declared_attr
|
|
7
|
+
from sqlalchemy.orm import Session, relationship
|
|
8
|
+
|
|
9
|
+
from fides.api.db.base_class import Base, FidesBase
|
|
10
|
+
from fides.api.db.util import EnumColumn
|
|
11
|
+
from fides.api.task.conditional_dependencies.schemas import (
|
|
12
|
+
ConditionGroup,
|
|
13
|
+
ConditionLeaf,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from fides.api.models.manual_task.manual_task import ManualTask
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ManualTaskConditionalDependencyType(str, Enum):
|
|
21
|
+
"""Enum for manual task conditional dependency types.
|
|
22
|
+
|
|
23
|
+
This enum defines the two types of nodes in a conditional dependency tree:
|
|
24
|
+
|
|
25
|
+
- leaf: A terminal node that represents a single condition (e.g., "user.age >= 18")
|
|
26
|
+
- group: A non-terminal node that groups multiple conditions with logical operators (AND/OR)
|
|
27
|
+
|
|
28
|
+
Examples:
|
|
29
|
+
leaf: Used for simple field comparisons like:
|
|
30
|
+
- "user.name exists"
|
|
31
|
+
- "user.age >= 18"
|
|
32
|
+
- "billing.subscription.status == 'active'"
|
|
33
|
+
|
|
34
|
+
group: Used to combine multiple conditions with logical operators:
|
|
35
|
+
- AND group: "user.age >= 18 AND user.active == true"
|
|
36
|
+
- OR group: "user.role == 'admin' OR user.verified == true"
|
|
37
|
+
- Nested groups: "(user.age >= 18 AND (user.role == 'admin' OR user.verified == true))"
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
leaf = "leaf"
|
|
41
|
+
group = "group"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ManualTaskConditionalDependency(Base):
|
|
45
|
+
"""Model for storing conditional dependencies."""
|
|
46
|
+
|
|
47
|
+
@declared_attr
|
|
48
|
+
def __tablename__(cls) -> str:
|
|
49
|
+
"""Overriding base class method to set the table name."""
|
|
50
|
+
return "manual_task_conditional_dependency"
|
|
51
|
+
|
|
52
|
+
# We need to redefine it here so that self-referential relationships
|
|
53
|
+
# can properly reference the `id` column instead of the built-in Python function.
|
|
54
|
+
id = Column(String(255), primary_key=True, default=FidesBase.generate_uuid)
|
|
55
|
+
|
|
56
|
+
# Foreign key relationships
|
|
57
|
+
manual_task_id = Column(
|
|
58
|
+
String, ForeignKey("manual_task.id", ondelete="CASCADE"), nullable=False
|
|
59
|
+
)
|
|
60
|
+
parent_id = Column(
|
|
61
|
+
String,
|
|
62
|
+
ForeignKey("manual_task_conditional_dependency.id", ondelete="CASCADE"),
|
|
63
|
+
nullable=True,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
# Condition metadata
|
|
67
|
+
condition_type = Column(
|
|
68
|
+
EnumColumn(ManualTaskConditionalDependencyType), nullable=False
|
|
69
|
+
) # leaf or group
|
|
70
|
+
field_address = Column(String, nullable=True) # For leaf conditions
|
|
71
|
+
operator = Column(String, nullable=True) # For leaf conditions
|
|
72
|
+
value = Column(JSONB, nullable=True) # For leaf conditions
|
|
73
|
+
logical_operator = Column(String, nullable=True) # 'and' or 'or' for groups
|
|
74
|
+
|
|
75
|
+
# Ordering
|
|
76
|
+
sort_order = Column(Integer, nullable=False, default=0)
|
|
77
|
+
|
|
78
|
+
__table_args__ = (
|
|
79
|
+
Index("ix_manual_task_conditional_dependency_manual_task_id", "manual_task_id"),
|
|
80
|
+
Index("ix_manual_task_conditional_dependency_parent_id", "parent_id"),
|
|
81
|
+
Index("ix_manual_task_conditional_dependency_condition_type", "condition_type"),
|
|
82
|
+
Index("ix_manual_task_conditional_dependency_sort_order", "sort_order"),
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# Relationships
|
|
86
|
+
task = relationship("ManualTask", back_populates="conditional_dependencies")
|
|
87
|
+
parent = relationship(
|
|
88
|
+
"ManualTaskConditionalDependency",
|
|
89
|
+
remote_side=[id],
|
|
90
|
+
back_populates="children",
|
|
91
|
+
foreign_keys=[parent_id],
|
|
92
|
+
)
|
|
93
|
+
children = relationship(
|
|
94
|
+
"ManualTaskConditionalDependency",
|
|
95
|
+
back_populates="parent",
|
|
96
|
+
cascade="all, delete-orphan",
|
|
97
|
+
foreign_keys=[parent_id],
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
def to_condition_leaf(self) -> ConditionLeaf:
|
|
101
|
+
"""Convert to ConditionLeaf if this is a leaf condition"""
|
|
102
|
+
if self.condition_type != "leaf":
|
|
103
|
+
raise ValueError("Cannot convert group condition to leaf")
|
|
104
|
+
|
|
105
|
+
return ConditionLeaf(
|
|
106
|
+
field_address=self.field_address, operator=self.operator, value=self.value
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
def to_condition_group(self) -> ConditionGroup:
|
|
110
|
+
"""Convert to ConditionGroup if this is a group condition"""
|
|
111
|
+
if self.condition_type != "group":
|
|
112
|
+
raise ValueError("Cannot convert leaf condition to group")
|
|
113
|
+
|
|
114
|
+
# Recursively build children
|
|
115
|
+
child_conditions = []
|
|
116
|
+
children_list = [child for child in self.children] # type: ignore[attr-defined]
|
|
117
|
+
for child in sorted(children_list, key=lambda x: x.sort_order):
|
|
118
|
+
if child.condition_type == "leaf":
|
|
119
|
+
child_conditions.append(child.to_condition_leaf())
|
|
120
|
+
else:
|
|
121
|
+
child_conditions.append(child.to_condition_group())
|
|
122
|
+
|
|
123
|
+
return ConditionGroup(
|
|
124
|
+
logical_operator=self.logical_operator, conditions=child_conditions
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
@classmethod
|
|
128
|
+
def get_root_condition(
|
|
129
|
+
cls, db: Session, manual_task_id: str
|
|
130
|
+
) -> Optional[Union[ConditionLeaf, ConditionGroup]]:
|
|
131
|
+
"""Get the root condition for a config"""
|
|
132
|
+
root = (
|
|
133
|
+
db.query(cls)
|
|
134
|
+
.filter(cls.manual_task_id == manual_task_id, cls.parent_id.is_(None))
|
|
135
|
+
.first()
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
if not root:
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
if root.condition_type == "leaf":
|
|
142
|
+
return root.to_condition_leaf()
|
|
143
|
+
|
|
144
|
+
return root.to_condition_group()
|
|
@@ -26,6 +26,9 @@ from fides.api.schemas.base_class import FidesSchema
|
|
|
26
26
|
if TYPE_CHECKING:
|
|
27
27
|
from fides.api.models.attachment import Attachment
|
|
28
28
|
from fides.api.models.fides_user import FidesUser
|
|
29
|
+
from fides.api.models.manual_task.conditional_dependency import (
|
|
30
|
+
ManualTaskConditionalDependency,
|
|
31
|
+
)
|
|
29
32
|
|
|
30
33
|
# ------------------------------------------------------------
|
|
31
34
|
# Enums
|
|
@@ -242,6 +245,13 @@ class ManualTask(Base):
|
|
|
242
245
|
viewonly=True, # No cascade delete - submissions are historical data
|
|
243
246
|
)
|
|
244
247
|
|
|
248
|
+
conditional_dependencies = relationship(
|
|
249
|
+
"ManualTaskConditionalDependency",
|
|
250
|
+
back_populates="task",
|
|
251
|
+
uselist=True,
|
|
252
|
+
cascade="all, delete-orphan",
|
|
253
|
+
)
|
|
254
|
+
|
|
245
255
|
# Properties
|
|
246
256
|
@property
|
|
247
257
|
def assigned_users(self) -> list[str]:
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Union
|
|
4
|
+
from urllib.parse import unquote_to_bytes
|
|
5
|
+
|
|
6
|
+
from sqlalchemy import Column, ForeignKey, String
|
|
7
|
+
from sqlalchemy.ext.declarative import declared_attr
|
|
8
|
+
from sqlalchemy.orm import Session, relationship
|
|
9
|
+
from sqlalchemy_utils.types.encrypted.encrypted_type import (
|
|
10
|
+
AesGcmEngine,
|
|
11
|
+
StringEncryptedType,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
from fides.api.db.base_class import Base, JSONTypeOverride
|
|
15
|
+
from fides.api.util.custom_json_encoder import ENCODED_BYTES_PREFIX
|
|
16
|
+
from fides.config import CONFIG
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from fides.api.models.privacy_request import PrivacyRequest
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class MaskingSecret(Base):
|
|
23
|
+
"""SQLAlchemy model for storing secret caching information"""
|
|
24
|
+
|
|
25
|
+
@declared_attr
|
|
26
|
+
def __tablename__(self) -> str:
|
|
27
|
+
return "masking_secret"
|
|
28
|
+
|
|
29
|
+
privacy_request_id = Column(
|
|
30
|
+
String, ForeignKey("privacyrequest.id", ondelete="CASCADE"), nullable=False
|
|
31
|
+
)
|
|
32
|
+
secret = Column(
|
|
33
|
+
StringEncryptedType(
|
|
34
|
+
JSONTypeOverride,
|
|
35
|
+
CONFIG.security.app_encryption_key,
|
|
36
|
+
AesGcmEngine,
|
|
37
|
+
"pkcs5",
|
|
38
|
+
),
|
|
39
|
+
nullable=False,
|
|
40
|
+
)
|
|
41
|
+
masking_strategy = Column(String, nullable=False)
|
|
42
|
+
secret_type = Column(String, nullable=False)
|
|
43
|
+
privacy_request = relationship("PrivacyRequest", back_populates="masking_secrets")
|
|
44
|
+
|
|
45
|
+
def set_secret(self, secret: Union[str, bytes]) -> None:
|
|
46
|
+
"""Set the secret value, handling both string and bytes types"""
|
|
47
|
+
if isinstance(secret, str):
|
|
48
|
+
self.secret = secret.encode("utf-8")
|
|
49
|
+
elif isinstance(secret, bytes):
|
|
50
|
+
self.secret = secret
|
|
51
|
+
else:
|
|
52
|
+
raise ValueError("Secret must be either string or bytes")
|
|
53
|
+
|
|
54
|
+
def get_secret(self) -> Union[str, bytes]:
|
|
55
|
+
"""Retrieve the secret in its original type"""
|
|
56
|
+
secret = self.secret
|
|
57
|
+
if isinstance(secret, str) and secret.startswith(ENCODED_BYTES_PREFIX):
|
|
58
|
+
return unquote_to_bytes(secret)[len(ENCODED_BYTES_PREFIX) :]
|
|
59
|
+
return secret
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def create(
|
|
63
|
+
cls,
|
|
64
|
+
db: Session,
|
|
65
|
+
*,
|
|
66
|
+
data: dict[str, Any],
|
|
67
|
+
check_name: bool = True,
|
|
68
|
+
) -> "MaskingSecret":
|
|
69
|
+
"""
|
|
70
|
+
Create a new masking secret. Handles both string and bytes secrets automatically, encrypting them for storage.
|
|
71
|
+
"""
|
|
72
|
+
return super().create(db=db, data=data, check_name=check_name)
|
fides/api/models/policy.py
CHANGED
|
@@ -25,7 +25,9 @@ from fides.api.models.client import ClientDetail
|
|
|
25
25
|
from fides.api.models.connectionconfig import ConnectionConfig
|
|
26
26
|
from fides.api.models.sql_models import DataCategory # type: ignore
|
|
27
27
|
from fides.api.models.storage import StorageConfig, get_active_default_storage_config
|
|
28
|
+
from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
|
|
28
29
|
from fides.api.schemas.policy import ActionType, DrpAction
|
|
30
|
+
from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
|
|
29
31
|
from fides.api.util.data_category import _validate_data_category
|
|
30
32
|
from fides.config import CONFIG
|
|
31
33
|
|
|
@@ -140,6 +142,27 @@ class Policy(Base):
|
|
|
140
142
|
except IndexError:
|
|
141
143
|
return None
|
|
142
144
|
|
|
145
|
+
def generate_masking_secrets(self) -> Optional[List[MaskingSecretCache]]:
|
|
146
|
+
"""
|
|
147
|
+
Returns a list of masking secrets for the masking strategies in the policy.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
masking_secrets: List[MaskingSecretCache] = []
|
|
151
|
+
erasure_rules = self.get_rules_for_action(action_type=ActionType.erasure)
|
|
152
|
+
unique_masking_strategies_by_name: Set[str] = set()
|
|
153
|
+
for rule in erasure_rules:
|
|
154
|
+
strategy_name: str = rule.masking_strategy["strategy"] # type: ignore
|
|
155
|
+
configuration = rule.masking_strategy["configuration"] # type: ignore
|
|
156
|
+
if strategy_name in unique_masking_strategies_by_name:
|
|
157
|
+
continue
|
|
158
|
+
unique_masking_strategies_by_name.add(strategy_name)
|
|
159
|
+
masking_strategy = MaskingStrategy.get_strategy(
|
|
160
|
+
strategy_name, configuration
|
|
161
|
+
)
|
|
162
|
+
if masking_strategy.secrets_required():
|
|
163
|
+
masking_secrets.extend(masking_strategy.generate_secrets_for_cache())
|
|
164
|
+
return masking_secrets
|
|
165
|
+
|
|
143
166
|
def applies_to(self, node: "TraversalNode") -> bool:
|
|
144
167
|
"""
|
|
145
168
|
Returns True if any data category in the traversal node starts with any of the policy's target categories.
|
|
@@ -50,6 +50,7 @@ from fides.api.models.comment import Comment, CommentReference, CommentReference
|
|
|
50
50
|
from fides.api.models.fides_user import FidesUser
|
|
51
51
|
from fides.api.models.field_types import EncryptedLargeDataDescriptor
|
|
52
52
|
from fides.api.models.manual_webhook import AccessManualWebhook
|
|
53
|
+
from fides.api.models.masking_secret import MaskingSecret
|
|
53
54
|
from fides.api.models.policy import (
|
|
54
55
|
Policy,
|
|
55
56
|
PolicyPreWebhook,
|
|
@@ -98,7 +99,6 @@ from fides.api.util.cache import (
|
|
|
98
99
|
get_drp_request_body_cache_key,
|
|
99
100
|
get_encryption_cache_key,
|
|
100
101
|
get_identity_cache_key,
|
|
101
|
-
get_masking_secret_cache_key,
|
|
102
102
|
)
|
|
103
103
|
from fides.api.util.collection_util import Row, extract_key_for_address
|
|
104
104
|
from fides.api.util.constants import API_DATE_FORMAT
|
|
@@ -145,6 +145,12 @@ class PrivacyRequest(
|
|
|
145
145
|
ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
|
|
146
146
|
nullable=True,
|
|
147
147
|
)
|
|
148
|
+
finalized_at = Column(DateTime(timezone=True), nullable=True)
|
|
149
|
+
finalized_by = Column(
|
|
150
|
+
String,
|
|
151
|
+
ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
|
|
152
|
+
nullable=True,
|
|
153
|
+
)
|
|
148
154
|
submitted_by = Column(
|
|
149
155
|
String,
|
|
150
156
|
ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"),
|
|
@@ -289,6 +295,13 @@ class PrivacyRequest(
|
|
|
289
295
|
order_by="RequestTask.created_at",
|
|
290
296
|
)
|
|
291
297
|
|
|
298
|
+
masking_secrets: "RelationshipProperty[List[MaskingSecret]]" = relationship(
|
|
299
|
+
"MaskingSecret",
|
|
300
|
+
back_populates="privacy_request",
|
|
301
|
+
uselist=True,
|
|
302
|
+
passive_deletes="all",
|
|
303
|
+
)
|
|
304
|
+
|
|
292
305
|
@property
|
|
293
306
|
def days_left(self: PrivacyRequest) -> Union[int, None]:
|
|
294
307
|
if self.due_date is None:
|
|
@@ -566,19 +579,24 @@ class PrivacyRequest(
|
|
|
566
579
|
encryption_key,
|
|
567
580
|
)
|
|
568
581
|
|
|
569
|
-
def
|
|
570
|
-
|
|
571
|
-
|
|
582
|
+
def persist_masking_secrets(
|
|
583
|
+
self, masking_secrets: List[MaskingSecretCache]
|
|
584
|
+
) -> None:
|
|
585
|
+
"""Persists masking encryption secrets to database."""
|
|
586
|
+
if not masking_secrets:
|
|
572
587
|
return
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
588
|
+
|
|
589
|
+
session = Session.object_session(self)
|
|
590
|
+
for masking_secret in masking_secrets:
|
|
591
|
+
MaskingSecret.create(
|
|
592
|
+
db=session,
|
|
593
|
+
data={
|
|
594
|
+
"privacy_request_id": self.id,
|
|
595
|
+
"secret": masking_secret.secret,
|
|
596
|
+
"masking_strategy": masking_secret.masking_strategy,
|
|
597
|
+
"secret_type": masking_secret.secret_type,
|
|
598
|
+
},
|
|
599
|
+
)
|
|
582
600
|
|
|
583
601
|
def get_cached_identity_data(self) -> Dict[str, Any]:
|
|
584
602
|
"""Retrieves any identity data pertaining to this request from the cache"""
|
fides/api/oauth/roles.py
CHANGED
|
@@ -47,6 +47,7 @@ from fides.common.api.scope_registry import (
|
|
|
47
47
|
SYSTEM_READ,
|
|
48
48
|
USER_PERMISSION_ASSIGN_OWNERS,
|
|
49
49
|
USER_READ,
|
|
50
|
+
USER_READ_OWN,
|
|
50
51
|
WEBHOOK_READ,
|
|
51
52
|
)
|
|
52
53
|
|
|
@@ -126,6 +127,7 @@ viewer_scopes = [ # Intentionally omitted USER_PERMISSION_READ and PRIVACY_REQU
|
|
|
126
127
|
|
|
127
128
|
respondent_scopes = [
|
|
128
129
|
PRIVACY_REQUEST_MANUAL_STEPS_RESPOND, # allows respondents to respond to assigned manual steps
|
|
130
|
+
USER_READ_OWN,
|
|
129
131
|
]
|
|
130
132
|
|
|
131
133
|
external_respondent_scopes = [
|
|
@@ -11,6 +11,14 @@ from fides.api.schemas.messaging.messaging import MessagingServiceType
|
|
|
11
11
|
from fides.config.admin_ui_settings import ErrorNotificationMode
|
|
12
12
|
|
|
13
13
|
|
|
14
|
+
class SqlDryRunMode(str, Enum):
|
|
15
|
+
"""SQL dry run mode for controlling execution of SQL statements in privacy requests"""
|
|
16
|
+
|
|
17
|
+
none = "none"
|
|
18
|
+
access = "access"
|
|
19
|
+
erasure = "erasure"
|
|
20
|
+
|
|
21
|
+
|
|
14
22
|
class StorageTypeApiAccepted(Enum):
|
|
15
23
|
"""Enum for storage destination types accepted in API updates"""
|
|
16
24
|
|
|
@@ -62,7 +70,9 @@ class ExecutionApplicationConfig(FidesSchema):
|
|
|
62
70
|
subject_identity_verification_required: Optional[bool] = None
|
|
63
71
|
disable_consent_identity_verification: Optional[bool] = None
|
|
64
72
|
require_manual_request_approval: Optional[bool] = None
|
|
65
|
-
|
|
73
|
+
sql_dry_run: Optional[SqlDryRunMode] = None
|
|
74
|
+
|
|
75
|
+
model_config = ConfigDict(use_enum_values=True, extra="forbid")
|
|
66
76
|
|
|
67
77
|
|
|
68
78
|
class AdminUIConfig(FidesSchema):
|