ethyca-fides 2.66.1b1__py2.py3-none-any.whl → 2.66.2b0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.66.1b1.dist-info → ethyca_fides-2.66.2b0.dist-info}/METADATA +1 -1
- {ethyca_fides-2.66.1b1.dist-info → ethyca_fides-2.66.2b0.dist-info}/RECORD +193 -191
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/7e9a2b52f498_adding_masking_secrets.py +60 -0
- fides/api/api/v1/endpoints/drp_endpoints.py +7 -1
- fides/api/app_setup.py +3 -2
- fides/api/common_exceptions.py +4 -0
- fides/api/db/database.py +1 -1
- fides/api/models/masking_secret.py +72 -0
- fides/api/models/policy.py +23 -0
- fides/api/models/privacy_request/privacy_request.py +25 -13
- fides/api/schemas/masking/masking_secrets.py +1 -1
- fides/api/service/connectors/query_configs/bigquery_query_config.py +48 -14
- fides/api/service/privacy_request/request_runner_service.py +26 -0
- fides/api/service/privacy_request/request_service.py +1 -22
- fides/api/task/graph_task.py +27 -3
- fides/api/util/cache.py +5 -0
- fides/api/util/encryption/secrets_util.py +48 -18
- fides/data/language/languages.yml +3 -1
- fides/service/privacy_request/privacy_request_service.py +6 -1
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/QGhYEMWDATmdofDzIl0_2/_buildManifest.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1817-6f35f58cd08b04ae.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-a152f52351c84ef4.js → _app-65e47e67bee20654.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-f18b2b83c4592f03.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-1daa805a3099c6d4.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-fe58cebba358119d.js +1 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +2 -2
- fides/ui-build/static/admin/lib/fides.js +2 -2
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/cUz9aQNEfv77_K6F0m_Ja/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1817-10f6ab6da28df4d2.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-444ec9b3fec6e428.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-7eb7d26fa4b5cf17.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/[id]-79fe68f1eb3add49.js +0 -1
- {ethyca_fides-2.66.1b1.dist-info → ethyca_fides-2.66.2b0.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.66.1b1.dist-info → ethyca_fides-2.66.2b0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.66.1b1.dist-info → ethyca_fides-2.66.2b0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.66.1b1.dist-info → ethyca_fides-2.66.2b0.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{cUz9aQNEfv77_K6F0m_Ja → QGhYEMWDATmdofDzIl0_2}/_ssgManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3450-24041e1b2a846739.js → 3450-f7bb8d46fbe4e78d.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3872-b97dd7695d6532bc.js → 3872-08c855f3edb230c4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{4121-c4747258599705d0.js → 4121-f50675521dfee6eb.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{431-0e77abba7e220e31.js → 431-ade3e312fac3430b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{4608-b2d8a76e3a680db4.js → 4608-a8e3100e2806dbff.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{5574-0b7c88e3f81b08fb.js → 5574-a7d8a753d273b11a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6084-7bc3756bd23c3a4c.js → 6084-a872a8bc704c980f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6662-8c43dd2b75e0264c.js → 6662-4a7805f74af51f1d.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6882-0fa957731918160e.js → 6882-85c6277d9531a83a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6954-d30c6dff440d46b0.js → 6954-34e062e4bffc7e71.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7476-2d2c80dfe64c0f90.js → 7476-cfe41e915f0c4f40.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7630-06f4207597225cbb.js → 7630-7a8039aa37893129.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{787-32ca132be974eaf9.js → 787-d05b25c73f49b6af.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{79-056290dba15480c2.js → 79-1952e8aab93b7209.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{796-00487f9036709b7b.js → 796-d2876f4d09a6d81f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{9226-081cb18771536d64.js → 9226-7a799c930b73f217.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{9826-e77d5e5b8458fe77.js → 9826-b7d5467e3a3225c8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-a539d34e7c9ab0ee.js → manual-acb59f8b5e97512a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-fa0c254854fbf79c.js → multiple-8ff7f37913ad736a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-9c86d99f81d128ac.js → add-systems-f28841affa791975.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-3a77a92d2a94426a.js → add-vendors-d00c9034cdeb0236.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-886518c906a28271.js → configure-50d21e23b607688b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-a333f503aa057e06.js → privacy-experience-09d4408014bcfe1c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-7a1c8073262ff0b4.js → [id]-d67542783ef5ddac.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-39ef289b3c88ebd4.js → new-3f20e8a316bb3d5b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-deeb59b8332261c1.js → privacy-notices-23e9dcd4590312d2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-7507764f89f9d9c8.js → consent-509691da6b06f834.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-11d52f1570759c4d.js → [resourceUrn]-99c9092d65c94807.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-6020cd361c57b387.js → [projectUrn]-80a6cc8e8573514a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-2fc76749940a06fd.js → projects-774fecea22ba8852.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-b83afa5565d0c84e.js → [resourceUrn]-f6bd6aff389cb9fe.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{resources-7648bbd4f6711e4d.js → resources-6c3714ee97a718c1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-7d1c53686a9a5d73.js → data-catalog-2f43cfefc869b85f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/{[systemId]-031d7589a07fde30.js → [systemId]-71579a199158952e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/{[monitorId]-efcbfd52186591bc.js → [monitorId]-7e63ac744c45f6da.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-94acf9cc27209714.js → action-center-409694b8441bd8fb.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-393e20924c83373e.js → [resourceUrn]-31e6c54794a9883e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-8733807dad4bc96e.js → detection-2822a423a7ad0550.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-14bd7500362ff224.js → [resourceUrn]-6421ce247549c5d6.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9e7dfd5a6acc2e8f.js → discovery-3eac407ac5181a3c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{datamap-e48f4ef032791c1d.js → datamap-e707dc714452498e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-24f87303adb9428e.js → [...subfieldNames]-1c98bd0959d9570a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-41aa4e51bd73d13e.js → [collectionName]-e548cabda7da32c9.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-b296323ed0922fd5.js → [datasetId]-a8e8b5f4ee7af86c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-cbb409d1bc89d10b.js → new-513c862c3a707735.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-07df9c4896721e9f.js → dataset-747b7a13289f1cd7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-0a60fe49678a3ece.js → [id]-3d22525b3c327b2e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-c903ab62611245d8.js → new-d2cad97495e86adb.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-33b01e2ab2d5ce7e.js → datastore-connection-6865aa958f3da342.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{index-ca1c9d4de5abec7a.js → index-f127ebaac4689d10.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/integrations/{[id]-2f9efedecc4b9273.js → [id]-8d83a5518c00fcfc.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-0f91eae539d47163.js → integrations-f10a7dcf7541c865.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-8b20a058d18181b9.js → [id]-5627d0d0668077f9.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-c7054125338aee6c.js → add-template-feca66ad5c5fe54a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{messaging-a4fdfa9047bb5770.js → messaging-c1bd3e7adbe8d2d3.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-8bfde56c86128ec3.js → table-migration-05616e2ae20ff4f8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{messaging-08a6d60bb9230d5f.js → messaging-f9be923340cd99cf.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{storage-29d47f1bdd08c45d.js → storage-eb1ccc8a5dbf0fe5.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-2806f3458fe85897.js → configure-295bfe6880b209f2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-568620fc7e239e7b.js → [id]-dd99183f93763ae4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-394043fd823d69f5.js → add-property-0bdbc1fcbf553b8f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-92d7f12a95a1adc5.js → alpha-8152e5828469cf91.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-70fb68b7519d5222.js → about-9839e544924ac1f2.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-7c8bda96813441c5.js → consent-f7a0f8367129cd70.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-b773691516488006.js → custom-fields-10df8d6fdc8bc149.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-21f14d9141a5ad8a.js → domain-records-4056a3323cdc7042.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-3022368d5438dfb7.js → domains-5a3691559262e874.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-89903d6f579b1aeb.js → email-templates-e1d3c3d0ab878812.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-14621f32ad4f9120.js → locations-6946e78a5d43e654.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-5106ff9a9403ee92.js → organization-26df33de21fb11b3.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-0927740524f79b02.js → regulations-102efd9199e87124.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-47284d49803f8b3b.js → test-datasets-170c9c55f65ffb0f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-7bc2d6b23d5bb71f.js → [id]-d4a57ea18935dd70.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{systems-c3a536901d6b445d.js → systems-2112c006765c66b6.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{taxonomy-ab4f4d260c01163b.js → taxonomy-a53e89319582ce58.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-a2524414e968f862.js → new-bc4eb541906781e6.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-6111bb65a2607dc5.js → user-management-45bfa04e45a7d13f.js} +0 -0
fides/_version.py
CHANGED
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2025-07-
|
|
11
|
+
"date": "2025-07-25T16:16:16-0700",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "2.66.
|
|
14
|
+
"full-revisionid": "3805a2cc72d6f998672d84b64fd0f2c78a64f81d",
|
|
15
|
+
"version": "2.66.2b0"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""adding masking secrets
|
|
2
|
+
|
|
3
|
+
Revision ID: 7e9a2b52f498
|
|
4
|
+
Revises: d0031087eacb
|
|
5
|
+
Create Date: 2024-12-23 23:29:53.557951
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import sqlalchemy as sa
|
|
10
|
+
import sqlalchemy_utils
|
|
11
|
+
from alembic import op
|
|
12
|
+
|
|
13
|
+
# revision identifiers, used by Alembic.
|
|
14
|
+
revision = "7e9a2b52f498"
|
|
15
|
+
down_revision = "d0031087eacb"
|
|
16
|
+
branch_labels = None
|
|
17
|
+
depends_on = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def upgrade():
|
|
21
|
+
op.create_table(
|
|
22
|
+
"masking_secret",
|
|
23
|
+
sa.Column("id", sa.String(length=255), nullable=False),
|
|
24
|
+
sa.Column(
|
|
25
|
+
"created_at",
|
|
26
|
+
sa.DateTime(timezone=True),
|
|
27
|
+
server_default=sa.text("now()"),
|
|
28
|
+
nullable=True,
|
|
29
|
+
),
|
|
30
|
+
sa.Column(
|
|
31
|
+
"updated_at",
|
|
32
|
+
sa.DateTime(timezone=True),
|
|
33
|
+
server_default=sa.text("now()"),
|
|
34
|
+
nullable=True,
|
|
35
|
+
),
|
|
36
|
+
sa.Column("privacy_request_id", sa.String(), nullable=False),
|
|
37
|
+
sa.Column(
|
|
38
|
+
"secret",
|
|
39
|
+
sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(),
|
|
40
|
+
nullable=False,
|
|
41
|
+
),
|
|
42
|
+
sa.Column("masking_strategy", sa.String(), nullable=False),
|
|
43
|
+
sa.Column(
|
|
44
|
+
"secret_type",
|
|
45
|
+
sa.String(),
|
|
46
|
+
nullable=False,
|
|
47
|
+
),
|
|
48
|
+
sa.ForeignKeyConstraint(
|
|
49
|
+
["privacy_request_id"], ["privacyrequest.id"], ondelete="CASCADE"
|
|
50
|
+
),
|
|
51
|
+
sa.PrimaryKeyConstraint("id"),
|
|
52
|
+
)
|
|
53
|
+
op.create_index(
|
|
54
|
+
op.f("ix_masking_secret_id"), "masking_secret", ["id"], unique=False
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def downgrade():
|
|
59
|
+
op.drop_index(op.f("ix_masking_secret_id"), table_name="masking_secret")
|
|
60
|
+
op.drop_table("masking_secret")
|
|
@@ -125,7 +125,13 @@ async def create_drp_privacy_request(
|
|
|
125
125
|
"Decrypting identity for DRP privacy request {}", privacy_request.id
|
|
126
126
|
)
|
|
127
127
|
|
|
128
|
-
cache_data(privacy_request,
|
|
128
|
+
cache_data(privacy_request, mapped_identity, None, data)
|
|
129
|
+
|
|
130
|
+
if masking_secrets := policy.generate_masking_secrets():
|
|
131
|
+
logger.info(
|
|
132
|
+
"Caching masking secrets for privacy request {}", privacy_request.id
|
|
133
|
+
)
|
|
134
|
+
privacy_request.persist_masking_secrets(masking_secrets)
|
|
129
135
|
|
|
130
136
|
queue_privacy_request(privacy_request.id)
|
|
131
137
|
|
fides/api/app_setup.py
CHANGED
|
@@ -171,8 +171,9 @@ async def run_database_startup(app: FastAPI) -> None:
|
|
|
171
171
|
if CONFIG.database.automigrate:
|
|
172
172
|
try:
|
|
173
173
|
configure_db(CONFIG.database.sync_database_uri)
|
|
174
|
-
|
|
175
|
-
|
|
174
|
+
if not CONFIG.test_mode:
|
|
175
|
+
with get_autoclose_db_session() as session:
|
|
176
|
+
seed_db(session)
|
|
176
177
|
if CONFIG.database.load_samples:
|
|
177
178
|
async with get_async_autoclose_db_session() as async_session:
|
|
178
179
|
await seed.load_samples(async_session)
|
fides/api/common_exceptions.py
CHANGED
|
@@ -283,6 +283,10 @@ class MalisciousUrlException(Exception):
|
|
|
283
283
|
"""Fides has detected a potentially maliscious URL."""
|
|
284
284
|
|
|
285
285
|
|
|
286
|
+
class MaskingSecretsExpired(BaseException):
|
|
287
|
+
"""The cached masking secrets have expired for the given privacy request."""
|
|
288
|
+
|
|
289
|
+
|
|
286
290
|
class AuthenticationError(HTTPException):
|
|
287
291
|
"""To be raised when attempting to fetch an access token using
|
|
288
292
|
invalid credentials.
|
fides/api/db/database.py
CHANGED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Union
|
|
4
|
+
from urllib.parse import unquote_to_bytes
|
|
5
|
+
|
|
6
|
+
from sqlalchemy import Column, ForeignKey, String
|
|
7
|
+
from sqlalchemy.ext.declarative import declared_attr
|
|
8
|
+
from sqlalchemy.orm import Session, relationship
|
|
9
|
+
from sqlalchemy_utils.types.encrypted.encrypted_type import (
|
|
10
|
+
AesGcmEngine,
|
|
11
|
+
StringEncryptedType,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
from fides.api.db.base_class import Base, JSONTypeOverride
|
|
15
|
+
from fides.api.util.custom_json_encoder import ENCODED_BYTES_PREFIX
|
|
16
|
+
from fides.config import CONFIG
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from fides.api.models.privacy_request import PrivacyRequest
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class MaskingSecret(Base):
|
|
23
|
+
"""SQLAlchemy model for storing secret caching information"""
|
|
24
|
+
|
|
25
|
+
@declared_attr
|
|
26
|
+
def __tablename__(self) -> str:
|
|
27
|
+
return "masking_secret"
|
|
28
|
+
|
|
29
|
+
privacy_request_id = Column(
|
|
30
|
+
String, ForeignKey("privacyrequest.id", ondelete="CASCADE"), nullable=False
|
|
31
|
+
)
|
|
32
|
+
secret = Column(
|
|
33
|
+
StringEncryptedType(
|
|
34
|
+
JSONTypeOverride,
|
|
35
|
+
CONFIG.security.app_encryption_key,
|
|
36
|
+
AesGcmEngine,
|
|
37
|
+
"pkcs5",
|
|
38
|
+
),
|
|
39
|
+
nullable=False,
|
|
40
|
+
)
|
|
41
|
+
masking_strategy = Column(String, nullable=False)
|
|
42
|
+
secret_type = Column(String, nullable=False)
|
|
43
|
+
privacy_request = relationship("PrivacyRequest", back_populates="masking_secrets")
|
|
44
|
+
|
|
45
|
+
def set_secret(self, secret: Union[str, bytes]) -> None:
|
|
46
|
+
"""Set the secret value, handling both string and bytes types"""
|
|
47
|
+
if isinstance(secret, str):
|
|
48
|
+
self.secret = secret.encode("utf-8")
|
|
49
|
+
elif isinstance(secret, bytes):
|
|
50
|
+
self.secret = secret
|
|
51
|
+
else:
|
|
52
|
+
raise ValueError("Secret must be either string or bytes")
|
|
53
|
+
|
|
54
|
+
def get_secret(self) -> Union[str, bytes]:
|
|
55
|
+
"""Retrieve the secret in its original type"""
|
|
56
|
+
secret = self.secret
|
|
57
|
+
if isinstance(secret, str) and secret.startswith(ENCODED_BYTES_PREFIX):
|
|
58
|
+
return unquote_to_bytes(secret)[len(ENCODED_BYTES_PREFIX) :]
|
|
59
|
+
return secret
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def create(
|
|
63
|
+
cls,
|
|
64
|
+
db: Session,
|
|
65
|
+
*,
|
|
66
|
+
data: dict[str, Any],
|
|
67
|
+
check_name: bool = True,
|
|
68
|
+
) -> "MaskingSecret":
|
|
69
|
+
"""
|
|
70
|
+
Create a new masking secret. Handles both string and bytes secrets automatically, encrypting them for storage.
|
|
71
|
+
"""
|
|
72
|
+
return super().create(db=db, data=data, check_name=check_name)
|
fides/api/models/policy.py
CHANGED
|
@@ -25,7 +25,9 @@ from fides.api.models.client import ClientDetail
|
|
|
25
25
|
from fides.api.models.connectionconfig import ConnectionConfig
|
|
26
26
|
from fides.api.models.sql_models import DataCategory # type: ignore
|
|
27
27
|
from fides.api.models.storage import StorageConfig, get_active_default_storage_config
|
|
28
|
+
from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
|
|
28
29
|
from fides.api.schemas.policy import ActionType, DrpAction
|
|
30
|
+
from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
|
|
29
31
|
from fides.api.util.data_category import _validate_data_category
|
|
30
32
|
from fides.config import CONFIG
|
|
31
33
|
|
|
@@ -140,6 +142,27 @@ class Policy(Base):
|
|
|
140
142
|
except IndexError:
|
|
141
143
|
return None
|
|
142
144
|
|
|
145
|
+
def generate_masking_secrets(self) -> Optional[List[MaskingSecretCache]]:
|
|
146
|
+
"""
|
|
147
|
+
Returns a list of masking secrets for the masking strategies in the policy.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
masking_secrets: List[MaskingSecretCache] = []
|
|
151
|
+
erasure_rules = self.get_rules_for_action(action_type=ActionType.erasure)
|
|
152
|
+
unique_masking_strategies_by_name: Set[str] = set()
|
|
153
|
+
for rule in erasure_rules:
|
|
154
|
+
strategy_name: str = rule.masking_strategy["strategy"] # type: ignore
|
|
155
|
+
configuration = rule.masking_strategy["configuration"] # type: ignore
|
|
156
|
+
if strategy_name in unique_masking_strategies_by_name:
|
|
157
|
+
continue
|
|
158
|
+
unique_masking_strategies_by_name.add(strategy_name)
|
|
159
|
+
masking_strategy = MaskingStrategy.get_strategy(
|
|
160
|
+
strategy_name, configuration
|
|
161
|
+
)
|
|
162
|
+
if masking_strategy.secrets_required():
|
|
163
|
+
masking_secrets.extend(masking_strategy.generate_secrets_for_cache())
|
|
164
|
+
return masking_secrets
|
|
165
|
+
|
|
143
166
|
def applies_to(self, node: "TraversalNode") -> bool:
|
|
144
167
|
"""
|
|
145
168
|
Returns True if any data category in the traversal node starts with any of the policy's target categories.
|
|
@@ -50,6 +50,7 @@ from fides.api.models.comment import Comment, CommentReference, CommentReference
|
|
|
50
50
|
from fides.api.models.fides_user import FidesUser
|
|
51
51
|
from fides.api.models.field_types import EncryptedLargeDataDescriptor
|
|
52
52
|
from fides.api.models.manual_webhook import AccessManualWebhook
|
|
53
|
+
from fides.api.models.masking_secret import MaskingSecret
|
|
53
54
|
from fides.api.models.policy import (
|
|
54
55
|
Policy,
|
|
55
56
|
PolicyPreWebhook,
|
|
@@ -98,7 +99,6 @@ from fides.api.util.cache import (
|
|
|
98
99
|
get_drp_request_body_cache_key,
|
|
99
100
|
get_encryption_cache_key,
|
|
100
101
|
get_identity_cache_key,
|
|
101
|
-
get_masking_secret_cache_key,
|
|
102
102
|
)
|
|
103
103
|
from fides.api.util.collection_util import Row, extract_key_for_address
|
|
104
104
|
from fides.api.util.constants import API_DATE_FORMAT
|
|
@@ -289,6 +289,13 @@ class PrivacyRequest(
|
|
|
289
289
|
order_by="RequestTask.created_at",
|
|
290
290
|
)
|
|
291
291
|
|
|
292
|
+
masking_secrets: "RelationshipProperty[List[MaskingSecret]]" = relationship(
|
|
293
|
+
"MaskingSecret",
|
|
294
|
+
back_populates="privacy_request",
|
|
295
|
+
uselist=True,
|
|
296
|
+
passive_deletes="all",
|
|
297
|
+
)
|
|
298
|
+
|
|
292
299
|
@property
|
|
293
300
|
def days_left(self: PrivacyRequest) -> Union[int, None]:
|
|
294
301
|
if self.due_date is None:
|
|
@@ -566,19 +573,24 @@ class PrivacyRequest(
|
|
|
566
573
|
encryption_key,
|
|
567
574
|
)
|
|
568
575
|
|
|
569
|
-
def
|
|
570
|
-
|
|
571
|
-
|
|
576
|
+
def persist_masking_secrets(
|
|
577
|
+
self, masking_secrets: List[MaskingSecretCache]
|
|
578
|
+
) -> None:
|
|
579
|
+
"""Persists masking encryption secrets to database."""
|
|
580
|
+
if not masking_secrets:
|
|
572
581
|
return
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
+
|
|
583
|
+
session = Session.object_session(self)
|
|
584
|
+
for masking_secret in masking_secrets:
|
|
585
|
+
MaskingSecret.create(
|
|
586
|
+
db=session,
|
|
587
|
+
data={
|
|
588
|
+
"privacy_request_id": self.id,
|
|
589
|
+
"secret": masking_secret.secret,
|
|
590
|
+
"masking_strategy": masking_secret.masking_strategy,
|
|
591
|
+
"secret_type": masking_secret.secret_type,
|
|
592
|
+
},
|
|
593
|
+
)
|
|
582
594
|
|
|
583
595
|
def get_cached_identity_data(self) -> Dict[str, Any]:
|
|
584
596
|
"""Retrieves any identity data pertaining to this request from the cache"""
|
|
@@ -199,9 +199,16 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
199
199
|
|
|
200
200
|
table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True)
|
|
201
201
|
where_clauses: List[ColumnElement] = [
|
|
202
|
-
|
|
202
|
+
table.c[k] == v for k, v in non_empty_reference_field_keys.items()
|
|
203
203
|
]
|
|
204
204
|
|
|
205
|
+
# Create update values using Column objects as keys to handle column names with spaces
|
|
206
|
+
update_values = {}
|
|
207
|
+
for column_name, value in final_update_map.items():
|
|
208
|
+
# Use bracket notation to access columns with spaces in their names
|
|
209
|
+
column = table.c[column_name]
|
|
210
|
+
update_values[column] = value
|
|
211
|
+
|
|
205
212
|
if self.partitioning:
|
|
206
213
|
partition_clauses = self.get_partition_clauses()
|
|
207
214
|
partitioned_queries = []
|
|
@@ -212,12 +219,12 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
212
219
|
partitioned_queries.append(
|
|
213
220
|
table.update()
|
|
214
221
|
.where(*(where_clauses + [text(partition_clause)]))
|
|
215
|
-
.values(
|
|
222
|
+
.values(update_values)
|
|
216
223
|
)
|
|
217
224
|
|
|
218
225
|
return partitioned_queries
|
|
219
226
|
|
|
220
|
-
return [table.update().where(*where_clauses).values(
|
|
227
|
+
return [table.update().where(*where_clauses).values(update_values)]
|
|
221
228
|
|
|
222
229
|
def generate_delete(
|
|
223
230
|
self,
|
|
@@ -255,9 +262,9 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
255
262
|
where_clauses: List[ColumnElement] = []
|
|
256
263
|
for column_name, values in filtered_data.items():
|
|
257
264
|
if len(values) == 1:
|
|
258
|
-
where_clauses.append(
|
|
265
|
+
where_clauses.append(table.c[column_name] == values[0])
|
|
259
266
|
else:
|
|
260
|
-
where_clauses.append(
|
|
267
|
+
where_clauses.append(table.c[column_name].in_(values))
|
|
261
268
|
|
|
262
269
|
# Combine reference clauses with OR instead of AND
|
|
263
270
|
combined_reference_clause = or_(*where_clauses)
|
|
@@ -306,6 +313,25 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
306
313
|
formatted_fields.append(field_path.levels[0])
|
|
307
314
|
return formatted_fields
|
|
308
315
|
|
|
316
|
+
def format_clause_for_query(
|
|
317
|
+
self, string_path: str, operator: str, operand: str
|
|
318
|
+
) -> str:
|
|
319
|
+
"""
|
|
320
|
+
Returns clauses with proper BigQuery backtick escaping for column names.
|
|
321
|
+
Handles column names with spaces and nested fields (dot-separated) by escaping each part individually.
|
|
322
|
+
"""
|
|
323
|
+
# For nested fields (containing dots), escape each part individually
|
|
324
|
+
if "." in string_path:
|
|
325
|
+
parts = string_path.split(".")
|
|
326
|
+
escaped_field = ".".join(f"`{part}`" for part in parts)
|
|
327
|
+
else:
|
|
328
|
+
# For simple fields, wrap the entire name in backticks
|
|
329
|
+
escaped_field = f"`{string_path}`"
|
|
330
|
+
|
|
331
|
+
if operator == "IN":
|
|
332
|
+
return f"{escaped_field} IN ({operand})"
|
|
333
|
+
return f"{escaped_field} {operator} :{operand}"
|
|
334
|
+
|
|
309
335
|
def generate_raw_query_without_tuples(
|
|
310
336
|
self, field_list: List[str], filters: Dict[str, List[Any]]
|
|
311
337
|
) -> Optional[TextClause]:
|
|
@@ -315,27 +341,35 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
315
341
|
|
|
316
342
|
This is an override of the base class method that supports nested fields for BigQuery.
|
|
317
343
|
|
|
318
|
-
Examples with
|
|
344
|
+
Examples with field names containing dots and spaces, notice these are replaced with underscores in the parameter bindings:
|
|
319
345
|
|
|
320
346
|
1. Single value filter:
|
|
321
347
|
field_list = ["id", "name", "email"]
|
|
322
348
|
filters = {"user.id": [123]}
|
|
323
349
|
|
|
324
|
-
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (user
|
|
350
|
+
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`user`.`id` = :user_id)
|
|
325
351
|
With parameter binding: user_id = 123
|
|
326
352
|
|
|
327
|
-
2.
|
|
353
|
+
2. Field with spaces:
|
|
354
|
+
field_list = ["id", "custom id", "email"]
|
|
355
|
+
filters = {"custom id": ["abc123"]}
|
|
356
|
+
|
|
357
|
+
Generates: SELECT id, `custom id`, email FROM `project_id.dataset_id.table_name` WHERE (`custom id` = :custom_id)
|
|
358
|
+
With parameter binding: custom_id = "abc123"
|
|
359
|
+
|
|
360
|
+
3. Multiple value filter with nested field:
|
|
328
361
|
field_list = ["id", "name", "email"]
|
|
329
|
-
filters = {"
|
|
362
|
+
filters = {"contact_info.primary_email": ["active", "pending"]}
|
|
330
363
|
|
|
331
|
-
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (
|
|
332
|
-
With parameter bindings:
|
|
364
|
+
Generates: SELECT id, name, email FROM `project_id.dataset_id.table_name` WHERE (`contact_info`.`primary_email` IN (:contact_info_primary_email_in_stmt_generated_0, :contact_info_primary_email_in_stmt_generated_1))
|
|
365
|
+
With parameter bindings: contact_info_primary_email_in_stmt_generated_0 = "active", contact_info_primary_email_in_stmt_generated_1 = "pending"
|
|
333
366
|
"""
|
|
334
367
|
clauses = []
|
|
335
368
|
query_data = {}
|
|
336
369
|
for field_name, field_value in filters.items():
|
|
337
|
-
# Replace dots with underscores in field names for parameter binding
|
|
338
|
-
|
|
370
|
+
# Replace dots and spaces with underscores in field names for parameter binding
|
|
371
|
+
# SQLAlchemy parameter names cannot contain spaces or special characters
|
|
372
|
+
field_binding_name = field_name.replace(".", "_").replace(" ", "_")
|
|
339
373
|
data = set(field_value)
|
|
340
374
|
if len(data) == 1:
|
|
341
375
|
clauses.append(
|
|
@@ -358,7 +392,7 @@ class BigQueryQueryConfig(QueryStringWithoutTuplesOverrideQueryConfig):
|
|
|
358
392
|
clauses.append(self.format_clause_for_query(field_name, "IN", operand))
|
|
359
393
|
|
|
360
394
|
if len(clauses) > 0:
|
|
361
|
-
formatted_fields = ", ".join(field_list)
|
|
395
|
+
formatted_fields = ", ".join([f"`{field}`" for field in field_list])
|
|
362
396
|
query_str = self.get_formatted_query_string(formatted_fields, clauses)
|
|
363
397
|
return text(query_str).params(query_data)
|
|
364
398
|
|
|
@@ -14,6 +14,7 @@ from fides.api.common_exceptions import (
|
|
|
14
14
|
ClientUnsuccessfulException,
|
|
15
15
|
IdentityNotFoundException,
|
|
16
16
|
ManualWebhookFieldsUnset,
|
|
17
|
+
MaskingSecretsExpired,
|
|
17
18
|
MessageDispatchException,
|
|
18
19
|
NoCachedManualWebhookEntry,
|
|
19
20
|
PrivacyRequestExit,
|
|
@@ -75,6 +76,7 @@ from fides.api.task.graph_task import (
|
|
|
75
76
|
from fides.api.task.manual.manual_task_utils import create_manual_task_artificial_graphs
|
|
76
77
|
from fides.api.tasks import DatabaseTask, celery_app
|
|
77
78
|
from fides.api.tasks.scheduled.scheduler import scheduler
|
|
79
|
+
from fides.api.util.cache import get_all_masking_secret_keys
|
|
78
80
|
from fides.api.util.collection_util import Row
|
|
79
81
|
from fides.api.util.logger import Pii, _log_exception, _log_warning
|
|
80
82
|
from fides.api.util.logger_context_utils import LoggerContextKeys, log_context
|
|
@@ -531,6 +533,8 @@ def run_privacy_request(
|
|
|
531
533
|
request_checkpoint=CurrentStep.erasure, from_checkpoint=resume_step
|
|
532
534
|
):
|
|
533
535
|
privacy_request.cache_failed_checkpoint_details(CurrentStep.erasure)
|
|
536
|
+
_verify_masking_secrets(policy, privacy_request, resume_step)
|
|
537
|
+
|
|
534
538
|
# We only need to run the erasure once until masking strategies are handled
|
|
535
539
|
erasure_runner(
|
|
536
540
|
privacy_request=privacy_request,
|
|
@@ -1000,3 +1004,25 @@ def run_webhooks_and_report_status(
|
|
|
1000
1004
|
return False
|
|
1001
1005
|
|
|
1002
1006
|
return True
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
def _verify_masking_secrets(
|
|
1010
|
+
policy: Policy, privacy_request: PrivacyRequest, resume_step: Optional[CurrentStep]
|
|
1011
|
+
) -> None:
|
|
1012
|
+
"""
|
|
1013
|
+
Checks that the required masking secrets are still cached for the given request.
|
|
1014
|
+
Raises an exception if masking secrets are needed for the given policy but they don't exist.
|
|
1015
|
+
"""
|
|
1016
|
+
|
|
1017
|
+
if resume_step is None:
|
|
1018
|
+
return
|
|
1019
|
+
|
|
1020
|
+
# if masking can be performed without any masking secrets, we skip the cache check
|
|
1021
|
+
if (
|
|
1022
|
+
policy.generate_masking_secrets()
|
|
1023
|
+
and not get_all_masking_secret_keys(privacy_request.id)
|
|
1024
|
+
and not privacy_request.masking_secrets
|
|
1025
|
+
):
|
|
1026
|
+
raise MaskingSecretsExpired(
|
|
1027
|
+
f"The masking secrets for privacy request ID '{privacy_request.id}' have expired. Please submit a new erasure request."
|
|
1028
|
+
)
|
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import json
|
|
4
4
|
from asyncio import sleep
|
|
5
5
|
from datetime import datetime, timedelta
|
|
6
|
-
from typing import Any, Dict,
|
|
6
|
+
from typing import Any, Dict, Optional, Set
|
|
7
7
|
|
|
8
8
|
from httpx import AsyncClient
|
|
9
9
|
from loguru import logger
|
|
@@ -11,7 +11,6 @@ from sqlalchemy import text
|
|
|
11
11
|
from sqlalchemy.sql.elements import TextClause
|
|
12
12
|
|
|
13
13
|
from fides.api.common_exceptions import PrivacyRequestNotFound
|
|
14
|
-
from fides.api.models.policy import Policy
|
|
15
14
|
from fides.api.models.privacy_request import (
|
|
16
15
|
EXITED_EXECUTION_LOG_STATUSES,
|
|
17
16
|
PrivacyRequest,
|
|
@@ -19,14 +18,12 @@ from fides.api.models.privacy_request import (
|
|
|
19
18
|
)
|
|
20
19
|
from fides.api.models.worker_task import ExecutionLogStatus
|
|
21
20
|
from fides.api.schemas.drp_privacy_request import DrpPrivacyRequestCreate
|
|
22
|
-
from fides.api.schemas.masking.masking_secrets import MaskingSecretCache
|
|
23
21
|
from fides.api.schemas.policy import ActionType
|
|
24
22
|
from fides.api.schemas.privacy_request import (
|
|
25
23
|
PrivacyRequestResponse,
|
|
26
24
|
PrivacyRequestStatus,
|
|
27
25
|
)
|
|
28
26
|
from fides.api.schemas.redis_cache import Identity
|
|
29
|
-
from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy
|
|
30
27
|
from fides.api.tasks import DSR_QUEUE_NAME, DatabaseTask, celery_app
|
|
31
28
|
from fides.api.tasks.scheduled.scheduler import scheduler
|
|
32
29
|
from fides.api.util.cache import (
|
|
@@ -69,7 +66,6 @@ def build_required_privacy_request_kwargs(
|
|
|
69
66
|
|
|
70
67
|
def cache_data(
|
|
71
68
|
privacy_request: PrivacyRequest,
|
|
72
|
-
policy: Policy,
|
|
73
69
|
identity: Identity,
|
|
74
70
|
encryption_key: Optional[str],
|
|
75
71
|
drp_request_body: Optional[DrpPrivacyRequestCreate],
|
|
@@ -82,23 +78,6 @@ def cache_data(
|
|
|
82
78
|
privacy_request.cache_custom_privacy_request_fields(custom_privacy_request_fields)
|
|
83
79
|
privacy_request.cache_encryption(encryption_key) # handles None already
|
|
84
80
|
|
|
85
|
-
# Store masking secrets in the cache
|
|
86
|
-
logger.info("Caching masking secrets for privacy request {}", privacy_request.id)
|
|
87
|
-
erasure_rules = policy.get_rules_for_action(action_type=ActionType.erasure)
|
|
88
|
-
unique_masking_strategies_by_name: Set[str] = set()
|
|
89
|
-
for rule in erasure_rules:
|
|
90
|
-
strategy_name: str = rule.masking_strategy["strategy"] # type: ignore
|
|
91
|
-
configuration = rule.masking_strategy["configuration"] # type: ignore
|
|
92
|
-
if strategy_name in unique_masking_strategies_by_name:
|
|
93
|
-
continue
|
|
94
|
-
unique_masking_strategies_by_name.add(strategy_name)
|
|
95
|
-
masking_strategy = MaskingStrategy.get_strategy(strategy_name, configuration)
|
|
96
|
-
if masking_strategy.secrets_required():
|
|
97
|
-
masking_secrets: List[MaskingSecretCache] = (
|
|
98
|
-
masking_strategy.generate_secrets_for_cache()
|
|
99
|
-
)
|
|
100
|
-
for masking_secret in masking_secrets:
|
|
101
|
-
privacy_request.cache_masking_secret(masking_secret)
|
|
102
81
|
if drp_request_body:
|
|
103
82
|
privacy_request.cache_drp_request_body(drp_request_body)
|
|
104
83
|
|
fides/api/task/graph_task.py
CHANGED
|
@@ -421,6 +421,7 @@ class GraphTask(ABC): # pylint: disable=too-many-instance-attributes
|
|
|
421
421
|
action_type: ActionType,
|
|
422
422
|
ex: Optional[BaseException] = None,
|
|
423
423
|
success_override_msg: Optional[str] = None,
|
|
424
|
+
record_count: Optional[int] = None,
|
|
424
425
|
) -> None:
|
|
425
426
|
"""On completion activities"""
|
|
426
427
|
if ex:
|
|
@@ -440,8 +441,23 @@ class GraphTask(ABC): # pylint: disable=too-many-instance-attributes
|
|
|
440
441
|
mark_current_and_downstream_nodes_as_failed(request_task, db)
|
|
441
442
|
else:
|
|
442
443
|
logger.info("Ending {}, {}", self.resources.request.id, self.key)
|
|
444
|
+
|
|
445
|
+
# Build standardized success message with record count
|
|
446
|
+
base_message = (
|
|
447
|
+
str(success_override_msg) if success_override_msg else "success"
|
|
448
|
+
)
|
|
449
|
+
if record_count is not None:
|
|
450
|
+
if action_type == ActionType.access:
|
|
451
|
+
message = f"{base_message} - retrieved {record_count} records"
|
|
452
|
+
elif action_type == ActionType.erasure:
|
|
453
|
+
message = f"{base_message} - masked {record_count} records"
|
|
454
|
+
else:
|
|
455
|
+
message = f"{base_message} - processed {record_count} records"
|
|
456
|
+
else:
|
|
457
|
+
message = base_message
|
|
458
|
+
|
|
443
459
|
self.update_status(
|
|
444
|
-
|
|
460
|
+
message,
|
|
445
461
|
build_affected_field_logs(
|
|
446
462
|
self.execution_node, self.resources.policy, action_type
|
|
447
463
|
),
|
|
@@ -637,7 +653,11 @@ class GraphTask(ABC): # pylint: disable=too-many-instance-attributes
|
|
|
637
653
|
if messages:
|
|
638
654
|
success_message = "\n".join(messages)
|
|
639
655
|
|
|
640
|
-
self.log_end(
|
|
656
|
+
self.log_end(
|
|
657
|
+
ActionType.access,
|
|
658
|
+
success_override_msg=success_message,
|
|
659
|
+
record_count=len(filtered_output),
|
|
660
|
+
)
|
|
641
661
|
return filtered_output
|
|
642
662
|
|
|
643
663
|
@retry(action_type=ActionType.erasure, default_return=0)
|
|
@@ -731,7 +751,11 @@ class GraphTask(ABC): # pylint: disable=too-many-instance-attributes
|
|
|
731
751
|
if messages:
|
|
732
752
|
success_message = "\n".join(messages)
|
|
733
753
|
|
|
734
|
-
self.log_end(
|
|
754
|
+
self.log_end(
|
|
755
|
+
ActionType.erasure,
|
|
756
|
+
success_override_msg=success_message,
|
|
757
|
+
record_count=output,
|
|
758
|
+
)
|
|
735
759
|
return output
|
|
736
760
|
|
|
737
761
|
@retry(action_type=ActionType.consent, default_return=False)
|
fides/api/util/cache.py
CHANGED
|
@@ -387,3 +387,8 @@ def get_queue_counts() -> Dict[str, int]:
|
|
|
387
387
|
logger.critical(exception)
|
|
388
388
|
queue_counts = {}
|
|
389
389
|
return queue_counts
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def get_all_masking_secret_keys(privacy_request_id: str) -> List[str]:
|
|
393
|
+
cache: FidesopsRedis = get_cache()
|
|
394
|
+
return cache.keys(f"id-{privacy_request_id}-masking-secret-*")
|