ethyca-fides 2.69.1b0__py2.py3-none-any.whl → 2.69.1b2__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/METADATA +1 -1
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/RECORD +226 -224
- fides/_version.py +3 -3
- fides/api/api/v1/endpoints/dsr_package_link.py +5 -3
- fides/api/api/v1/endpoints/oauth_endpoints.py +1 -1
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +3 -5
- fides/api/api/v1/endpoints/user_endpoints.py +1 -24
- fides/api/app_setup.py +16 -2
- fides/api/main.py +22 -0
- fides/api/models/client.py +5 -9
- fides/api/models/fides_user.py +2 -1
- fides/api/oauth/utils.py +11 -27
- fides/api/service/privacy_request/request_service.py +19 -2
- fides/api/service/storage/storage_uploader_service.py +1 -21
- fides/api/service/storage/streaming/dsr_storage.py +1 -5
- fides/api/service/storage/streaming/s3/s3_storage_client.py +78 -40
- fides/api/service/storage/streaming/s3/streaming_s3.py +9 -21
- fides/api/service/storage/streaming/smart_open_client.py +8 -7
- fides/api/service/storage/streaming/smart_open_streaming_storage.py +4 -28
- fides/api/service/storage/streaming/storage_client_factory.py +7 -3
- fides/api/task/graph_runners.py +2 -32
- fides/api/task/graph_task.py +4 -2
- fides/api/task/scheduler_utils.py +39 -0
- fides/api/util/endpoint_utils.py +0 -13
- fides/api/util/rate_limit.py +194 -0
- fides/config/execution_settings.py +0 -4
- fides/config/redis_settings.py +27 -3
- fides/config/security_settings.py +24 -6
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/0agWtBSaxTBxQfxPA99Ra/_buildManifest.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{1345-04e37a66c0d40dc1.js → 1345-5e1c5b66e25c566e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3729-f5f2976904dce90d.js → 3729-a1ca1608efc11ac4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3847-2c0126e6eb54c526.js → 3847-230bf61b053bc706.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3855-9dd54ded74f4036b.js → 3855-ef5194cdb228beb6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4121-c8d5d717e31899e1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4164-355644b916ae0094.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4608-d101417a3abb4ad6.js → 4608-23bbd4c3c4a59f42.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4786-7aafb744445445b2.js → 4786-0827aae7aceadd22.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4808-357ca7ea7bbd24c6.js → 4808-78ca630f2d2503cd.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4844-707b20a6c4b127cc.js → 4844-46324c3d848b8b6a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9046-058a4d8f0b5e08f9.js → 9046-712156d461165f56.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9676.b7d5d1d90b9da224.js → 9676.9fd9552ef744c717.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9951-b954027a046ce553.js → 9951-a88367a129b724ba.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-f18537fd2c4288e7.js → _app-ef8e1c986bc5b795.js} +2 -2
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-20253dd047fb9736.js → manual-9dc7e70ab5b05723.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-169099ff7b305cf5.js → add-systems-1632a59203fe8eab.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-28b192e2c074b0f3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-adc500a03e239857.js → [resourceUrn]-da1a48336daff6f8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-c8b3d090e4ba60d3.js → [resourceUrn]-470da05db63767cd.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/{[systemId]-cfb0b1200bc1a2c9.js → [systemId]-2f0a33ef9ba1f1da.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/{[monitorId]-fed8b879c13c2bf3.js → [monitorId]-e9d4f25b20ff6781.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-58a110542d6bcd0f.js → activity-b6ae7adb8ef0b525.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-22eec362dfbb1d2a.js → [resourceUrn]-c3a97e6721ca0abe.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-4decce5ef996e563.js → detection-a0a7de552ef71f5b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-01acdd1ad492fd89.js → [resourceUrn]-109754fec0755339.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-85fdbf4cde60d910.js → discovery-88654783b06b3b21.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{datamap-60e27b673c68bd27.js → datamap-89136e6800dc9369.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-2c7b1213b6604d30.js → new-97f06e21580f1f6a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-7ed3f05700dc397b.js → [id]-6f77d8647fca71e0.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-95f6d64f84fc6bf3.js → new-821dd1269834cfa2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-3a4cd3fe9094fba3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-0f12d5b658c98c9f.js → integrations-57e618d7b16ac69a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{[id]-b379873a5771e55b.js → [id]-0d0bb9eb004a3336.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{messaging-a9bb257906dcd7e0.js → messaging-f9320a58f489f5b7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{storage-de0036c74b78e9b7.js → storage-d0cfa8aeddd43a40.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{privacy-requests-91f578139548652c.js → privacy-requests-5a5edc8a4aa7c30a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/{datamap-83019a6753e14857.js → datamap-6903f42a0412bfa6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-e5331508e81222fc.js → consent-be47008304106395.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-74f0fe9656f4a1f6.js → custom-fields-ae1b57589da7b175.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-e91da49c0681a300.js → [id]-5a43f108d8047d5b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{taxonomy-78c3a5200d362cff.js → taxonomy-1b3f2d4bcb0e164d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{webpack-b5eb3e1da37616d2.js → webpack-678e89d68dbcd94f.js} +1 -1
- fides/ui-build/static/admin/_next/static/css/650df9c348000a26.css +1 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/4lPKe7mco0KEv09aOQH9A/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4121-bb71a24d41d04a28.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/768-7eac4b30d7477b0a.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-60e3394c887f3d5e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-3d24c1510aa4c555.js +0 -1
- fides/ui-build/static/admin/_next/static/css/abf2e162b1cd0e61.css +0 -1
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{4lPKe7mco0KEv09aOQH9A → 0agWtBSaxTBxQfxPA99Ra}/_ssgManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{1099-c34f76b4da5f3d15.js → 1099-79646e64f26d62fa.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{1817-e6934e258111a961.js → 1817-3d9e110e007853f0.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{2921-0696287bb8de1f0b.js → 2921-52328140bc420d0f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3620-8c0ee3d0b19c342d.js → 3620-31ebb43dba84cbbd.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3872-72ea3eb040366277.js → 3872-a91143aa35fa8ef8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3923-c4f2b03706ddbe39.js → 3923-bb2417b8dcade7a4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{401-959a15ed18a8abdf.js → 401-4af0a912e249d30f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{5258-1a8b9f66b97761fc.js → 5258-e880b606a2293803.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{5487-fd9724519f31caff.js → 5487-8c635883dcaa9c2a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{549-d3bef0990071230c.js → 549-38ea1d91ee2addaa.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6084-487d27d017c45e05.js → 6084-0096d7de64ef8015.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6853-f7ab74e30abbdeaf.js → 6853-b17673391117c976.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6954-b0a7b8ac6db238f8.js → 6954-9d46e2276c461c26.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7476-9a57db910472b48e.js → 7476-d1b0af9ade392e5b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7630-46807321449479c7.js → 7630-da0a7ce4e3a0d62c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{787-79e8e558bd80ece6.js → 787-3499983fa346b380.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{79-3e5047415bee9391.js → 79-f197fc4db8d530e5.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{796-fb2af44165f37ecc.js → 796-db1e30119ea973c7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{8002-c1f66179adabece8.js → 8002-971e29181f72edd1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{9826-0d9a7f61c08ed88a.js → 9826-b0b3d3cfb13bfbc1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{404-9c9efb820bb6b432.js → 404-471a6b18e712f050.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-caadc3c0e86a7bfe.js → multiple-4b79a1652297ed9a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-acefb9e08c6c4082.js → add-vendors-1ca9df7ca91bd101.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-558fbb1667473abb.js → configure-07bdbc9ae4137db4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-b1ff1c9683841815.js → [id]-f80cf2d3966816fd.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-674906d2d9a0a3a6.js → privacy-experience-2795cd4115a77c94.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-5b7aa7971f070513.js → [id]-e02921dc82dccbb1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-285958a12a66575e.js → new-98f9e4ba3610628a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-dc56bbdb6dd7a670.js → privacy-notices-17ed82777810d1c6.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-0843757f00eeaaec.js → properties-226efa1dcd41437f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-5cba58ebecb4511f.js → consent-09610b10923d9268.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-0f90cac9f190130c.js → [projectUrn]-d8e776f1e64e4ba8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-f9117645c941342c.js → projects-75b9629b0d9cdf96.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-c566640eecc6f3fe.js → data-catalog-6984c033b8fe3a13.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-991659e916ad60b1.js → action-center-9c428d3ef0985915.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-3f4ba87513e030a2.js → [...subfieldNames]-8f58192dcb54883d.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-a6cd31103deba465.js → [collectionName]-dcb4ab380a77aa1e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-491d2c8dd559d0cb.js → [datasetId]-6f16d43071fb9c11.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-c8bcd568d3b0ee7f.js → dataset-674bb3940f088ecc.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-9a33a412a5f36960.js → datastore-connection-23e4caf79faa8106.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{index-1299410f671fac23.js → index-23eb64eed81dcb69.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-ba232c4b17576ccb.js → [id]-c9a323eb6a929476.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-b97883026fbc5b1e.js → add-template-b9bb09e46921a590.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{messaging-38189c1058aa4acf.js → messaging-82c631a12b5a008c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-3119bdb3811409bf.js → ant-components-bc0e2adf6e0d3ac7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-8f9d0434dc3eb8cf.js → AntForm-86ffcc1ad3fa912e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-a14e51c7f22f3395.js → FormikAntFormItem-ec04f595465bdf69.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-a40645b57822684d.js → FormikControlled-41d309754ff0c1de.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-c55df6527b700701.js → FormikField-cab1f78cec7808f9.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-292d761616b162a0.js → forms-eb6058221403b156.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-135bcf384b81820a.js → table-migration-38360083348c3d6c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-70efccbe0732786b.js → configure-72ca94ec5ed85733.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-63c6f0193634add5.js → [id]-5ec775c4904fdbfe.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-e76567f0374d5912.js → add-property-a6812c0916f2949e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{properties-2050b7dac0413c11.js → properties-da734840e4f9d04b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-913f8eab62460f31.js → alpha-3e72e9f91991c119.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-2ed1ee6017c0656a.js → about-6aab092f4871cecb.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-bffd6292e7e0302a.js → [purpose_id]-9495e2eb506606c7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-a649ca07ce51b0fe.js → domain-records-23a6d7a921150188.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-eb675ba600cea2a8.js → domains-2a9e8859ab4d9de6.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-4db2f42f90867890.js → email-templates-4f9f0fdf9925ae90.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-89b404989b4ea21c.js → locations-46f7af35cee4a8bb.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-eb1ecff37fd85c72.js → organization-a596a96cb8d0aa8e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-07d883b5aaec58f2.js → regulations-6ed5fc2410e00857.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-61ecb7546830c345.js → test-datasets-86811e3cda277e77.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{systems-44d714017dd87e62.js → systems-045a841e22e85ea8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/{[id]-4f9001870e2b3aff.js → [id]-05d61c80a556b2d5.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-583a7f0c1bead240.js → user-management-2cab41659f1ee7da.js} +0 -0
|
@@ -54,7 +54,8 @@ def upload_to_s3_streaming(
|
|
|
54
54
|
return response
|
|
55
55
|
|
|
56
56
|
try:
|
|
57
|
-
|
|
57
|
+
logger.debug("Creating SmartOpenStorageClient with formatted secrets")
|
|
58
|
+
storage_client = SmartOpenStorageClient("s3", auth_method, formatted_secrets)
|
|
58
59
|
|
|
59
60
|
# Create upload config for the streaming interface
|
|
60
61
|
upload_config = StorageUploadConfig(
|
|
@@ -92,6 +93,10 @@ def _process_storage_secrets_input(
|
|
|
92
93
|
"""Process input and convert to string-keyed dictionary."""
|
|
93
94
|
final_secrets: dict[str, Any] = {}
|
|
94
95
|
|
|
96
|
+
logger.debug(
|
|
97
|
+
f"Processing storage secrets input of type: {type(storage_secrets).__name__}"
|
|
98
|
+
)
|
|
99
|
+
|
|
95
100
|
if isinstance(storage_secrets, StorageSecretsS3):
|
|
96
101
|
# Convert StorageSecretsS3 model directly to string keys
|
|
97
102
|
for key, value in storage_secrets.model_dump().items():
|
|
@@ -133,6 +138,7 @@ def _validate_aws_credentials(final_secrets: dict[str, Any]) -> None:
|
|
|
133
138
|
else:
|
|
134
139
|
# AUTOMATIC authentication - check if region is provided
|
|
135
140
|
has_region = "region_name" in final_secrets and final_secrets["region_name"]
|
|
141
|
+
|
|
136
142
|
if not has_region:
|
|
137
143
|
raise ValueError(
|
|
138
144
|
"Missing required region_name for AUTOMATIC authentication. "
|
|
@@ -140,13 +146,6 @@ def _validate_aws_credentials(final_secrets: dict[str, Any]) -> None:
|
|
|
140
146
|
)
|
|
141
147
|
|
|
142
148
|
|
|
143
|
-
def _set_default_region(final_secrets: dict[str, Any]) -> None:
|
|
144
|
-
"""Set default region if missing."""
|
|
145
|
-
if "region_name" not in final_secrets or not final_secrets["region_name"]:
|
|
146
|
-
logger.debug("Setting default region to 'us-east-1'")
|
|
147
|
-
final_secrets["region_name"] = "us-east-1"
|
|
148
|
-
|
|
149
|
-
|
|
150
149
|
def format_secrets(
|
|
151
150
|
storage_secrets: Union[StorageSecretsS3, dict[StorageSecrets, Any]] # type: ignore[misc]
|
|
152
151
|
) -> dict[str, Any]:
|
|
@@ -156,8 +155,8 @@ def format_secrets(
|
|
|
156
155
|
This function handles multiple credential formats and processes them through several stages:
|
|
157
156
|
1. Input processing: Accepts either StorageSecretsS3 models (from API) or raw dicts (from database)
|
|
158
157
|
2. Key normalization: Converts all keys to string format for consistency
|
|
159
|
-
3.
|
|
160
|
-
4.
|
|
158
|
+
3. Default setting: Sets default region if missing (before validation for better automatic auth support)
|
|
159
|
+
4. Validation: Ensures required AWS credentials are present based on auth method
|
|
161
160
|
5. Return: Returns string-keyed dict ready for S3StorageClient
|
|
162
161
|
|
|
163
162
|
Input formats:
|
|
@@ -179,18 +178,7 @@ def format_secrets(
|
|
|
179
178
|
Raises:
|
|
180
179
|
ValueError: If required AWS credentials are missing for the chosen auth method
|
|
181
180
|
"""
|
|
182
|
-
logger.debug("format_secrets called with type: {}", type(storage_secrets).__name__)
|
|
183
|
-
|
|
184
|
-
# Stage 1: Process input and create final format directly
|
|
185
181
|
final_secrets = _process_storage_secrets_input(storage_secrets)
|
|
186
|
-
|
|
187
|
-
# Stage 2: Validate required credentials BEFORE setting defaults
|
|
188
182
|
_validate_aws_credentials(final_secrets)
|
|
189
183
|
|
|
190
|
-
# Stage 3: Set default region if missing (after validation)
|
|
191
|
-
_set_default_region(final_secrets)
|
|
192
|
-
|
|
193
|
-
logger.debug(
|
|
194
|
-
"format_secrets completed successfully with {} keys", len(final_secrets)
|
|
195
|
-
)
|
|
196
184
|
return final_secrets
|
|
@@ -29,7 +29,12 @@ class SmartOpenStorageClient:
|
|
|
29
29
|
|
|
30
30
|
min_part_size: int = MIN_PART_SIZE
|
|
31
31
|
|
|
32
|
-
def __init__(
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
storage_type: str,
|
|
35
|
+
auth_method: Optional[str],
|
|
36
|
+
storage_secrets: Any,
|
|
37
|
+
):
|
|
33
38
|
"""Initialize the smart-open storage client.
|
|
34
39
|
|
|
35
40
|
Args:
|
|
@@ -38,9 +43,10 @@ class SmartOpenStorageClient:
|
|
|
38
43
|
Will be passed to the specific storage client implementation.
|
|
39
44
|
"""
|
|
40
45
|
self.storage_type = StorageClientFactory._normalize_storage_type(storage_type)
|
|
46
|
+
self.auth_method = auth_method
|
|
41
47
|
self.storage_secrets = storage_secrets
|
|
42
48
|
self._provider_client = StorageClientFactory.create_client(
|
|
43
|
-
storage_type, storage_secrets
|
|
49
|
+
storage_type, auth_method, storage_secrets
|
|
44
50
|
)
|
|
45
51
|
|
|
46
52
|
def _build_uri(self, bucket: str, key: str) -> str:
|
|
@@ -216,7 +222,6 @@ class SmartOpenStorageClient:
|
|
|
216
222
|
self,
|
|
217
223
|
bucket: str,
|
|
218
224
|
key: str,
|
|
219
|
-
content_type: Optional[str] = None,
|
|
220
225
|
) -> Any:
|
|
221
226
|
"""Get a writable stream for uploading data.
|
|
222
227
|
|
|
@@ -225,7 +230,6 @@ class SmartOpenStorageClient:
|
|
|
225
230
|
Args:
|
|
226
231
|
bucket: Storage bucket/container name
|
|
227
232
|
key: Object key/path
|
|
228
|
-
content_type: MIME type of the object
|
|
229
233
|
|
|
230
234
|
Returns:
|
|
231
235
|
Writable file-like object
|
|
@@ -233,9 +237,6 @@ class SmartOpenStorageClient:
|
|
|
233
237
|
uri = self._build_uri(bucket, key)
|
|
234
238
|
transport_params = self._get_transport_params()
|
|
235
239
|
|
|
236
|
-
if content_type:
|
|
237
|
-
transport_params["content_type"] = content_type
|
|
238
|
-
|
|
239
240
|
return smart_open.open(uri, "wb", transport_params=transport_params)
|
|
240
241
|
|
|
241
242
|
@retry_cloud_storage_operation(
|
|
@@ -244,7 +244,6 @@ class SmartOpenStreamingStorage:
|
|
|
244
244
|
all_attachments = []
|
|
245
245
|
|
|
246
246
|
for key, value in data.items():
|
|
247
|
-
logger.debug(f"Processing key '{key}' with value type: {type(value)}")
|
|
248
247
|
|
|
249
248
|
if not isinstance(value, list) or not value:
|
|
250
249
|
continue
|
|
@@ -270,10 +269,6 @@ class SmartOpenStreamingStorage:
|
|
|
270
269
|
"""
|
|
271
270
|
direct_attachments = []
|
|
272
271
|
|
|
273
|
-
logger.debug(
|
|
274
|
-
f"Found 'attachments' key with {len(attachments_list)} items - processing as direct attachments"
|
|
275
|
-
)
|
|
276
|
-
|
|
277
272
|
for idx, attachment in enumerate(attachments_list):
|
|
278
273
|
if not isinstance(attachment, dict):
|
|
279
274
|
continue
|
|
@@ -431,9 +426,6 @@ class SmartOpenStreamingStorage:
|
|
|
431
426
|
Iterator that yields chunks of the attachment content
|
|
432
427
|
"""
|
|
433
428
|
try:
|
|
434
|
-
logger.debug(
|
|
435
|
-
f"Starting streaming read of {storage_key} from bucket: {bucket}, key: {key}"
|
|
436
|
-
)
|
|
437
429
|
with self.storage_client.stream_read(bucket, key) as content_stream:
|
|
438
430
|
# Stream in chunks instead of reading entire file
|
|
439
431
|
chunk_count = 0
|
|
@@ -478,9 +470,6 @@ class SmartOpenStreamingStorage:
|
|
|
478
470
|
if validated:
|
|
479
471
|
validated_attachments.append(validated)
|
|
480
472
|
|
|
481
|
-
logger.debug(
|
|
482
|
-
f"Successfully validated {len(validated_attachments)} out of {len(raw_attachments)} attachments"
|
|
483
|
-
)
|
|
484
473
|
return validated_attachments
|
|
485
474
|
|
|
486
475
|
@retry_cloud_storage_operation(
|
|
@@ -667,7 +656,7 @@ class SmartOpenStreamingStorage:
|
|
|
667
656
|
)
|
|
668
657
|
except Exception as e:
|
|
669
658
|
logger.error(
|
|
670
|
-
f"Failed to generate presigned URL for {config.
|
|
659
|
+
f"Failed to generate presigned URL for {config.file_key}: {e}"
|
|
671
660
|
)
|
|
672
661
|
raise StorageUploadError(
|
|
673
662
|
f"Failed to generate presigned URL: {e}"
|
|
@@ -693,7 +682,6 @@ class SmartOpenStreamingStorage:
|
|
|
693
682
|
with self.storage_client.stream_upload(
|
|
694
683
|
config.bucket_name,
|
|
695
684
|
config.file_key,
|
|
696
|
-
content_type="application/zip",
|
|
697
685
|
) as upload_stream:
|
|
698
686
|
for chunk in stream_zip(combined_entries):
|
|
699
687
|
upload_stream.write(chunk)
|
|
@@ -708,9 +696,7 @@ class SmartOpenStreamingStorage:
|
|
|
708
696
|
config.bucket_name, config.file_key
|
|
709
697
|
)
|
|
710
698
|
except Exception as e:
|
|
711
|
-
logger.error(
|
|
712
|
-
f"Failed to generate presigned URL for {config.bucket_name}/{config.file_key}: {e}"
|
|
713
|
-
)
|
|
699
|
+
logger.error(f"Failed to generate presigned URL for {config.file_key}: {e}")
|
|
714
700
|
raise StorageUploadError(f"Failed to generate presigned URL: {e}") from e
|
|
715
701
|
|
|
716
702
|
@retry_cloud_storage_operation(
|
|
@@ -770,9 +756,7 @@ class SmartOpenStreamingStorage:
|
|
|
770
756
|
)
|
|
771
757
|
|
|
772
758
|
# Use smart-open's streaming upload capability
|
|
773
|
-
with self.storage_client.stream_upload(
|
|
774
|
-
bucket_name, file_key, content_type="application/zip"
|
|
775
|
-
) as upload_stream:
|
|
759
|
+
with self.storage_client.stream_upload(bucket_name, file_key) as upload_stream:
|
|
776
760
|
for chunk in stream_zip(zip_generator):
|
|
777
761
|
upload_stream.write(chunk)
|
|
778
762
|
|
|
@@ -800,9 +784,7 @@ class SmartOpenStreamingStorage:
|
|
|
800
784
|
zip_generator = self._convert_to_stream_zip_format(data_files_generator)
|
|
801
785
|
|
|
802
786
|
# Use smart-open streaming upload
|
|
803
|
-
with self.storage_client.stream_upload(
|
|
804
|
-
bucket_name, file_key, content_type="application/zip"
|
|
805
|
-
) as upload_stream:
|
|
787
|
+
with self.storage_client.stream_upload(bucket_name, file_key) as upload_stream:
|
|
806
788
|
for chunk in stream_zip(zip_generator):
|
|
807
789
|
upload_stream.write(chunk)
|
|
808
790
|
|
|
@@ -838,12 +820,10 @@ class SmartOpenStreamingStorage:
|
|
|
838
820
|
data_files_generator = self._create_data_files(
|
|
839
821
|
data, resp_format, all_attachments
|
|
840
822
|
)
|
|
841
|
-
logger.debug("Yielding data files for ZIP")
|
|
842
823
|
yield from self._convert_to_stream_zip_format(data_files_generator)
|
|
843
824
|
|
|
844
825
|
# Then, yield attachment files (already in stream_zip format, stream directly)
|
|
845
826
|
attachment_files_generator = self._create_attachment_files(all_attachments)
|
|
846
|
-
logger.debug("Yielding attachment files for ZIP")
|
|
847
827
|
yield from attachment_files_generator
|
|
848
828
|
|
|
849
829
|
def _create_data_files(
|
|
@@ -958,11 +938,7 @@ class SmartOpenStreamingStorage:
|
|
|
958
938
|
|
|
959
939
|
try:
|
|
960
940
|
source_bucket, source_key = self._parse_storage_url(storage_key)
|
|
961
|
-
logger.debug(
|
|
962
|
-
f"Parsed storage URL - bucket: {source_bucket}, key: {source_key}"
|
|
963
|
-
)
|
|
964
941
|
except ValueError as e:
|
|
965
|
-
logger.error(f"Could not parse storage URL: {storage_key} - {e}")
|
|
966
942
|
raise StorageUploadError(
|
|
967
943
|
f"Could not parse storage URL: {storage_key} - {e}"
|
|
968
944
|
) from e
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any, Optional
|
|
6
6
|
|
|
7
7
|
from fides.api.service.storage.streaming.base_storage_client import BaseStorageClient
|
|
8
8
|
from fides.api.service.storage.streaming.s3.s3_storage_client import S3StorageClient
|
|
@@ -17,7 +17,9 @@ class StorageClientFactory:
|
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
19
|
@staticmethod
|
|
20
|
-
def create_client(
|
|
20
|
+
def create_client(
|
|
21
|
+
storage_type: str, auth_method: Optional[str], storage_secrets: Any
|
|
22
|
+
) -> BaseStorageClient:
|
|
21
23
|
"""Create a provider-specific storage client.
|
|
22
24
|
|
|
23
25
|
Args:
|
|
@@ -34,7 +36,9 @@ class StorageClientFactory:
|
|
|
34
36
|
normalized_type = StorageClientFactory._normalize_storage_type(storage_type)
|
|
35
37
|
|
|
36
38
|
if normalized_type == "s3":
|
|
37
|
-
|
|
39
|
+
if not auth_method:
|
|
40
|
+
raise ValueError("auth_method is required for S3 storage")
|
|
41
|
+
return S3StorageClient(auth_method, storage_secrets)
|
|
38
42
|
if normalized_type == "gcs":
|
|
39
43
|
raise NotImplementedError("GCS storage is not yet supported")
|
|
40
44
|
raise ValueError(f"Unsupported storage type: {storage_type}")
|
fides/api/task/graph_runners.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
from typing import Any, Dict, List
|
|
1
|
+
from typing import Any, Dict, List
|
|
2
2
|
|
|
3
|
-
from loguru import logger
|
|
4
3
|
from sqlalchemy.orm import Session
|
|
5
4
|
|
|
6
5
|
from fides.api.common_exceptions import PrivacyRequestExit
|
|
@@ -19,37 +18,8 @@ from fides.api.task.deprecated_graph_task import (
|
|
|
19
18
|
run_consent_request_deprecated,
|
|
20
19
|
run_erasure_request_deprecated,
|
|
21
20
|
)
|
|
21
|
+
from fides.api.task.scheduler_utils import use_dsr_3_0_scheduler
|
|
22
22
|
from fides.api.util.collection_util import Row
|
|
23
|
-
from fides.config import CONFIG
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def use_dsr_3_0_scheduler(
|
|
27
|
-
privacy_request: PrivacyRequest, action_type: ActionType
|
|
28
|
-
) -> bool:
|
|
29
|
-
"""Return whether we should use the DSR 3.0 scheduler.
|
|
30
|
-
|
|
31
|
-
Override if we have a partially processed Privacy Request that was already run on
|
|
32
|
-
DSR 2.0 so we can finish processing it on 2.0.
|
|
33
|
-
|
|
34
|
-
"""
|
|
35
|
-
use_dsr_3_0 = CONFIG.execution.use_dsr_3_0
|
|
36
|
-
|
|
37
|
-
prev_results: Dict[str, Optional[List[Row]]] = (
|
|
38
|
-
privacy_request.get_raw_access_results()
|
|
39
|
-
)
|
|
40
|
-
existing_tasks_count: int = privacy_request.get_tasks_by_action(action_type).count()
|
|
41
|
-
|
|
42
|
-
if prev_results and use_dsr_3_0 and not existing_tasks_count:
|
|
43
|
-
# If we've previously tried to process this Privacy Request using DSR 2.0, continue doing so
|
|
44
|
-
# for access and erasure requests
|
|
45
|
-
logger.info(
|
|
46
|
-
"Overriding scheduler to run privacy request {} using DSR 2.0 as it's "
|
|
47
|
-
"already partially processed",
|
|
48
|
-
privacy_request.id,
|
|
49
|
-
)
|
|
50
|
-
use_dsr_3_0 = False
|
|
51
|
-
|
|
52
|
-
return use_dsr_3_0
|
|
53
23
|
|
|
54
24
|
|
|
55
25
|
def access_runner(
|
fides/api/task/graph_task.py
CHANGED
|
@@ -47,6 +47,7 @@ from fides.api.service.execution_context import collect_execution_log_messages
|
|
|
47
47
|
from fides.api.task.consolidate_query_matches import consolidate_query_matches
|
|
48
48
|
from fides.api.task.filter_element_match import filter_element_match
|
|
49
49
|
from fides.api.task.refine_target_path import FieldPathNodeInput
|
|
50
|
+
from fides.api.task.scheduler_utils import use_dsr_3_0_scheduler
|
|
50
51
|
from fides.api.task.task_resources import TaskResources
|
|
51
52
|
from fides.api.util.cache import get_cache
|
|
52
53
|
from fides.api.util.collection_util import (
|
|
@@ -589,7 +590,7 @@ class GraphTask(ABC): # pylint: disable=too-many-instance-attributes
|
|
|
589
590
|
# TODO Remove when we stop support for DSR 2.0
|
|
590
591
|
# Save data to build masking requests for DSR 2.0 in Redis.
|
|
591
592
|
# Results saved with matching array elements preserved
|
|
592
|
-
if not
|
|
593
|
+
if not use_dsr_3_0_scheduler(self.resources.request, ActionType.access):
|
|
593
594
|
self.resources.cache_results_with_placeholders(
|
|
594
595
|
f"access_request__{self.key}", placeholder_output
|
|
595
596
|
)
|
|
@@ -619,7 +620,8 @@ class GraphTask(ABC): # pylint: disable=too-many-instance-attributes
|
|
|
619
620
|
|
|
620
621
|
# TODO Remove when we stop support for DSR 2.0
|
|
621
622
|
# Saves intermediate access results for DSR 2.0 in Redis
|
|
622
|
-
|
|
623
|
+
# Only cache for existing DSR 2.0 requests
|
|
624
|
+
if not use_dsr_3_0_scheduler(self.resources.request, ActionType.access):
|
|
623
625
|
self.resources.cache_object(f"access_request__{self.key}", output)
|
|
624
626
|
|
|
625
627
|
# Return filtered rows with non-matched array data removed.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Utilities for determining which DSR scheduler to use."""
|
|
2
|
+
|
|
3
|
+
from typing import Dict, List, Optional
|
|
4
|
+
|
|
5
|
+
from loguru import logger
|
|
6
|
+
|
|
7
|
+
from fides.api.models.privacy_request import PrivacyRequest
|
|
8
|
+
from fides.api.schemas.policy import ActionType
|
|
9
|
+
from fides.api.util.collection_util import Row
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def use_dsr_3_0_scheduler(
|
|
13
|
+
privacy_request: PrivacyRequest, action_type: ActionType
|
|
14
|
+
) -> bool:
|
|
15
|
+
"""Return whether we should use the DSR 3.0 scheduler.
|
|
16
|
+
|
|
17
|
+
DSR 3.0 is now the default for all new privacy requests. Only allow DSR 2.0 for
|
|
18
|
+
existing requests that were already partially processed with DSR 2.0.
|
|
19
|
+
|
|
20
|
+
"""
|
|
21
|
+
# DSR 3.0 is now the default
|
|
22
|
+
use_dsr_3_0 = True
|
|
23
|
+
|
|
24
|
+
# Check if this is an existing DSR 2.0 request that should continue with DSR 2.0
|
|
25
|
+
prev_results: Dict[str, Optional[List[Row]]] = (
|
|
26
|
+
privacy_request.get_raw_access_results()
|
|
27
|
+
)
|
|
28
|
+
existing_tasks_count: int = privacy_request.get_tasks_by_action(action_type).count()
|
|
29
|
+
|
|
30
|
+
# Only allow DSR 2.0 for requests that have already been partially processed with DSR 2.0
|
|
31
|
+
if prev_results and not existing_tasks_count:
|
|
32
|
+
logger.info(
|
|
33
|
+
"Overriding scheduler to run privacy request {} using DSR 2.0 as it's "
|
|
34
|
+
"already partially processed",
|
|
35
|
+
privacy_request.id,
|
|
36
|
+
)
|
|
37
|
+
use_dsr_3_0 = False
|
|
38
|
+
|
|
39
|
+
return use_dsr_3_0
|
fides/api/util/endpoint_utils.py
CHANGED
|
@@ -5,8 +5,6 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
|
5
5
|
|
|
6
6
|
from fastapi import HTTPException
|
|
7
7
|
from fideslang import FidesModelType
|
|
8
|
-
from slowapi import Limiter
|
|
9
|
-
from slowapi.util import get_remote_address # type: ignore
|
|
10
8
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
11
9
|
from starlette.status import HTTP_400_BAD_REQUEST
|
|
12
10
|
|
|
@@ -23,7 +21,6 @@ from fides.common.api.scope_registry import (
|
|
|
23
21
|
ORGANIZATION,
|
|
24
22
|
SYSTEM,
|
|
25
23
|
)
|
|
26
|
-
from fides.config import CONFIG
|
|
27
24
|
|
|
28
25
|
from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip
|
|
29
26
|
ModelWithDefaultField,
|
|
@@ -44,16 +41,6 @@ CLI_SCOPE_PREFIX_MAPPING: Dict[str, str] = {
|
|
|
44
41
|
"system": SYSTEM,
|
|
45
42
|
}
|
|
46
43
|
|
|
47
|
-
# Used for rate limiting with Slow API
|
|
48
|
-
# Decorate individual routes to deviate from the default rate limits
|
|
49
|
-
fides_limiter = Limiter(
|
|
50
|
-
default_limits=[CONFIG.security.request_rate_limit],
|
|
51
|
-
headers_enabled=True,
|
|
52
|
-
key_prefix=CONFIG.security.rate_limit_prefix,
|
|
53
|
-
key_func=get_remote_address,
|
|
54
|
-
retry_after="http-date",
|
|
55
|
-
)
|
|
56
|
-
|
|
57
44
|
|
|
58
45
|
async def forbid_if_editing_is_default(
|
|
59
46
|
sql_model: Base,
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from ipaddress import ip_address
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
from fastapi import Request
|
|
7
|
+
from fastapi.responses import JSONResponse
|
|
8
|
+
from loguru import logger
|
|
9
|
+
from slowapi import Limiter
|
|
10
|
+
from slowapi.util import get_remote_address # type: ignore
|
|
11
|
+
from starlette.middleware.base import BaseHTTPMiddleware
|
|
12
|
+
|
|
13
|
+
from fides.config import CONFIG
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class InvalidClientIPError(Exception):
|
|
17
|
+
def __init__(self, detail: str, header_value: str, header_name: str):
|
|
18
|
+
self.detail = detail
|
|
19
|
+
self.header_value = header_value
|
|
20
|
+
self.header_name = header_name
|
|
21
|
+
super().__init__(detail)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def validate_client_ip(ip: Optional[str]) -> bool:
|
|
25
|
+
"""
|
|
26
|
+
Returns true if the provided ip is valid and not from a reserved range.
|
|
27
|
+
Returns false otherwise.
|
|
28
|
+
"""
|
|
29
|
+
if not ip:
|
|
30
|
+
return False
|
|
31
|
+
try:
|
|
32
|
+
ip_obj = ip_address(ip)
|
|
33
|
+
if (
|
|
34
|
+
ip_obj.is_loopback
|
|
35
|
+
or ip_obj.is_link_local
|
|
36
|
+
or ip_obj.is_reserved
|
|
37
|
+
or ip_obj.is_multicast
|
|
38
|
+
or ip_obj.is_private
|
|
39
|
+
):
|
|
40
|
+
return False
|
|
41
|
+
return True
|
|
42
|
+
except ValueError:
|
|
43
|
+
return False
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _extract_hostname_from_ip(ip: str) -> Optional[str]:
|
|
47
|
+
"""
|
|
48
|
+
Extract hostname/IP address from header value, stripping port if present.
|
|
49
|
+
|
|
50
|
+
Simple string-based approach following the reference implementation pattern.
|
|
51
|
+
Does not validate whether the result is a valid IP address.
|
|
52
|
+
|
|
53
|
+
Examples:
|
|
54
|
+
# IPv4 cases
|
|
55
|
+
_extract_hostname_from_ip("192.168.1.1") -> "192.168.1.1"
|
|
56
|
+
_extract_hostname_from_ip("192.168.1.1:8080") -> "192.168.1.1"
|
|
57
|
+
|
|
58
|
+
# IPv6 cases
|
|
59
|
+
_extract_hostname_from_ip("2001:db8::1") -> "2001:db8::1"
|
|
60
|
+
_extract_hostname_from_ip("[2001:db8::1]:8080") -> "2001:db8::1"
|
|
61
|
+
|
|
62
|
+
# Edge cases (alidation will later reject)
|
|
63
|
+
_extract_hostname_from_ip("192.168.1.1, 192.168.1.2") -> "192.168.1.1, 192.168.1.2"
|
|
64
|
+
_extract_hostname_from_ip("not-an-ip:8080") -> "not-an-ip"
|
|
65
|
+
|
|
66
|
+
# Error
|
|
67
|
+
_extract_hostname_from_ip("") -> raises ValueError
|
|
68
|
+
|
|
69
|
+
Raises:
|
|
70
|
+
ValueError: If no hostname can be extracted from the input
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
clean_ip = ip.strip()
|
|
74
|
+
|
|
75
|
+
if not clean_ip:
|
|
76
|
+
raise ValueError("Could not parse IP from header value")
|
|
77
|
+
|
|
78
|
+
# Handle IPv6 with port: [IPv6]:port
|
|
79
|
+
if "]:" in clean_ip:
|
|
80
|
+
return clean_ip.split("]:")[0].replace("[", "").strip()
|
|
81
|
+
|
|
82
|
+
# Handle IPv4 with port: IPv4:port
|
|
83
|
+
if ":" in clean_ip and "::" not in clean_ip:
|
|
84
|
+
return clean_ip.split(":")[0].strip()
|
|
85
|
+
|
|
86
|
+
# Return as-is (IPv6 without port, IPv4 without port, or other values)
|
|
87
|
+
return clean_ip
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _resolve_client_ip_from_header(request: Request, strict: bool) -> str:
|
|
91
|
+
"""Shared resolver for client IP from the configured header.
|
|
92
|
+
|
|
93
|
+
- When strict=True: raise InvalidClientIPError on invalid/malformed header values.
|
|
94
|
+
- When strict=False: never raise; fall back to the connection source IP.
|
|
95
|
+
"""
|
|
96
|
+
header_name = CONFIG.security.rate_limit_client_ip_header
|
|
97
|
+
if not header_name:
|
|
98
|
+
# This line should never be reached when rate limiting is enabled
|
|
99
|
+
logger.warning(
|
|
100
|
+
"Rate limit client IP header not configured. Falling back to source IP.",
|
|
101
|
+
header_name,
|
|
102
|
+
)
|
|
103
|
+
return get_remote_address(request)
|
|
104
|
+
|
|
105
|
+
ip_address_from_header = request.headers.get(header_name)
|
|
106
|
+
if not ip_address_from_header:
|
|
107
|
+
logger.debug(
|
|
108
|
+
"Rate limit header '{}' not found. Falling back to source IP.",
|
|
109
|
+
header_name,
|
|
110
|
+
)
|
|
111
|
+
return get_remote_address(request)
|
|
112
|
+
|
|
113
|
+
# Extract and validate IP
|
|
114
|
+
try:
|
|
115
|
+
extracted_ip = _extract_hostname_from_ip(ip_address_from_header)
|
|
116
|
+
if extracted_ip and validate_client_ip(extracted_ip):
|
|
117
|
+
return extracted_ip
|
|
118
|
+
raise ValueError("IP failed validation")
|
|
119
|
+
except ValueError:
|
|
120
|
+
if strict:
|
|
121
|
+
logger.error(
|
|
122
|
+
"Invalid IP '{}' in header '{}'. Rejecting request.",
|
|
123
|
+
ip_address_from_header,
|
|
124
|
+
header_name,
|
|
125
|
+
)
|
|
126
|
+
raise InvalidClientIPError(
|
|
127
|
+
detail="Invalid IP address format",
|
|
128
|
+
header_value=ip_address_from_header,
|
|
129
|
+
header_name=header_name,
|
|
130
|
+
)
|
|
131
|
+
# Non-strict path: fall back silently to source IP
|
|
132
|
+
return get_remote_address(request)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def get_client_ip_from_header(request: Request) -> str:
|
|
136
|
+
"""
|
|
137
|
+
Extracts the client IP from the configured CDN header.
|
|
138
|
+
|
|
139
|
+
If the header is not configured or is missing, it falls back to the
|
|
140
|
+
source IP on the request.
|
|
141
|
+
|
|
142
|
+
Raises InvalidClientIPError if header contains invalid IP format.
|
|
143
|
+
"""
|
|
144
|
+
return _resolve_client_ip_from_header(request, strict=True)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def safe_rate_limit_key(request: Request) -> str:
|
|
148
|
+
"""
|
|
149
|
+
Safe key function for SlowAPI limiter.
|
|
150
|
+
|
|
151
|
+
Must never raise. If the configured header is missing or malformed,
|
|
152
|
+
fall back to the connection source IP for rate limiting purposes.
|
|
153
|
+
"""
|
|
154
|
+
return _resolve_client_ip_from_header(request, strict=False)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class RateLimitIPValidationMiddleware(BaseHTTPMiddleware):
|
|
158
|
+
"""
|
|
159
|
+
Pre-validate the configured client IP header when rate limiting is enabled.
|
|
160
|
+
|
|
161
|
+
If the header is present but invalid, short-circuit the request with 422.
|
|
162
|
+
This keeps SlowAPI's middleware path free of exceptions from the key function.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
async def dispatch(self, request: Request, call_next): # type: ignore
|
|
166
|
+
if is_rate_limit_enabled:
|
|
167
|
+
try:
|
|
168
|
+
# Triggers parsing/validation; raises on invalid header
|
|
169
|
+
get_client_ip_from_header(request)
|
|
170
|
+
except InvalidClientIPError:
|
|
171
|
+
return JSONResponse(
|
|
172
|
+
status_code=422, content={"detail": "Invalid client IP header"}
|
|
173
|
+
)
|
|
174
|
+
return await call_next(request)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# Used for rate limiting with Slow API
|
|
178
|
+
# Decorate individual routes to deviate from the default rate limits
|
|
179
|
+
is_rate_limit_enabled = (
|
|
180
|
+
CONFIG.security.rate_limit_client_ip_header is not None
|
|
181
|
+
and CONFIG.security.rate_limit_client_ip_header != ""
|
|
182
|
+
)
|
|
183
|
+
fides_limiter = Limiter(
|
|
184
|
+
storage_uri=CONFIG.redis.connection_url_unencoded,
|
|
185
|
+
application_limits=[
|
|
186
|
+
CONFIG.security.request_rate_limit
|
|
187
|
+
], # Creates ONE shared bucket for all endpoints
|
|
188
|
+
headers_enabled=True,
|
|
189
|
+
key_prefix=CONFIG.security.rate_limit_prefix,
|
|
190
|
+
key_func=safe_rate_limit_key,
|
|
191
|
+
retry_after="http-date",
|
|
192
|
+
in_memory_fallback_enabled=False, # Fall back to no rate limiting if Redis unavailable
|
|
193
|
+
enabled=is_rate_limit_enabled,
|
|
194
|
+
)
|
|
@@ -65,10 +65,6 @@ class ExecutionSettings(FidesSettings):
|
|
|
65
65
|
default=3600,
|
|
66
66
|
description="Seconds between polling for async tasks to requeue",
|
|
67
67
|
)
|
|
68
|
-
use_dsr_3_0: bool = Field(
|
|
69
|
-
default=False,
|
|
70
|
-
description="Temporary flag to switch to using DSR 3.0 to process your tasks.",
|
|
71
|
-
)
|
|
72
68
|
erasure_request_finalization_required: bool = Field(
|
|
73
69
|
default=False,
|
|
74
70
|
description="Whether erasure requests require an additional finalization step after all collections have been executed.",
|
fides/config/redis_settings.py
CHANGED
|
@@ -181,8 +181,24 @@ class RedisSettings(FidesSettings):
|
|
|
181
181
|
description="A full connection URL to the read-only Redis cache. If not specified, this URL is automatically assembled from the read_only_host, read_only_port, read_only_password and read_only_db_index specified above.",
|
|
182
182
|
exclude=True,
|
|
183
183
|
)
|
|
184
|
+
connection_url_unencoded: Optional[str] = Field(
|
|
185
|
+
default=None,
|
|
186
|
+
description="A full connection URL to the Redis cache with the password unencoded. If not specified, this URL is automatically assembled from the host, port, password and db_index specified above.",
|
|
187
|
+
exclude=True,
|
|
188
|
+
)
|
|
189
|
+
read_only_connection_url_unencoded: Optional[str] = Field(
|
|
190
|
+
default=None,
|
|
191
|
+
description="A full connection URL to the read-only Redis cache with the password unencoded. If not specified, this URL is automatically assembled from the read_only_host, read_only_port, read_only_password and read_only_db_index specified above.",
|
|
192
|
+
exclude=True,
|
|
193
|
+
)
|
|
184
194
|
|
|
185
|
-
@field_validator(
|
|
195
|
+
@field_validator(
|
|
196
|
+
"connection_url",
|
|
197
|
+
"read_only_connection_url",
|
|
198
|
+
"connection_url_unencoded",
|
|
199
|
+
"read_only_connection_url_unencoded",
|
|
200
|
+
mode="before",
|
|
201
|
+
)
|
|
186
202
|
@classmethod
|
|
187
203
|
def assemble_connection_url(
|
|
188
204
|
cls,
|
|
@@ -195,7 +211,14 @@ class RedisSettings(FidesSettings):
|
|
|
195
211
|
return v
|
|
196
212
|
|
|
197
213
|
# Determine which set of settings to use based on field name
|
|
198
|
-
is_read_only = info.field_name
|
|
214
|
+
is_read_only = info.field_name in (
|
|
215
|
+
"read_only_connection_url",
|
|
216
|
+
"read_only_connection_url_unencoded",
|
|
217
|
+
)
|
|
218
|
+
is_unencoded = info.field_name in (
|
|
219
|
+
"connection_url_unencoded",
|
|
220
|
+
"read_only_connection_url_unencoded",
|
|
221
|
+
)
|
|
199
222
|
|
|
200
223
|
# Extract settings - fallbacks already resolved by field validators for read-only fields
|
|
201
224
|
user = (
|
|
@@ -244,7 +267,8 @@ class RedisSettings(FidesSettings):
|
|
|
244
267
|
# redis://<user>:<password>@<host>
|
|
245
268
|
auth_prefix = ""
|
|
246
269
|
if password or user:
|
|
247
|
-
|
|
270
|
+
encoded_password = password if is_unencoded else quote_plus(password)
|
|
271
|
+
auth_prefix = f"{quote_plus(user)}:{encoded_password}@"
|
|
248
272
|
|
|
249
273
|
# For host, we don't have a fallback - read replica should be a different host
|
|
250
274
|
host = (
|