ethyca-fides 2.69.1b0__py2.py3-none-any.whl → 2.69.1b2__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/METADATA +1 -1
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/RECORD +226 -224
- fides/_version.py +3 -3
- fides/api/api/v1/endpoints/dsr_package_link.py +5 -3
- fides/api/api/v1/endpoints/oauth_endpoints.py +1 -1
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +3 -5
- fides/api/api/v1/endpoints/user_endpoints.py +1 -24
- fides/api/app_setup.py +16 -2
- fides/api/main.py +22 -0
- fides/api/models/client.py +5 -9
- fides/api/models/fides_user.py +2 -1
- fides/api/oauth/utils.py +11 -27
- fides/api/service/privacy_request/request_service.py +19 -2
- fides/api/service/storage/storage_uploader_service.py +1 -21
- fides/api/service/storage/streaming/dsr_storage.py +1 -5
- fides/api/service/storage/streaming/s3/s3_storage_client.py +78 -40
- fides/api/service/storage/streaming/s3/streaming_s3.py +9 -21
- fides/api/service/storage/streaming/smart_open_client.py +8 -7
- fides/api/service/storage/streaming/smart_open_streaming_storage.py +4 -28
- fides/api/service/storage/streaming/storage_client_factory.py +7 -3
- fides/api/task/graph_runners.py +2 -32
- fides/api/task/graph_task.py +4 -2
- fides/api/task/scheduler_utils.py +39 -0
- fides/api/util/endpoint_utils.py +0 -13
- fides/api/util/rate_limit.py +194 -0
- fides/config/execution_settings.py +0 -4
- fides/config/redis_settings.py +27 -3
- fides/config/security_settings.py +24 -6
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/0agWtBSaxTBxQfxPA99Ra/_buildManifest.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{1345-04e37a66c0d40dc1.js → 1345-5e1c5b66e25c566e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3729-f5f2976904dce90d.js → 3729-a1ca1608efc11ac4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3847-2c0126e6eb54c526.js → 3847-230bf61b053bc706.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3855-9dd54ded74f4036b.js → 3855-ef5194cdb228beb6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4121-c8d5d717e31899e1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4164-355644b916ae0094.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4608-d101417a3abb4ad6.js → 4608-23bbd4c3c4a59f42.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4786-7aafb744445445b2.js → 4786-0827aae7aceadd22.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4808-357ca7ea7bbd24c6.js → 4808-78ca630f2d2503cd.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{4844-707b20a6c4b127cc.js → 4844-46324c3d848b8b6a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9046-058a4d8f0b5e08f9.js → 9046-712156d461165f56.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9676.b7d5d1d90b9da224.js → 9676.9fd9552ef744c717.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9951-b954027a046ce553.js → 9951-a88367a129b724ba.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-f18537fd2c4288e7.js → _app-ef8e1c986bc5b795.js} +2 -2
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-20253dd047fb9736.js → manual-9dc7e70ab5b05723.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-169099ff7b305cf5.js → add-systems-1632a59203fe8eab.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-28b192e2c074b0f3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-adc500a03e239857.js → [resourceUrn]-da1a48336daff6f8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-c8b3d090e4ba60d3.js → [resourceUrn]-470da05db63767cd.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/{[systemId]-cfb0b1200bc1a2c9.js → [systemId]-2f0a33ef9ba1f1da.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/{[monitorId]-fed8b879c13c2bf3.js → [monitorId]-e9d4f25b20ff6781.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-58a110542d6bcd0f.js → activity-b6ae7adb8ef0b525.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-22eec362dfbb1d2a.js → [resourceUrn]-c3a97e6721ca0abe.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-4decce5ef996e563.js → detection-a0a7de552ef71f5b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-01acdd1ad492fd89.js → [resourceUrn]-109754fec0755339.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-85fdbf4cde60d910.js → discovery-88654783b06b3b21.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{datamap-60e27b673c68bd27.js → datamap-89136e6800dc9369.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{new-2c7b1213b6604d30.js → new-97f06e21580f1f6a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-7ed3f05700dc397b.js → [id]-6f77d8647fca71e0.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-95f6d64f84fc6bf3.js → new-821dd1269834cfa2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-3a4cd3fe9094fba3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{integrations-0f12d5b658c98c9f.js → integrations-57e618d7b16ac69a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{[id]-b379873a5771e55b.js → [id]-0d0bb9eb004a3336.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{messaging-a9bb257906dcd7e0.js → messaging-f9320a58f489f5b7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/{storage-de0036c74b78e9b7.js → storage-d0cfa8aeddd43a40.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{privacy-requests-91f578139548652c.js → privacy-requests-5a5edc8a4aa7c30a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/{datamap-83019a6753e14857.js → datamap-6903f42a0412bfa6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{consent-e5331508e81222fc.js → consent-be47008304106395.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-74f0fe9656f4a1f6.js → custom-fields-ae1b57589da7b175.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-e91da49c0681a300.js → [id]-5a43f108d8047d5b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{taxonomy-78c3a5200d362cff.js → taxonomy-1b3f2d4bcb0e164d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{webpack-b5eb3e1da37616d2.js → webpack-678e89d68dbcd94f.js} +1 -1
- fides/ui-build/static/admin/_next/static/css/650df9c348000a26.css +1 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/4lPKe7mco0KEv09aOQH9A/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4121-bb71a24d41d04a28.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/768-7eac4b30d7477b0a.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-60e3394c887f3d5e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-3d24c1510aa4c555.js +0 -1
- fides/ui-build/static/admin/_next/static/css/abf2e162b1cd0e61.css +0 -1
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.69.1b0.dist-info → ethyca_fides-2.69.1b2.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{4lPKe7mco0KEv09aOQH9A → 0agWtBSaxTBxQfxPA99Ra}/_ssgManifest.js +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{1099-c34f76b4da5f3d15.js → 1099-79646e64f26d62fa.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{1817-e6934e258111a961.js → 1817-3d9e110e007853f0.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{2921-0696287bb8de1f0b.js → 2921-52328140bc420d0f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3620-8c0ee3d0b19c342d.js → 3620-31ebb43dba84cbbd.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3872-72ea3eb040366277.js → 3872-a91143aa35fa8ef8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{3923-c4f2b03706ddbe39.js → 3923-bb2417b8dcade7a4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{401-959a15ed18a8abdf.js → 401-4af0a912e249d30f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{5258-1a8b9f66b97761fc.js → 5258-e880b606a2293803.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{5487-fd9724519f31caff.js → 5487-8c635883dcaa9c2a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{549-d3bef0990071230c.js → 549-38ea1d91ee2addaa.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6084-487d27d017c45e05.js → 6084-0096d7de64ef8015.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6853-f7ab74e30abbdeaf.js → 6853-b17673391117c976.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{6954-b0a7b8ac6db238f8.js → 6954-9d46e2276c461c26.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7476-9a57db910472b48e.js → 7476-d1b0af9ade392e5b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{7630-46807321449479c7.js → 7630-da0a7ce4e3a0d62c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{787-79e8e558bd80ece6.js → 787-3499983fa346b380.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{79-3e5047415bee9391.js → 79-f197fc4db8d530e5.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{796-fb2af44165f37ecc.js → 796-db1e30119ea973c7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{8002-c1f66179adabece8.js → 8002-971e29181f72edd1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/{9826-0d9a7f61c08ed88a.js → 9826-b0b3d3cfb13bfbc1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{404-9c9efb820bb6b432.js → 404-471a6b18e712f050.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-caadc3c0e86a7bfe.js → multiple-4b79a1652297ed9a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-acefb9e08c6c4082.js → add-vendors-1ca9df7ca91bd101.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-558fbb1667473abb.js → configure-07bdbc9ae4137db4.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-b1ff1c9683841815.js → [id]-f80cf2d3966816fd.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-674906d2d9a0a3a6.js → privacy-experience-2795cd4115a77c94.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-5b7aa7971f070513.js → [id]-e02921dc82dccbb1.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-285958a12a66575e.js → new-98f9e4ba3610628a.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-dc56bbdb6dd7a670.js → privacy-notices-17ed82777810d1c6.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-0843757f00eeaaec.js → properties-226efa1dcd41437f.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{consent-5cba58ebecb4511f.js → consent-09610b10923d9268.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-0f90cac9f190130c.js → [projectUrn]-d8e776f1e64e4ba8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-f9117645c941342c.js → projects-75b9629b0d9cdf96.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-c566640eecc6f3fe.js → data-catalog-6984c033b8fe3a13.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-991659e916ad60b1.js → action-center-9c428d3ef0985915.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-3f4ba87513e030a2.js → [...subfieldNames]-8f58192dcb54883d.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-a6cd31103deba465.js → [collectionName]-dcb4ab380a77aa1e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-491d2c8dd559d0cb.js → [datasetId]-6f16d43071fb9c11.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-c8bcd568d3b0ee7f.js → dataset-674bb3940f088ecc.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{datastore-connection-9a33a412a5f36960.js → datastore-connection-23e4caf79faa8106.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{index-1299410f671fac23.js → index-23eb64eed81dcb69.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-ba232c4b17576ccb.js → [id]-c9a323eb6a929476.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-b97883026fbc5b1e.js → add-template-b9bb09e46921a590.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{messaging-38189c1058aa4acf.js → messaging-82c631a12b5a008c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{ant-components-3119bdb3811409bf.js → ant-components-bc0e2adf6e0d3ac7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-8f9d0434dc3eb8cf.js → AntForm-86ffcc1ad3fa912e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-a14e51c7f22f3395.js → FormikAntFormItem-ec04f595465bdf69.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-a40645b57822684d.js → FormikControlled-41d309754ff0c1de.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-c55df6527b700701.js → FormikField-cab1f78cec7808f9.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-292d761616b162a0.js → forms-eb6058221403b156.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/poc/{table-migration-135bcf384b81820a.js → table-migration-38360083348c3d6c.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-70efccbe0732786b.js → configure-72ca94ec5ed85733.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-63c6f0193634add5.js → [id]-5ec775c4904fdbfe.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-e76567f0374d5912.js → add-property-a6812c0916f2949e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{properties-2050b7dac0413c11.js → properties-da734840e4f9d04b.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-913f8eab62460f31.js → alpha-3e72e9f91991c119.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-2ed1ee6017c0656a.js → about-6aab092f4871cecb.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-bffd6292e7e0302a.js → [purpose_id]-9495e2eb506606c7.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-a649ca07ce51b0fe.js → domain-records-23a6d7a921150188.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-eb675ba600cea2a8.js → domains-2a9e8859ab4d9de6.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-4db2f42f90867890.js → email-templates-4f9f0fdf9925ae90.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-89b404989b4ea21c.js → locations-46f7af35cee4a8bb.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-eb1ecff37fd85c72.js → organization-a596a96cb8d0aa8e.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-07d883b5aaec58f2.js → regulations-6ed5fc2410e00857.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/{test-datasets-61ecb7546830c345.js → test-datasets-86811e3cda277e77.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{systems-44d714017dd87e62.js → systems-045a841e22e85ea8.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/{[id]-4f9001870e2b3aff.js → [id]-05d61c80a556b2d5.js} +0 -0
- /fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-583a7f0c1bead240.js → user-management-2cab41659f1ee7da.js} +0 -0
fides/_version.py
CHANGED
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2025-
|
|
11
|
+
"date": "2025-09-02T19:44:54+0200",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "2.69.
|
|
14
|
+
"full-revisionid": "be844cb784cc2f806cb1fe1d7d92cb8cc4c91ad4",
|
|
15
|
+
"version": "2.69.1b2"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -21,7 +21,7 @@ from fides.api.schemas.privacy_request import PrivacyRequestStatus
|
|
|
21
21
|
from fides.api.schemas.storage.storage import StorageType
|
|
22
22
|
from fides.api.service.storage.streaming.s3 import S3StorageClient
|
|
23
23
|
from fides.api.util.api_router import APIRouter
|
|
24
|
-
from fides.api.util.
|
|
24
|
+
from fides.api.util.rate_limit import fides_limiter
|
|
25
25
|
from fides.common.api.v1.urn_registry import PRIVACY_CENTER_DSR_PACKAGE, V1_URL_PREFIX
|
|
26
26
|
from fides.config import CONFIG
|
|
27
27
|
|
|
@@ -62,7 +62,7 @@ def raise_error(status_code: int, detail: str) -> None:
|
|
|
62
62
|
PRIVACY_CENTER_DSR_PACKAGE,
|
|
63
63
|
status_code=HTTP_302_FOUND,
|
|
64
64
|
)
|
|
65
|
-
@fides_limiter.limit(CONFIG.security.
|
|
65
|
+
@fides_limiter.limit(CONFIG.security.request_rate_limit)
|
|
66
66
|
def get_access_results_urls(
|
|
67
67
|
privacy_request_id: str,
|
|
68
68
|
token: str,
|
|
@@ -155,7 +155,9 @@ def get_access_results_urls(
|
|
|
155
155
|
|
|
156
156
|
try:
|
|
157
157
|
# Use S3StorageClient for cleaner presigned URL generation
|
|
158
|
-
s3_storage_client = S3StorageClient(
|
|
158
|
+
s3_storage_client = S3StorageClient(
|
|
159
|
+
storage_config.details.get("auth_method"), storage_config.secrets
|
|
160
|
+
)
|
|
159
161
|
result_url = s3_storage_client.generate_presigned_url(
|
|
160
162
|
bucket=bucket_name,
|
|
161
163
|
key=file_name,
|
|
@@ -37,7 +37,7 @@ from fides.api.service.authentication.authentication_strategy_oauth2_authorizati
|
|
|
37
37
|
)
|
|
38
38
|
from fides.api.util.api_router import APIRouter
|
|
39
39
|
from fides.api.util.connection_util import connection_status
|
|
40
|
-
from fides.api.util.
|
|
40
|
+
from fides.api.util.rate_limit import fides_limiter
|
|
41
41
|
from fides.common.api.scope_registry import (
|
|
42
42
|
CLIENT_CREATE,
|
|
43
43
|
CLIENT_DELETE,
|
|
@@ -1667,17 +1667,15 @@ def privacy_request_data_transfer(
|
|
|
1667
1667
|
detail=f"Rule key {rule_key} not found",
|
|
1668
1668
|
)
|
|
1669
1669
|
|
|
1670
|
-
|
|
1671
|
-
|
|
1670
|
+
access_result: Dict[str, Optional[List[Row]]] = (
|
|
1671
|
+
privacy_request.get_raw_access_results()
|
|
1672
1672
|
)
|
|
1673
1673
|
|
|
1674
|
-
if not
|
|
1674
|
+
if not access_result:
|
|
1675
1675
|
raise HTTPException(
|
|
1676
1676
|
status_code=HTTP_404_NOT_FOUND,
|
|
1677
1677
|
detail=f"No access request information found for privacy request id {privacy_request_id}",
|
|
1678
1678
|
)
|
|
1679
|
-
|
|
1680
|
-
access_result = {k.split("__")[-1]: v for k, v in value_dict.items()}
|
|
1681
1679
|
datasets = DatasetConfig.all(db=db)
|
|
1682
1680
|
if not datasets:
|
|
1683
1681
|
raise HTTPException(
|
|
@@ -59,7 +59,7 @@ from fides.api.schemas.user import (
|
|
|
59
59
|
)
|
|
60
60
|
from fides.api.service.deps import get_user_service
|
|
61
61
|
from fides.api.util.api_router import APIRouter
|
|
62
|
-
from fides.api.util.
|
|
62
|
+
from fides.api.util.rate_limit import fides_limiter
|
|
63
63
|
from fides.common.api.scope_registry import (
|
|
64
64
|
SCOPE_REGISTRY,
|
|
65
65
|
SYSTEM_MANAGER_DELETE,
|
|
@@ -208,17 +208,6 @@ def update_user_password(
|
|
|
208
208
|
|
|
209
209
|
current_user.update_password(db=db, new_password=data.new_password)
|
|
210
210
|
|
|
211
|
-
# Delete the user's associated OAuth client to invalidate all existing sessions
|
|
212
|
-
if current_user.client:
|
|
213
|
-
try:
|
|
214
|
-
current_user.client.delete(db)
|
|
215
|
-
except Exception as exc:
|
|
216
|
-
logger.exception(
|
|
217
|
-
"Unable to delete user client during password reset for user {}: {}",
|
|
218
|
-
current_user.id,
|
|
219
|
-
exc,
|
|
220
|
-
)
|
|
221
|
-
|
|
222
211
|
logger.info("Updated user with id: '{}'.", current_user.id)
|
|
223
212
|
return current_user
|
|
224
213
|
|
|
@@ -247,18 +236,6 @@ def force_update_password(
|
|
|
247
236
|
)
|
|
248
237
|
|
|
249
238
|
user.update_password(db=db, new_password=data.new_password)
|
|
250
|
-
|
|
251
|
-
# Delete the user's associated OAuth client to invalidate all existing sessions
|
|
252
|
-
if user.client:
|
|
253
|
-
try:
|
|
254
|
-
user.client.delete(db)
|
|
255
|
-
except Exception as exc:
|
|
256
|
-
logger.exception(
|
|
257
|
-
"Unable to delete user client during admin-forced password reset for user {}: {}",
|
|
258
|
-
user.id,
|
|
259
|
-
exc,
|
|
260
|
-
)
|
|
261
|
-
|
|
262
239
|
logger.info("Updated user with id: '{}'.", user.id)
|
|
263
240
|
return user
|
|
264
241
|
|
fides/api/app_setup.py
CHANGED
|
@@ -48,9 +48,13 @@ from fides.api.service.saas_request.override_implementations import *
|
|
|
48
48
|
from fides.api.util.api_router import APIRouter
|
|
49
49
|
from fides.api.util.cache import get_cache
|
|
50
50
|
from fides.api.util.consent_util import create_default_tcf_purpose_overrides_on_startup
|
|
51
|
-
from fides.api.util.endpoint_utils import fides_limiter
|
|
52
51
|
from fides.api.util.errors import FidesError
|
|
53
52
|
from fides.api.util.logger import setup as setup_logging
|
|
53
|
+
from fides.api.util.rate_limit import (
|
|
54
|
+
RateLimitIPValidationMiddleware,
|
|
55
|
+
fides_limiter,
|
|
56
|
+
is_rate_limit_enabled,
|
|
57
|
+
)
|
|
54
58
|
from fides.config import CONFIG
|
|
55
59
|
from fides.config.config_proxy import ConfigProxy
|
|
56
60
|
|
|
@@ -88,7 +92,17 @@ def create_fides_app(
|
|
|
88
92
|
for handler in ExceptionHandlers.get_handlers():
|
|
89
93
|
# Starlette bug causing this to fail mypy
|
|
90
94
|
fastapi_app.add_exception_handler(RedisNotConfigured, handler) # type: ignore
|
|
91
|
-
|
|
95
|
+
|
|
96
|
+
if is_rate_limit_enabled:
|
|
97
|
+
# Validate header before SlowAPI processes the request
|
|
98
|
+
fastapi_app.add_middleware(RateLimitIPValidationMiddleware)
|
|
99
|
+
# Required for default rate limiting to work
|
|
100
|
+
fastapi_app.add_middleware(SlowAPIMiddleware)
|
|
101
|
+
else:
|
|
102
|
+
logger.warning(
|
|
103
|
+
"Rate limiting client IPs is disabled because the FIDES__SECURITY__RATE_LIMIT_CLIENT_IP_HEADER env var is not configured."
|
|
104
|
+
)
|
|
105
|
+
|
|
92
106
|
fastapi_app.add_middleware(
|
|
93
107
|
GZipMiddleware, minimum_size=1000, compresslevel=5
|
|
94
108
|
) # minimum_size is in bytes
|
fides/api/main.py
CHANGED
|
@@ -18,6 +18,8 @@ from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
|
|
18
18
|
from fideslog.sdk.python.event import AnalyticsEvent
|
|
19
19
|
from loguru import logger
|
|
20
20
|
from pyinstrument import Profiler
|
|
21
|
+
from slowapi import _rate_limit_exceeded_handler
|
|
22
|
+
from slowapi.errors import RateLimitExceeded
|
|
21
23
|
from starlette.background import BackgroundTask
|
|
22
24
|
from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
|
|
23
25
|
from uvicorn import Config, Server
|
|
@@ -60,6 +62,7 @@ from fides.api.ui import (
|
|
|
60
62
|
)
|
|
61
63
|
from fides.api.util.endpoint_utils import API_PREFIX
|
|
62
64
|
from fides.api.util.logger import _log_exception
|
|
65
|
+
from fides.api.util.rate_limit import safe_rate_limit_key
|
|
63
66
|
from fides.cli.utils import FIDES_ASCII_ART
|
|
64
67
|
from fides.config import CONFIG, check_required_webserver_config_values
|
|
65
68
|
|
|
@@ -388,3 +391,22 @@ async def request_validation_exception_handler(
|
|
|
388
391
|
"detail": jsonable_encoder(exc.errors(), exclude={"input", "url", "ctx"})
|
|
389
392
|
},
|
|
390
393
|
)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
@app.exception_handler(RateLimitExceeded)
|
|
397
|
+
async def rate_limit_handler(request: Request, exc: RateLimitExceeded) -> Response:
|
|
398
|
+
"""Log rate limit violations and delegate to default handler."""
|
|
399
|
+
client_ip = safe_rate_limit_key(
|
|
400
|
+
request
|
|
401
|
+
) # non exception-raising, falls back to source IP
|
|
402
|
+
|
|
403
|
+
# Log the rate limit event
|
|
404
|
+
logger.warning(
|
|
405
|
+
"Rate limit exceeded - IP: %s, Path: %s, Method: %s",
|
|
406
|
+
client_ip,
|
|
407
|
+
request.url.path,
|
|
408
|
+
request.method,
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
# Use the default handler to generate the proper response
|
|
412
|
+
return _rate_limit_exceeded_handler(request, exc)
|
fides/api/models/client.py
CHANGED
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
from datetime import datetime
|
|
5
|
-
from typing import
|
|
5
|
+
from typing import Any, Optional
|
|
6
6
|
|
|
7
7
|
from sqlalchemy import ARRAY, Column, ForeignKey, String
|
|
8
8
|
from sqlalchemy.ext.declarative import declared_attr
|
|
@@ -22,12 +22,10 @@ from fides.api.cryptography.schemas.jwt import (
|
|
|
22
22
|
JWE_PAYLOAD_SYSTEMS,
|
|
23
23
|
)
|
|
24
24
|
from fides.api.db.base_class import Base
|
|
25
|
+
from fides.api.models.fides_user import FidesUser
|
|
25
26
|
from fides.api.oauth.jwt import generate_jwe
|
|
26
27
|
from fides.config import FidesConfig
|
|
27
28
|
|
|
28
|
-
if TYPE_CHECKING:
|
|
29
|
-
from fides.api.models.fides_user import FidesUser
|
|
30
|
-
|
|
31
29
|
DEFAULT_SCOPES: list[str] = []
|
|
32
30
|
DEFAULT_ROLES: list[str] = []
|
|
33
31
|
DEFAULT_SYSTEMS: list[str] = []
|
|
@@ -50,11 +48,9 @@ class ClientDetail(Base):
|
|
|
50
48
|
ARRAY(String), nullable=False, server_default="{}", default=dict
|
|
51
49
|
)
|
|
52
50
|
fides_key = Column(String, index=True, unique=True, nullable=True)
|
|
53
|
-
user_id = Column(
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
# Explicitly annotate the backref relationship for mypy
|
|
57
|
-
user: Optional["FidesUser"]
|
|
51
|
+
user_id = Column(
|
|
52
|
+
String, ForeignKey(FidesUser.id_field_path), nullable=True, unique=True
|
|
53
|
+
)
|
|
58
54
|
|
|
59
55
|
@classmethod
|
|
60
56
|
def create_client_and_secret(
|
fides/api/models/fides_user.py
CHANGED
|
@@ -22,7 +22,7 @@ from fides.api.cryptography.cryptographic_util import (
|
|
|
22
22
|
from fides.api.db.base_class import Base
|
|
23
23
|
from fides.api.models.audit_log import AuditLog
|
|
24
24
|
|
|
25
|
-
# SystemManager
|
|
25
|
+
# Intentionally importing SystemManager here to build the FidesUser.systems relationship
|
|
26
26
|
from fides.api.schemas.user import DisabledReason
|
|
27
27
|
from fides.config import CONFIG
|
|
28
28
|
|
|
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
|
|
|
32
32
|
FidesUserRespondentEmailVerification,
|
|
33
33
|
)
|
|
34
34
|
from fides.api.models.sql_models import System # type: ignore[attr-defined]
|
|
35
|
+
from fides.api.models.system_manager import SystemManager
|
|
35
36
|
|
|
36
37
|
|
|
37
38
|
class FidesUser(Base):
|
fides/api/oauth/utils.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
-
from datetime import datetime
|
|
4
|
+
from datetime import datetime
|
|
5
5
|
from functools import update_wrapper
|
|
6
6
|
from types import FunctionType
|
|
7
7
|
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
@@ -67,14 +67,17 @@ def is_token_expired(
|
|
|
67
67
|
"""Check if a token has expired based on its issued_at timestamp and duration."""
|
|
68
68
|
if issued_at is None:
|
|
69
69
|
return True
|
|
70
|
-
|
|
71
|
-
return datetime.now() > expiration_time
|
|
70
|
+
return (datetime.now() - issued_at).total_seconds() / 60.0 > token_duration_minutes
|
|
72
71
|
|
|
73
72
|
|
|
74
|
-
def is_callback_token_expired(issued_at: datetime) -> bool:
|
|
73
|
+
def is_callback_token_expired(issued_at: Optional[datetime]) -> bool:
|
|
75
74
|
"""Check if a callback token has expired (24 hours)."""
|
|
76
|
-
|
|
77
|
-
|
|
75
|
+
if not issued_at:
|
|
76
|
+
return True
|
|
77
|
+
|
|
78
|
+
return (
|
|
79
|
+
datetime.now() - issued_at
|
|
80
|
+
).total_seconds() / 60.0 > CONFIG.execution.privacy_request_delay_timeout
|
|
78
81
|
|
|
79
82
|
|
|
80
83
|
def _get_webhook_jwe_or_error(
|
|
@@ -222,9 +225,7 @@ async def get_current_user(
|
|
|
222
225
|
created_at=datetime.utcnow(),
|
|
223
226
|
)
|
|
224
227
|
|
|
225
|
-
|
|
226
|
-
raise AuthorizationError(detail="Client has no associated user")
|
|
227
|
-
return client.user
|
|
228
|
+
return client.user # type: ignore[attr-defined]
|
|
228
229
|
|
|
229
230
|
|
|
230
231
|
def verify_callback_oauth_policy_pre_webhook(
|
|
@@ -369,10 +370,8 @@ def extract_token_and_load_client(
|
|
|
369
370
|
logger.debug("Auth token expired.")
|
|
370
371
|
raise AuthorizationError(detail="Not Authorized for this action")
|
|
371
372
|
|
|
372
|
-
issued_at_dt = datetime.fromisoformat(issued_at)
|
|
373
|
-
|
|
374
373
|
if is_token_expired(
|
|
375
|
-
|
|
374
|
+
datetime.fromisoformat(issued_at),
|
|
376
375
|
token_duration_override or CONFIG.security.oauth_access_token_expire_minutes,
|
|
377
376
|
):
|
|
378
377
|
raise AuthorizationError(detail="Not Authorized for this action")
|
|
@@ -395,21 +394,6 @@ def extract_token_and_load_client(
|
|
|
395
394
|
logger.debug("Auth token belongs to an invalid client_id.")
|
|
396
395
|
raise AuthorizationError(detail="Not Authorized for this action")
|
|
397
396
|
|
|
398
|
-
# Invalidate tokens issued prior to the user's most recent password reset.
|
|
399
|
-
# This ensures any existing sessions are expired immediately after a password change.
|
|
400
|
-
try:
|
|
401
|
-
if client.user is not None and client.user.password_reset_at is not None:
|
|
402
|
-
password_reset_at = client.user.password_reset_at
|
|
403
|
-
if password_reset_at and issued_at_dt < password_reset_at:
|
|
404
|
-
logger.debug("Auth token issued before latest password reset.")
|
|
405
|
-
raise AuthorizationError(detail="Not Authorized for this action")
|
|
406
|
-
except (
|
|
407
|
-
Exception
|
|
408
|
-
) as exc: # pragma: no cover - defensive: never block auth on relationship issues
|
|
409
|
-
logger.exception(
|
|
410
|
-
"Unable to evaluate password reset timestamp for client user: {}", exc
|
|
411
|
-
)
|
|
412
|
-
|
|
413
397
|
# Populate request-scoped context with the authenticated user identifier.
|
|
414
398
|
# Prefer the linked user_id; fall back to the client id when this is the
|
|
415
399
|
# special root client (which has no associated FidesUser row).
|
|
@@ -340,7 +340,7 @@ def remove_saved_dsr_data(self: DatabaseTask) -> None:
|
|
|
340
340
|
def initiate_interrupted_task_requeue_poll() -> None:
|
|
341
341
|
"""Initiates scheduler to check for and requeue interrupted tasks"""
|
|
342
342
|
|
|
343
|
-
if CONFIG.test_mode
|
|
343
|
+
if CONFIG.test_mode:
|
|
344
344
|
return
|
|
345
345
|
|
|
346
346
|
assert (
|
|
@@ -540,6 +540,7 @@ def _get_request_task_ids_in_progress(
|
|
|
540
540
|
|
|
541
541
|
|
|
542
542
|
# pylint: disable=too-many-branches
|
|
543
|
+
# pylint: disable=too-many-statements
|
|
543
544
|
@celery_app.task(base=DatabaseTask, bind=True)
|
|
544
545
|
def requeue_interrupted_tasks(self: DatabaseTask) -> None:
|
|
545
546
|
"""
|
|
@@ -660,8 +661,24 @@ def requeue_interrupted_tasks(self: DatabaseTask) -> None:
|
|
|
660
661
|
break
|
|
661
662
|
|
|
662
663
|
# If the task ID is not cached, we can't check if it's running
|
|
663
|
-
# This means the subtask is stuck -
|
|
664
|
+
# This means the subtask is stuck - but we need to handle this differently
|
|
665
|
+
# based on the privacy request status
|
|
664
666
|
if not subtask_id:
|
|
667
|
+
if (
|
|
668
|
+
privacy_request.status
|
|
669
|
+
== PrivacyRequestStatus.requires_input
|
|
670
|
+
):
|
|
671
|
+
# For requires_input status, don't automatically error the request
|
|
672
|
+
# as it's intentionally waiting for user input
|
|
673
|
+
logger.warning(
|
|
674
|
+
f"No task ID found for request task {request_task_id} "
|
|
675
|
+
f"(privacy request {privacy_request.id}) in requires_input status - "
|
|
676
|
+
f"keeping request in current status as it may be waiting for manual input"
|
|
677
|
+
)
|
|
678
|
+
should_requeue = False
|
|
679
|
+
break
|
|
680
|
+
|
|
681
|
+
# For other statuses, cancel the entire privacy request
|
|
665
682
|
_cancel_interrupted_tasks_and_error_privacy_request(
|
|
666
683
|
db,
|
|
667
684
|
privacy_request,
|
|
@@ -51,13 +51,7 @@ def upload(
|
|
|
51
51
|
config.secrets is not None,
|
|
52
52
|
)
|
|
53
53
|
|
|
54
|
-
if config.secrets:
|
|
55
|
-
logger.debug("Storage config secrets type: {}", type(config.secrets))
|
|
56
|
-
if isinstance(config.secrets, dict):
|
|
57
|
-
logger.debug("Storage config secrets keys: {}", list(config.secrets.keys()))
|
|
58
|
-
else:
|
|
59
|
-
logger.debug("Storage config secrets is not a dict: {}", config.secrets)
|
|
60
|
-
else:
|
|
54
|
+
if not config.secrets:
|
|
61
55
|
logger.warning("Storage config has no secrets!")
|
|
62
56
|
|
|
63
57
|
uploader: Any = _get_uploader_from_config_type(config.type) # type: ignore
|
|
@@ -130,19 +124,6 @@ def _s3_uploader(
|
|
|
130
124
|
config.secrets is not None,
|
|
131
125
|
)
|
|
132
126
|
|
|
133
|
-
if config.secrets:
|
|
134
|
-
logger.debug(
|
|
135
|
-
"Config secrets keys: {}",
|
|
136
|
-
(
|
|
137
|
-
list(config.secrets.keys())
|
|
138
|
-
if isinstance(config.secrets, dict)
|
|
139
|
-
else "Not a dict"
|
|
140
|
-
),
|
|
141
|
-
)
|
|
142
|
-
logger.debug("Config secrets type: {}", type(config.secrets))
|
|
143
|
-
else:
|
|
144
|
-
logger.warning("Config secrets is None or empty!")
|
|
145
|
-
|
|
146
127
|
enable_streaming = config.details.get(StorageDetails.ENABLE_STREAMING.value, False)
|
|
147
128
|
file_key: str = _construct_file_key(privacy_request.id, config, enable_streaming)
|
|
148
129
|
|
|
@@ -154,7 +135,6 @@ def _s3_uploader(
|
|
|
154
135
|
file_key = f"{privacy_request.id}.zip"
|
|
155
136
|
# Use streaming upload for better memory efficiency
|
|
156
137
|
logger.debug("Using streaming S3 upload for {}", file_key)
|
|
157
|
-
logger.debug("Calling upload_to_s3_streaming with secrets: {}", config.secrets)
|
|
158
138
|
return upload_to_s3_streaming(
|
|
159
139
|
config.secrets, # type: ignore
|
|
160
140
|
data,
|
|
@@ -17,7 +17,6 @@ def stream_dsr_buffer_to_storage(
|
|
|
17
17
|
bucket_name: str,
|
|
18
18
|
file_key: str,
|
|
19
19
|
dsr_buffer: BytesIO,
|
|
20
|
-
content_type: str = "application/zip",
|
|
21
20
|
) -> None:
|
|
22
21
|
"""Stream DSR buffer to storage using smart-open streaming.
|
|
23
22
|
|
|
@@ -29,15 +28,12 @@ def stream_dsr_buffer_to_storage(
|
|
|
29
28
|
bucket_name: Storage bucket name
|
|
30
29
|
file_key: File key in storage
|
|
31
30
|
dsr_buffer: Pre-generated DSR report buffer (BytesIO)
|
|
32
|
-
content_type: MIME type for the uploaded file (defaults to application/zip)
|
|
33
31
|
"""
|
|
34
32
|
# Get the content from the buffer
|
|
35
33
|
content = dsr_buffer.getvalue()
|
|
36
34
|
try:
|
|
37
35
|
# Use smart-open's streaming upload for efficient memory usage
|
|
38
|
-
with storage_client.stream_upload(
|
|
39
|
-
bucket_name, file_key, content_type=content_type
|
|
40
|
-
) as upload_stream:
|
|
36
|
+
with storage_client.stream_upload(bucket_name, file_key) as upload_stream:
|
|
41
37
|
upload_stream.write(content)
|
|
42
38
|
|
|
43
39
|
except Exception as e:
|
|
@@ -7,7 +7,7 @@ from typing import Any, Optional
|
|
|
7
7
|
from fideslang.validation import AnyHttpUrlString
|
|
8
8
|
from loguru import logger
|
|
9
9
|
|
|
10
|
-
from fides.api.schemas.storage.storage import AWSAuthMethod
|
|
10
|
+
from fides.api.schemas.storage.storage import AWSAuthMethod
|
|
11
11
|
from fides.api.service.storage.s3 import create_presigned_url_for_s3
|
|
12
12
|
from fides.api.service.storage.streaming.base_storage_client import BaseStorageClient
|
|
13
13
|
from fides.api.util.aws_util import get_s3_client
|
|
@@ -20,17 +20,19 @@ class S3StorageClient(BaseStorageClient):
|
|
|
20
20
|
generation for the smart-open storage client.
|
|
21
21
|
"""
|
|
22
22
|
|
|
23
|
-
def __init__(self, storage_secrets: dict[
|
|
23
|
+
def __init__(self, auth_method: str, storage_secrets: dict[str, Any]):
|
|
24
24
|
"""Initialize the storage client with secrets.
|
|
25
25
|
|
|
26
26
|
Args:
|
|
27
|
-
storage_secrets: Provider-specific storage credentials and configuration using
|
|
27
|
+
storage_secrets: Provider-specific storage credentials and configuration using string keys
|
|
28
|
+
(e.g., "aws_access_key_id", "region_name") from format_secrets()
|
|
28
29
|
"""
|
|
29
30
|
super().__init__(storage_secrets)
|
|
30
|
-
self.storage_secrets: dict[
|
|
31
|
+
self.storage_secrets: dict[str, Any] = storage_secrets
|
|
32
|
+
self.auth_method = auth_method
|
|
31
33
|
|
|
32
34
|
def build_uri(self, bucket: str, key: str) -> str:
|
|
33
|
-
"""Build
|
|
35
|
+
"""Build S3 URI for the given bucket and key.
|
|
34
36
|
|
|
35
37
|
Args:
|
|
36
38
|
bucket: S3 bucket name
|
|
@@ -45,26 +47,75 @@ class S3StorageClient(BaseStorageClient):
|
|
|
45
47
|
|
|
46
48
|
def get_transport_params(self) -> dict[str, Any]:
|
|
47
49
|
"""Get S3-specific transport parameters for smart-open.
|
|
50
|
+
Type annotation: get_s3_client returns a boto3 S3 client object, not a Session
|
|
51
|
+
This is what smart-open expects for the "client" transport parameter
|
|
48
52
|
|
|
49
53
|
Returns:
|
|
50
54
|
Dictionary of S3 transport parameters for smart-open
|
|
51
55
|
"""
|
|
52
|
-
params = {}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
56
|
+
params: dict[str, Any] = {}
|
|
57
|
+
|
|
58
|
+
# Create S3 client for smart-open
|
|
59
|
+
try:
|
|
60
|
+
# Determine auth method based on available credentials
|
|
61
|
+
if self.auth_method == AWSAuthMethod.AUTOMATIC.value:
|
|
62
|
+
|
|
63
|
+
# For automatic authentication, check if region is available
|
|
64
|
+
if not self.storage_secrets.get("region_name", None):
|
|
65
|
+
logger.warning(
|
|
66
|
+
"No region specified in storage secrets for automatic authentication"
|
|
67
|
+
"This may cause credential issues - consider setting a default region"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
# Extract assume_role_arn if present
|
|
71
|
+
assume_role_arn = None
|
|
72
|
+
if (
|
|
73
|
+
"assume_role_arn" in self.storage_secrets
|
|
74
|
+
and self.storage_secrets["assume_role_arn"]
|
|
75
|
+
):
|
|
76
|
+
assume_role_arn = self.storage_secrets["assume_role_arn"]
|
|
77
|
+
logger.debug(f"Using assume role ARN: {assume_role_arn}")
|
|
78
|
+
|
|
79
|
+
# Create S3 client using existing utility
|
|
80
|
+
# get_s3_client returns a boto3 S3 client, not a Session
|
|
81
|
+
s3_client: Any = None
|
|
82
|
+
try:
|
|
83
|
+
s3_client = get_s3_client(
|
|
84
|
+
self.auth_method, self.storage_secrets, assume_role_arn # type: ignore
|
|
85
|
+
)
|
|
86
|
+
logger.debug("Successfully created S3 client")
|
|
87
|
+
except Exception as e:
|
|
88
|
+
# For automatic authentication, try to provide more helpful error messages
|
|
89
|
+
if self.auth_method == AWSAuthMethod.AUTOMATIC.value:
|
|
90
|
+
logger.error(
|
|
91
|
+
f"Failed to create S3 client with automatic authentication: {e}. "
|
|
92
|
+
"This usually means AWS credentials are not available in the environment"
|
|
93
|
+
"Please ensure AWS credentials are configured via environment variables, IAM roles, or AWS profiles"
|
|
94
|
+
)
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"Automatic AWS authentication failed: {e}. Please check your AWS credential configuration."
|
|
97
|
+
)
|
|
98
|
+
raise
|
|
99
|
+
|
|
100
|
+
params["client"] = s3_client
|
|
101
|
+
|
|
102
|
+
except Exception as e:
|
|
103
|
+
logger.error(f"Failed to create S3 client for smart-open: {e}")
|
|
104
|
+
raise
|
|
105
|
+
|
|
106
|
+
# Include credentials at top level for compatibility
|
|
107
|
+
# Note: When using an S3 client, these credential parameters are not needed
|
|
108
|
+
# and will be ignored by smart-open, causing warnings
|
|
109
|
+
# Only include them if no S3 client is provided (fallback scenario)
|
|
110
|
+
if not params.get("client"):
|
|
111
|
+
for key, transport_key in [
|
|
112
|
+
("aws_access_key_id", "access_key"),
|
|
113
|
+
("aws_secret_access_key", "secret_key"),
|
|
114
|
+
("region_name", "region"),
|
|
115
|
+
("assume_role_arn", "assume_role_arn"),
|
|
116
|
+
]:
|
|
117
|
+
if key in self.storage_secrets and self.storage_secrets[key]:
|
|
118
|
+
params[transport_key] = self.storage_secrets[key]
|
|
68
119
|
|
|
69
120
|
return params
|
|
70
121
|
|
|
@@ -85,28 +136,15 @@ class S3StorageClient(BaseStorageClient):
|
|
|
85
136
|
Exception: If presigned URL generation fails
|
|
86
137
|
"""
|
|
87
138
|
try:
|
|
88
|
-
# Storage secrets are already in the right format for get_s3_client
|
|
89
|
-
# get_s3_client expects dict[StorageSecrets, Any] with enum keys
|
|
90
|
-
s3_secrets = self.storage_secrets
|
|
91
|
-
|
|
92
|
-
# Determine auth method based on available credentials
|
|
93
|
-
# If AWS credentials are present, use SECRET_KEYS, otherwise use AUTOMATIC
|
|
94
|
-
if (
|
|
95
|
-
StorageSecrets.AWS_ACCESS_KEY_ID in self.storage_secrets
|
|
96
|
-
and StorageSecrets.AWS_SECRET_ACCESS_KEY in self.storage_secrets
|
|
97
|
-
and self.storage_secrets[StorageSecrets.AWS_ACCESS_KEY_ID]
|
|
98
|
-
and self.storage_secrets[StorageSecrets.AWS_SECRET_ACCESS_KEY]
|
|
99
|
-
):
|
|
100
|
-
auth_method = AWSAuthMethod.SECRET_KEYS.value
|
|
101
|
-
else:
|
|
102
|
-
auth_method = AWSAuthMethod.AUTOMATIC.value
|
|
103
|
-
|
|
104
139
|
# Extract assume_role_arn if present for role assumption
|
|
105
140
|
assume_role_arn = None
|
|
106
|
-
if
|
|
107
|
-
assume_role_arn = self.storage_secrets[
|
|
141
|
+
if "assume_role_arn" in self.storage_secrets:
|
|
142
|
+
assume_role_arn = self.storage_secrets["assume_role_arn"]
|
|
108
143
|
|
|
109
|
-
|
|
144
|
+
# get_s3_client returns a boto3 S3 client, not a Session
|
|
145
|
+
s3_client: Any = get_s3_client(
|
|
146
|
+
self.auth_method, self.storage_secrets, assume_role_arn # type: ignore
|
|
147
|
+
)
|
|
110
148
|
return create_presigned_url_for_s3(s3_client, bucket, key, ttl_seconds)
|
|
111
149
|
except Exception as e:
|
|
112
150
|
logger.error(f"Failed to generate S3 presigned URL: {e}")
|