ethyca-fides 2.59.2rc0__py2.py3-none-any.whl → 2.59.3b0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ethyca_fides-2.59.2rc0.dist-info → ethyca_fides-2.59.3b0.dist-info}/METADATA +2 -2
- {ethyca_fides-2.59.2rc0.dist-info → ethyca_fides-2.59.3b0.dist-info}/RECORD +214 -211
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/c9c72b3d550b_data_migration_cookie_asset.py +218 -0
- fides/api/api/v1/endpoints/storage_endpoints.py +7 -6
- fides/api/db/system.py +1 -121
- fides/api/models/asset.py +1 -2
- fides/api/models/privacy_notice.py +14 -19
- fides/api/models/sql_models.py +0 -51
- fides/api/schemas/application_config.py +1 -0
- fides/api/schemas/connection_configuration/connection_secrets_datahub.py +3 -3
- fides/api/schemas/connection_configuration/connection_secrets_mysql.py +12 -0
- fides/api/schemas/connection_configuration/connection_secrets_website.py +4 -2
- fides/api/schemas/consentable_item.py +5 -8
- fides/api/schemas/messaging/messaging.py +9 -2
- fides/api/schemas/storage/storage.py +40 -4
- fides/api/schemas/storage/storage_secrets_docs_only.py +8 -2
- fides/api/schemas/system.py +1 -2
- fides/api/service/connectors/mysql_connector.py +13 -1
- fides/api/service/connectors/saas_connector.py +18 -13
- fides/api/service/saas_request/saas_request_override_factory.py +3 -3
- fides/api/service/storage/gcs.py +38 -0
- fides/api/service/storage/storage_authenticator_service.py +39 -16
- fides/api/service/storage/storage_uploader_service.py +25 -1
- fides/api/task/graph_task.py +2 -3
- fides/api/tasks/storage.py +50 -2
- fides/api/util/aws_util.py +28 -17
- fides/api/util/consent_util.py +10 -4
- fides/api/util/storage_util.py +2 -0
- fides/service/messaging/aws_ses_service.py +25 -16
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/-PblD_AgOnSlhLxTLZi_9/_buildManifest.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1100-0046887f49e9e7c7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1817-a3a32e6b9dbb1fa3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{1904-281183a117e26585.js → 1904-e1034674c6eef004.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{2479-b4f7a1c7c711cb65.js → 2479-b4f7654c27a15c05.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/3503-d8e375550e56f343.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3513-98745ae0f615fba1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3702-3d814671c31eaca3.js → 3702-8a6360b2deef2a87.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/3855-7a43acdb1f8de2f5.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3872-a650869600244910.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/401-a15fcc78cc272c09.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4060-93514991b84ca46c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4121-eb6970efae35eefc.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4481-c19a0c5e273a3ec7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/5487-61ad3f5d4f08a602.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{5826-1f4f74bf3b5348e4.js → 5826-1ab083b9ceb98988.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/5835-283ca343b5b4cef3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/5973-25b9c987c0a9ae4b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6277-9d4e419633574bee.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6372-2aee10b79089521a.js → 6372-3d6d19a89e1d7a37.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6853-45e8c48ad0fdf1db.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6954-6de30beefefac64a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/7751-851b3ad999abc3de.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/79-4882c5dcf25370ee.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/8433-234a6feeb457cbb8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/8934-296cdc5dd1118b47.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/9282-dc4848a62893b25e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/9327-6243fdc967120bc6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9392.da35a8e778812d91.js → 9392.9178313b7a01d889.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9572-c34e18bd40467909.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9676.f297a1347ac09c56.js → 9676.d1e42fd1d03c4a76.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9719-d3de30714871f267.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/9767-ee5b0af0e65d3802.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9951-66aad0602aadfbe2.js → 9951-c61c4ff3597a481d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9965-dbb07277c85ac53a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{404-29560aa2e6a60963.js → 404-b482dd9e6ba3397d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-1ce747bb4aa843da.js → _app-c9fca18ff48ecfc5.js} +48 -48
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/manual-e16fcedb2215db13.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-2392e3a101fae073.js → multiple-892fc00510cddef8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems-fc8a5f443dd81085.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/ant-poc-2bade915860dd327.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-7dfe37b9f0ff9a3c.js → add-vendors-cc58c1e18966fcfd.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-dba7848b760ba227.js → configure-8bd11c8df37a90ed.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/[id]-44b53932c42a2f9b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-c02b14c50b19bd91.js → new-300fd2800c417d0b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience-397b121b78bf8ea5.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-75c2ed6ba3de48cd.js → [id]-72dd15bfd87616c9.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-487ae57dc7e2ded2.js → new-8bbccdd747f4100c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices-1f402b7525f78140.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-f67fda6a71d0a46b.js → properties-55cc394ebc428e59.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-de9118ec446ce994.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{consent-75395dfc224cf44b.js → consent-3b4b4a0d2cb9b62b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-fa65c67ec8c4b5e6.js → [projectUrn]-b0b976e8dd5e4cd6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-e76a07ee6ee8d55f.js → projects-4ea56a1c3010ba99.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-f8760e1b908d1f4c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-cdd23dc94e1fbf89.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-94308fff1468b5e6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-96ed37484e58ea12.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/activity-7e25ddfe45048b35.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-1bcaa606739ea1c5.js → [resourceUrn]-10f789d4ab2b8927.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-22f55dc12354b4c6.js → detection-f473b2ee1c67be3c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-86111c25dc30ef2b.js → [resourceUrn]-2569551ae75ea5a2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-853be75f08b9ab5c.js → discovery-ad077f30849fdc0a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{datamap-423172d31de8e9c6.js → datamap-830dda342f368353.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-704553f5329fb9d4.js → [...subfieldNames]-da407b63cd3f5276.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-06c19dca941edb14.js → [collectionName]-76205fa5ddc5c3ae.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-eac517f43d5f53a8.js → [datasetId]-ab7c88da427b6396.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/new-444b87ad53458367.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-f4d57b3432c57112.js → dataset-39d8340d9554fe48.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-b2b1168d0361baa8.js → [id]-e39d66c8dfdc4c6f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-c804349a417a888b.js → new-8f2cfc79022cb6d0.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-3cabb50677e47db9.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/index-e1eb64f593b0ccc3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-0fc3ffd11ba163f9.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-d584110d81e5b395.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-66f5fbadd8455805.js → [id]-1d18e28fa55a149b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-d441abb1f045940f.js → add-template-11bdab1b292c6f89.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-de5efa1ff6d0e9ea.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{[id]-54ea39f203182c53.js → [id]-051412abae59451b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-c69436a231dd7cf1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-8a2e12ddfa5a423e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure-52bf8401c60d2847.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-60e356e69eb1ed16.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-f4fb941df069b7bf.js → [id]-835b7c4dd2240d16.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-bccb6ffab25aa214.js → add-property-984851804558be53.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{properties-9a7ac623370b7c00.js → properties-d2b95b0ebae06804.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-5d1aededc7523146.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/about-83c33e9ff98ad009.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/[purpose_id]-52879ce4c099bdd0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent-5ea84b682427181e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-ae05fb2ea8c30974.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-d9088f5cd9fb2822.js → domain-records-888b6e4374fc766e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/domains-08b1f8e454931f20.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/email-templates-b6824ffc25575c84.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-759564ca0ae62840.js → locations-a8cd013bdd3aef75.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/organization-1a35f97f2290e117.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-2dce4501fca920b3.js → regulations-5abb227df116c9c4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/test-datasets-da1b2dd3ec61a08e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]-5bdfee002e18a8b2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/systems-1397c0ab23bb780c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-e9f8f8cb92dac79b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-082c3156175f9267.js → new-65ef1bdebb679666.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/{[id]-d527d1d1635541df.js → [id]-28abb07c95f2c7c2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management-4eb69d5c6af8581b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{webpack-03e375f6d6b2c71b.js → webpack-cf42f5a0e9af0ace.js} +1 -1
- fides/ui-build/static/admin/_next/static/css/a063d3d67fe688f6.css +1 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/ant-poc.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -0
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/P3RkpoojridGfvvbDI91a/_buildManifest.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1100-45c0634b4f51d10c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1376-92890c17ce39ca0d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/146-d1820217dc36d46d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1817-6abbe957a53026d7.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3086-d1ba90bc6ac9174b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3244-3d94bf3393366412.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3855-237afbbea7f54707.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3872-c53d74aebd12cfd3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/401-50974f107c2712fc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4060-52fd85fb1e15a2b8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4121-476f657b03a78965.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4481-288d74a33473cff7.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5102-626b9ff42e904276.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5480-1623dbacdf4e6794.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5487-9a2aecb0ec63d567.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5973-942ce645a4279587.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6395-4224d6d26d1e8bb7.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6853-ca5dacd25c9ccb7c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-127745bfdc5bc0b3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7751-3913b8c055f599e5.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/79-77cfed02164241ca.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/8433-c4c765833ab9cc07.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/8934-ffa2b0509bc7a845.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9282-1a48b10b114d01f4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9327-4970d356f7000c0b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9572-2b9b10e146130c85.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9767-bf415d0daea960cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/manual-58e9256e86916ecd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems-a6926f7ec7ad10cf.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/ant-poc-404f3c9018952800.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/[id]-78de4bde88e18b0f.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience-7eeeee3769e73f78.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices-65091908d6296afc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-7a1976c0d1aca8b6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-75989f9732d90793.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-6b3d1fe762c747d8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-61eafb8444bffb76.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-38476c697da53480.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/activity-e7869f658e3017b9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/new-5aff1d01c6c30ad6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-0faa9b3c4555e585.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/index-e72904e316ede1b0.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-17191a759e167ca8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-88e51c81a380ebad.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-eef56c95b08aa24c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-d889076067104e56.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-3355b4803b2916f3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure-b8c94b10ab90b061.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-3ac47981f1e2b0aa.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-fe743440d7eb007b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/about-4412a7b468b6d4bf.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent-cdc866af93898716.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-a7dc8113067dff42.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/domains-2f03e981234c40ad.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/email-templates-cf09ad896c7396a6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/organization-5bcc7a6976e01a56.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/test-datasets-0b47ff26897c1d9a.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]-dd053a3bf2a9ca6c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems-820893393f1516ee.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-8fccd670220dbf60.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management-facb8c0128a44cb1.js +0 -1
- fides/ui-build/static/admin/_next/static/css/687135955af5b7e1.css +0 -1
- {ethyca_fides-2.59.2rc0.dist-info → ethyca_fides-2.59.3b0.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.59.2rc0.dist-info → ethyca_fides-2.59.3b0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.59.2rc0.dist-info → ethyca_fides-2.59.3b0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.59.2rc0.dist-info → ethyca_fides-2.59.3b0.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{P3RkpoojridGfvvbDI91a → -PblD_AgOnSlhLxTLZi_9}/_ssgManifest.js +0 -0
@@ -3,7 +3,6 @@ from typing import Dict, List, Literal, Optional
|
|
3
3
|
from pydantic import BaseModel, Field
|
4
4
|
|
5
5
|
from fides.api.models.consent_automation import ConsentableItem as ConsentableItemModel
|
6
|
-
from fides.api.models.privacy_notice import UserConsentPreference
|
7
6
|
from fides.api.schemas.base_class import FidesSchema
|
8
7
|
|
9
8
|
|
@@ -85,10 +84,8 @@ class ConsentWebhookResult(BaseModel):
|
|
85
84
|
|
86
85
|
identity_map: Dict[
|
87
86
|
Literal["email", "phone_number", "fides_user_device", "external_id"], str
|
88
|
-
] =
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
"""Returns true if both the identity map and notice map are not empty."""
|
94
|
-
return bool(self.identity_map) and bool(self.notice_map)
|
87
|
+
] = Field(default_factory=dict, description="The identity of the user.")
|
88
|
+
notice_id_map: Dict[str, str] = Field(
|
89
|
+
default_factory=dict,
|
90
|
+
description="A map of privacy notice IDs to user consent preferences.",
|
91
|
+
)
|
@@ -286,12 +286,19 @@ class MessagingServiceDetailsTwilioEmail(BaseModel):
|
|
286
286
|
class MessagingServiceDetailsAWS_SES(BaseModel):
|
287
287
|
"""The details required to represent an AWS SES email configuration."""
|
288
288
|
|
289
|
-
email_from: str
|
290
|
-
domain: str
|
289
|
+
email_from: Optional[str] = None
|
290
|
+
domain: Optional[str] = None
|
291
291
|
aws_region: str
|
292
292
|
|
293
293
|
model_config = ConfigDict(extra="forbid")
|
294
294
|
|
295
|
+
@model_validator(mode="before")
|
296
|
+
@classmethod
|
297
|
+
def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]:
|
298
|
+
if not values.get("domain") and not values.get("email_from"):
|
299
|
+
raise ValueError("Either 'email_from' or 'domain' must be provided.")
|
300
|
+
return values
|
301
|
+
|
295
302
|
|
296
303
|
class MessagingServiceSecrets(Enum):
|
297
304
|
"""Enum for message service secrets"""
|
@@ -61,6 +61,20 @@ class StorageDetailsS3(FileBasedStorageDetails):
|
|
61
61
|
model_config = ConfigDict(use_enum_values=True)
|
62
62
|
|
63
63
|
|
64
|
+
class GCSAuthMethod(str, Enum):
|
65
|
+
ADC = "adc" # Application Default Credentials
|
66
|
+
SERVICE_ACCOUNT_KEYS = "service_account_keys"
|
67
|
+
|
68
|
+
|
69
|
+
class StorageDetailsGCS(FileBasedStorageDetails):
|
70
|
+
"""The details required to represent a Google Cloud Storage bucket."""
|
71
|
+
|
72
|
+
auth_method: GCSAuthMethod
|
73
|
+
bucket: str
|
74
|
+
max_retries: Optional[int] = 0
|
75
|
+
model_config = ConfigDict(use_enum_values=True)
|
76
|
+
|
77
|
+
|
64
78
|
class StorageDetailsLocal(FileBasedStorageDetails):
|
65
79
|
"""The details required to configurate local storage configuration"""
|
66
80
|
|
@@ -71,6 +85,8 @@ class StorageSecrets(Enum):
|
|
71
85
|
# s3-specific
|
72
86
|
AWS_ACCESS_KEY_ID = "aws_access_key_id"
|
73
87
|
AWS_SECRET_ACCESS_KEY = "aws_secret_access_key"
|
88
|
+
REGION_NAME = "region_name"
|
89
|
+
AWS_ASSUME_ROLE = "assume_role_arn"
|
74
90
|
|
75
91
|
|
76
92
|
class StorageSecretsLocal(BaseModel):
|
@@ -80,10 +96,27 @@ class StorageSecretsLocal(BaseModel):
|
|
80
96
|
|
81
97
|
|
82
98
|
class StorageSecretsS3(BaseModel):
|
83
|
-
|
99
|
+
aws_access_key_id: Optional[str] = None
|
100
|
+
aws_secret_access_key: Optional[str] = None
|
101
|
+
region_name: Optional[str] = None
|
102
|
+
assume_role_arn: Optional[str] = None
|
103
|
+
model_config = ConfigDict(extra="forbid")
|
104
|
+
|
84
105
|
|
85
|
-
|
86
|
-
|
106
|
+
class StorageSecretsGCS(BaseModel):
|
107
|
+
"""The secrets required to connect to a Google Cloud Storage bucket."""
|
108
|
+
|
109
|
+
type: str = "service_account"
|
110
|
+
project_id: str
|
111
|
+
private_key_id: str
|
112
|
+
private_key: str
|
113
|
+
client_email: str
|
114
|
+
client_id: str
|
115
|
+
auth_uri: str
|
116
|
+
token_uri: str
|
117
|
+
auth_provider_x509_cert_url: str
|
118
|
+
client_x509_cert_url: str
|
119
|
+
universe_domain: str
|
87
120
|
model_config = ConfigDict(extra="forbid")
|
88
121
|
|
89
122
|
|
@@ -99,6 +132,7 @@ class StorageType(Enum):
|
|
99
132
|
|
100
133
|
FULLY_CONFIGURED_STORAGE_TYPES = (
|
101
134
|
StorageType.s3,
|
135
|
+
StorageType.gcs,
|
102
136
|
) # storage types that are considered "fully configured"
|
103
137
|
|
104
138
|
|
@@ -108,6 +142,7 @@ class StorageDestinationBase(BaseModel):
|
|
108
142
|
type: StorageType
|
109
143
|
details: Union[
|
110
144
|
StorageDetailsS3,
|
145
|
+
StorageDetailsGCS,
|
111
146
|
StorageDetailsLocal,
|
112
147
|
] = Field(validate_default=True)
|
113
148
|
format: Optional[ResponseFormat] = ResponseFormat.json.value # type: ignore
|
@@ -145,6 +180,7 @@ class StorageDestinationBase(BaseModel):
|
|
145
180
|
try:
|
146
181
|
schema = {
|
147
182
|
StorageType.s3.value: StorageDetailsS3,
|
183
|
+
StorageType.gcs.value: StorageDetailsGCS,
|
148
184
|
StorageType.local.value: StorageDetailsLocal,
|
149
185
|
}[storage_type]
|
150
186
|
except KeyError:
|
@@ -209,7 +245,7 @@ class BulkPutStorageConfigResponse(BulkResponse):
|
|
209
245
|
failed: List[BulkUpdateFailed] = []
|
210
246
|
|
211
247
|
|
212
|
-
SUPPORTED_STORAGE_SECRETS = StorageSecretsS3
|
248
|
+
SUPPORTED_STORAGE_SECRETS = Union[StorageSecretsS3, StorageSecretsGCS]
|
213
249
|
|
214
250
|
|
215
251
|
class StorageConfigStatus(Enum):
|
@@ -1,9 +1,15 @@
|
|
1
|
+
from typing import Union
|
2
|
+
|
1
3
|
from fides.api.schemas.base_class import NoValidationSchema
|
2
|
-
from fides.api.schemas.storage.storage import StorageSecretsS3
|
4
|
+
from fides.api.schemas.storage.storage import StorageSecretsGCS, StorageSecretsS3
|
3
5
|
|
4
6
|
|
5
7
|
class StorageSecretsS3Docs(StorageSecretsS3, NoValidationSchema):
|
6
8
|
"""The secrets required to connect to S3, for documentation"""
|
7
9
|
|
8
10
|
|
9
|
-
|
11
|
+
class StorageSecretsGCSDocs(StorageSecretsGCS, NoValidationSchema):
|
12
|
+
"""The secrets required to connect to Google Cloud Storage, for documentation"""
|
13
|
+
|
14
|
+
|
15
|
+
possible_storage_secrets = Union[StorageSecretsS3Docs, StorageSecretsGCSDocs]
|
fides/api/schemas/system.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
from datetime import datetime
|
2
2
|
from typing import Any, Dict, List, Optional, Sequence
|
3
3
|
|
4
|
-
from fideslang.models import
|
4
|
+
from fideslang.models import PrivacyDeclaration, System
|
5
5
|
from pydantic import ConfigDict, Field
|
6
6
|
from pydantic.main import BaseModel
|
7
7
|
|
@@ -17,7 +17,6 @@ class PrivacyDeclarationResponse(PrivacyDeclaration):
|
|
17
17
|
id: str = Field(
|
18
18
|
description="The database-assigned ID of the privacy declaration on the system. This is meant to be a read-only field, returned only in API responses"
|
19
19
|
)
|
20
|
-
cookies: Optional[List[Cookies]] = []
|
21
20
|
|
22
21
|
|
23
22
|
class BasicSystemResponse(System):
|
@@ -1,10 +1,11 @@
|
|
1
|
-
from typing import List
|
1
|
+
from typing import Dict, List
|
2
2
|
|
3
3
|
from sqlalchemy.engine import Engine, LegacyCursorResult, create_engine # type: ignore
|
4
4
|
|
5
5
|
from fides.api.graph.execution import ExecutionNode
|
6
6
|
from fides.api.schemas.connection_configuration.connection_secrets_mysql import (
|
7
7
|
MySQLSchema,
|
8
|
+
MySQLSSLMode,
|
8
9
|
)
|
9
10
|
from fides.api.service.connectors.query_configs.mysql_query_config import (
|
10
11
|
MySQLQueryConfig,
|
@@ -69,16 +70,27 @@ class MySQLConnector(SQLConnector):
|
|
69
70
|
uri = self.build_ssh_uri(local_address=self.ssh_server.local_bind_address)
|
70
71
|
else:
|
71
72
|
uri = (self.configuration.secrets or {}).get("url") or self.build_uri()
|
73
|
+
connect_args = self.get_connect_args()
|
72
74
|
return create_engine(
|
73
75
|
uri,
|
74
76
|
hide_parameters=self.hide_parameters,
|
75
77
|
echo=not self.hide_parameters,
|
78
|
+
connect_args=connect_args,
|
76
79
|
)
|
77
80
|
|
78
81
|
def query_config(self, node: ExecutionNode) -> SQLQueryConfig:
|
79
82
|
"""Query wrapper corresponding to the input execution_node."""
|
80
83
|
return MySQLQueryConfig(node)
|
81
84
|
|
85
|
+
def get_connect_args(self) -> Dict[str, Dict[str, MySQLSSLMode]]:
|
86
|
+
"""Get connection arguments for the engine"""
|
87
|
+
ssl_mode = self.configuration.secrets.get("ssl_mode", MySQLSSLMode.preferred)
|
88
|
+
return {
|
89
|
+
"ssl": {
|
90
|
+
"mode": ssl_mode,
|
91
|
+
}
|
92
|
+
}
|
93
|
+
|
82
94
|
@staticmethod
|
83
95
|
def cursor_result_to_rows(results: LegacyCursorResult) -> List[Row]:
|
84
96
|
"""
|
@@ -4,6 +4,7 @@ from json import JSONDecodeError
|
|
4
4
|
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
5
5
|
|
6
6
|
import pydash
|
7
|
+
from fideslang.validation import FidesKey
|
7
8
|
from loguru import logger
|
8
9
|
from requests import Response
|
9
10
|
from sqlalchemy.orm import Session
|
@@ -18,7 +19,6 @@ from fides.api.common_exceptions import (
|
|
18
19
|
from fides.api.graph.execution import ExecutionNode
|
19
20
|
from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
|
20
21
|
from fides.api.models.policy import Policy
|
21
|
-
from fides.api.models.privacy_notice import UserConsentPreference
|
22
22
|
from fides.api.models.privacy_request import PrivacyRequest, RequestTask
|
23
23
|
from fides.api.schemas.consentable_item import (
|
24
24
|
ConsentableItem,
|
@@ -687,15 +687,16 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
|
|
687
687
|
|
688
688
|
if notice_based_override_function:
|
689
689
|
# follow the notice-based SaaS consent flow
|
690
|
-
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
690
|
+
(
|
691
|
+
notice_preference_map,
|
692
|
+
filtered_preferences,
|
693
|
+
) = build_user_consent_and_filtered_preferences_for_service(
|
694
|
+
self.configuration.system,
|
695
|
+
privacy_request,
|
696
|
+
session,
|
697
|
+
True,
|
697
698
|
)
|
698
|
-
if not
|
699
|
+
if not notice_preference_map:
|
699
700
|
logger.info(
|
700
701
|
"Skipping consent requests on node {}: No actionable consent preferences to propagate",
|
701
702
|
node.address.value,
|
@@ -723,12 +724,13 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
|
|
723
724
|
)
|
724
725
|
consent_propagation_status = self._invoke_consent_request_override(
|
725
726
|
notice_based_override_function,
|
727
|
+
self.configuration.key,
|
726
728
|
self.create_client(),
|
727
729
|
policy,
|
728
730
|
privacy_request,
|
729
731
|
self.secrets,
|
730
732
|
identity_data,
|
731
|
-
|
733
|
+
notice_preference_map, # type: ignore[arg-type]
|
732
734
|
notice_based_consentable_item_hierarchy,
|
733
735
|
)
|
734
736
|
if consent_propagation_status == ConsentPropagationStatus.no_update_needed:
|
@@ -800,6 +802,7 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
|
|
800
802
|
)
|
801
803
|
consent_propagation_status = self._invoke_consent_request_override(
|
802
804
|
override_function,
|
805
|
+
self.configuration.key,
|
803
806
|
self.create_client(),
|
804
807
|
policy,
|
805
808
|
privacy_request,
|
@@ -997,12 +1000,13 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
|
|
997
1000
|
@staticmethod
|
998
1001
|
def _invoke_consent_request_override(
|
999
1002
|
override_function: RequestOverrideFunction,
|
1003
|
+
connection_key: FidesKey,
|
1000
1004
|
client: AuthenticatedClient,
|
1001
1005
|
policy: Policy,
|
1002
1006
|
privacy_request: PrivacyRequest,
|
1003
1007
|
secrets: Any,
|
1004
1008
|
identity_data: Optional[Dict[str, Any]] = None,
|
1005
|
-
|
1009
|
+
notice_preference_map: Optional[Dict[str, Dict[str, Any]]] = None,
|
1006
1010
|
consentable_items_hierarchy: Optional[List[ConsentableItem]] = None,
|
1007
1011
|
) -> ConsentPropagationStatus:
|
1008
1012
|
"""
|
@@ -1011,13 +1015,14 @@ class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
|
|
1011
1015
|
"""
|
1012
1016
|
try:
|
1013
1017
|
logger.info("Invoking consent request override function...")
|
1014
|
-
if
|
1018
|
+
if notice_preference_map:
|
1015
1019
|
# At this point, we've already validated the override function signature to take these params
|
1016
1020
|
return override_function(
|
1021
|
+
connection_key,
|
1017
1022
|
client,
|
1018
1023
|
secrets,
|
1019
1024
|
identity_data,
|
1020
|
-
|
1025
|
+
notice_preference_map,
|
1021
1026
|
consentable_items_hierarchy,
|
1022
1027
|
) # type: ignore
|
1023
1028
|
return override_function(
|
@@ -248,7 +248,7 @@ def validate_update_consent_function(f: Callable) -> None:
|
|
248
248
|
)
|
249
249
|
if len(sig.parameters) < 5:
|
250
250
|
raise InvalidSaaSRequestOverrideException(
|
251
|
-
"Provided SaaS update consent function must declare at least
|
251
|
+
"Provided SaaS update consent function must declare at least 5 parameters"
|
252
252
|
)
|
253
253
|
|
254
254
|
|
@@ -258,9 +258,9 @@ def validate_process_consent_webhook_function(f: Callable) -> None:
|
|
258
258
|
raise InvalidSaaSRequestOverrideException(
|
259
259
|
"Provided SaaS process consent webhook function must return a ConsentWebhookResult"
|
260
260
|
)
|
261
|
-
if len(sig.parameters) <
|
261
|
+
if len(sig.parameters) < 4:
|
262
262
|
raise InvalidSaaSRequestOverrideException(
|
263
|
-
"Provided SaaS process consent webhook function must declare at least
|
263
|
+
"Provided SaaS process consent webhook function must declare at least 4 parameters"
|
264
264
|
)
|
265
265
|
|
266
266
|
|
@@ -0,0 +1,38 @@
|
|
1
|
+
from typing import Dict, Optional
|
2
|
+
|
3
|
+
from google.cloud.storage import Client # type: ignore
|
4
|
+
from google.oauth2 import service_account
|
5
|
+
from loguru import logger
|
6
|
+
|
7
|
+
from fides.api.common_exceptions import StorageUploadError
|
8
|
+
from fides.api.schemas.storage.storage import GCSAuthMethod
|
9
|
+
|
10
|
+
|
11
|
+
def get_gcs_client(
|
12
|
+
auth_method: str,
|
13
|
+
storage_secrets: Optional[Dict],
|
14
|
+
) -> Client:
|
15
|
+
"""
|
16
|
+
Abstraction to retrieve a GCS client using secrets.
|
17
|
+
"""
|
18
|
+
if auth_method == GCSAuthMethod.ADC.value:
|
19
|
+
storage_client = Client()
|
20
|
+
|
21
|
+
elif auth_method == GCSAuthMethod.SERVICE_ACCOUNT_KEYS.value:
|
22
|
+
if not storage_secrets:
|
23
|
+
err_msg = "Storage secrets not found for Google Cloud Storage."
|
24
|
+
logger.warning(err_msg)
|
25
|
+
raise StorageUploadError(err_msg)
|
26
|
+
|
27
|
+
credentials = service_account.Credentials.from_service_account_info(
|
28
|
+
dict(storage_secrets)
|
29
|
+
)
|
30
|
+
storage_client = Client(credentials=credentials)
|
31
|
+
|
32
|
+
else:
|
33
|
+
logger.error("Google Cloud Storage auth method not supported: {}", auth_method)
|
34
|
+
raise ValueError(
|
35
|
+
f"Google Cloud Storage auth method not supported: {auth_method}"
|
36
|
+
)
|
37
|
+
|
38
|
+
return storage_client
|
@@ -1,7 +1,12 @@
|
|
1
|
-
from typing import Any
|
1
|
+
from typing import Any
|
2
2
|
|
3
3
|
from botocore.exceptions import ClientError
|
4
|
+
from google.auth.exceptions import GoogleAuthError
|
5
|
+
from google.auth.transport.requests import Request
|
6
|
+
from google.oauth2 import service_account
|
7
|
+
from loguru import logger
|
4
8
|
|
9
|
+
from fides.api.models.storage import StorageConfig
|
5
10
|
from fides.api.schemas.storage.storage import (
|
6
11
|
SUPPORTED_STORAGE_SECRETS,
|
7
12
|
AWSAuthMethod,
|
@@ -13,32 +18,50 @@ from fides.api.util.aws_util import get_aws_session
|
|
13
18
|
|
14
19
|
def secrets_are_valid(
|
15
20
|
secrets: SUPPORTED_STORAGE_SECRETS,
|
16
|
-
|
21
|
+
storage_config: StorageConfig,
|
17
22
|
) -> bool:
|
18
23
|
"""Authenticates upload destination with appropriate upload method"""
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
)
|
27
|
-
uploader: Any = _get_authenticator_from_config(storage_type)
|
28
|
-
return uploader(secrets)
|
29
|
-
|
30
|
-
|
31
|
-
def _s3_authenticator(secrets: Dict[StorageSecrets, Any]) -> bool:
|
24
|
+
uploader: Any = _get_authenticator_from_config(storage_config.type) # type: ignore
|
25
|
+
return uploader(storage_config, secrets)
|
26
|
+
|
27
|
+
|
28
|
+
def _s3_authenticator(
|
29
|
+
config: StorageConfig, secrets: dict[StorageSecrets, Any]
|
30
|
+
) -> bool:
|
32
31
|
"""Authenticates secrets for s3, returns true if secrets are valid"""
|
33
32
|
try:
|
34
|
-
get_aws_session(AWSAuthMethod.SECRET_KEYS.value, secrets.model_dump(mode="json")) # type: ignore
|
33
|
+
get_aws_session(config.details["auth_method"] or AWSAuthMethod.SECRET_KEYS.value, secrets.model_dump(mode="json")) # type: ignore
|
35
34
|
return True
|
36
35
|
except ClientError:
|
37
36
|
return False
|
38
37
|
|
39
38
|
|
39
|
+
def _gcs_authenticator(storage_config: StorageConfig, secrets: dict) -> bool:
|
40
|
+
"""Autenticates secrets for Google Cloud Storage, returns true if secrets are valid"""
|
41
|
+
try:
|
42
|
+
credentials = service_account.Credentials.from_service_account_info(
|
43
|
+
dict(secrets),
|
44
|
+
scopes=["https://www.googleapis.com/auth/devstorage.read_only"],
|
45
|
+
)
|
46
|
+
# To validate the credentials, it is necessary to make a request to Google Cloud API
|
47
|
+
credentials.refresh(Request())
|
48
|
+
return True
|
49
|
+
|
50
|
+
except GoogleAuthError as auth_error:
|
51
|
+
logger.warning(
|
52
|
+
"Google authentication error trying to authenticate GCS secrets: {}",
|
53
|
+
auth_error,
|
54
|
+
)
|
55
|
+
return False
|
56
|
+
|
57
|
+
except Exception as e:
|
58
|
+
logger.warning("Unexpected error authenticating GCS secrets: {}", e)
|
59
|
+
return False
|
60
|
+
|
61
|
+
|
40
62
|
def _get_authenticator_from_config(storage_type: StorageType) -> Any:
|
41
63
|
"""Determines which uploader method to use based on storage type"""
|
42
64
|
return {
|
43
65
|
StorageType.s3.value: _s3_authenticator,
|
66
|
+
StorageType.gcs.value: _gcs_authenticator,
|
44
67
|
}[storage_type.value]
|
@@ -14,7 +14,7 @@ from fides.api.schemas.storage.storage import (
|
|
14
14
|
StorageDetails,
|
15
15
|
StorageType,
|
16
16
|
)
|
17
|
-
from fides.api.tasks.storage import upload_to_local, upload_to_s3
|
17
|
+
from fides.api.tasks.storage import upload_to_gcs, upload_to_local, upload_to_s3
|
18
18
|
|
19
19
|
|
20
20
|
def upload(
|
@@ -78,6 +78,7 @@ def _get_uploader_from_config_type(storage_type: StorageType) -> Any:
|
|
78
78
|
return {
|
79
79
|
StorageType.s3.value: _s3_uploader,
|
80
80
|
StorageType.local.value: _local_uploader,
|
81
|
+
StorageType.gcs.value: _gcs_uploader,
|
81
82
|
}[storage_type.value]
|
82
83
|
|
83
84
|
|
@@ -106,6 +107,29 @@ def _s3_uploader(
|
|
106
107
|
)
|
107
108
|
|
108
109
|
|
110
|
+
def _gcs_uploader(
|
111
|
+
_: Session,
|
112
|
+
config: StorageConfig,
|
113
|
+
data: Dict,
|
114
|
+
privacy_request: PrivacyRequest,
|
115
|
+
) -> str:
|
116
|
+
"""Constructs necessary info needed for Google Cloud Storage before calling upload"""
|
117
|
+
file_key: str = _construct_file_key(privacy_request.id, config)
|
118
|
+
|
119
|
+
bucket_name = config.details[StorageDetails.BUCKET.value]
|
120
|
+
auth_method = config.details[StorageDetails.AUTH_METHOD.value]
|
121
|
+
|
122
|
+
return upload_to_gcs(
|
123
|
+
config.secrets,
|
124
|
+
data,
|
125
|
+
bucket_name,
|
126
|
+
file_key,
|
127
|
+
config.format.value, # type: ignore
|
128
|
+
privacy_request,
|
129
|
+
auth_method,
|
130
|
+
)
|
131
|
+
|
132
|
+
|
109
133
|
def _local_uploader(
|
110
134
|
_: Session,
|
111
135
|
config: StorageConfig,
|
fides/api/task/graph_task.py
CHANGED
@@ -125,16 +125,15 @@ def retry(
|
|
125
125
|
ActionDisabled,
|
126
126
|
NotSupportedForCollection,
|
127
127
|
) as exc:
|
128
|
-
traceback.print_exc()
|
129
128
|
logger.warning(
|
130
|
-
"Skipping collection {} for privacy_request: {}",
|
129
|
+
"{} - Skipping collection {} for privacy_request: {}",
|
130
|
+
exc.__class__.__name__,
|
131
131
|
self.execution_node.address,
|
132
132
|
self.resources.request.id,
|
133
133
|
)
|
134
134
|
self.log_skipped(action_type, exc)
|
135
135
|
return default_return
|
136
136
|
except SkippingConsentPropagation as exc:
|
137
|
-
traceback.print_exc()
|
138
137
|
logger.warning(
|
139
138
|
"Skipping consent propagation on collection {} for privacy_request: {}",
|
140
139
|
self.execution_node.address,
|
fides/api/tasks/storage.py
CHANGED
@@ -16,6 +16,7 @@ from fides.api.schemas.storage.storage import ResponseFormat, StorageSecrets
|
|
16
16
|
from fides.api.service.privacy_request.dsr_package.dsr_report_builder import (
|
17
17
|
DsrReportBuilder,
|
18
18
|
)
|
19
|
+
from fides.api.service.storage.gcs import get_gcs_client
|
19
20
|
from fides.api.service.storage.s3 import (
|
20
21
|
create_presigned_url_for_s3,
|
21
22
|
generic_upload_to_s3,
|
@@ -64,7 +65,7 @@ def encrypt_access_request_results(data: Union[str, bytes], request_id: str) ->
|
|
64
65
|
def write_to_in_memory_buffer(
|
65
66
|
resp_format: str, data: Dict[str, Any], privacy_request: PrivacyRequest
|
66
67
|
) -> BytesIO:
|
67
|
-
"""Write JSON/CSV data to in-memory file-like object to be passed to S3. Encrypt data if encryption key/nonce
|
68
|
+
"""Write JSON/CSV data to in-memory file-like object to be passed to S3 or GCS. Encrypt data if encryption key/nonce
|
68
69
|
has been cached for the given privacy request id
|
69
70
|
|
70
71
|
:param resp_format: str, should be one of ResponseFormat
|
@@ -130,7 +131,11 @@ def upload_to_s3( # pylint: disable=R0913
|
|
130
131
|
raise ValueError("Privacy request must be provided")
|
131
132
|
|
132
133
|
try:
|
133
|
-
s3_client = get_s3_client(
|
134
|
+
s3_client = get_s3_client(
|
135
|
+
auth_method,
|
136
|
+
storage_secrets,
|
137
|
+
assume_role_arn=CONFIG.credentials["storage"].get("aws_s3_assume_role_arn"),
|
138
|
+
)
|
134
139
|
|
135
140
|
# handles file chunking
|
136
141
|
try:
|
@@ -157,6 +162,49 @@ def upload_to_s3( # pylint: disable=R0913
|
|
157
162
|
raise ValueError(f"The parameters you provided are incorrect: {e}")
|
158
163
|
|
159
164
|
|
165
|
+
def upload_to_gcs(
|
166
|
+
storage_secrets: Dict,
|
167
|
+
data: Dict,
|
168
|
+
bucket_name: str,
|
169
|
+
file_key: str,
|
170
|
+
resp_format: str,
|
171
|
+
privacy_request: PrivacyRequest,
|
172
|
+
auth_method: str,
|
173
|
+
) -> str:
|
174
|
+
"""Uploads access request data to a Google Cloud Storage bucket"""
|
175
|
+
logger.info("Starting Google Cloud Storage upload of {}", file_key)
|
176
|
+
|
177
|
+
try:
|
178
|
+
storage_client = get_gcs_client(auth_method, storage_secrets)
|
179
|
+
bucket = storage_client.bucket(bucket_name)
|
180
|
+
|
181
|
+
blob = bucket.blob(file_key)
|
182
|
+
in_memory_file = write_to_in_memory_buffer(resp_format, data, privacy_request)
|
183
|
+
content_type = {
|
184
|
+
ResponseFormat.json.value: "application/json",
|
185
|
+
ResponseFormat.csv.value: "application/zip",
|
186
|
+
ResponseFormat.html.value: "application/zip",
|
187
|
+
}
|
188
|
+
blob.upload_from_string(
|
189
|
+
in_memory_file.getvalue(), content_type=content_type[resp_format]
|
190
|
+
)
|
191
|
+
|
192
|
+
logger.info("File {} uploaded to {}", file_key, blob.public_url)
|
193
|
+
|
194
|
+
presigned_url = blob.generate_signed_url(
|
195
|
+
version="v4",
|
196
|
+
expiration=CONFIG.security.subject_request_download_link_ttl_seconds,
|
197
|
+
method="GET",
|
198
|
+
)
|
199
|
+
return presigned_url
|
200
|
+
except Exception as e:
|
201
|
+
logger.error(
|
202
|
+
"Encountered error while uploading and generating link for Google Cloud Storage object: {}",
|
203
|
+
e,
|
204
|
+
)
|
205
|
+
raise e
|
206
|
+
|
207
|
+
|
160
208
|
def upload_to_local(
|
161
209
|
data: Dict,
|
162
210
|
file_key: str,
|
fides/api/util/aws_util.py
CHANGED
@@ -22,6 +22,10 @@ def get_aws_session(
|
|
22
22
|
if storage_secrets is None:
|
23
23
|
# set to an empty dict to allow for more dynamic code downstream
|
24
24
|
storage_secrets = {}
|
25
|
+
|
26
|
+
stored_region_name = storage_secrets.get("region_name") # type: ignore
|
27
|
+
stored_assume_role_arn: Optional[str] = storage_secrets.get("assume_role_arn") # type: ignore
|
28
|
+
|
25
29
|
if auth_method == AWSAuthMethod.SECRET_KEYS.value:
|
26
30
|
if not storage_secrets:
|
27
31
|
err_msg = "Storage secrets not found for S3 storage."
|
@@ -33,12 +37,10 @@ def get_aws_session(
|
|
33
37
|
aws_secret_access_key=storage_secrets[
|
34
38
|
StorageSecrets.AWS_SECRET_ACCESS_KEY.value # type: ignore
|
35
39
|
],
|
36
|
-
region_name=
|
40
|
+
region_name=stored_region_name,
|
37
41
|
)
|
38
42
|
elif auth_method == AWSAuthMethod.AUTOMATIC.value:
|
39
|
-
session = Session(
|
40
|
-
region_name=storage_secrets.get("region_name"), # type: ignore
|
41
|
-
)
|
43
|
+
session = Session(region_name=stored_region_name)
|
42
44
|
logger.info("Successfully created automatic session")
|
43
45
|
else:
|
44
46
|
logger.error("AWS auth method not supported: {}", auth_method)
|
@@ -48,20 +50,13 @@ def get_aws_session(
|
|
48
50
|
sts_client = session.client("sts")
|
49
51
|
sts_client.get_caller_identity()
|
50
52
|
|
51
|
-
|
53
|
+
target_assume_role_arn = assume_role_arn or stored_assume_role_arn
|
54
|
+
if target_assume_role_arn:
|
52
55
|
try:
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
logger.info(
|
58
|
-
f"Assumed role {assume_role_arn} and got temporary credentials."
|
59
|
-
)
|
60
|
-
return Session(
|
61
|
-
aws_access_key_id=temp_credentials["AccessKeyId"],
|
62
|
-
aws_secret_access_key=temp_credentials["SecretAccessKey"],
|
63
|
-
aws_session_token=temp_credentials["SessionToken"],
|
64
|
-
region_name=storage_secrets.get("region_name"), # type: ignore
|
56
|
+
return get_assumed_role_session(
|
57
|
+
target_assume_role_arn,
|
58
|
+
sts_client,
|
59
|
+
stored_region_name,
|
65
60
|
)
|
66
61
|
except ClientError as error:
|
67
62
|
logger.exception(
|
@@ -72,6 +67,22 @@ def get_aws_session(
|
|
72
67
|
return session
|
73
68
|
|
74
69
|
|
70
|
+
def get_assumed_role_session(
|
71
|
+
assume_role_arn: str, sts_client: Any, region_name: Optional[str] = None
|
72
|
+
) -> Session:
|
73
|
+
response = sts_client.assume_role(
|
74
|
+
RoleArn=assume_role_arn, RoleSessionName="FidesAssumeRoleSession"
|
75
|
+
)
|
76
|
+
temp_credentials = response["Credentials"]
|
77
|
+
logger.info(f"Assumed role {assume_role_arn} and got temporary credentials.")
|
78
|
+
return Session(
|
79
|
+
aws_access_key_id=temp_credentials["AccessKeyId"],
|
80
|
+
aws_secret_access_key=temp_credentials["SecretAccessKey"],
|
81
|
+
aws_session_token=temp_credentials["SessionToken"],
|
82
|
+
region_name=region_name,
|
83
|
+
)
|
84
|
+
|
85
|
+
|
75
86
|
def get_s3_client(
|
76
87
|
auth_method: str,
|
77
88
|
storage_secrets: Optional[Dict[StorageSecrets, Any]],
|