ethyca-fides 2.59.2rc1__py2.py3-none-any.whl → 2.60.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ethyca_fides-2.59.2rc1.dist-info → ethyca_fides-2.60.0.dist-info}/METADATA +2 -2
- {ethyca_fides-2.59.2rc1.dist-info → ethyca_fides-2.60.0.dist-info}/RECORD +217 -212
- {ethyca_fides-2.59.2rc1.dist-info → ethyca_fides-2.60.0.dist-info}/WHEEL +1 -1
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/c9c72b3d550b_data_migration_cookie_asset.py +218 -0
- fides/api/api/v1/endpoints/storage_endpoints.py +7 -6
- fides/api/db/system.py +1 -121
- fides/api/models/asset.py +1 -2
- fides/api/models/privacy_notice.py +14 -19
- fides/api/models/sql_models.py +0 -51
- fides/api/schemas/application_config.py +1 -0
- fides/api/schemas/connection_configuration/connection_secrets_datahub.py +3 -3
- fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_mysql.py +17 -0
- fides/api/schemas/connection_configuration/connection_secrets_google_cloud_sql_postgres.py +17 -0
- fides/api/schemas/connection_configuration/connection_secrets_website.py +4 -2
- fides/api/schemas/connection_configuration/enums/google_cloud_sql_ip_type.py +9 -0
- fides/api/schemas/messaging/messaging.py +9 -2
- fides/api/schemas/storage/storage.py +40 -4
- fides/api/schemas/storage/storage_secrets_docs_only.py +8 -2
- fides/api/schemas/system.py +1 -2
- fides/api/service/connectors/google_cloud_mysql_connector.py +4 -0
- fides/api/service/connectors/google_cloud_postgres_connector.py +4 -0
- fides/api/service/connectors/sql_connector.py +2 -2
- fides/api/service/storage/gcs.py +38 -0
- fides/api/service/storage/storage_authenticator_service.py +39 -16
- fides/api/service/storage/storage_uploader_service.py +25 -1
- fides/api/tasks/storage.py +52 -2
- fides/api/util/aws_util.py +28 -17
- fides/api/util/storage_util.py +2 -0
- fides/service/messaging/aws_ses_service.py +25 -16
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/chunks/1100-c3d52c9d7d6d0a39.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1817-3d9a78857dd5a9f4.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{1904-281183a117e26585.js → 1904-00404c45e69d3225.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{2479-b4f7a1c7c711cb65.js → 2479-064aefb3d187d6a2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/3503-296468f82b067ec4.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3513-d03549cb201e7938.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3702-3d814671c31eaca3.js → 3702-929a386a64d55f19.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/3855-3345160a78baa978.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3872-d7045d15aafffa12.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/401-1ba7e037992ac460.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4060-dd7b6e0472703a2e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4121-01c43bbbfc6b8fd1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4481-cd91353711218b86.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/5487-fd9b930f4246520a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{5826-1f4f74bf3b5348e4.js → 5826-5f6e7a91e9ff71d1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/5835-057c2dec5117817c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/5973-9199898696748d0c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6277-6c465012cef1bd3d.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6372-2aee10b79089521a.js → 6372-e35fc0a8874fdecf.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6853-df71f47693b139d3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6954-00d6683243cc75d8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/7751-b3af4819eac0bc9c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/79-110e5199495a81c7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/8433-2c6947927b1e74f5.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/8934-41284e609c329cdd.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/9282-6b14898de61ba6f0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/9327-2be9f27fbce039b0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9392.da35a8e778812d91.js → 9392.9178313b7a01d889.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9572-c3e70eb702e9d879.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9676.f297a1347ac09c56.js → 9676.d1e42fd1d03c4a76.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9719-8ce33427015f12ae.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/9767-e8b0b48d602451d8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9951-66aad0602aadfbe2.js → 9951-d8d1174b00f9dac1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9965-023779db7a1b4aa2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{404-29560aa2e6a60963.js → 404-73e79d3760ef2658.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-13a604eb8d39dffc.js → _app-79807dd0b151eaa4.js} +48 -48
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/manual-2edda5eb5412ffb2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-2392e3a101fae073.js → multiple-27c3bdcc05a1f18e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems-2c64d0a87784019b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/ant-poc-3c256e928748177a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-7dfe37b9f0ff9a3c.js → add-vendors-b3d62016149e5aa4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-dba7848b760ba227.js → configure-790c7c61ff9500f1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/[id]-6641f24adf72b00c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-c02b14c50b19bd91.js → new-300fd2800c417d0b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience-f0ba90152c43c742.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-75c2ed6ba3de48cd.js → [id]-669b1039a5f0dcb6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-487ae57dc7e2ded2.js → new-e24975eaeedc7c51.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices-90b331a666216e38.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-f67fda6a71d0a46b.js → properties-b9919fa183e9308a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-1762ed46d98cebad.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{consent-75395dfc224cf44b.js → consent-07bc6a18cbad414a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-fa65c67ec8c4b5e6.js → [projectUrn]-864af917f60d8f35.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-e76a07ee6ee8d55f.js → projects-45ac2c54f21a57d5.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-8e0bc6d01176081a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-a1259aa44bf99194.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-a8b44307dfd5c23e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-333964fc8ce20616.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/activity-438ab93c6cd5c2a6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-1bcaa606739ea1c5.js → [resourceUrn]-10f789d4ab2b8927.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-22f55dc12354b4c6.js → detection-f473b2ee1c67be3c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-86111c25dc30ef2b.js → [resourceUrn]-2569551ae75ea5a2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-853be75f08b9ab5c.js → discovery-ad077f30849fdc0a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{datamap-423172d31de8e9c6.js → datamap-05144da6383ba6a4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-704553f5329fb9d4.js → [...subfieldNames]-a3311bdf11e5bb1b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-06c19dca941edb14.js → [collectionName]-b584029b66084b2a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-eac517f43d5f53a8.js → [datasetId]-2ababde59fc9efb4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/new-60a80195eafe9839.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-f4d57b3432c57112.js → dataset-5190a42b4bddac7d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-b2b1168d0361baa8.js → [id]-63af795429688cac.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-c804349a417a888b.js → new-32ff2757c1fb5a00.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-92dce084869b6a94.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/index-7220a947bb52c5a8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-5a600a16f2ad5fa0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-372906eb228e73b7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-66f5fbadd8455805.js → [id]-34e5d096311c5aab.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-d441abb1f045940f.js → add-template-722590a322ea866b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-77726c8d9ad39a64.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{[id]-54ea39f203182c53.js → [id]-12037a323158a7be.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-df490b611f1360fa.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-8575fd1d25457b4d.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure-66c6d3725fc1861f.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-ff5856fc531097b2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-f4fb941df069b7bf.js → [id]-7f8cc582fc11b46d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-bccb6ffab25aa214.js → add-property-b739361d2b5714db.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{properties-9a7ac623370b7c00.js → properties-febd16a18011fd1e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-6d046c2bf03ce067.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/about-95e9953afaad9260.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/[purpose_id]-0d593a93ef57dbd7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent-41e6213e69789c1b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-d6e18187d6412a34.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-d9088f5cd9fb2822.js → domain-records-0392d1bc2b8ec616.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/domains-db42cce5ec0ca55b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/email-templates-297f558258a46f74.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-759564ca0ae62840.js → locations-38ce7883fd27b61f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/organization-3dc30591cc0a8881.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-2dce4501fca920b3.js → regulations-22d97471513e835d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/test-datasets-f408ebdc01892ff0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]-69e582763772aec9.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/systems-d7a63f851cc99583.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-244453d6d5413d6c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-082c3156175f9267.js → new-65ef1bdebb679666.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/{[id]-d527d1d1635541df.js → [id]-f29f0fbd62e10b6d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management-f2dc8220e4aab9d7.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{webpack-03e375f6d6b2c71b.js → webpack-32c43a8d709ca5c6.js} +1 -1
- fides/ui-build/static/admin/_next/static/css/a063d3d67fe688f6.css +1 -0
- fides/ui-build/static/admin/_next/static/jixd0oMcDeyH0tzSCfcXV/_buildManifest.js +1 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/ant-poc.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -0
- fides/ui-build/static/admin/lib/fides-tcf.js +3 -3
- fides/ui-build/static/admin/lib/fides.js +2 -2
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -0
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/ui-build/static/admin/_next/static/chunks/1100-45c0634b4f51d10c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1376-92890c17ce39ca0d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/146-d1820217dc36d46d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1817-6abbe957a53026d7.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3086-d1ba90bc6ac9174b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3244-3d94bf3393366412.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3855-237afbbea7f54707.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3872-c53d74aebd12cfd3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/401-50974f107c2712fc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4060-52fd85fb1e15a2b8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4121-476f657b03a78965.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4481-288d74a33473cff7.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5102-626b9ff42e904276.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5480-1623dbacdf4e6794.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5487-9a2aecb0ec63d567.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5973-ae645d0f2db4d579.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6395-4224d6d26d1e8bb7.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6853-ca5dacd25c9ccb7c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-127745bfdc5bc0b3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7751-3913b8c055f599e5.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/79-77cfed02164241ca.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/8433-c4c765833ab9cc07.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/8934-ffa2b0509bc7a845.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9282-1a48b10b114d01f4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9327-4970d356f7000c0b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9572-2b9b10e146130c85.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9767-bf415d0daea960cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/manual-58e9256e86916ecd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems-a6926f7ec7ad10cf.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/ant-poc-404f3c9018952800.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/[id]-78de4bde88e18b0f.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience-7eeeee3769e73f78.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices-65091908d6296afc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-7a1976c0d1aca8b6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-75989f9732d90793.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/[systemId]-6b3d1fe762c747d8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]-61eafb8444bffb76.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-38476c697da53480.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/activity-e7869f658e3017b9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/new-5aff1d01c6c30ad6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-0faa9b3c4555e585.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/index-e72904e316ede1b0.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-17191a759e167ca8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-88e51c81a380ebad.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging-eef56c95b08aa24c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-d889076067104e56.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-3355b4803b2916f3.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure-b8c94b10ab90b061.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-3ac47981f1e2b0aa.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-fe743440d7eb007b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/about-4412a7b468b6d4bf.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent-cdc866af93898716.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-a7dc8113067dff42.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/domains-2f03e981234c40ad.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/email-templates-cf09ad896c7396a6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/organization-5bcc7a6976e01a56.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/test-datasets-0b47ff26897c1d9a.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]-dd053a3bf2a9ca6c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems-820893393f1516ee.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-8fccd670220dbf60.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management-facb8c0128a44cb1.js +0 -1
- fides/ui-build/static/admin/_next/static/css/687135955af5b7e1.css +0 -1
- fides/ui-build/static/admin/_next/static/oEiv2CthgHT2QEn0X5ZRN/_buildManifest.js +0 -1
- {ethyca_fides-2.59.2rc1.dist-info → ethyca_fides-2.60.0.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.59.2rc1.dist-info → ethyca_fides-2.60.0.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.59.2rc1.dist-info → ethyca_fides-2.60.0.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{oEiv2CthgHT2QEn0X5ZRN → jixd0oMcDeyH0tzSCfcXV}/_ssgManifest.js +0 -0
@@ -8,6 +8,9 @@ from fides.api.schemas.base_class import NoValidationSchema
|
|
8
8
|
from fides.api.schemas.connection_configuration.connection_secrets import (
|
9
9
|
ConnectionConfigSecretsSchema,
|
10
10
|
)
|
11
|
+
from fides.api.schemas.connection_configuration.enums.google_cloud_sql_ip_type import (
|
12
|
+
GoogleCloudSQLIPType,
|
13
|
+
)
|
11
14
|
|
12
15
|
|
13
16
|
class KeyfileCreds(BaseModel):
|
@@ -57,6 +60,11 @@ class GoogleCloudSQLPostgresSchema(ConnectionConfigSecretsSchema):
|
|
57
60
|
json_schema_extra={"sensitive": True},
|
58
61
|
description="The contents of the key file that contains authentication credentials for a service account in GCP.",
|
59
62
|
)
|
63
|
+
ip_type: Optional[GoogleCloudSQLIPType] = Field(
|
64
|
+
default=None,
|
65
|
+
title="IP type",
|
66
|
+
description="Specify the IP Address type required for your database (defaults to public). See the Google Cloud documentation for more information about connection options: https://cloud.google.com/sql/docs/postgres/connect-overview",
|
67
|
+
)
|
60
68
|
|
61
69
|
_required_components: ClassVar[List[str]] = [
|
62
70
|
"db_iam_user",
|
@@ -71,6 +79,15 @@ class GoogleCloudSQLPostgresSchema(ConnectionConfigSecretsSchema):
|
|
71
79
|
v = json.loads(v)
|
72
80
|
return KeyfileCreds.model_validate(v)
|
73
81
|
|
82
|
+
@field_validator("ip_type", mode="before")
|
83
|
+
@classmethod
|
84
|
+
def empty_string_to_none(cls, v: Optional[str]) -> Optional[GoogleCloudSQLIPType]:
|
85
|
+
if v == "":
|
86
|
+
return None
|
87
|
+
if v is not None:
|
88
|
+
return GoogleCloudSQLIPType(v)
|
89
|
+
return v
|
90
|
+
|
74
91
|
|
75
92
|
class GoogleCloudSQLPostgresDocsSchema(
|
76
93
|
GoogleCloudSQLPostgresSchema, NoValidationSchema
|
@@ -1,5 +1,5 @@
|
|
1
1
|
from fideslang.validation import AnyHttpUrlString
|
2
|
-
from pydantic import BaseModel
|
2
|
+
from pydantic import BaseModel, Field
|
3
3
|
|
4
4
|
from fides.api.schemas.base_class import NoValidationSchema
|
5
5
|
|
@@ -7,7 +7,9 @@ from fides.api.schemas.base_class import NoValidationSchema
|
|
7
7
|
class WebsiteSchema(BaseModel):
|
8
8
|
"""Schema to validate the secrets needed for a generic website connector"""
|
9
9
|
|
10
|
-
url: AnyHttpUrlString
|
10
|
+
url: AnyHttpUrlString = Field(
|
11
|
+
title="URL",
|
12
|
+
)
|
11
13
|
|
12
14
|
|
13
15
|
class WebsiteDocsScehma(WebsiteSchema, NoValidationSchema):
|
@@ -286,12 +286,19 @@ class MessagingServiceDetailsTwilioEmail(BaseModel):
|
|
286
286
|
class MessagingServiceDetailsAWS_SES(BaseModel):
|
287
287
|
"""The details required to represent an AWS SES email configuration."""
|
288
288
|
|
289
|
-
email_from: str
|
290
|
-
domain: str
|
289
|
+
email_from: Optional[str] = None
|
290
|
+
domain: Optional[str] = None
|
291
291
|
aws_region: str
|
292
292
|
|
293
293
|
model_config = ConfigDict(extra="forbid")
|
294
294
|
|
295
|
+
@model_validator(mode="before")
|
296
|
+
@classmethod
|
297
|
+
def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]:
|
298
|
+
if not values.get("domain") and not values.get("email_from"):
|
299
|
+
raise ValueError("Either 'email_from' or 'domain' must be provided.")
|
300
|
+
return values
|
301
|
+
|
295
302
|
|
296
303
|
class MessagingServiceSecrets(Enum):
|
297
304
|
"""Enum for message service secrets"""
|
@@ -61,6 +61,20 @@ class StorageDetailsS3(FileBasedStorageDetails):
|
|
61
61
|
model_config = ConfigDict(use_enum_values=True)
|
62
62
|
|
63
63
|
|
64
|
+
class GCSAuthMethod(str, Enum):
|
65
|
+
ADC = "adc" # Application Default Credentials
|
66
|
+
SERVICE_ACCOUNT_KEYS = "service_account_keys"
|
67
|
+
|
68
|
+
|
69
|
+
class StorageDetailsGCS(FileBasedStorageDetails):
|
70
|
+
"""The details required to represent a Google Cloud Storage bucket."""
|
71
|
+
|
72
|
+
auth_method: GCSAuthMethod
|
73
|
+
bucket: str
|
74
|
+
max_retries: Optional[int] = 0
|
75
|
+
model_config = ConfigDict(use_enum_values=True)
|
76
|
+
|
77
|
+
|
64
78
|
class StorageDetailsLocal(FileBasedStorageDetails):
|
65
79
|
"""The details required to configurate local storage configuration"""
|
66
80
|
|
@@ -71,6 +85,8 @@ class StorageSecrets(Enum):
|
|
71
85
|
# s3-specific
|
72
86
|
AWS_ACCESS_KEY_ID = "aws_access_key_id"
|
73
87
|
AWS_SECRET_ACCESS_KEY = "aws_secret_access_key"
|
88
|
+
REGION_NAME = "region_name"
|
89
|
+
AWS_ASSUME_ROLE = "assume_role_arn"
|
74
90
|
|
75
91
|
|
76
92
|
class StorageSecretsLocal(BaseModel):
|
@@ -80,10 +96,27 @@ class StorageSecretsLocal(BaseModel):
|
|
80
96
|
|
81
97
|
|
82
98
|
class StorageSecretsS3(BaseModel):
|
83
|
-
|
99
|
+
aws_access_key_id: Optional[str] = None
|
100
|
+
aws_secret_access_key: Optional[str] = None
|
101
|
+
region_name: Optional[str] = None
|
102
|
+
assume_role_arn: Optional[str] = None
|
103
|
+
model_config = ConfigDict(extra="forbid")
|
104
|
+
|
84
105
|
|
85
|
-
|
86
|
-
|
106
|
+
class StorageSecretsGCS(BaseModel):
|
107
|
+
"""The secrets required to connect to a Google Cloud Storage bucket."""
|
108
|
+
|
109
|
+
type: str = "service_account"
|
110
|
+
project_id: str
|
111
|
+
private_key_id: str
|
112
|
+
private_key: str
|
113
|
+
client_email: str
|
114
|
+
client_id: str
|
115
|
+
auth_uri: str
|
116
|
+
token_uri: str
|
117
|
+
auth_provider_x509_cert_url: str
|
118
|
+
client_x509_cert_url: str
|
119
|
+
universe_domain: str
|
87
120
|
model_config = ConfigDict(extra="forbid")
|
88
121
|
|
89
122
|
|
@@ -99,6 +132,7 @@ class StorageType(Enum):
|
|
99
132
|
|
100
133
|
FULLY_CONFIGURED_STORAGE_TYPES = (
|
101
134
|
StorageType.s3,
|
135
|
+
StorageType.gcs,
|
102
136
|
) # storage types that are considered "fully configured"
|
103
137
|
|
104
138
|
|
@@ -108,6 +142,7 @@ class StorageDestinationBase(BaseModel):
|
|
108
142
|
type: StorageType
|
109
143
|
details: Union[
|
110
144
|
StorageDetailsS3,
|
145
|
+
StorageDetailsGCS,
|
111
146
|
StorageDetailsLocal,
|
112
147
|
] = Field(validate_default=True)
|
113
148
|
format: Optional[ResponseFormat] = ResponseFormat.json.value # type: ignore
|
@@ -145,6 +180,7 @@ class StorageDestinationBase(BaseModel):
|
|
145
180
|
try:
|
146
181
|
schema = {
|
147
182
|
StorageType.s3.value: StorageDetailsS3,
|
183
|
+
StorageType.gcs.value: StorageDetailsGCS,
|
148
184
|
StorageType.local.value: StorageDetailsLocal,
|
149
185
|
}[storage_type]
|
150
186
|
except KeyError:
|
@@ -209,7 +245,7 @@ class BulkPutStorageConfigResponse(BulkResponse):
|
|
209
245
|
failed: List[BulkUpdateFailed] = []
|
210
246
|
|
211
247
|
|
212
|
-
SUPPORTED_STORAGE_SECRETS = StorageSecretsS3
|
248
|
+
SUPPORTED_STORAGE_SECRETS = Union[StorageSecretsS3, StorageSecretsGCS]
|
213
249
|
|
214
250
|
|
215
251
|
class StorageConfigStatus(Enum):
|
@@ -1,9 +1,15 @@
|
|
1
|
+
from typing import Union
|
2
|
+
|
1
3
|
from fides.api.schemas.base_class import NoValidationSchema
|
2
|
-
from fides.api.schemas.storage.storage import StorageSecretsS3
|
4
|
+
from fides.api.schemas.storage.storage import StorageSecretsGCS, StorageSecretsS3
|
3
5
|
|
4
6
|
|
5
7
|
class StorageSecretsS3Docs(StorageSecretsS3, NoValidationSchema):
|
6
8
|
"""The secrets required to connect to S3, for documentation"""
|
7
9
|
|
8
10
|
|
9
|
-
|
11
|
+
class StorageSecretsGCSDocs(StorageSecretsGCS, NoValidationSchema):
|
12
|
+
"""The secrets required to connect to Google Cloud Storage, for documentation"""
|
13
|
+
|
14
|
+
|
15
|
+
possible_storage_secrets = Union[StorageSecretsS3Docs, StorageSecretsGCSDocs]
|
fides/api/schemas/system.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
from datetime import datetime
|
2
2
|
from typing import Any, Dict, List, Optional, Sequence
|
3
3
|
|
4
|
-
from fideslang.models import
|
4
|
+
from fideslang.models import PrivacyDeclaration, System
|
5
5
|
from pydantic import ConfigDict, Field
|
6
6
|
from pydantic.main import BaseModel
|
7
7
|
|
@@ -17,7 +17,6 @@ class PrivacyDeclarationResponse(PrivacyDeclaration):
|
|
17
17
|
id: str = Field(
|
18
18
|
description="The database-assigned ID of the privacy declaration on the system. This is meant to be a read-only field, returned only in API responses"
|
19
19
|
)
|
20
|
-
cookies: Optional[List[Cookies]] = []
|
21
20
|
|
22
21
|
|
23
22
|
class BasicSystemResponse(System):
|
@@ -8,6 +8,9 @@ from sqlalchemy.engine import Engine, LegacyCursorResult, create_engine # type:
|
|
8
8
|
from fides.api.schemas.connection_configuration.connection_secrets_google_cloud_sql_mysql import (
|
9
9
|
GoogleCloudSQLMySQLSchema,
|
10
10
|
)
|
11
|
+
from fides.api.schemas.connection_configuration.enums.google_cloud_sql_ip_type import (
|
12
|
+
GoogleCloudSQLIPType,
|
13
|
+
)
|
11
14
|
from fides.api.service.connectors.sql_connector import SQLConnector
|
12
15
|
from fides.api.util.collection_util import Row
|
13
16
|
from fides.config import get_config
|
@@ -37,6 +40,7 @@ class GoogleCloudSQLMySQLConnector(SQLConnector):
|
|
37
40
|
conn: pymysql.connections.Connection = connector.connect(
|
38
41
|
config.instance_connection_name,
|
39
42
|
"pymysql",
|
43
|
+
ip_type=config.ip_type or GoogleCloudSQLIPType.public,
|
40
44
|
user=config.db_iam_user,
|
41
45
|
db=config.dbname,
|
42
46
|
enable_iam_auth=True,
|
@@ -16,6 +16,9 @@ from fides.api.graph.execution import ExecutionNode
|
|
16
16
|
from fides.api.schemas.connection_configuration.connection_secrets_google_cloud_sql_postgres import (
|
17
17
|
GoogleCloudSQLPostgresSchema,
|
18
18
|
)
|
19
|
+
from fides.api.schemas.connection_configuration.enums.google_cloud_sql_ip_type import (
|
20
|
+
GoogleCloudSQLIPType,
|
21
|
+
)
|
19
22
|
from fides.api.service.connectors.query_configs.google_cloud_postgres_query_config import (
|
20
23
|
GoogleCloudSQLPostgresQueryConfig,
|
21
24
|
)
|
@@ -53,6 +56,7 @@ class GoogleCloudSQLPostgresConnector(SQLConnector):
|
|
53
56
|
conn: pg8000.dbapi.Connection = connector.connect(
|
54
57
|
config.instance_connection_name,
|
55
58
|
"pg8000",
|
59
|
+
ip_type=config.ip_type or GoogleCloudSQLIPType.public,
|
56
60
|
user=config.db_iam_user,
|
57
61
|
db=config.dbname or self.default_db_name,
|
58
62
|
enable_iam_auth=True,
|
@@ -119,8 +119,8 @@ class SQLConnector(BaseConnector[Engine]):
|
|
119
119
|
)
|
120
120
|
except ClientResponseError as e:
|
121
121
|
raise ConnectionException(f"Connection error: {e.message}")
|
122
|
-
except Exception:
|
123
|
-
raise ConnectionException("Connection error
|
122
|
+
except Exception as exc:
|
123
|
+
raise ConnectionException(f"Connection error: {exc}")
|
124
124
|
|
125
125
|
return ConnectionTestStatus.succeeded
|
126
126
|
|
@@ -0,0 +1,38 @@
|
|
1
|
+
from typing import Dict, Optional
|
2
|
+
|
3
|
+
from google.cloud.storage import Client # type: ignore
|
4
|
+
from google.oauth2 import service_account
|
5
|
+
from loguru import logger
|
6
|
+
|
7
|
+
from fides.api.common_exceptions import StorageUploadError
|
8
|
+
from fides.api.schemas.storage.storage import GCSAuthMethod
|
9
|
+
|
10
|
+
|
11
|
+
def get_gcs_client(
|
12
|
+
auth_method: str,
|
13
|
+
storage_secrets: Optional[Dict],
|
14
|
+
) -> Client:
|
15
|
+
"""
|
16
|
+
Abstraction to retrieve a GCS client using secrets.
|
17
|
+
"""
|
18
|
+
if auth_method == GCSAuthMethod.ADC.value:
|
19
|
+
storage_client = Client()
|
20
|
+
|
21
|
+
elif auth_method == GCSAuthMethod.SERVICE_ACCOUNT_KEYS.value:
|
22
|
+
if not storage_secrets:
|
23
|
+
err_msg = "Storage secrets not found for Google Cloud Storage."
|
24
|
+
logger.warning(err_msg)
|
25
|
+
raise StorageUploadError(err_msg)
|
26
|
+
|
27
|
+
credentials = service_account.Credentials.from_service_account_info(
|
28
|
+
dict(storage_secrets)
|
29
|
+
)
|
30
|
+
storage_client = Client(credentials=credentials)
|
31
|
+
|
32
|
+
else:
|
33
|
+
logger.error("Google Cloud Storage auth method not supported: {}", auth_method)
|
34
|
+
raise ValueError(
|
35
|
+
f"Google Cloud Storage auth method not supported: {auth_method}"
|
36
|
+
)
|
37
|
+
|
38
|
+
return storage_client
|
@@ -1,7 +1,12 @@
|
|
1
|
-
from typing import Any
|
1
|
+
from typing import Any
|
2
2
|
|
3
3
|
from botocore.exceptions import ClientError
|
4
|
+
from google.auth.exceptions import GoogleAuthError
|
5
|
+
from google.auth.transport.requests import Request
|
6
|
+
from google.oauth2 import service_account
|
7
|
+
from loguru import logger
|
4
8
|
|
9
|
+
from fides.api.models.storage import StorageConfig
|
5
10
|
from fides.api.schemas.storage.storage import (
|
6
11
|
SUPPORTED_STORAGE_SECRETS,
|
7
12
|
AWSAuthMethod,
|
@@ -13,32 +18,50 @@ from fides.api.util.aws_util import get_aws_session
|
|
13
18
|
|
14
19
|
def secrets_are_valid(
|
15
20
|
secrets: SUPPORTED_STORAGE_SECRETS,
|
16
|
-
|
21
|
+
storage_config: StorageConfig,
|
17
22
|
) -> bool:
|
18
23
|
"""Authenticates upload destination with appropriate upload method"""
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
)
|
27
|
-
uploader: Any = _get_authenticator_from_config(storage_type)
|
28
|
-
return uploader(secrets)
|
29
|
-
|
30
|
-
|
31
|
-
def _s3_authenticator(secrets: Dict[StorageSecrets, Any]) -> bool:
|
24
|
+
uploader: Any = _get_authenticator_from_config(storage_config.type) # type: ignore
|
25
|
+
return uploader(storage_config, secrets)
|
26
|
+
|
27
|
+
|
28
|
+
def _s3_authenticator(
|
29
|
+
config: StorageConfig, secrets: dict[StorageSecrets, Any]
|
30
|
+
) -> bool:
|
32
31
|
"""Authenticates secrets for s3, returns true if secrets are valid"""
|
33
32
|
try:
|
34
|
-
get_aws_session(AWSAuthMethod.SECRET_KEYS.value, secrets.model_dump(mode="json")) # type: ignore
|
33
|
+
get_aws_session(config.details["auth_method"] or AWSAuthMethod.SECRET_KEYS.value, secrets.model_dump(mode="json")) # type: ignore
|
35
34
|
return True
|
36
35
|
except ClientError:
|
37
36
|
return False
|
38
37
|
|
39
38
|
|
39
|
+
def _gcs_authenticator(storage_config: StorageConfig, secrets: dict) -> bool:
|
40
|
+
"""Autenticates secrets for Google Cloud Storage, returns true if secrets are valid"""
|
41
|
+
try:
|
42
|
+
credentials = service_account.Credentials.from_service_account_info(
|
43
|
+
dict(secrets),
|
44
|
+
scopes=["https://www.googleapis.com/auth/devstorage.read_only"],
|
45
|
+
)
|
46
|
+
# To validate the credentials, it is necessary to make a request to Google Cloud API
|
47
|
+
credentials.refresh(Request())
|
48
|
+
return True
|
49
|
+
|
50
|
+
except GoogleAuthError as auth_error:
|
51
|
+
logger.warning(
|
52
|
+
"Google authentication error trying to authenticate GCS secrets: {}",
|
53
|
+
auth_error,
|
54
|
+
)
|
55
|
+
return False
|
56
|
+
|
57
|
+
except Exception as e:
|
58
|
+
logger.warning("Unexpected error authenticating GCS secrets: {}", e)
|
59
|
+
return False
|
60
|
+
|
61
|
+
|
40
62
|
def _get_authenticator_from_config(storage_type: StorageType) -> Any:
|
41
63
|
"""Determines which uploader method to use based on storage type"""
|
42
64
|
return {
|
43
65
|
StorageType.s3.value: _s3_authenticator,
|
66
|
+
StorageType.gcs.value: _gcs_authenticator,
|
44
67
|
}[storage_type.value]
|
@@ -14,7 +14,7 @@ from fides.api.schemas.storage.storage import (
|
|
14
14
|
StorageDetails,
|
15
15
|
StorageType,
|
16
16
|
)
|
17
|
-
from fides.api.tasks.storage import upload_to_local, upload_to_s3
|
17
|
+
from fides.api.tasks.storage import upload_to_gcs, upload_to_local, upload_to_s3
|
18
18
|
|
19
19
|
|
20
20
|
def upload(
|
@@ -78,6 +78,7 @@ def _get_uploader_from_config_type(storage_type: StorageType) -> Any:
|
|
78
78
|
return {
|
79
79
|
StorageType.s3.value: _s3_uploader,
|
80
80
|
StorageType.local.value: _local_uploader,
|
81
|
+
StorageType.gcs.value: _gcs_uploader,
|
81
82
|
}[storage_type.value]
|
82
83
|
|
83
84
|
|
@@ -106,6 +107,29 @@ def _s3_uploader(
|
|
106
107
|
)
|
107
108
|
|
108
109
|
|
110
|
+
def _gcs_uploader(
|
111
|
+
_: Session,
|
112
|
+
config: StorageConfig,
|
113
|
+
data: Dict,
|
114
|
+
privacy_request: PrivacyRequest,
|
115
|
+
) -> str:
|
116
|
+
"""Constructs necessary info needed for Google Cloud Storage before calling upload"""
|
117
|
+
file_key: str = _construct_file_key(privacy_request.id, config)
|
118
|
+
|
119
|
+
bucket_name = config.details[StorageDetails.BUCKET.value]
|
120
|
+
auth_method = config.details[StorageDetails.AUTH_METHOD.value]
|
121
|
+
|
122
|
+
return upload_to_gcs(
|
123
|
+
config.secrets,
|
124
|
+
data,
|
125
|
+
bucket_name,
|
126
|
+
file_key,
|
127
|
+
config.format.value, # type: ignore
|
128
|
+
privacy_request,
|
129
|
+
auth_method,
|
130
|
+
)
|
131
|
+
|
132
|
+
|
109
133
|
def _local_uploader(
|
110
134
|
_: Session,
|
111
135
|
config: StorageConfig,
|
fides/api/tasks/storage.py
CHANGED
@@ -16,6 +16,7 @@ from fides.api.schemas.storage.storage import ResponseFormat, StorageSecrets
|
|
16
16
|
from fides.api.service.privacy_request.dsr_package.dsr_report_builder import (
|
17
17
|
DsrReportBuilder,
|
18
18
|
)
|
19
|
+
from fides.api.service.storage.gcs import get_gcs_client
|
19
20
|
from fides.api.service.storage.s3 import (
|
20
21
|
create_presigned_url_for_s3,
|
21
22
|
generic_upload_to_s3,
|
@@ -64,7 +65,7 @@ def encrypt_access_request_results(data: Union[str, bytes], request_id: str) ->
|
|
64
65
|
def write_to_in_memory_buffer(
|
65
66
|
resp_format: str, data: Dict[str, Any], privacy_request: PrivacyRequest
|
66
67
|
) -> BytesIO:
|
67
|
-
"""Write JSON/CSV data to in-memory file-like object to be passed to S3. Encrypt data if encryption key/nonce
|
68
|
+
"""Write JSON/CSV data to in-memory file-like object to be passed to S3 or GCS. Encrypt data if encryption key/nonce
|
68
69
|
has been cached for the given privacy request id
|
69
70
|
|
70
71
|
:param resp_format: str, should be one of ResponseFormat
|
@@ -130,7 +131,13 @@ def upload_to_s3( # pylint: disable=R0913
|
|
130
131
|
raise ValueError("Privacy request must be provided")
|
131
132
|
|
132
133
|
try:
|
133
|
-
s3_client = get_s3_client(
|
134
|
+
s3_client = get_s3_client(
|
135
|
+
auth_method,
|
136
|
+
storage_secrets,
|
137
|
+
assume_role_arn=CONFIG.credentials.get( # pylint: disable=no-member
|
138
|
+
"storage", {}
|
139
|
+
).get("aws_s3_assume_role_arn"),
|
140
|
+
)
|
134
141
|
|
135
142
|
# handles file chunking
|
136
143
|
try:
|
@@ -157,6 +164,49 @@ def upload_to_s3( # pylint: disable=R0913
|
|
157
164
|
raise ValueError(f"The parameters you provided are incorrect: {e}")
|
158
165
|
|
159
166
|
|
167
|
+
def upload_to_gcs(
|
168
|
+
storage_secrets: Dict,
|
169
|
+
data: Dict,
|
170
|
+
bucket_name: str,
|
171
|
+
file_key: str,
|
172
|
+
resp_format: str,
|
173
|
+
privacy_request: PrivacyRequest,
|
174
|
+
auth_method: str,
|
175
|
+
) -> str:
|
176
|
+
"""Uploads access request data to a Google Cloud Storage bucket"""
|
177
|
+
logger.info("Starting Google Cloud Storage upload of {}", file_key)
|
178
|
+
|
179
|
+
try:
|
180
|
+
storage_client = get_gcs_client(auth_method, storage_secrets)
|
181
|
+
bucket = storage_client.bucket(bucket_name)
|
182
|
+
|
183
|
+
blob = bucket.blob(file_key)
|
184
|
+
in_memory_file = write_to_in_memory_buffer(resp_format, data, privacy_request)
|
185
|
+
content_type = {
|
186
|
+
ResponseFormat.json.value: "application/json",
|
187
|
+
ResponseFormat.csv.value: "application/zip",
|
188
|
+
ResponseFormat.html.value: "application/zip",
|
189
|
+
}
|
190
|
+
blob.upload_from_string(
|
191
|
+
in_memory_file.getvalue(), content_type=content_type[resp_format]
|
192
|
+
)
|
193
|
+
|
194
|
+
logger.info("File {} uploaded to {}", file_key, blob.public_url)
|
195
|
+
|
196
|
+
presigned_url = blob.generate_signed_url(
|
197
|
+
version="v4",
|
198
|
+
expiration=CONFIG.security.subject_request_download_link_ttl_seconds,
|
199
|
+
method="GET",
|
200
|
+
)
|
201
|
+
return presigned_url
|
202
|
+
except Exception as e:
|
203
|
+
logger.error(
|
204
|
+
"Encountered error while uploading and generating link for Google Cloud Storage object: {}",
|
205
|
+
e,
|
206
|
+
)
|
207
|
+
raise e
|
208
|
+
|
209
|
+
|
160
210
|
def upload_to_local(
|
161
211
|
data: Dict,
|
162
212
|
file_key: str,
|
fides/api/util/aws_util.py
CHANGED
@@ -22,6 +22,10 @@ def get_aws_session(
|
|
22
22
|
if storage_secrets is None:
|
23
23
|
# set to an empty dict to allow for more dynamic code downstream
|
24
24
|
storage_secrets = {}
|
25
|
+
|
26
|
+
stored_region_name = storage_secrets.get("region_name") # type: ignore
|
27
|
+
stored_assume_role_arn: Optional[str] = storage_secrets.get("assume_role_arn") # type: ignore
|
28
|
+
|
25
29
|
if auth_method == AWSAuthMethod.SECRET_KEYS.value:
|
26
30
|
if not storage_secrets:
|
27
31
|
err_msg = "Storage secrets not found for S3 storage."
|
@@ -33,12 +37,10 @@ def get_aws_session(
|
|
33
37
|
aws_secret_access_key=storage_secrets[
|
34
38
|
StorageSecrets.AWS_SECRET_ACCESS_KEY.value # type: ignore
|
35
39
|
],
|
36
|
-
region_name=
|
40
|
+
region_name=stored_region_name,
|
37
41
|
)
|
38
42
|
elif auth_method == AWSAuthMethod.AUTOMATIC.value:
|
39
|
-
session = Session(
|
40
|
-
region_name=storage_secrets.get("region_name"), # type: ignore
|
41
|
-
)
|
43
|
+
session = Session(region_name=stored_region_name)
|
42
44
|
logger.info("Successfully created automatic session")
|
43
45
|
else:
|
44
46
|
logger.error("AWS auth method not supported: {}", auth_method)
|
@@ -48,20 +50,13 @@ def get_aws_session(
|
|
48
50
|
sts_client = session.client("sts")
|
49
51
|
sts_client.get_caller_identity()
|
50
52
|
|
51
|
-
|
53
|
+
target_assume_role_arn = assume_role_arn or stored_assume_role_arn
|
54
|
+
if target_assume_role_arn:
|
52
55
|
try:
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
logger.info(
|
58
|
-
f"Assumed role {assume_role_arn} and got temporary credentials."
|
59
|
-
)
|
60
|
-
return Session(
|
61
|
-
aws_access_key_id=temp_credentials["AccessKeyId"],
|
62
|
-
aws_secret_access_key=temp_credentials["SecretAccessKey"],
|
63
|
-
aws_session_token=temp_credentials["SessionToken"],
|
64
|
-
region_name=storage_secrets.get("region_name"), # type: ignore
|
56
|
+
return get_assumed_role_session(
|
57
|
+
target_assume_role_arn,
|
58
|
+
sts_client,
|
59
|
+
stored_region_name,
|
65
60
|
)
|
66
61
|
except ClientError as error:
|
67
62
|
logger.exception(
|
@@ -72,6 +67,22 @@ def get_aws_session(
|
|
72
67
|
return session
|
73
68
|
|
74
69
|
|
70
|
+
def get_assumed_role_session(
|
71
|
+
assume_role_arn: str, sts_client: Any, region_name: Optional[str] = None
|
72
|
+
) -> Session:
|
73
|
+
response = sts_client.assume_role(
|
74
|
+
RoleArn=assume_role_arn, RoleSessionName="FidesAssumeRoleSession"
|
75
|
+
)
|
76
|
+
temp_credentials = response["Credentials"]
|
77
|
+
logger.info(f"Assumed role {assume_role_arn} and got temporary credentials.")
|
78
|
+
return Session(
|
79
|
+
aws_access_key_id=temp_credentials["AccessKeyId"],
|
80
|
+
aws_secret_access_key=temp_credentials["SecretAccessKey"],
|
81
|
+
aws_session_token=temp_credentials["SessionToken"],
|
82
|
+
region_name=region_name,
|
83
|
+
)
|
84
|
+
|
85
|
+
|
75
86
|
def get_s3_client(
|
76
87
|
auth_method: str,
|
77
88
|
storage_secrets: Optional[Dict[StorageSecrets, Any]],
|
fides/api/util/storage_util.py
CHANGED
@@ -6,6 +6,7 @@ from pydantic import ValidationError
|
|
6
6
|
|
7
7
|
from fides.api.schemas.storage.storage import (
|
8
8
|
SUPPORTED_STORAGE_SECRETS,
|
9
|
+
StorageSecretsGCS,
|
9
10
|
StorageSecretsS3,
|
10
11
|
StorageType,
|
11
12
|
)
|
@@ -31,6 +32,7 @@ def get_schema_for_secrets(
|
|
31
32
|
try:
|
32
33
|
schema = {
|
33
34
|
StorageType.s3: StorageSecretsS3,
|
35
|
+
StorageType.gcs: StorageSecretsGCS,
|
34
36
|
}[storage_type]
|
35
37
|
except KeyError:
|
36
38
|
raise ValueError(
|
@@ -97,27 +97,29 @@ class AWS_SES_Service:
|
|
97
97
|
|
98
98
|
def validate_email_and_domain_status(self) -> None:
|
99
99
|
"""
|
100
|
-
Validate that the email
|
100
|
+
Validate that either the email or domain (or both) are verified in SES.
|
101
101
|
"""
|
102
102
|
email = self.messaging_config_details.email_from
|
103
103
|
domain = self.messaging_config_details.domain
|
104
|
+
identities = list(filter(None, (email, domain)))
|
105
|
+
|
106
|
+
# Defensive code just in case, there should always be email_from or domain
|
107
|
+
if not identities:
|
108
|
+
raise AWS_SESException(
|
109
|
+
"No identity (email_from or domain) configured for SES validation."
|
110
|
+
)
|
104
111
|
|
105
112
|
ses_client = self.get_ses_client()
|
106
113
|
response = ses_client.get_identity_verification_attributes(
|
107
|
-
Identities=
|
108
|
-
)
|
109
|
-
email_status = (
|
110
|
-
response["VerificationAttributes"].get(email, {}).get("VerificationStatus")
|
111
|
-
)
|
112
|
-
domain_status = (
|
113
|
-
response["VerificationAttributes"].get(domain, {}).get("VerificationStatus")
|
114
|
+
Identities=identities
|
114
115
|
)
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
116
|
+
attributes = response.get("VerificationAttributes", {})
|
117
|
+
|
118
|
+
for identity in identities:
|
119
|
+
status = attributes.get(identity, {}).get("VerificationStatus")
|
120
|
+
if status != "Success":
|
121
|
+
logger.error(f"{identity} is not verified in SES.")
|
122
|
+
raise AWS_SESException(f"{identity} is not verified in SES.")
|
121
123
|
|
122
124
|
def send_email(
|
123
125
|
self,
|
@@ -127,13 +129,20 @@ class AWS_SES_Service:
|
|
127
129
|
) -> None:
|
128
130
|
"""
|
129
131
|
Send an email using AWS SES.
|
130
|
-
|
132
|
+
Either `email_from` or `domain` must be verified in SES.
|
131
133
|
"""
|
132
134
|
self.validate_email_and_domain_status()
|
133
135
|
ses_client = self.get_ses_client()
|
134
136
|
|
137
|
+
from_address = self.messaging_config_details.email_from
|
138
|
+
if not from_address:
|
139
|
+
# If there is no email_from, there is a domain, and the domain was verified against SES.
|
140
|
+
# When you verify a domain identity, you can send email from any subdomain or email address of the
|
141
|
+
# verified domain without having to verify each one individually.
|
142
|
+
from_address = f"noreply@{self.messaging_config_details.domain}"
|
143
|
+
|
135
144
|
ses_client.send_email(
|
136
|
-
Source=
|
145
|
+
Source=from_address,
|
137
146
|
Destination={"ToAddresses": [to.strip()]},
|
138
147
|
Message={
|
139
148
|
"Subject": {"Data": subject},
|