ethyca-fides 2.73.2b0__py2.py3-none-any.whl → 2.76.0rc2__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ethyca_fides-2.73.2b0.dist-info → ethyca_fides-2.76.0rc2.dist-info}/METADATA +2 -1
- {ethyca_fides-2.73.2b0.dist-info → ethyca_fides-2.76.0rc2.dist-info}/RECORD +349 -336
- fides/_version.py +3 -3
- fides/api/alembic/migrations/versions/303287c70600_migrate_resource_type_from_enum_to_.py +99 -0
- fides/api/alembic/migrations/versions/795f46f656c0_migrate_field_type_from_enum_to_string.py +61 -0
- fides/api/alembic/migrations/versions/xx_2025_10_29_1659_80d28dea3b6b_added_duplicate_group_table.py +79 -0
- fides/api/alembic/migrations/versions/xx_2025_11_05_0200_f1a2b3c4d5e6_create_staged_resource_error_table.py +82 -0
- fides/api/alembic/migrations/versions/xx_2025_11_07_1709_56fe6fad2d89_add_notice_display_order.py +47 -0
- fides/api/alembic/migrations/versions/xx_2025_11_10_1200_a1b2c3d4e5f6_add_test_datastore_to_connectiontype.py +53 -0
- fides/api/alembic/migrations/versions/xx_2025_11_11_1317_7d82c8fc4c34_store_saas_template_datasets.py +68 -0
- fides/api/alembic/migrations/versions/xx_2025_11_12_1430_b2c3d4e5f6a7_add_default_identity_definitions.py +81 -0
- fides/api/alembic/migrations/versions/xx_2025_11_25_1854_3ff6449c099e_add_index_on_providedidentity_privacy_.py +50 -0
- fides/api/api/v1/api.py +2 -0
- fides/api/api/v1/endpoints/admin.py +39 -0
- fides/api/api/v1/endpoints/connector_template_endpoints.py +167 -0
- fides/api/api/v1/endpoints/generic_overrides.py +6 -6
- fides/api/api/v1/endpoints/pre_approval_webhook_endpoints.py +5 -5
- fides/api/api/v1/endpoints/privacy_request_endpoints.py +101 -21
- fides/api/api/v1/endpoints/saas_config_endpoints.py +3 -44
- fides/api/api/v1/endpoints/storage_endpoints.py +5 -1
- fides/api/api/v1/endpoints/system.py +2 -2
- fides/api/api/v1/endpoints/user_endpoints.py +10 -4
- fides/api/db/base.py +1 -0
- fides/api/db/database.py +228 -1
- fides/api/db/seed.py +1 -1
- fides/api/email_templates/templates/external_user_welcome.html +9 -5
- fides/api/migrations/post_upgrade_index_creation.py +10 -0
- fides/api/models/connectionconfig.py +4 -1
- fides/api/models/detection_discovery/__init__.py +2 -0
- fides/api/models/detection_discovery/core.py +10 -0
- fides/api/models/detection_discovery/staged_resource_error.py +25 -0
- fides/api/models/experience_notices.py +6 -1
- fides/api/models/identity_definition.py +2 -1
- fides/api/models/location_regulation_selections.py +22 -0
- fides/api/models/manual_webhook.py +17 -6
- fides/api/models/messaging_template.py +7 -0
- fides/api/models/policy.py +3 -0
- fides/api/models/privacy_experience.py +20 -0
- fides/api/models/privacy_notice.py +45 -1
- fides/api/models/privacy_preference.py +1 -0
- fides/api/models/privacy_request/duplicate_group.py +84 -0
- fides/api/models/privacy_request/privacy_request.py +84 -9
- fides/api/models/privacy_request/provided_identity.py +22 -1
- fides/api/models/saas_template_dataset.py +63 -0
- fides/api/models/sql_models.py +150 -7
- fides/api/models/taxonomy.py +61 -1
- fides/api/oauth/roles.py +2 -0
- fides/api/schemas/connection_configuration/connection_secrets_mssql.py +5 -0
- fides/api/schemas/messaging/messaging.py +1 -0
- fides/api/schemas/oauth.py +2 -1
- fides/api/schemas/privacy_request.py +38 -6
- fides/api/schemas/saas/connector_template.py +14 -0
- fides/api/service/async_dsr/strategies/async_dsr_strategy_polling.py +19 -7
- fides/api/service/connectors/base_erasure_email_connector.py +7 -0
- fides/api/service/connectors/microsoft_sql_server_connector.py +6 -1
- fides/api/service/connectors/saas/connector_registry_service.py +45 -1
- fides/api/service/messaging/message_dispatch_service.py +44 -4
- fides/api/service/privacy_request/dsr_package/dsr_report_builder.py +3 -3
- fides/api/service/privacy_request/dsr_package/templates/attachments_index.html +41 -30
- fides/api/service/privacy_request/dsr_package/templates/clickme.html +68 -0
- fides/api/service/privacy_request/dsr_package/templates/dataset_index.html +30 -27
- fides/api/service/privacy_request/duplication_detection.py +439 -0
- fides/api/service/privacy_request/email_batch_service.py +2 -1
- fides/api/service/privacy_request/request_runner_service.py +31 -5
- fides/api/service/privacy_request/request_service.py +4 -2
- fides/api/service/saas_request/saas_request_override_factory.py +7 -2
- fides/api/task/conditional_dependencies/privacy_request/__init__.py +0 -0
- fides/api/task/conditional_dependencies/privacy_request/convenience_fields.py +155 -0
- fides/api/task/conditional_dependencies/privacy_request/privacy_request_data.py +140 -0
- fides/api/task/conditional_dependencies/privacy_request/schemas.py +212 -0
- fides/api/task/conditional_dependencies/util.py +111 -0
- fides/api/task/execute_request_tasks.py +17 -4
- fides/api/task/manual/manual_task_conditional_evaluation.py +32 -27
- fides/api/task/manual/manual_task_graph_task.py +4 -1
- fides/api/task/manual/manual_task_utils.py +63 -20
- fides/api/tasks/__init__.py +14 -0
- fides/api/util/cache.py +4 -0
- fides/api/util/connection_util.py +2 -3
- fides/api/util/fuzzy_search_utils.py +7 -1
- fides/api/util/lock.py +13 -3
- fides/api/util/logger.py +58 -4
- fides/api/util/logger_context_utils.py +3 -1
- fides/api/util/memory_watchdog.py +118 -0
- fides/api/util/saas_config_updater.py +12 -2
- fides/api/worker/__init__.py +4 -0
- fides/common/api/scope_registry.py +6 -0
- fides/common/api/v1/urn_registry.py +11 -2
- fides/config/__init__.py +8 -2
- fides/config/celery_settings.py +42 -0
- fides/service/messaging/messaging_service.py +75 -74
- fides/service/privacy_request/privacy_request_service.py +151 -19
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/chunks/1099-688fa865621531cc.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{1115-26393b586775ea29.js → 1115-7fd171dac1eb0e51.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/1276.deb10ae2643f8463.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1438-8a33b3834d6e43f3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6148-59a59d5c5925344f.js → 1533-84e250d1f26e6d7d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/1821-c1daa160f492aacf.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/1840-359ee056e4cf6629.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{1975.afe8cad52f904fcf.js → 1975.bef017bc80e2012c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{2040-7eed8491ca7276ed.js → 2040-70972e15960d9afe.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/2121.321b0fd3932164d4.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{2397-083fc511acb6105d.js → 2397-3434603a97f3f5d6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/2921-49ed0ed897832958.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3150-da5406b80d25fe6d.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3214-90ce0a366b0f461a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3377-02bf9780fd383d94.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3615-5e2d062d684b8fa1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3655-5e1ba5dd68b5ec48.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3729-31ff8ba51491bf21.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3872-cff30ca0844fe2b1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3931-8bedde156fe83564.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{401-7c345d019bb9bcbd.js → 401-582d9970d89deefe.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4093-7e47408c28de5375.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4259.d1507e0db19cbed7.js → 4259.05038c9b78467244.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4322-f6aeff6880726c83.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4339-04a715ab07122744.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4496-ccbce2459174e0d6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/454-d5c2c84f1a14e4f1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4589-c1d83c6a8dab4d30.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4809-a8f4a108a42f53ed.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4903-19c0bc07a956dfa8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/4910-d990773601f794c1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{5185-e7f8b81dd3dfbe0b.js → 5185-96423702fba70ced.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/5258-4e308cca01d59367.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/5487-5c3501754bf027ba.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/549-1bdc3e6f3264c020.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/5505-43b9c39491b88e08.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{5783-21775c232dce7af7.js → 5783-016dfcee8e49bf61.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/5826-4db99ea4e5077911.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/590-be447cacf12419dd.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6084-91badbc6569a0efb.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6344-026cb323c1d49926.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6362-ba0e12f2fc4cad94.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6853-de9905d28e5b19b3.js → 6853-882889659769d7b4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6882-bb1b469d7d7f5335.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6954-0dcf22a9aabe39c5.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/7170-46db82bb5b55e856.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/7218-2ace8c82e3e7eb74.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{7245-c9bc628d078c2170.js → 7245-1cdafb35f289861b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/7630-7f75ab7b8df42eb3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/7654-2e9a8be02e41769a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{8212-393420e5a9751791.js → 8212-348ddd2b6933db70.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9014-eeae6f581158e645.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{5596-29a7c8322530b7cf.js → 9195-da717d324917b049.js} +3 -3
- fides/ui-build/static/admin/_next/static/chunks/9450-b7b7bb1d755ecf57.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9676.7d029a5383595b69.js → 9676.1f395eeb9cc34968.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9826-a737a9956c1d0905.js → 9826-1078e46f3ac0b688.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9911-ece086f2230e34f0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{404-eb019192ce498f32.js → 404-800be6996aaa999c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-ee588a308812715d.js → _app-de4b578c904df772.js} +70 -70
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-7081e0e49f67716c.js → manual-f12020b82dd4bd1a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-a188f84239f4b2a8.js → multiple-a911b7990371704d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{add-systems-ee9df33ebd471099.js → add-systems-ad585b79953c2753.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-4f9cf087fcee87e6.js → add-vendors-bb263d394ca1c8fa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure-6907c368d8611c44.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-1b02a4991201b7e4.js → [id]-c32f381af358149b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-a5e738a234dadc7e.js → new-6efb3c069d8b47dd.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-b08abefec298ccf1.js → privacy-experience-1975c529905eea9b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-8c164c4b8310214e.js → [id]-b05331178928ab52.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-db789002d84c8829.js → new-516834e930bb0d0d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices-985717f2565f9d9d.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-5cba30eba1e97e56.js → properties-2be773e08498e40c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-850afb74f4192e87.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{consent-614af0a2c2ba966c.js → consent-b63d1e395d879b86.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn]-5b28f0f674ea87bd.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]-676177e2f3c9c2a2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects-794906929efb8e1d.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/[resourceUrn]-3ccbf7c0d06507b9.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources-81d1b50585468fb0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-1ea0b24d306b1e67.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/datastore/[monitorId]-d4861a4a218bb65a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/infrastructure/[monitorId]-66bd265044daf97d.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/infrastructure-c9c79fa8576a4f77.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/website/[monitorId]/[systemId]-60cacc3232c2eead.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/website/[monitorId]-437bd64a3016de36.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-8349248c2da970a6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/activity-6919a1d6cadaa46b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-b072cf25aefc98f6.js → [resourceUrn]-1a50d421897d3da1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-fd3e8817d8e6dee4.js → detection-49509414a15e8393.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-146624cf59792bf7.js → [resourceUrn]-1a1bb80b586d0c0f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9695cc9c85592ec5.js → discovery-49de61df1e8e7fba.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-c2719f5cff20c0f8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-5f88280db168083e.js → [...subfieldNames]-415015aebab60436.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-bfbcf19c28c794ae.js → [collectionName]-5accb09715b5122d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]-aebecca1d8ec5f98.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/new-9551a82ddec9f909.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset-72f8fe47beef0f09.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/[id]-3a8aa3f633528e88.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/new-87512616f35ec6da.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-2f1bf4eac7aa55fd.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{fides-js-docs-1f4335dca5c09860.js → fides-js-docs-5235760b3e508d7d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{index-1343fa525a206571.js → index-8d67cd2872cb682a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-7a631df29cd0e31a.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-ea3bc43cdaf273de.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/new-privacy-requests-5d8632bba1b81cd4.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/digests/{[id]-63b3be660fb12c0f.js → [id]-92e01822ecde8fb8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/digests/{new-d3b577962dd33266.js → new-1256cf6d3f6794e0.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/digests-0daac00911d27617.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/providers/{[key]-720cde29f81db47f.js → [key]-2d976fe5e8ba0a3a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/providers/{new-3668866076b53016.js → new-7766ba497b863740.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/providers-f2880d2ed4734270.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/templates/{[id]-8063dceb32310c85.js → [id]-669f585c3458faff.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/templates/{add-template-4931c70bee62232f.js → add-template-c79e7724e4bc3899.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/templates-891654e8dc13965c.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{notifications-93af719dab3bd053.js → notifications-40c8148244c5d347.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/ant-components-1e86f3e28bd23ed6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/AntForm-9b7bd8c38f02c091.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/FormikAntFormItem-b18a53a940cf9e19.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/FormikControlled-6d028d7450e77578.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/FormikField-8399083ee2cd8cc2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/forms-0a910125cdb2b3b8.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-acc90b6f7fe915cb.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-bbc42026f2685438.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-fa82cffba448ccd8.js → configure-c08ca6ad21c99799.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-77c2db582f8823bc.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-bc1c289647e52c48.js → [id]-74ccea4868408e3d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-cbfaa23d96f5ed0b.js → add-property-7d9f09bfe9d44dfc.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{properties-a15a3fd0ed88f39c.js → properties-cd77bc30672bb1fa.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-cc3bd9540132d5ed.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/sandbox/privacy-notices-7ce7d720107ab4b5.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-19de0024418a4924.js → [purpose_id]-ae789892343c24f5.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent-13ee1b331ced0846.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields/[id]-ee4e43692336a330.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields/new-ca51d794abfcbf25.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-ecd1dc5db8e81409.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-3b87002753b23ca5.js → domain-records-df06f7e2f668c540.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domains-aacd9d0ad47082d4.js → domains-da0c77bd510c6c51.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{email-templates-3cdd0b39901190ba.js → email-templates-950b0c115bf673d8.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-61076eedbfd137b9.js → locations-7e36cb4756973a9d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{organization-b07b11d6002f8c8c.js → organization-a0e5ed486d24ccf3.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/privacy-requests-3cdebafb6870d3ad.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-b7c0d3b1b754e70f.js → regulations-159aad34f1021320.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/test-datasets-dbd1a64090ad0946.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-59c89489fa32a4cb.js → [id]-7d042497a57a3788.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems-21f1172e73dfc9f0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-e553161e6338ee48.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-7dce2916cc589c54.js → new-efc4af017723e57a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/{[id]-5d5a68e555d18693.js → [id]-b152319d67372ee4.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management-dba8692491f7935e.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/webpack-09ca52bc7beb0b43.js +1 -0
- fides/ui-build/static/admin/_next/static/css/3d66bb57ddcb0978.css +1 -0
- fides/ui-build/static/admin/_next/static/css/a1e4be9466578ef1.css +1 -0
- fides/ui-build/static/admin/_next/static/l2vgGUHB04Fi4oEVlVMrc/_buildManifest.js +1 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/datastore/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/datastore.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/infrastructure/[monitorId].html +1 -0
- fides/ui-build/static/admin/data-discovery/action-center/infrastructure.html +1 -0
- fides/ui-build/static/admin/data-discovery/action-center/website/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/website/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/website.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +2 -2
- fides/ui-build/static/admin/lib/fides.js +2 -2
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/new-privacy-requests.html +1 -1
- fides/ui-build/static/admin/notifications/digests/[id].html +1 -1
- fides/ui-build/static/admin/notifications/digests/new.html +1 -1
- fides/ui-build/static/admin/notifications/digests.html +1 -1
- fides/ui-build/static/admin/notifications/providers/[key].html +1 -1
- fides/ui-build/static/admin/notifications/providers/new.html +1 -1
- fides/ui-build/static/admin/notifications/providers.html +1 -1
- fides/ui-build/static/admin/notifications/templates/[id].html +1 -1
- fides/ui-build/static/admin/notifications/templates/add-template.html +1 -1
- fides/ui-build/static/admin/notifications/templates.html +1 -1
- fides/ui-build/static/admin/notifications.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/sandbox/privacy-notices.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields/[id].html +1 -1
- fides/ui-build/static/admin/settings/custom-fields/new.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/privacy-requests.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/api/service/privacy_request/dsr_package/templates/welcome.html +0 -66
- fides/ui-build/static/admin/_next/static/chunks/1099-31f9d973bc287df8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1316-2606e19807c08aa5.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1437-8b1f6c8797c68bfd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1467-8808ec8836e033f9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/155-f6302d32cba4cab6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1817-2d5cf537a2992c79.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2121.1de2fa060eae34aa.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2921-34a43f2f8f5e5e69.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/2962-342ad1b4ab402ded.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3377-988ac2f3a2e8d5d4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3446-f40c352c43ac950c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/346-aa3b88efb85f2e28.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3696-6f90e41a53d22920.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3700-f695f2f6b8251971.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3772-9f1713f9d5f97a10.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3873-d18e47b327445db5.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3923-f0a85dc5c3684fa0.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4496-4ff19366c597ec16.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4817-1f3e6ea38625d8d5.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5163-e682273cd76a7d07.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5258-62d6bc19add60aa6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5279-bd6cccabdd6ca447.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/549-28537a6de666944b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5619-9b50cec521203989.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/5643-3459282d296a3c59.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6277-3759894435cb8569.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6315-e2fb5ea77179a871.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6344-ca66a6e10d128179.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6419-11d67f7fd4e2f247.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6954-84789a4e4fb04eb9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/699-8ca44b0de9fa20f0.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7059-2bb7c38578549703.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7202.3febe9e89640c712.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7476-a02d970ea4d3f7d0.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7488-cf92601852e3c509.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7630-a11610c2b31ab2ca.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7773-9ae233109bc64ec2.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/796-e36d610066135f8c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/8011-75af8b480fa114e6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/8373-22b4d20e8cc06b3a.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/8765-f622a35b40a7ec63.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9037-453224ba3ee65b13.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9046-d9c6498368b993d1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9187-7438242f0d380bb0.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9278-08cc704317fe535e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9330-f753636a31c4ea04.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9682-d1a3afa1394f8304.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9729-fcf6ff4e3534e4a8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9899-d6437facac926264.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9965-25621dd507e0cfd6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure-9af75caefc74eaca.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices-6528eb24165aceb6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/reporting-11f1683aa15e1a62.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn]-32bd7a7c990e5bf6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]-d60761c20382b259.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects-07e7d38ce34e1e6c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/[resourceUrn]-b07a0707f8c2ec0d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources-aed94957009eb3fd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog-127c114dd8f102ed.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/datastore/[monitorId]-5aa7a9fa96160de8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/website/[monitorId]/[systemId]-899bf30dde8b3292.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/website/[monitorId]-73085f50abb775c0.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center-823d0dd77e66585b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/activity-24a82e07a0008516.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datamap-3b100c44ea9e3988.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]-6fbe2b584a509226.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/new-3d1e48f4b95d7f6b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset-d3c6ecf7f29bea6e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/[id]-0a4aa42be2da0255.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/new-14313e441a13192c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-f139d1ce26404f30.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/[id]-b2d3d28b10a758e6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-a733e5d7c3ce9bb8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/new-privacy-requests-df0c95e408c54c7e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/digests-aed9afd988a48acf.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/providers-36a0ac36062abd02.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/notifications/templates-a14c876b49422597.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/ant-components-9cfb469de7b4aeab.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/AntForm-e715cc654fb6a5cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/FormikAntFormItem-72ae299bcb6adae6.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/FormikControlled-2337f8c81a766eb0.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/FormikField-0af454f55494f6fa.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/forms-d1b90ffa996fbd89.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/table-migration-19724b9e0581b96d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-cdd3754289a28317.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-cc41ae605f2b55ae.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-6e4c535b6d614596.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/datamap-632b3ee563d070f2.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/sandbox/privacy-notices-afe921f6e2a526fb.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent-5b6807dced8d03c5.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields/[id]-8634aae3259def37.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields/new-2d9dcca17965dc57.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/custom-fields-7dce52bfc1b2652c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/privacy-requests-084a2b4431d35322.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/test-datasets-b72d36243a0a545c.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems-cfaa37a0df83674b.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-56a5434969cbe9ba.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management-e6a211d8a0401086.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/webpack-c2c11809187b9f84.js +0 -1
- fides/ui-build/static/admin/_next/static/css/d41a048a166d50e4.css +0 -1
- fides/ui-build/static/admin/_next/static/css/d78390d6134d8328.css +0 -1
- fides/ui-build/static/admin/_next/static/wCNFtmYQhEDMaMPeBB4BM/_buildManifest.js +0 -1
- fides/ui-build/static/admin/poc/table-migration.html +0 -1
- {ethyca_fides-2.73.2b0.dist-info → ethyca_fides-2.76.0rc2.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.73.2b0.dist-info → ethyca_fides-2.76.0rc2.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.73.2b0.dist-info → ethyca_fides-2.76.0rc2.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.73.2b0.dist-info → ethyca_fides-2.76.0rc2.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{wCNFtmYQhEDMaMPeBB4BM → l2vgGUHB04Fi4oEVlVMrc}/_ssgManifest.js +0 -0
|
@@ -3,7 +3,7 @@ from typing import Callable, List, Optional, Tuple
|
|
|
3
3
|
from celery.app.task import Task
|
|
4
4
|
from loguru import logger
|
|
5
5
|
from sqlalchemy.exc import OperationalError
|
|
6
|
-
from sqlalchemy.orm import Query, Session
|
|
6
|
+
from sqlalchemy.orm import Query, Session, selectinload
|
|
7
7
|
from tenacity import (
|
|
8
8
|
RetryCallState,
|
|
9
9
|
retry,
|
|
@@ -21,6 +21,7 @@ from fides.api.common_exceptions import (
|
|
|
21
21
|
)
|
|
22
22
|
from fides.api.graph.config import TERMINATOR_ADDRESS, CollectionAddress
|
|
23
23
|
from fides.api.models.connectionconfig import ConnectionConfig
|
|
24
|
+
from fides.api.models.policy import Policy
|
|
24
25
|
from fides.api.models.privacy_request import ExecutionLog, PrivacyRequest, RequestTask
|
|
25
26
|
from fides.api.models.worker_task import (
|
|
26
27
|
RESUMABLE_EXECUTION_LOG_STATUSES,
|
|
@@ -48,11 +49,23 @@ def get_privacy_request_and_task(
|
|
|
48
49
|
session: Session, privacy_request_id: str, privacy_request_task_id: str
|
|
49
50
|
) -> Tuple[PrivacyRequest, RequestTask]:
|
|
50
51
|
"""
|
|
51
|
-
Retrieves and validates a privacy request and its associated task
|
|
52
|
+
Retrieves and validates a privacy request and its associated task.
|
|
53
|
+
Eager loads policy.rules to avoid N+1 queries during manual task execution.
|
|
52
54
|
"""
|
|
53
55
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
+
# Eager load relationships to avoid N+1 queries during manual task execution:
|
|
57
|
+
# - policy.rules: accessed by get_policy_convenience_fields()
|
|
58
|
+
# - provided_identities: accessed by get_persisted_identity()
|
|
59
|
+
# - custom_fields: accessed by get_persisted_custom_privacy_request_fields()
|
|
60
|
+
privacy_request: Optional[PrivacyRequest] = (
|
|
61
|
+
session.query(PrivacyRequest)
|
|
62
|
+
.options(
|
|
63
|
+
selectinload(PrivacyRequest.policy).selectinload(Policy.rules), # type: ignore[attr-defined]
|
|
64
|
+
selectinload(PrivacyRequest.provided_identities), # type: ignore[attr-defined]
|
|
65
|
+
selectinload(PrivacyRequest.custom_fields), # type: ignore[attr-defined]
|
|
66
|
+
)
|
|
67
|
+
.filter(PrivacyRequest.id == privacy_request_id)
|
|
68
|
+
.first()
|
|
56
69
|
)
|
|
57
70
|
request_task: Optional[RequestTask] = RequestTask.get(
|
|
58
71
|
db=session, object_id=privacy_request_task_id
|
|
@@ -8,13 +8,21 @@ from fides.api.models.manual_task import ManualTask
|
|
|
8
8
|
from fides.api.models.manual_task.conditional_dependency import (
|
|
9
9
|
ManualTaskConditionalDependency,
|
|
10
10
|
)
|
|
11
|
+
from fides.api.models.privacy_request import PrivacyRequest
|
|
11
12
|
from fides.api.task.conditional_dependencies.evaluator import ConditionEvaluator
|
|
13
|
+
from fides.api.task.conditional_dependencies.privacy_request.privacy_request_data import (
|
|
14
|
+
PrivacyRequestDataTransformer,
|
|
15
|
+
)
|
|
12
16
|
from fides.api.task.conditional_dependencies.schemas import EvaluationResult
|
|
17
|
+
from fides.api.task.conditional_dependencies.util import extract_nested_field_value
|
|
13
18
|
from fides.api.util.collection_util import Row
|
|
14
19
|
|
|
15
20
|
|
|
16
21
|
def extract_conditional_dependency_data_from_inputs(
|
|
17
|
-
*inputs: list[Row],
|
|
22
|
+
*inputs: list[Row],
|
|
23
|
+
manual_task: ManualTask,
|
|
24
|
+
input_keys: list[CollectionAddress],
|
|
25
|
+
privacy_request: PrivacyRequest,
|
|
18
26
|
) -> dict[str, Any]:
|
|
19
27
|
"""
|
|
20
28
|
Extract data for conditional dependency field addresses from input data.
|
|
@@ -33,14 +41,35 @@ def extract_conditional_dependency_data_from_inputs(
|
|
|
33
41
|
|
|
34
42
|
conditional_data: dict[str, Any] = {}
|
|
35
43
|
|
|
36
|
-
# Get all conditional dependencies field addresses
|
|
44
|
+
# Get all task input conditional dependencies field addresses
|
|
37
45
|
field_addresses = [
|
|
38
46
|
dependency.field_address
|
|
39
47
|
for dependency in manual_task.conditional_dependencies
|
|
40
48
|
if dependency.field_address
|
|
49
|
+
and not dependency.field_address.startswith("privacy_request.")
|
|
41
50
|
]
|
|
42
51
|
|
|
43
|
-
#
|
|
52
|
+
# Get all privacy request conditional dependencies field addresses
|
|
53
|
+
privacy_request_field_addresses = {
|
|
54
|
+
dependency.field_address
|
|
55
|
+
for dependency in manual_task.conditional_dependencies
|
|
56
|
+
if dependency.field_address
|
|
57
|
+
and dependency.field_address.startswith("privacy_request.")
|
|
58
|
+
}
|
|
59
|
+
# If there are any privacy request conditional dependencies field addresses,
|
|
60
|
+
# transform the privacy request data into a dictionary structure for evaluation
|
|
61
|
+
if privacy_request_field_addresses:
|
|
62
|
+
privacy_request_data_transformer = PrivacyRequestDataTransformer(
|
|
63
|
+
privacy_request
|
|
64
|
+
)
|
|
65
|
+
conditional_privacy_request_data = (
|
|
66
|
+
privacy_request_data_transformer.to_evaluation_data(
|
|
67
|
+
privacy_request_field_addresses
|
|
68
|
+
)
|
|
69
|
+
)
|
|
70
|
+
conditional_data = {**conditional_privacy_request_data}
|
|
71
|
+
|
|
72
|
+
# if no field addresses, return conditional data which may contain privacy request data or be empty
|
|
44
73
|
# This will allow the manual task to be executed if there are no conditional dependencies
|
|
45
74
|
if not field_addresses:
|
|
46
75
|
return conditional_data
|
|
@@ -103,30 +132,6 @@ def parse_field_address(field_address: str) -> tuple[str, list[str]]:
|
|
|
103
132
|
return source_collection_key, field_path
|
|
104
133
|
|
|
105
134
|
|
|
106
|
-
def extract_nested_field_value(data: Any, field_path: list[str]) -> Any:
|
|
107
|
-
"""
|
|
108
|
-
Extract a nested field value by traversing the field path.
|
|
109
|
-
|
|
110
|
-
Args:
|
|
111
|
-
data: The data to extract from (usually a dict)
|
|
112
|
-
field_path: List of field names to traverse (e.g., ["profile", "preferences", "theme"])
|
|
113
|
-
|
|
114
|
-
Returns:
|
|
115
|
-
The value at the end of the field path, or None if not found
|
|
116
|
-
"""
|
|
117
|
-
if not field_path:
|
|
118
|
-
return data
|
|
119
|
-
|
|
120
|
-
current = data
|
|
121
|
-
for field_name in field_path:
|
|
122
|
-
if isinstance(current, dict) and field_name in current:
|
|
123
|
-
current = current[field_name]
|
|
124
|
-
else:
|
|
125
|
-
return None
|
|
126
|
-
|
|
127
|
-
return current
|
|
128
|
-
|
|
129
|
-
|
|
130
135
|
def set_nested_value(field_address: str, value: Any) -> dict[str, Any]:
|
|
131
136
|
"""
|
|
132
137
|
Set a field value in the conditional data structure.
|
|
@@ -158,7 +158,10 @@ class ManualTaskGraphTask(GraphTask):
|
|
|
158
158
|
|
|
159
159
|
# Extract conditional dependency data from inputs
|
|
160
160
|
conditional_data = extract_conditional_dependency_data_from_inputs(
|
|
161
|
-
*inputs,
|
|
161
|
+
*inputs,
|
|
162
|
+
manual_task=manual_task,
|
|
163
|
+
input_keys=self.execution_node.input_keys,
|
|
164
|
+
privacy_request=self.resources.request,
|
|
162
165
|
)
|
|
163
166
|
# Evaluate conditional dependencies
|
|
164
167
|
evaluation_result = evaluate_conditional_dependencies(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
-
from sqlalchemy.orm import Session
|
|
3
|
+
from sqlalchemy.orm import Session, selectinload
|
|
4
4
|
|
|
5
5
|
from fides.api.graph.config import (
|
|
6
6
|
Collection,
|
|
@@ -65,6 +65,12 @@ def get_manual_task_for_connection_config(
|
|
|
65
65
|
return (
|
|
66
66
|
db.query(ManualTask)
|
|
67
67
|
.join(ConnectionConfig, ManualTask.parent_entity_id == ConnectionConfig.id)
|
|
68
|
+
.options(
|
|
69
|
+
selectinload(ManualTask.configs).selectinload(
|
|
70
|
+
"field_definitions"
|
|
71
|
+
), # type: ignore[attr-defined]
|
|
72
|
+
selectinload(ManualTask.conditional_dependencies),
|
|
73
|
+
)
|
|
68
74
|
.filter(
|
|
69
75
|
ConnectionConfig.key == connection_config_key,
|
|
70
76
|
ManualTask.parent_entity_type == "connection_config",
|
|
@@ -121,21 +127,17 @@ def create_conditional_dependency_scalar_fields(
|
|
|
121
127
|
return fields
|
|
122
128
|
|
|
123
129
|
|
|
124
|
-
def
|
|
125
|
-
|
|
130
|
+
def _create_collection_from_manual_task(
|
|
131
|
+
manual_task: ManualTask,
|
|
126
132
|
) -> Optional[Collection]:
|
|
127
|
-
|
|
128
|
-
manual_task = get_manual_task_for_connection_config(db, connection_key)
|
|
129
|
-
|
|
130
|
-
if not manual_task:
|
|
131
|
-
return None
|
|
132
|
-
|
|
133
|
+
"""Create a Collection from a ManualTask. Helper function to avoid duplication."""
|
|
133
134
|
# Get conditional dependency field addresses - raw field data
|
|
134
135
|
conditional_field_addresses: set[str] = {
|
|
135
136
|
dependency.field_address
|
|
136
137
|
for dependency in manual_task.conditional_dependencies
|
|
137
138
|
if dependency.condition_type == ConditionalDependencyType.leaf
|
|
138
139
|
and dependency.field_address is not None
|
|
140
|
+
and not dependency.field_address.startswith("privacy_request.")
|
|
139
141
|
}
|
|
140
142
|
|
|
141
143
|
# Create scalar fields for data category fields and conditional dependency field addresses
|
|
@@ -152,6 +154,18 @@ def create_collection_for_connection_key(
|
|
|
152
154
|
return Collection(name=ManualTaskAddress.MANUAL_DATA_COLLECTION, fields=fields)
|
|
153
155
|
|
|
154
156
|
|
|
157
|
+
def create_collection_for_connection_key(
|
|
158
|
+
db: Session, connection_key: str
|
|
159
|
+
) -> Optional[Collection]:
|
|
160
|
+
# Get the manual task for this connection config
|
|
161
|
+
manual_task = get_manual_task_for_connection_config(db, connection_key)
|
|
162
|
+
|
|
163
|
+
if not manual_task:
|
|
164
|
+
return None
|
|
165
|
+
|
|
166
|
+
return _create_collection_from_manual_task(manual_task)
|
|
167
|
+
|
|
168
|
+
|
|
155
169
|
def create_manual_task_artificial_graphs(db: Session) -> list[GraphDataset]:
|
|
156
170
|
"""
|
|
157
171
|
Create artificial GraphDataset objects for manual tasks that can be included
|
|
@@ -167,23 +181,52 @@ def create_manual_task_artificial_graphs(db: Session) -> list[GraphDataset]:
|
|
|
167
181
|
Returns:
|
|
168
182
|
List of GraphDataset objects representing manual tasks as individual collections
|
|
169
183
|
"""
|
|
170
|
-
manual_task_graphs = []
|
|
184
|
+
manual_task_graphs: list[GraphDataset] = []
|
|
171
185
|
manual_addresses = get_manual_task_addresses(db)
|
|
172
186
|
|
|
187
|
+
if not manual_addresses:
|
|
188
|
+
return manual_task_graphs
|
|
189
|
+
|
|
190
|
+
# Batch load all manual tasks with their relationships to avoid N+1 queries
|
|
191
|
+
connection_keys = [address.dataset for address in manual_addresses]
|
|
192
|
+
manual_tasks = (
|
|
193
|
+
db.query(ManualTask, ConnectionConfig.key)
|
|
194
|
+
.join(ConnectionConfig, ManualTask.parent_entity_id == ConnectionConfig.id)
|
|
195
|
+
.options(
|
|
196
|
+
selectinload(ManualTask.configs).selectinload(
|
|
197
|
+
"field_definitions"
|
|
198
|
+
), # type: ignore[attr-defined]
|
|
199
|
+
selectinload(ManualTask.conditional_dependencies),
|
|
200
|
+
)
|
|
201
|
+
.filter(
|
|
202
|
+
ConnectionConfig.key.in_(connection_keys),
|
|
203
|
+
ManualTask.parent_entity_type == "connection_config",
|
|
204
|
+
)
|
|
205
|
+
.all()
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
# Create a lookup map by connection key
|
|
209
|
+
manual_task_map = {connection_key: task for task, connection_key in manual_tasks}
|
|
210
|
+
|
|
173
211
|
for address in manual_addresses:
|
|
174
212
|
connection_key = address.dataset
|
|
213
|
+
manual_task = manual_task_map.get(connection_key)
|
|
175
214
|
|
|
176
|
-
|
|
177
|
-
|
|
215
|
+
if not manual_task:
|
|
216
|
+
continue
|
|
178
217
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
218
|
+
# Create collection using the helper function to avoid duplication
|
|
219
|
+
collection = _create_collection_from_manual_task(manual_task)
|
|
220
|
+
if not collection:
|
|
221
|
+
continue
|
|
222
|
+
|
|
223
|
+
# Create a synthetic GraphDataset with all manual task collections
|
|
224
|
+
graph_dataset = GraphDataset(
|
|
225
|
+
name=connection_key,
|
|
226
|
+
collections=[collection],
|
|
227
|
+
connection_key=connection_key,
|
|
228
|
+
)
|
|
186
229
|
|
|
187
|
-
|
|
230
|
+
manual_task_graphs.append(graph_dataset)
|
|
188
231
|
|
|
189
232
|
return manual_task_graphs
|
fides/api/tasks/__init__.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from typing import Any, ContextManager, Dict, List, Optional
|
|
2
2
|
|
|
3
3
|
from celery import Celery, Task
|
|
4
|
+
from celery.signals import setup_logging as celery_setup_logging
|
|
4
5
|
from loguru import logger
|
|
5
6
|
from sqlalchemy.exc import OperationalError
|
|
6
7
|
from sqlalchemy.orm import Session
|
|
@@ -18,6 +19,8 @@ from fides.config import CONFIG, FidesConfig
|
|
|
18
19
|
|
|
19
20
|
MESSAGING_QUEUE_NAME = "fidesops.messaging"
|
|
20
21
|
PRIVACY_PREFERENCES_QUEUE_NAME = "fides.privacy_preferences" # This queue is used in Fidesplus for saving privacy preferences and notices served
|
|
22
|
+
PRIVACY_PREFERENCES_EXPORT_JOB_QUEUE_NAME = "fides.privacy_request_exports"
|
|
23
|
+
PRIVACY_PREFERENCES_INGESTION_JOB_QUEUE_NAME = "fides.privacy_request_ingestion"
|
|
21
24
|
DSR_QUEUE_NAME = "fides.dsr" # This queue is used for running data subject requests
|
|
22
25
|
CONSENT_WEBHOOK_QUEUE_NAME = "fidesplus.consent_webhooks" # This queue is used for processing consent webhooks from 3rd-party APIs
|
|
23
26
|
DISCOVERY_MONITORS_DETECTION_QUEUE_NAME = "fidesplus.discovery_monitors_detection" # This queue is used for running discovery monitors detection tasks
|
|
@@ -123,6 +126,17 @@ def _create_celery(config: FidesConfig = CONFIG) -> Celery:
|
|
|
123
126
|
celery_app = _create_celery(CONFIG)
|
|
124
127
|
|
|
125
128
|
|
|
129
|
+
@celery_setup_logging.connect
|
|
130
|
+
def configure_celery_logging(**kwargs: Any) -> None:
|
|
131
|
+
"""
|
|
132
|
+
Prevent Celery from configuring logging on worker startup.
|
|
133
|
+
|
|
134
|
+
By connecting to the setup_logging signal and doing nothing, we prevent Celery
|
|
135
|
+
from overriding our Loguru logging configuration. Our logging setup in _create_celery
|
|
136
|
+
has already configured logging with InterceptHandler to capture all stdlib logs.
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
|
|
126
140
|
def get_worker_ids() -> List[Optional[str]]:
|
|
127
141
|
"""
|
|
128
142
|
Returns a list of the connected healthy worker UUIDs.
|
fides/api/util/cache.py
CHANGED
|
@@ -17,6 +17,8 @@ from fides.api.tasks import (
|
|
|
17
17
|
DISCOVERY_MONITORS_PROMOTION_QUEUE_NAME,
|
|
18
18
|
DSR_QUEUE_NAME,
|
|
19
19
|
MESSAGING_QUEUE_NAME,
|
|
20
|
+
PRIVACY_PREFERENCES_EXPORT_JOB_QUEUE_NAME,
|
|
21
|
+
PRIVACY_PREFERENCES_INGESTION_JOB_QUEUE_NAME,
|
|
20
22
|
PRIVACY_PREFERENCES_QUEUE_NAME,
|
|
21
23
|
celery_app,
|
|
22
24
|
)
|
|
@@ -432,6 +434,8 @@ def get_queue_counts() -> Dict[str, int]:
|
|
|
432
434
|
for queue in [
|
|
433
435
|
MESSAGING_QUEUE_NAME,
|
|
434
436
|
PRIVACY_PREFERENCES_QUEUE_NAME,
|
|
437
|
+
PRIVACY_PREFERENCES_EXPORT_JOB_QUEUE_NAME,
|
|
438
|
+
PRIVACY_PREFERENCES_INGESTION_JOB_QUEUE_NAME,
|
|
435
439
|
DSR_QUEUE_NAME,
|
|
436
440
|
DISCOVERY_MONITORS_DETECTION_QUEUE_NAME,
|
|
437
441
|
DISCOVERY_MONITORS_CLASSIFICATION_QUEUE_NAME,
|
|
@@ -55,9 +55,8 @@ def requeue_requires_input_requests(db: Session) -> None:
|
|
|
55
55
|
lingering in a "requires_input" state.
|
|
56
56
|
"""
|
|
57
57
|
if not AccessManualWebhook.get_enabled(db):
|
|
58
|
-
for pr in PrivacyRequest.filter(
|
|
59
|
-
|
|
60
|
-
conditions=(PrivacyRequest.status == PrivacyRequestStatus.requires_input),
|
|
58
|
+
for pr in PrivacyRequest.query_without_large_columns(db).filter(
|
|
59
|
+
PrivacyRequest.status == PrivacyRequestStatus.requires_input
|
|
61
60
|
):
|
|
62
61
|
logger.info(
|
|
63
62
|
"Queuing privacy request '{} with '{}' status now that manual inputs are no longer required.",
|
|
@@ -66,7 +66,13 @@ def build_automaton(
|
|
|
66
66
|
|
|
67
67
|
logger.debug("Creating new automaton...")
|
|
68
68
|
automaton = ahocorasick.Automaton() # pylint: disable=c-extension-no-member
|
|
69
|
-
|
|
69
|
+
# Use query_without_large_columns to prevent OOM errors when building automaton
|
|
70
|
+
# We only need identity data, not filtered results or access result URLs
|
|
71
|
+
all_privacy_requests: List[
|
|
72
|
+
PrivacyRequest
|
|
73
|
+
] = PrivacyRequest.query_without_large_columns(db).yield_per(
|
|
74
|
+
1000
|
|
75
|
+
) # type: ignore
|
|
70
76
|
for request in all_privacy_requests:
|
|
71
77
|
_add_decrypted_identities_to_automaton(request.get_persisted_identity().__dict__, request.id, automaton) # type: ignore
|
|
72
78
|
set_automaton_cache_signal()
|
fides/api/util/lock.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from contextlib import contextmanager
|
|
4
|
-
from typing import Generator
|
|
4
|
+
from typing import Generator, Optional
|
|
5
5
|
|
|
6
6
|
from loguru import logger
|
|
7
7
|
from redis.lock import Lock
|
|
@@ -18,7 +18,12 @@ def get_redis_lock(lock_key: str, timeout: int) -> Lock:
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
@contextmanager
|
|
21
|
-
def redis_lock(
|
|
21
|
+
def redis_lock(
|
|
22
|
+
lock_key: str,
|
|
23
|
+
timeout: int,
|
|
24
|
+
blocking: bool = False,
|
|
25
|
+
blocking_timeout: Optional[int] = None,
|
|
26
|
+
) -> Generator[Lock | None, None, None]:
|
|
22
27
|
"""
|
|
23
28
|
A context manager for acquiring a Redis lock.
|
|
24
29
|
If the lock is acquired, it yields the lock object.
|
|
@@ -30,7 +35,12 @@ def redis_lock(lock_key: str, timeout: int) -> Generator[Lock | None, None, None
|
|
|
30
35
|
"""
|
|
31
36
|
lock = get_redis_lock(lock_key, timeout)
|
|
32
37
|
|
|
33
|
-
|
|
38
|
+
# If we're blocking but no blocking timeout is provided
|
|
39
|
+
# fall back to the lock timeout as the blocking timeout
|
|
40
|
+
if blocking and blocking_timeout is None:
|
|
41
|
+
blocking_timeout = timeout
|
|
42
|
+
|
|
43
|
+
if not lock.acquire(blocking=blocking, blocking_timeout=blocking_timeout):
|
|
34
44
|
logger.info(
|
|
35
45
|
f"Another process is already running for lock '{lock_key}'. Skipping this execution."
|
|
36
46
|
)
|
fides/api/util/logger.py
CHANGED
|
@@ -7,8 +7,10 @@ Defines the logging format and other helper functions to be used throughout the
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import logging
|
|
10
|
+
import os
|
|
10
11
|
import sys
|
|
11
12
|
from contextlib import contextmanager
|
|
13
|
+
from types import FrameType
|
|
12
14
|
from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional
|
|
13
15
|
|
|
14
16
|
from loguru import logger
|
|
@@ -24,6 +26,37 @@ if TYPE_CHECKING:
|
|
|
24
26
|
MASKED = "MASKED"
|
|
25
27
|
|
|
26
28
|
|
|
29
|
+
class InterceptHandler(logging.Handler):
|
|
30
|
+
"""
|
|
31
|
+
Intercept standard library logging and redirect to Loguru.
|
|
32
|
+
|
|
33
|
+
This handler is added to the root logger to capture logs from libraries
|
|
34
|
+
that use standard library logging (SQLAlchemy, Alembic, Celery, etc.)
|
|
35
|
+
and route them through Loguru for consistent formatting and serialization.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
39
|
+
"""Emit a log record by routing it through Loguru."""
|
|
40
|
+
try:
|
|
41
|
+
level = logger.level(record.levelname).name
|
|
42
|
+
except ValueError:
|
|
43
|
+
level = str(record.levelno)
|
|
44
|
+
|
|
45
|
+
frame: Optional[FrameType] = logging.currentframe()
|
|
46
|
+
depth = 2
|
|
47
|
+
# Get the directory path of the logging module to match all files within it
|
|
48
|
+
logging_module_path = os.path.dirname(logging.__file__) + os.sep
|
|
49
|
+
while frame is not None and frame.f_code.co_filename.startswith(
|
|
50
|
+
logging_module_path
|
|
51
|
+
):
|
|
52
|
+
frame = frame.f_back
|
|
53
|
+
depth += 1
|
|
54
|
+
|
|
55
|
+
logger.opt(depth=depth, exception=record.exc_info).log(
|
|
56
|
+
level, record.getMessage()
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
27
60
|
class RedisSink:
|
|
28
61
|
"""A sink that writes log messages to Redis."""
|
|
29
62
|
|
|
@@ -134,18 +167,30 @@ def create_handler_dicts(
|
|
|
134
167
|
"catch": True,
|
|
135
168
|
}
|
|
136
169
|
|
|
170
|
+
# Helper function to check if a log record has custom extra context
|
|
171
|
+
# Loguru always includes an 'extra' dict, so we need to check if it has any keys
|
|
172
|
+
def has_custom_extra(log_record: Dict) -> bool:
|
|
173
|
+
"""Check if log record has custom extra context beyond Loguru's defaults."""
|
|
174
|
+
extra = log_record.get("extra", {})
|
|
175
|
+
return len(extra) > 0
|
|
176
|
+
|
|
177
|
+
# Helper to filter logs without custom extra
|
|
178
|
+
def filter_standard(log_record: Dict) -> bool:
|
|
179
|
+
"""Filter for logs without custom extra context."""
|
|
180
|
+
return not has_custom_extra(log_record)
|
|
181
|
+
|
|
137
182
|
# Configure handler
|
|
138
183
|
standard_dict = {
|
|
139
184
|
**base_config,
|
|
140
185
|
"sink": sink,
|
|
141
|
-
"filter":
|
|
186
|
+
"filter": filter_standard,
|
|
142
187
|
}
|
|
143
188
|
|
|
144
189
|
# Create extra dict with additional formatting for logs with extra context
|
|
145
190
|
extra_dict = {
|
|
146
191
|
**standard_dict,
|
|
147
192
|
"format": log_format + " | <dim>{extra}</dim>",
|
|
148
|
-
"filter":
|
|
193
|
+
"filter": has_custom_extra,
|
|
149
194
|
}
|
|
150
195
|
|
|
151
196
|
return [standard_dict, extra_dict]
|
|
@@ -171,13 +216,14 @@ def setup(config: FidesConfig) -> None:
|
|
|
171
216
|
# Configure main sink from config
|
|
172
217
|
destination = config.logging.destination
|
|
173
218
|
main_sink = sys.stdout if not destination else destination
|
|
219
|
+
is_json_serialization = config.logging.serialization == "json"
|
|
174
220
|
handlers.extend(
|
|
175
221
|
create_handler_dicts(
|
|
176
222
|
level=config.logging.level,
|
|
177
223
|
include_called_from=config.dev_mode,
|
|
178
224
|
sink=main_sink,
|
|
179
|
-
serialize=
|
|
180
|
-
colorize=config.logging.colorize,
|
|
225
|
+
serialize=is_json_serialization,
|
|
226
|
+
colorize=config.logging.colorize and not is_json_serialization,
|
|
181
227
|
)
|
|
182
228
|
)
|
|
183
229
|
|
|
@@ -196,6 +242,14 @@ def setup(config: FidesConfig) -> None:
|
|
|
196
242
|
|
|
197
243
|
logger.configure(handlers=handlers)
|
|
198
244
|
|
|
245
|
+
# Add InterceptHandler to root logger to capture standard library logs
|
|
246
|
+
# This intercepts logs from SQLAlchemy, Alembic, Celery, etc.
|
|
247
|
+
logging.root.handlers = [InterceptHandler()]
|
|
248
|
+
logging.root.setLevel(config.logging.level)
|
|
249
|
+
|
|
250
|
+
# Capture Python warnings and redirect them to logging so they get formatted by Loguru
|
|
251
|
+
logging.captureWarnings(True)
|
|
252
|
+
|
|
199
253
|
|
|
200
254
|
def obfuscate_message(message: str) -> str:
|
|
201
255
|
"""
|
|
@@ -165,7 +165,9 @@ def request_details(
|
|
|
165
165
|
|
|
166
166
|
if response is not None:
|
|
167
167
|
if CONFIG.dev_mode and response.content:
|
|
168
|
-
details[LoggerContextKeys.response.value] = response.content.decode(
|
|
168
|
+
details[LoggerContextKeys.response.value] = response.content.decode(
|
|
169
|
+
"utf-8", errors="replace"
|
|
170
|
+
)
|
|
169
171
|
|
|
170
172
|
details[LoggerContextKeys.status_code.value] = response.status_code
|
|
171
173
|
|
|
@@ -13,6 +13,7 @@ killed by the system's OOM killer which would result in a WorkerLostError and in
|
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
+
import gc
|
|
16
17
|
import os
|
|
17
18
|
import signal
|
|
18
19
|
import threading
|
|
@@ -21,6 +22,7 @@ from functools import wraps
|
|
|
21
22
|
from types import FrameType, TracebackType
|
|
22
23
|
from typing import Any, Callable, Literal, Optional, Type, TypeVar, Union
|
|
23
24
|
|
|
25
|
+
import objgraph # type: ignore
|
|
24
26
|
import psutil # type: ignore
|
|
25
27
|
from loguru import logger
|
|
26
28
|
|
|
@@ -56,6 +58,118 @@ class MemoryLimitExceeded(RuntimeError):
|
|
|
56
58
|
self.memory_percent: Optional[float] = memory_percent
|
|
57
59
|
|
|
58
60
|
|
|
61
|
+
def _capture_heap_dump() -> None:
|
|
62
|
+
"""Capture and log actionable heap dump information for memory leak diagnosis.
|
|
63
|
+
|
|
64
|
+
This function builds and logs a comprehensive memory profiling report as a single
|
|
65
|
+
message to help identify memory leaks. It's designed to be called when memory
|
|
66
|
+
threshold is exceeded, just before task termination.
|
|
67
|
+
|
|
68
|
+
The report includes:
|
|
69
|
+
- Process memory stats (RSS, VMS)
|
|
70
|
+
- Top 10 object type counts (spot anomalies like 100k of a custom class)
|
|
71
|
+
- Garbage collector stats (uncollectable objects indicate leaks)
|
|
72
|
+
|
|
73
|
+
All exceptions are caught to ensure heap dump failures don't prevent graceful
|
|
74
|
+
task termination.
|
|
75
|
+
"""
|
|
76
|
+
try:
|
|
77
|
+
# Force garbage collection to get accurate counts
|
|
78
|
+
gc.collect()
|
|
79
|
+
|
|
80
|
+
report_lines = [
|
|
81
|
+
"", # Leading newline for visual separation
|
|
82
|
+
"=" * 80,
|
|
83
|
+
"MEMORY DUMP",
|
|
84
|
+
"=" * 80,
|
|
85
|
+
"",
|
|
86
|
+
]
|
|
87
|
+
|
|
88
|
+
# Section 1: Process Memory Stats
|
|
89
|
+
try:
|
|
90
|
+
process = psutil.Process()
|
|
91
|
+
mem_info = process.memory_info()
|
|
92
|
+
report_lines.extend(
|
|
93
|
+
[
|
|
94
|
+
"PROCESS MEMORY STATS",
|
|
95
|
+
"-" * 80,
|
|
96
|
+
f"RSS (Resident Set Size): {mem_info.rss / 1024 / 1024:.1f} MiB",
|
|
97
|
+
f"VMS (Virtual Memory): {mem_info.vms / 1024 / 1024:.1f} MiB",
|
|
98
|
+
"",
|
|
99
|
+
]
|
|
100
|
+
)
|
|
101
|
+
except Exception as exc:
|
|
102
|
+
logger.debug("Failed to get process memory stats: {}", exc)
|
|
103
|
+
|
|
104
|
+
# Section 2: Object Type Counts (most actionable for finding leaks)
|
|
105
|
+
type_stats = objgraph.most_common_types(limit=10)
|
|
106
|
+
|
|
107
|
+
report_lines.extend(
|
|
108
|
+
[
|
|
109
|
+
"OBJECT TYPE COUNTS (Top 10)",
|
|
110
|
+
"-" * 80,
|
|
111
|
+
]
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
if type_stats:
|
|
115
|
+
max_typename_len = max(len(typename) for typename, _ in type_stats)
|
|
116
|
+
|
|
117
|
+
for typename, count in type_stats:
|
|
118
|
+
# Right-align count for easy scanning
|
|
119
|
+
report_lines.append(f"{typename:<{max_typename_len}} {count:>10,}")
|
|
120
|
+
else:
|
|
121
|
+
report_lines.append("No object type data available")
|
|
122
|
+
|
|
123
|
+
report_lines.append("")
|
|
124
|
+
|
|
125
|
+
# Section 3: Garbage Collector Stats
|
|
126
|
+
gc_count = gc.get_count()
|
|
127
|
+
|
|
128
|
+
report_lines.extend(
|
|
129
|
+
[
|
|
130
|
+
"GARBAGE COLLECTOR STATS",
|
|
131
|
+
"-" * 80,
|
|
132
|
+
f"Collections: gen0={gc_count[0]}, gen1={gc_count[1]}, gen2={gc_count[2]}",
|
|
133
|
+
f"Tracked objects: {len(gc.get_objects()):,}",
|
|
134
|
+
]
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
# Show uncollectable objects (strong indicator of memory leaks)
|
|
138
|
+
uncollectable = gc.garbage
|
|
139
|
+
if uncollectable:
|
|
140
|
+
report_lines.append(
|
|
141
|
+
f"⚠ UNCOLLECTABLE OBJECTS: {len(uncollectable)} (LEAK DETECTED!)"
|
|
142
|
+
)
|
|
143
|
+
# Show types of uncollectable objects
|
|
144
|
+
uncollectable_types: dict[str, int] = {}
|
|
145
|
+
for obj in uncollectable[:50]: # Limit to first 50
|
|
146
|
+
obj_type = type(obj).__name__
|
|
147
|
+
uncollectable_types[obj_type] = uncollectable_types.get(obj_type, 0) + 1
|
|
148
|
+
for obj_type, count in sorted(
|
|
149
|
+
uncollectable_types.items(), key=lambda x: x[1], reverse=True
|
|
150
|
+
):
|
|
151
|
+
report_lines.append(f" {obj_type}: {count}")
|
|
152
|
+
else:
|
|
153
|
+
report_lines.append("No uncollectable objects detected")
|
|
154
|
+
|
|
155
|
+
report_lines.extend(
|
|
156
|
+
[
|
|
157
|
+
"",
|
|
158
|
+
"=" * 80,
|
|
159
|
+
"END OF MEMORY DUMP",
|
|
160
|
+
"=" * 80,
|
|
161
|
+
"",
|
|
162
|
+
]
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
# Log the entire report as a single message
|
|
166
|
+
heap_dump_report = "\n".join(report_lines)
|
|
167
|
+
logger.error(heap_dump_report)
|
|
168
|
+
|
|
169
|
+
except Exception as exc: # pragma: no cover
|
|
170
|
+
logger.exception("Failed to capture heap dump: {}", exc)
|
|
171
|
+
|
|
172
|
+
|
|
59
173
|
class MemoryWatchdog:
|
|
60
174
|
"""Background memory monitor that enables graceful task termination.
|
|
61
175
|
|
|
@@ -138,6 +252,10 @@ class MemoryWatchdog:
|
|
|
138
252
|
"""Convert the signal into an exception on the main thread."""
|
|
139
253
|
current_percent = _system_memory_percent()
|
|
140
254
|
logger.error("Memory limit exceeded: {}%", current_percent)
|
|
255
|
+
|
|
256
|
+
# Capture heap dump before terminating
|
|
257
|
+
_capture_heap_dump()
|
|
258
|
+
|
|
141
259
|
self.stop()
|
|
142
260
|
raise MemoryLimitExceeded(
|
|
143
261
|
"Memory usage exceeded threshold", memory_percent=current_percent
|