udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30382__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of udata might be problematic. Click here for more details.
- tasks/__init__.py +109 -107
- tasks/helpers.py +18 -18
- udata/__init__.py +4 -4
- udata/admin/views.py +5 -5
- udata/api/__init__.py +135 -124
- udata/api/commands.py +45 -37
- udata/api/errors.py +5 -4
- udata/api/fields.py +23 -21
- udata/api/oauth2.py +55 -74
- udata/api/parsers.py +15 -15
- udata/api/signals.py +1 -1
- udata/api_fields.py +137 -89
- udata/app.py +56 -54
- udata/assets.py +5 -5
- udata/auth/__init__.py +37 -26
- udata/auth/forms.py +23 -15
- udata/auth/helpers.py +1 -1
- udata/auth/mails.py +3 -3
- udata/auth/password_validation.py +19 -15
- udata/auth/views.py +94 -68
- udata/commands/__init__.py +71 -69
- udata/commands/cache.py +7 -7
- udata/commands/db.py +201 -140
- udata/commands/dcat.py +36 -30
- udata/commands/fixtures.py +100 -84
- udata/commands/images.py +21 -20
- udata/commands/info.py +17 -20
- udata/commands/init.py +10 -10
- udata/commands/purge.py +12 -13
- udata/commands/serve.py +41 -29
- udata/commands/static.py +16 -18
- udata/commands/test.py +20 -20
- udata/commands/tests/fixtures.py +26 -24
- udata/commands/worker.py +31 -33
- udata/core/__init__.py +12 -12
- udata/core/activity/__init__.py +0 -1
- udata/core/activity/api.py +59 -49
- udata/core/activity/models.py +28 -26
- udata/core/activity/signals.py +1 -1
- udata/core/activity/tasks.py +16 -10
- udata/core/badges/api.py +6 -6
- udata/core/badges/commands.py +14 -13
- udata/core/badges/fields.py +8 -5
- udata/core/badges/forms.py +7 -4
- udata/core/badges/models.py +16 -31
- udata/core/badges/permissions.py +1 -3
- udata/core/badges/signals.py +2 -2
- udata/core/badges/tasks.py +3 -2
- udata/core/badges/tests/test_commands.py +10 -10
- udata/core/badges/tests/test_model.py +24 -31
- udata/core/contact_point/api.py +19 -18
- udata/core/contact_point/api_fields.py +21 -14
- udata/core/contact_point/factories.py +2 -2
- udata/core/contact_point/forms.py +7 -6
- udata/core/contact_point/models.py +3 -5
- udata/core/dataservices/api.py +26 -21
- udata/core/dataservices/factories.py +13 -11
- udata/core/dataservices/models.py +35 -40
- udata/core/dataservices/permissions.py +4 -4
- udata/core/dataservices/rdf.py +40 -17
- udata/core/dataservices/tasks.py +4 -3
- udata/core/dataset/actions.py +10 -10
- udata/core/dataset/activities.py +21 -23
- udata/core/dataset/api.py +321 -298
- udata/core/dataset/api_fields.py +443 -271
- udata/core/dataset/apiv2.py +305 -229
- udata/core/dataset/commands.py +38 -36
- udata/core/dataset/constants.py +61 -54
- udata/core/dataset/csv.py +70 -74
- udata/core/dataset/events.py +39 -32
- udata/core/dataset/exceptions.py +8 -4
- udata/core/dataset/factories.py +57 -65
- udata/core/dataset/forms.py +87 -63
- udata/core/dataset/models.py +336 -280
- udata/core/dataset/permissions.py +9 -6
- udata/core/dataset/preview.py +15 -17
- udata/core/dataset/rdf.py +156 -122
- udata/core/dataset/search.py +92 -77
- udata/core/dataset/signals.py +1 -1
- udata/core/dataset/tasks.py +63 -54
- udata/core/discussions/actions.py +5 -5
- udata/core/discussions/api.py +124 -120
- udata/core/discussions/factories.py +2 -2
- udata/core/discussions/forms.py +9 -7
- udata/core/discussions/metrics.py +1 -3
- udata/core/discussions/models.py +25 -24
- udata/core/discussions/notifications.py +18 -14
- udata/core/discussions/permissions.py +3 -3
- udata/core/discussions/signals.py +4 -4
- udata/core/discussions/tasks.py +24 -28
- udata/core/followers/api.py +32 -33
- udata/core/followers/models.py +9 -9
- udata/core/followers/signals.py +3 -3
- udata/core/jobs/actions.py +7 -7
- udata/core/jobs/api.py +99 -92
- udata/core/jobs/commands.py +48 -49
- udata/core/jobs/forms.py +11 -11
- udata/core/jobs/models.py +6 -6
- udata/core/metrics/__init__.py +2 -2
- udata/core/metrics/commands.py +34 -30
- udata/core/metrics/models.py +2 -4
- udata/core/metrics/signals.py +1 -1
- udata/core/metrics/tasks.py +3 -3
- udata/core/organization/activities.py +12 -15
- udata/core/organization/api.py +167 -174
- udata/core/organization/api_fields.py +183 -124
- udata/core/organization/apiv2.py +32 -32
- udata/core/organization/commands.py +20 -22
- udata/core/organization/constants.py +11 -11
- udata/core/organization/csv.py +17 -15
- udata/core/organization/factories.py +8 -11
- udata/core/organization/forms.py +32 -26
- udata/core/organization/metrics.py +2 -1
- udata/core/organization/models.py +87 -67
- udata/core/organization/notifications.py +18 -14
- udata/core/organization/permissions.py +10 -11
- udata/core/organization/rdf.py +14 -14
- udata/core/organization/search.py +30 -28
- udata/core/organization/signals.py +7 -7
- udata/core/organization/tasks.py +42 -61
- udata/core/owned.py +38 -27
- udata/core/post/api.py +82 -81
- udata/core/post/constants.py +8 -5
- udata/core/post/factories.py +4 -4
- udata/core/post/forms.py +13 -14
- udata/core/post/models.py +20 -22
- udata/core/post/tests/test_api.py +30 -32
- udata/core/reports/api.py +8 -7
- udata/core/reports/constants.py +1 -3
- udata/core/reports/models.py +10 -10
- udata/core/reuse/activities.py +15 -19
- udata/core/reuse/api.py +123 -126
- udata/core/reuse/api_fields.py +120 -85
- udata/core/reuse/apiv2.py +11 -10
- udata/core/reuse/constants.py +23 -23
- udata/core/reuse/csv.py +18 -18
- udata/core/reuse/factories.py +5 -9
- udata/core/reuse/forms.py +24 -21
- udata/core/reuse/models.py +55 -51
- udata/core/reuse/permissions.py +2 -2
- udata/core/reuse/search.py +49 -46
- udata/core/reuse/signals.py +1 -1
- udata/core/reuse/tasks.py +4 -5
- udata/core/site/api.py +47 -50
- udata/core/site/factories.py +2 -2
- udata/core/site/forms.py +4 -5
- udata/core/site/models.py +94 -63
- udata/core/site/rdf.py +14 -14
- udata/core/spam/api.py +16 -9
- udata/core/spam/constants.py +4 -4
- udata/core/spam/fields.py +13 -7
- udata/core/spam/models.py +27 -20
- udata/core/spam/signals.py +1 -1
- udata/core/spam/tests/test_spam.py +6 -5
- udata/core/spatial/api.py +72 -80
- udata/core/spatial/api_fields.py +73 -58
- udata/core/spatial/commands.py +67 -64
- udata/core/spatial/constants.py +3 -3
- udata/core/spatial/factories.py +37 -54
- udata/core/spatial/forms.py +27 -26
- udata/core/spatial/geoids.py +17 -17
- udata/core/spatial/models.py +43 -47
- udata/core/spatial/tasks.py +2 -1
- udata/core/spatial/tests/test_api.py +115 -130
- udata/core/spatial/tests/test_fields.py +74 -77
- udata/core/spatial/tests/test_geoid.py +22 -22
- udata/core/spatial/tests/test_models.py +5 -7
- udata/core/spatial/translations.py +16 -16
- udata/core/storages/__init__.py +16 -18
- udata/core/storages/api.py +66 -64
- udata/core/storages/tasks.py +7 -7
- udata/core/storages/utils.py +15 -15
- udata/core/storages/views.py +5 -6
- udata/core/tags/api.py +17 -14
- udata/core/tags/csv.py +4 -4
- udata/core/tags/models.py +8 -5
- udata/core/tags/tasks.py +11 -13
- udata/core/tags/views.py +4 -4
- udata/core/topic/api.py +84 -73
- udata/core/topic/apiv2.py +157 -127
- udata/core/topic/factories.py +3 -4
- udata/core/topic/forms.py +12 -14
- udata/core/topic/models.py +14 -19
- udata/core/topic/parsers.py +26 -26
- udata/core/user/activities.py +30 -29
- udata/core/user/api.py +151 -152
- udata/core/user/api_fields.py +132 -100
- udata/core/user/apiv2.py +7 -7
- udata/core/user/commands.py +38 -38
- udata/core/user/factories.py +8 -9
- udata/core/user/forms.py +14 -11
- udata/core/user/metrics.py +2 -2
- udata/core/user/models.py +68 -69
- udata/core/user/permissions.py +4 -5
- udata/core/user/rdf.py +7 -8
- udata/core/user/tasks.py +2 -2
- udata/core/user/tests/test_user_model.py +24 -16
- udata/db/tasks.py +2 -1
- udata/entrypoints.py +35 -31
- udata/errors.py +2 -1
- udata/event/values.py +6 -6
- udata/factories.py +2 -2
- udata/features/identicon/api.py +5 -6
- udata/features/identicon/backends.py +48 -55
- udata/features/identicon/tests/test_backends.py +4 -5
- udata/features/notifications/__init__.py +0 -1
- udata/features/notifications/actions.py +9 -9
- udata/features/notifications/api.py +17 -13
- udata/features/territories/__init__.py +12 -10
- udata/features/territories/api.py +14 -15
- udata/features/territories/models.py +23 -28
- udata/features/transfer/actions.py +8 -11
- udata/features/transfer/api.py +84 -77
- udata/features/transfer/factories.py +2 -1
- udata/features/transfer/models.py +11 -12
- udata/features/transfer/notifications.py +19 -15
- udata/features/transfer/permissions.py +5 -5
- udata/forms/__init__.py +5 -2
- udata/forms/fields.py +164 -172
- udata/forms/validators.py +19 -22
- udata/forms/widgets.py +9 -13
- udata/frontend/__init__.py +31 -26
- udata/frontend/csv.py +68 -58
- udata/frontend/markdown.py +40 -44
- udata/harvest/actions.py +89 -77
- udata/harvest/api.py +294 -238
- udata/harvest/backends/__init__.py +4 -4
- udata/harvest/backends/base.py +128 -111
- udata/harvest/backends/dcat.py +80 -66
- udata/harvest/commands.py +56 -60
- udata/harvest/csv.py +8 -8
- udata/harvest/exceptions.py +6 -3
- udata/harvest/filters.py +24 -23
- udata/harvest/forms.py +27 -28
- udata/harvest/models.py +88 -80
- udata/harvest/notifications.py +15 -10
- udata/harvest/signals.py +13 -13
- udata/harvest/tasks.py +11 -10
- udata/harvest/tests/factories.py +23 -24
- udata/harvest/tests/test_actions.py +136 -166
- udata/harvest/tests/test_api.py +220 -214
- udata/harvest/tests/test_base_backend.py +117 -112
- udata/harvest/tests/test_dcat_backend.py +380 -308
- udata/harvest/tests/test_filters.py +33 -22
- udata/harvest/tests/test_models.py +11 -14
- udata/harvest/tests/test_notifications.py +6 -7
- udata/harvest/tests/test_tasks.py +7 -6
- udata/i18n.py +237 -78
- udata/linkchecker/backends.py +5 -11
- udata/linkchecker/checker.py +23 -22
- udata/linkchecker/commands.py +4 -6
- udata/linkchecker/models.py +6 -6
- udata/linkchecker/tasks.py +18 -20
- udata/mail.py +21 -21
- udata/migrations/2020-07-24-remove-s-from-scope-oauth.py +9 -8
- udata/migrations/2020-08-24-add-fs-filename.py +9 -8
- udata/migrations/2020-09-28-update-reuses-datasets-metrics.py +5 -4
- udata/migrations/2020-10-16-migrate-ods-resources.py +9 -10
- udata/migrations/2021-04-08-update-schema-with-new-structure.py +8 -7
- udata/migrations/2021-05-27-fix-default-schema-name.py +7 -6
- udata/migrations/2021-07-05-remove-unused-badges.py +17 -15
- udata/migrations/2021-07-07-update-schema-for-community-resources.py +7 -6
- udata/migrations/2021-08-17-follow-integrity.py +5 -4
- udata/migrations/2021-08-17-harvest-integrity.py +13 -12
- udata/migrations/2021-08-17-oauth2client-integrity.py +5 -4
- udata/migrations/2021-08-17-transfer-integrity.py +5 -4
- udata/migrations/2021-08-17-users-integrity.py +9 -8
- udata/migrations/2021-12-14-reuse-topics.py +7 -6
- udata/migrations/2022-04-21-improve-extension-detection.py +8 -7
- udata/migrations/2022-09-22-clean-inactive-harvest-datasets.py +16 -14
- udata/migrations/2022-10-10-add-fs_uniquifier-to-user-model.py +6 -6
- udata/migrations/2022-10-10-migrate-harvest-extras.py +36 -26
- udata/migrations/2023-02-08-rename-internal-dates.py +46 -28
- udata/migrations/2024-01-29-fix-reuse-and-dataset-with-private-None.py +10 -8
- udata/migrations/2024-03-22-migrate-activity-kwargs-to-extras.py +6 -4
- udata/migrations/2024-06-11-fix-reuse-datasets-references.py +7 -6
- udata/migrations/__init__.py +123 -105
- udata/models/__init__.py +4 -4
- udata/mongo/__init__.py +13 -11
- udata/mongo/badges_field.py +3 -2
- udata/mongo/datetime_fields.py +13 -12
- udata/mongo/document.py +17 -16
- udata/mongo/engine.py +15 -16
- udata/mongo/errors.py +2 -1
- udata/mongo/extras_fields.py +30 -20
- udata/mongo/queryset.py +12 -12
- udata/mongo/slug_fields.py +38 -28
- udata/mongo/taglist_field.py +1 -2
- udata/mongo/url_field.py +5 -5
- udata/mongo/uuid_fields.py +4 -3
- udata/notifications/__init__.py +1 -1
- udata/notifications/mattermost.py +10 -9
- udata/rdf.py +167 -188
- udata/routing.py +40 -45
- udata/search/__init__.py +18 -19
- udata/search/adapter.py +17 -16
- udata/search/commands.py +44 -51
- udata/search/fields.py +13 -20
- udata/search/query.py +23 -18
- udata/search/result.py +9 -10
- udata/sentry.py +21 -19
- udata/settings.py +262 -198
- udata/sitemap.py +8 -6
- udata/static/chunks/{11.e9b9ca1f3e03d4020377.js → 11.52e531c19f8de80c00cf.js} +3 -3
- udata/static/chunks/{11.e9b9ca1f3e03d4020377.js.map → 11.52e531c19f8de80c00cf.js.map} +1 -1
- udata/static/chunks/{13.038c0d9aa0dfa0181c4b.js → 13.c3343a7f1070061c0e10.js} +2 -2
- udata/static/chunks/{13.038c0d9aa0dfa0181c4b.js.map → 13.c3343a7f1070061c0e10.js.map} +1 -1
- udata/static/chunks/{16.0baa2b64a74a2dcde25c.js → 16.8fa42440ad75ca172e6d.js} +2 -2
- udata/static/chunks/{16.0baa2b64a74a2dcde25c.js.map → 16.8fa42440ad75ca172e6d.js.map} +1 -1
- udata/static/chunks/{19.350a9f150b074b4ecefa.js → 19.9c6c8412729cd6d59cfa.js} +3 -3
- udata/static/chunks/{19.350a9f150b074b4ecefa.js.map → 19.9c6c8412729cd6d59cfa.js.map} +1 -1
- udata/static/chunks/{5.6ebbce2b9b3e696d3da5.js → 5.71d15c2e4f21feee2a9a.js} +3 -3
- udata/static/chunks/{5.6ebbce2b9b3e696d3da5.js.map → 5.71d15c2e4f21feee2a9a.js.map} +1 -1
- udata/static/chunks/{6.d8a5f7b017bcbd083641.js → 6.9139dc098b8ea640b890.js} +3 -3
- udata/static/chunks/{6.d8a5f7b017bcbd083641.js.map → 6.9139dc098b8ea640b890.js.map} +1 -1
- udata/static/common.js +1 -1
- udata/static/common.js.map +1 -1
- udata/storage/s3.py +20 -13
- udata/tags.py +4 -5
- udata/tasks.py +43 -42
- udata/tests/__init__.py +9 -6
- udata/tests/api/__init__.py +5 -6
- udata/tests/api/test_auth_api.py +395 -321
- udata/tests/api/test_base_api.py +31 -33
- udata/tests/api/test_contact_points.py +7 -9
- udata/tests/api/test_dataservices_api.py +211 -158
- udata/tests/api/test_datasets_api.py +823 -812
- udata/tests/api/test_follow_api.py +13 -15
- udata/tests/api/test_me_api.py +95 -112
- udata/tests/api/test_organizations_api.py +301 -339
- udata/tests/api/test_reports_api.py +35 -25
- udata/tests/api/test_reuses_api.py +134 -139
- udata/tests/api/test_swagger.py +5 -5
- udata/tests/api/test_tags_api.py +18 -25
- udata/tests/api/test_topics_api.py +94 -94
- udata/tests/api/test_transfer_api.py +53 -48
- udata/tests/api/test_user_api.py +128 -141
- udata/tests/apiv2/test_datasets.py +290 -198
- udata/tests/apiv2/test_me_api.py +10 -11
- udata/tests/apiv2/test_organizations.py +56 -74
- udata/tests/apiv2/test_swagger.py +5 -5
- udata/tests/apiv2/test_topics.py +69 -87
- udata/tests/cli/test_cli_base.py +8 -8
- udata/tests/cli/test_db_cli.py +21 -19
- udata/tests/dataservice/test_dataservice_tasks.py +8 -12
- udata/tests/dataset/test_csv_adapter.py +44 -35
- udata/tests/dataset/test_dataset_actions.py +2 -3
- udata/tests/dataset/test_dataset_commands.py +7 -8
- udata/tests/dataset/test_dataset_events.py +36 -29
- udata/tests/dataset/test_dataset_model.py +224 -217
- udata/tests/dataset/test_dataset_rdf.py +142 -131
- udata/tests/dataset/test_dataset_tasks.py +15 -15
- udata/tests/dataset/test_resource_preview.py +10 -13
- udata/tests/features/territories/__init__.py +9 -13
- udata/tests/features/territories/test_territories_api.py +71 -91
- udata/tests/forms/test_basic_fields.py +7 -7
- udata/tests/forms/test_current_user_field.py +39 -66
- udata/tests/forms/test_daterange_field.py +31 -39
- udata/tests/forms/test_dict_field.py +28 -26
- udata/tests/forms/test_extras_fields.py +102 -76
- udata/tests/forms/test_form_field.py +8 -8
- udata/tests/forms/test_image_field.py +33 -26
- udata/tests/forms/test_model_field.py +134 -123
- udata/tests/forms/test_model_list_field.py +7 -7
- udata/tests/forms/test_nested_model_list_field.py +117 -79
- udata/tests/forms/test_publish_as_field.py +36 -65
- udata/tests/forms/test_reference_field.py +34 -53
- udata/tests/forms/test_user_forms.py +23 -21
- udata/tests/forms/test_uuid_field.py +6 -10
- udata/tests/frontend/__init__.py +9 -6
- udata/tests/frontend/test_auth.py +7 -6
- udata/tests/frontend/test_csv.py +81 -96
- udata/tests/frontend/test_hooks.py +43 -43
- udata/tests/frontend/test_markdown.py +211 -191
- udata/tests/helpers.py +32 -37
- udata/tests/models.py +2 -2
- udata/tests/organization/test_csv_adapter.py +21 -16
- udata/tests/organization/test_notifications.py +11 -18
- udata/tests/organization/test_organization_model.py +13 -13
- udata/tests/organization/test_organization_rdf.py +29 -22
- udata/tests/organization/test_organization_tasks.py +16 -17
- udata/tests/plugin.py +76 -73
- udata/tests/reuse/test_reuse_model.py +21 -21
- udata/tests/reuse/test_reuse_task.py +11 -13
- udata/tests/search/__init__.py +11 -12
- udata/tests/search/test_adapter.py +60 -70
- udata/tests/search/test_query.py +16 -16
- udata/tests/search/test_results.py +10 -7
- udata/tests/site/test_site_api.py +11 -16
- udata/tests/site/test_site_metrics.py +20 -30
- udata/tests/site/test_site_model.py +4 -5
- udata/tests/site/test_site_rdf.py +94 -78
- udata/tests/test_activity.py +17 -17
- udata/tests/test_discussions.py +292 -299
- udata/tests/test_i18n.py +37 -40
- udata/tests/test_linkchecker.py +91 -85
- udata/tests/test_mail.py +13 -17
- udata/tests/test_migrations.py +219 -180
- udata/tests/test_model.py +164 -157
- udata/tests/test_notifications.py +17 -17
- udata/tests/test_owned.py +14 -14
- udata/tests/test_rdf.py +25 -23
- udata/tests/test_routing.py +89 -93
- udata/tests/test_storages.py +137 -128
- udata/tests/test_tags.py +44 -46
- udata/tests/test_topics.py +7 -7
- udata/tests/test_transfer.py +42 -49
- udata/tests/test_uris.py +160 -161
- udata/tests/test_utils.py +79 -71
- udata/tests/user/test_user_rdf.py +5 -9
- udata/tests/workers/test_jobs_commands.py +57 -58
- udata/tests/workers/test_tasks_routing.py +23 -29
- udata/tests/workers/test_workers_api.py +125 -131
- udata/tests/workers/test_workers_helpers.py +6 -6
- udata/tracking.py +4 -6
- udata/uris.py +45 -46
- udata/utils.py +68 -66
- udata/wsgi.py +1 -1
- {udata-9.1.2.dev30355.dist-info → udata-9.1.2.dev30382.dist-info}/METADATA +3 -2
- udata-9.1.2.dev30382.dist-info/RECORD +704 -0
- udata-9.1.2.dev30355.dist-info/RECORD +0 -704
- {udata-9.1.2.dev30355.dist-info → udata-9.1.2.dev30382.dist-info}/LICENSE +0 -0
- {udata-9.1.2.dev30355.dist-info → udata-9.1.2.dev30382.dist-info}/WHEEL +0 -0
- {udata-9.1.2.dev30355.dist-info → udata-9.1.2.dev30382.dist-info}/entry_points.txt +0 -0
- {udata-9.1.2.dev30355.dist-info → udata-9.1.2.dev30382.dist-info}/top_level.txt +0 -0
udata/mongo/engine.py
CHANGED
|
@@ -1,24 +1,24 @@
|
|
|
1
|
-
from bson import
|
|
1
|
+
from bson import DBRef, ObjectId
|
|
2
2
|
from flask_mongoengine import MongoEngine, MongoEngineSessionInterface
|
|
3
|
+
from flask_storage.mongo import FileField, ImageField
|
|
3
4
|
from mongoengine.base import TopLevelDocumentMetaclass, get_document
|
|
4
5
|
from mongoengine.errors import ValidationError
|
|
5
|
-
from mongoengine.signals import
|
|
6
|
-
|
|
7
|
-
from flask_storage.mongo import FileField, ImageField
|
|
6
|
+
from mongoengine.signals import post_save, pre_save
|
|
8
7
|
|
|
9
8
|
from .badges_field import BadgesField
|
|
10
|
-
from .taglist_field import TagListField
|
|
11
9
|
from .datetime_fields import DateField, DateRange, Datetimed
|
|
10
|
+
from .document import DomainModel, UDataDocument
|
|
12
11
|
from .extras_fields import ExtrasField, OrganizationExtrasField
|
|
12
|
+
from .queryset import UDataQuerySet
|
|
13
13
|
from .slug_fields import SlugField
|
|
14
|
+
from .taglist_field import TagListField
|
|
14
15
|
from .url_field import URLField
|
|
15
16
|
from .uuid_fields import AutoUUIDField
|
|
16
|
-
from .queryset import UDataQuerySet
|
|
17
|
-
from .document import UDataDocument, DomainModel
|
|
18
17
|
|
|
19
18
|
|
|
20
19
|
class UDataMongoEngine(MongoEngine):
|
|
21
|
-
|
|
20
|
+
"""Customized mongoengine with extra fields types and helpers"""
|
|
21
|
+
|
|
22
22
|
def __init__(self, app=None):
|
|
23
23
|
super(UDataMongoEngine, self).__init__(app)
|
|
24
24
|
self.BadgesField = BadgesField
|
|
@@ -44,19 +44,19 @@ class UDataMongoEngine(MongoEngine):
|
|
|
44
44
|
self.pre_save = pre_save
|
|
45
45
|
|
|
46
46
|
def resolve_model(self, model):
|
|
47
|
-
|
|
47
|
+
"""
|
|
48
48
|
Resolve a model given a name or dict with `class` entry.
|
|
49
49
|
|
|
50
50
|
:raises ValueError: model specification is wrong or does not exists
|
|
51
|
-
|
|
51
|
+
"""
|
|
52
52
|
if not model:
|
|
53
|
-
raise ValueError(
|
|
53
|
+
raise ValueError("Unsupported model specifications")
|
|
54
54
|
if isinstance(model, str):
|
|
55
55
|
classname = model
|
|
56
|
-
elif isinstance(model, dict) and
|
|
57
|
-
classname = model[
|
|
56
|
+
elif isinstance(model, dict) and "class" in model:
|
|
57
|
+
classname = model["class"]
|
|
58
58
|
else:
|
|
59
|
-
raise ValueError(
|
|
59
|
+
raise ValueError("Unsupported model specifications")
|
|
60
60
|
|
|
61
61
|
try:
|
|
62
62
|
return get_document(classname)
|
|
@@ -65,6 +65,5 @@ class UDataMongoEngine(MongoEngine):
|
|
|
65
65
|
raise ValueError(message)
|
|
66
66
|
|
|
67
67
|
|
|
68
|
-
|
|
69
68
|
db = UDataMongoEngine()
|
|
70
|
-
session_interface = MongoEngineSessionInterface(db)
|
|
69
|
+
session_interface = MongoEngineSessionInterface(db)
|
udata/mongo/errors.py
CHANGED
udata/mongo/extras_fields.py
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
|
|
3
2
|
from datetime import date, datetime
|
|
4
3
|
|
|
5
4
|
from mongoengine import EmbeddedDocument
|
|
6
|
-
from mongoengine.fields import
|
|
5
|
+
from mongoengine.fields import BaseField, DictField
|
|
7
6
|
|
|
8
7
|
log = logging.getLogger(__name__)
|
|
9
8
|
|
|
@@ -17,9 +16,9 @@ class ExtrasField(DictField):
|
|
|
17
16
|
super(ExtrasField, self).__init__()
|
|
18
17
|
|
|
19
18
|
def register(self, key, dbtype):
|
|
20
|
-
|
|
19
|
+
"""Register a DB type to add constraint on a given extra key"""
|
|
21
20
|
if not issubclass(dbtype, (BaseField, EmbeddedDocument)):
|
|
22
|
-
msg =
|
|
21
|
+
msg = "ExtrasField can only register MongoEngine fields"
|
|
23
22
|
raise TypeError(msg)
|
|
24
23
|
self.registered[key] = dbtype
|
|
25
24
|
|
|
@@ -32,28 +31,31 @@ class ExtrasField(DictField):
|
|
|
32
31
|
|
|
33
32
|
if not extra_cls:
|
|
34
33
|
if not isinstance(value, ALLOWED_TYPES):
|
|
35
|
-
types =
|
|
36
|
-
msg =
|
|
34
|
+
types = ", ".join(t.__name__ for t in ALLOWED_TYPES)
|
|
35
|
+
msg = "Value should be an instance of: {types}"
|
|
37
36
|
errors[key] = msg.format(types=types)
|
|
38
37
|
continue
|
|
39
38
|
|
|
40
39
|
try:
|
|
41
40
|
if issubclass(extra_cls, EmbeddedDocument):
|
|
42
|
-
(
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
(
|
|
42
|
+
value.validate()
|
|
43
|
+
if isinstance(value, extra_cls)
|
|
44
|
+
else extra_cls(**value).validate()
|
|
45
|
+
)
|
|
45
46
|
else:
|
|
46
47
|
extra_cls().validate(value)
|
|
47
48
|
except Exception as e:
|
|
48
|
-
errors[key] = getattr(e,
|
|
49
|
+
errors[key] = getattr(e, "message", str(e))
|
|
49
50
|
|
|
50
51
|
if errors:
|
|
51
|
-
self.error(
|
|
52
|
+
self.error("Unsupported types", errors=errors)
|
|
52
53
|
|
|
53
54
|
def __call__(self, key):
|
|
54
55
|
def inner(cls):
|
|
55
56
|
self.register(key, cls)
|
|
56
57
|
return cls
|
|
58
|
+
|
|
57
59
|
return inner
|
|
58
60
|
|
|
59
61
|
def to_python(self, value):
|
|
@@ -75,29 +77,37 @@ class OrganizationExtrasField(ExtrasField):
|
|
|
75
77
|
optional_keys = ["choices"]
|
|
76
78
|
valid_types = ["str", "int", "float", "bool", "datetime", "date", "choice"]
|
|
77
79
|
|
|
78
|
-
for elem in values.get(
|
|
80
|
+
for elem in values.get("custom", []):
|
|
79
81
|
# Check if all mandatory keys are in the dictionary
|
|
80
82
|
if not all(key in elem for key in mandatory_keys):
|
|
81
|
-
errors[
|
|
83
|
+
errors["custom"] = (
|
|
84
|
+
"The dictionary does not contain the mandatory keys: 'title', 'description', 'type'."
|
|
85
|
+
)
|
|
82
86
|
|
|
83
87
|
# Check if the dictionary contains only keys that are either mandatory or optional
|
|
84
88
|
if not all(key in mandatory_keys + optional_keys for key in elem):
|
|
85
|
-
errors[
|
|
89
|
+
errors["custom"] = "The dictionary does contains extra keys than allowed ones."
|
|
86
90
|
|
|
87
91
|
# Check if the "type" value is one of the valid types
|
|
88
92
|
if elem.get("type") not in valid_types:
|
|
89
|
-
errors[
|
|
90
|
-
|
|
93
|
+
errors["type"] = "Type '{type}' of '{title}' should be one of: {types}".format(
|
|
94
|
+
type=elem.get("type"), title=elem.get("title"), types=valid_types
|
|
95
|
+
)
|
|
91
96
|
|
|
92
97
|
# Check if the "choices" key is present only if the type is "choice" and it's not an empty list
|
|
93
98
|
is_choices_valid = True
|
|
94
99
|
if elem.get("type") == "choice":
|
|
95
|
-
is_choices_valid =
|
|
96
|
-
|
|
100
|
+
is_choices_valid = (
|
|
101
|
+
"choices" in elem
|
|
102
|
+
and isinstance(elem["choices"], list)
|
|
103
|
+
and len(elem["choices"]) > 0
|
|
104
|
+
)
|
|
97
105
|
elif "choices" in elem:
|
|
98
106
|
is_choices_valid = False
|
|
99
107
|
if not is_choices_valid:
|
|
100
|
-
errors[
|
|
108
|
+
errors["choices"] = (
|
|
109
|
+
"The 'choices' key must be an non empty list and can only be present when type 'choice' is selected."
|
|
110
|
+
)
|
|
101
111
|
|
|
102
112
|
if errors:
|
|
103
|
-
self.error(
|
|
113
|
+
self.error("Custom extras error", errors=errors)
|
udata/mongo/queryset.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
|
-
from bson import
|
|
3
|
+
from bson import DBRef, ObjectId
|
|
4
4
|
from flask_mongoengine import BaseQuerySet
|
|
5
5
|
|
|
6
6
|
from udata.utils import Paginable
|
|
@@ -9,7 +9,8 @@ log = logging.getLogger(__name__)
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class DBPaginator(Paginable):
|
|
12
|
-
|
|
12
|
+
"""A simple paginable implementation"""
|
|
13
|
+
|
|
13
14
|
def __init__(self, queryset):
|
|
14
15
|
self.queryset = queryset
|
|
15
16
|
|
|
@@ -45,8 +46,7 @@ class UDataQuerySet(BaseQuerySet):
|
|
|
45
46
|
data = self.in_bulk(ids)
|
|
46
47
|
return [data[id] for id in ids]
|
|
47
48
|
|
|
48
|
-
def get_or_create(self, write_concern=None, auto_save=True,
|
|
49
|
-
*q_objs, **query):
|
|
49
|
+
def get_or_create(self, write_concern=None, auto_save=True, *q_objs, **query):
|
|
50
50
|
"""Retrieve unique object or create, if it doesn't exist.
|
|
51
51
|
|
|
52
52
|
Returns a tuple of ``(object, created)``, where ``object`` is
|
|
@@ -58,7 +58,7 @@ class UDataQuerySet(BaseQuerySet):
|
|
|
58
58
|
https://github.com/MongoEngine/mongoengine/
|
|
59
59
|
pull/1029/files#diff-05c70acbd0634d6d05e4a6e3a9b7d66b
|
|
60
60
|
"""
|
|
61
|
-
defaults = query.pop(
|
|
61
|
+
defaults = query.pop("defaults", {})
|
|
62
62
|
try:
|
|
63
63
|
doc = self.get(*q_objs, **query)
|
|
64
64
|
return doc, False
|
|
@@ -71,7 +71,7 @@ class UDataQuerySet(BaseQuerySet):
|
|
|
71
71
|
return doc, True
|
|
72
72
|
|
|
73
73
|
def generic_in(self, **kwargs):
|
|
74
|
-
|
|
74
|
+
"""Bypass buggy GenericReferenceField querying issue"""
|
|
75
75
|
query = {}
|
|
76
76
|
for key, value in kwargs.items():
|
|
77
77
|
if not value:
|
|
@@ -82,15 +82,15 @@ class UDataQuerySet(BaseQuerySet):
|
|
|
82
82
|
if isinstance(value, (list, tuple)):
|
|
83
83
|
if all(isinstance(v, str) for v in value):
|
|
84
84
|
ids = [ObjectId(v) for v in value]
|
|
85
|
-
query[
|
|
85
|
+
query["{0}._ref.$id".format(key)] = {"$in": ids}
|
|
86
86
|
elif all(isinstance(v, DBRef) for v in value):
|
|
87
|
-
query[
|
|
87
|
+
query["{0}._ref".format(key)] = {"$in": value}
|
|
88
88
|
elif all(isinstance(v, ObjectId) for v in value):
|
|
89
|
-
query[
|
|
89
|
+
query["{0}._ref.$id".format(key)] = {"$in": value}
|
|
90
90
|
elif isinstance(value, ObjectId):
|
|
91
|
-
query[
|
|
91
|
+
query["{0}._ref.$id".format(key)] = value
|
|
92
92
|
elif isinstance(value, str):
|
|
93
|
-
query[
|
|
93
|
+
query["{0}._ref.$id".format(key)] = ObjectId(value)
|
|
94
94
|
else:
|
|
95
|
-
self.error(
|
|
95
|
+
self.error("expect a list of string, ObjectId or DBRef")
|
|
96
96
|
return self(__raw__=query)
|
udata/mongo/slug_fields.py
CHANGED
|
@@ -1,27 +1,36 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
import slugify
|
|
3
2
|
|
|
3
|
+
import slugify
|
|
4
4
|
from flask_mongoengine import Document
|
|
5
5
|
from mongoengine.fields import StringField
|
|
6
|
-
from mongoengine.signals import
|
|
6
|
+
from mongoengine.signals import post_delete, pre_save
|
|
7
7
|
|
|
8
|
-
from .queryset import UDataQuerySet
|
|
9
8
|
from udata.utils import is_uuid
|
|
10
9
|
|
|
10
|
+
from .queryset import UDataQuerySet
|
|
11
|
+
|
|
11
12
|
log = logging.getLogger(__name__)
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
class SlugField(StringField):
|
|
15
|
-
|
|
16
|
+
"""
|
|
16
17
|
A field that that produces a slug from the inputs and auto-
|
|
17
18
|
increments the slug if the value already exists.
|
|
18
|
-
|
|
19
|
+
"""
|
|
20
|
+
|
|
19
21
|
# Do not remove, this is required to trigger field population
|
|
20
22
|
_auto_gen = True
|
|
21
23
|
|
|
22
|
-
def __init__(
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
populate_from=None,
|
|
27
|
+
update=False,
|
|
28
|
+
lower_case=True,
|
|
29
|
+
separator="-",
|
|
30
|
+
follow=False,
|
|
31
|
+
**kwargs,
|
|
32
|
+
):
|
|
33
|
+
kwargs.setdefault("unique", True)
|
|
25
34
|
self.populate_from = populate_from
|
|
26
35
|
self.update = update
|
|
27
36
|
self.lower_case = lower_case
|
|
@@ -33,7 +42,7 @@ class SlugField(StringField):
|
|
|
33
42
|
def __get__(self, instance, owner):
|
|
34
43
|
# mongoengine calls this after document initialization
|
|
35
44
|
# We register signals handlers here to have a owner reference
|
|
36
|
-
if not hasattr(self,
|
|
45
|
+
if not hasattr(self, "owner"):
|
|
37
46
|
self.owner = owner
|
|
38
47
|
pre_save.connect(self.populate_on_pre_save, sender=owner)
|
|
39
48
|
if self.follow:
|
|
@@ -51,20 +60,20 @@ class SlugField(StringField):
|
|
|
51
60
|
pass
|
|
52
61
|
|
|
53
62
|
def slugify(self, value):
|
|
54
|
-
|
|
63
|
+
"""
|
|
55
64
|
Apply slugification according to specified field rules
|
|
56
|
-
|
|
65
|
+
"""
|
|
57
66
|
if value is None:
|
|
58
67
|
return
|
|
59
68
|
|
|
60
|
-
return slugify.slugify(
|
|
61
|
-
|
|
62
|
-
|
|
69
|
+
return slugify.slugify(
|
|
70
|
+
value, max_length=self.max_length, separator=self.separator, to_lower=self.lower_case
|
|
71
|
+
)
|
|
63
72
|
|
|
64
73
|
def latest(self, value):
|
|
65
|
-
|
|
74
|
+
"""
|
|
66
75
|
Get the latest object for a given old slug
|
|
67
|
-
|
|
76
|
+
"""
|
|
68
77
|
namespace = self.owner_document.__name__
|
|
69
78
|
follow = SlugFollow.objects(namespace=namespace, old_slug=value).first()
|
|
70
79
|
if follow:
|
|
@@ -72,9 +81,9 @@ class SlugField(StringField):
|
|
|
72
81
|
return None
|
|
73
82
|
|
|
74
83
|
def cleanup_on_delete(self, sender, document, **kwargs):
|
|
75
|
-
|
|
84
|
+
"""
|
|
76
85
|
Clean up slug redirections on object deletion
|
|
77
|
-
|
|
86
|
+
"""
|
|
78
87
|
if not self.follow or sender is not self.owner_document:
|
|
79
88
|
return
|
|
80
89
|
slug = getattr(document, self.db_field)
|
|
@@ -88,30 +97,31 @@ class SlugField(StringField):
|
|
|
88
97
|
|
|
89
98
|
|
|
90
99
|
class SlugFollow(Document):
|
|
91
|
-
|
|
100
|
+
"""
|
|
92
101
|
Keeps track of slug changes for a given namespace/class.
|
|
93
102
|
Fields are:
|
|
94
103
|
* namespace - A namespace under which this slug falls
|
|
95
104
|
(e.g. match, team, user etc)
|
|
96
105
|
* old_slug - Before change slug.
|
|
97
106
|
* new_slug - After change slug
|
|
98
|
-
|
|
107
|
+
"""
|
|
108
|
+
|
|
99
109
|
namespace = StringField(required=True)
|
|
100
110
|
old_slug = StringField(required=True)
|
|
101
111
|
new_slug = StringField(required=True)
|
|
102
112
|
|
|
103
113
|
meta = {
|
|
104
|
-
|
|
105
|
-
(
|
|
114
|
+
"indexes": [
|
|
115
|
+
("namespace", "old_slug"),
|
|
106
116
|
],
|
|
107
|
-
|
|
117
|
+
"queryset_class": UDataQuerySet,
|
|
108
118
|
}
|
|
109
119
|
|
|
110
120
|
|
|
111
121
|
def populate_slug(instance, field):
|
|
112
|
-
|
|
122
|
+
"""
|
|
113
123
|
Populate a slug field if needed.
|
|
114
|
-
|
|
124
|
+
"""
|
|
115
125
|
value = getattr(instance, field.db_field)
|
|
116
126
|
|
|
117
127
|
try:
|
|
@@ -164,14 +174,14 @@ def populate_slug(instance, field):
|
|
|
164
174
|
|
|
165
175
|
while exists(slug):
|
|
166
176
|
# keep space for index suffix, trim slug if needed
|
|
167
|
-
slug_overflow = len(
|
|
177
|
+
slug_overflow = len("{0}-{1}".format(base_slug, index)) - field.max_length
|
|
168
178
|
if slug_overflow >= 1:
|
|
169
179
|
base_slug = base_slug[:-slug_overflow]
|
|
170
|
-
slug =
|
|
180
|
+
slug = "{0}-{1}".format(base_slug, index)
|
|
171
181
|
index += 1
|
|
172
182
|
|
|
173
183
|
if is_uuid(slug):
|
|
174
|
-
slug =
|
|
184
|
+
slug = "{0}-uuid".format(slug)
|
|
175
185
|
|
|
176
186
|
# Track old slugs for this class
|
|
177
187
|
if field.follow and old_slug != slug:
|
udata/mongo/taglist_field.py
CHANGED
udata/mongo/url_field.py
CHANGED
|
@@ -4,7 +4,7 @@ from udata import uris
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class URLField(StringField):
|
|
7
|
-
|
|
7
|
+
"""
|
|
8
8
|
An URL field using the udata URL normalization and validation rules.
|
|
9
9
|
|
|
10
10
|
The URL spaces are automatically stripped.
|
|
@@ -18,9 +18,9 @@ class URLField(StringField):
|
|
|
18
18
|
:params list schemes: List of allowed schemes
|
|
19
19
|
:params list tlds: List of allowed TLDs
|
|
20
20
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, private=None, local=None, schemes=None, tlds=None, **kwargs):
|
|
24
24
|
super(URLField, self).__init__(**kwargs)
|
|
25
25
|
self.private = private
|
|
26
26
|
self.local = local
|
|
@@ -36,7 +36,7 @@ class URLField(StringField):
|
|
|
36
36
|
super(URLField, self).validate(value)
|
|
37
37
|
kwargs = {
|
|
38
38
|
a: getattr(self, a)
|
|
39
|
-
for a in (
|
|
39
|
+
for a in ("private", "local", "schemes", "tlds")
|
|
40
40
|
if getattr(self, a) is not None
|
|
41
41
|
}
|
|
42
42
|
try:
|
udata/mongo/uuid_fields.py
CHANGED
|
@@ -7,11 +7,12 @@ log = logging.getLogger(__name__)
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class AutoUUIDField(UUIDField):
|
|
10
|
-
|
|
10
|
+
"""
|
|
11
11
|
An autopopulated UUID field.
|
|
12
|
-
|
|
12
|
+
"""
|
|
13
|
+
|
|
13
14
|
def __init__(self, **kwargs):
|
|
14
|
-
kwargs.setdefault(
|
|
15
|
+
kwargs.setdefault("binary", False)
|
|
15
16
|
super(AutoUUIDField, self).__init__(**kwargs)
|
|
16
17
|
|
|
17
18
|
def __set__(self, instance, value):
|
udata/notifications/__init__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
from .mattermost import notify_potential_spam
|
|
1
|
+
from .mattermost import notify_potential_spam # noqa
|
|
@@ -1,21 +1,22 @@
|
|
|
1
1
|
import requests
|
|
2
|
-
from udata.core.spam.signals import on_new_potential_spam
|
|
3
2
|
from flask import current_app
|
|
4
3
|
|
|
4
|
+
from udata.core.spam.signals import on_new_potential_spam
|
|
5
|
+
|
|
5
6
|
|
|
6
7
|
@on_new_potential_spam.connect
|
|
7
8
|
def notify_potential_spam(sender, **kwargs):
|
|
8
|
-
message = kwargs.get(
|
|
9
|
-
reason = kwargs.get(
|
|
10
|
-
text = kwargs.get(
|
|
9
|
+
message = kwargs.get("message")
|
|
10
|
+
reason = kwargs.get("reason")
|
|
11
|
+
text = kwargs.get("text")
|
|
11
12
|
|
|
12
|
-
message = f
|
|
13
|
+
message = f":warning: @all {message}"
|
|
13
14
|
|
|
14
15
|
if reason:
|
|
15
|
-
message += f
|
|
16
|
+
message += f" ({reason})"
|
|
16
17
|
|
|
17
18
|
if text:
|
|
18
|
-
message += f
|
|
19
|
+
message += f"\n\n> {text}"
|
|
19
20
|
|
|
20
21
|
send_message(message)
|
|
21
22
|
|
|
@@ -26,11 +27,11 @@ def send_message(text: str):
|
|
|
26
27
|
Args:
|
|
27
28
|
text (str): Text to send to a channel
|
|
28
29
|
"""
|
|
29
|
-
webhook = current_app.config.get(
|
|
30
|
+
webhook = current_app.config.get("MATTERMOST_WEBHOOK")
|
|
30
31
|
if not webhook:
|
|
31
32
|
return
|
|
32
33
|
|
|
33
|
-
data = {
|
|
34
|
+
data = {"text": text}
|
|
34
35
|
|
|
35
36
|
r = requests.post(webhook, json=data)
|
|
36
37
|
r.raise_for_status()
|