udata 9.1.4__py2.py3-none-any.whl → 9.1.4.dev30965__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of udata might be problematic. Click here for more details.

Files changed (112) hide show
  1. tasks/__init__.py +2 -2
  2. udata/__init__.py +1 -1
  3. udata/api/__init__.py +3 -2
  4. udata/api/commands.py +1 -0
  5. udata/api/fields.py +1 -22
  6. udata/api_fields.py +37 -140
  7. udata/app.py +1 -1
  8. udata/auth/__init__.py +12 -8
  9. udata/commands/db.py +3 -3
  10. udata/commands/dcat.py +1 -1
  11. udata/commands/fixtures.py +40 -60
  12. udata/commands/purge.py +1 -2
  13. udata/commands/tests/test_fixtures.py +11 -44
  14. udata/core/activity/api.py +1 -14
  15. udata/core/activity/tasks.py +1 -1
  16. udata/core/badges/models.py +2 -6
  17. udata/core/contact_point/api.py +3 -1
  18. udata/core/dataservices/api.py +1 -37
  19. udata/core/dataservices/models.py +0 -38
  20. udata/core/dataservices/tasks.py +1 -1
  21. udata/core/dataset/events.py +1 -4
  22. udata/core/dataset/forms.py +2 -0
  23. udata/core/dataset/models.py +10 -12
  24. udata/core/dataset/rdf.py +1 -1
  25. udata/core/discussions/api.py +1 -1
  26. udata/core/discussions/models.py +2 -2
  27. udata/core/discussions/tasks.py +1 -1
  28. udata/core/metrics/models.py +1 -4
  29. udata/core/organization/api.py +7 -11
  30. udata/core/organization/api_fields.py +4 -10
  31. udata/core/organization/apiv2.py +1 -1
  32. udata/core/organization/csv.py +0 -1
  33. udata/core/organization/rdf.py +1 -4
  34. udata/core/owned.py +2 -4
  35. udata/core/post/api.py +2 -2
  36. udata/core/reuse/api.py +25 -32
  37. udata/core/reuse/api_fields.py +101 -2
  38. udata/core/reuse/apiv2.py +4 -4
  39. udata/core/reuse/forms.py +45 -0
  40. udata/core/reuse/models.py +16 -98
  41. udata/core/site/api.py +29 -3
  42. udata/core/spatial/commands.py +3 -3
  43. udata/core/spatial/factories.py +1 -1
  44. udata/core/spatial/forms.py +1 -1
  45. udata/core/spatial/models.py +2 -2
  46. udata/core/spatial/tests/test_models.py +1 -1
  47. udata/core/spatial/translations.py +1 -3
  48. udata/core/topic/api.py +2 -2
  49. udata/core/topic/apiv2.py +2 -1
  50. udata/core/user/api.py +8 -28
  51. udata/core/user/metrics.py +1 -1
  52. udata/cors.py +4 -4
  53. udata/features/transfer/api.py +2 -1
  54. udata/harvest/actions.py +1 -1
  55. udata/harvest/backends/__init__.py +1 -1
  56. udata/harvest/tasks.py +1 -0
  57. udata/harvest/tests/factories.py +2 -0
  58. udata/harvest/tests/test_base_backend.py +1 -0
  59. udata/harvest/tests/test_dcat_backend.py +17 -16
  60. udata/migrations/2020-07-24-remove-s-from-scope-oauth.py +1 -1
  61. udata/migrations/2021-07-05-remove-unused-badges.py +1 -0
  62. udata/migrations/2023-02-08-rename-internal-dates.py +2 -0
  63. udata/migrations/2024-06-11-fix-reuse-datasets-references.py +1 -0
  64. udata/mongo/datetime_fields.py +4 -11
  65. udata/mongo/document.py +0 -2
  66. udata/mongo/taglist_field.py +0 -26
  67. udata/search/commands.py +1 -1
  68. udata/search/query.py +1 -1
  69. udata/settings.py +0 -1
  70. udata/static/admin.js +36 -36
  71. udata/static/admin.js.map +1 -1
  72. udata/static/chunks/{12.576e63b7a990f8eab784.js → 12.5b900cac4417e10ef3a0.js} +2 -2
  73. udata/static/chunks/12.5b900cac4417e10ef3a0.js.map +1 -0
  74. udata/static/chunks/{28.1ef31a46255dc2bf56d1.js → 28.1759a7f57d526e6db574.js} +2 -2
  75. udata/static/chunks/28.1759a7f57d526e6db574.js.map +1 -0
  76. udata/static/common.js +1 -1
  77. udata/static/common.js.map +1 -1
  78. udata/tests/api/test_base_api.py +1 -1
  79. udata/tests/api/test_contact_points.py +4 -4
  80. udata/tests/api/test_dataservices_api.py +0 -59
  81. udata/tests/api/test_datasets_api.py +10 -10
  82. udata/tests/api/test_organizations_api.py +39 -39
  83. udata/tests/api/test_reuses_api.py +0 -49
  84. udata/tests/api/test_tags_api.py +4 -4
  85. udata/tests/api/test_transfer_api.py +1 -1
  86. udata/tests/api/test_user_api.py +0 -11
  87. udata/tests/apiv2/test_datasets.py +4 -4
  88. udata/tests/dataset/test_dataset_events.py +0 -28
  89. udata/tests/dataset/test_dataset_model.py +3 -3
  90. udata/tests/frontend/__init__.py +2 -0
  91. udata/tests/frontend/test_auth.py +1 -0
  92. udata/tests/organization/test_csv_adapter.py +2 -0
  93. udata/tests/organization/test_notifications.py +3 -3
  94. udata/tests/organization/test_organization_rdf.py +6 -31
  95. udata/tests/reuse/test_reuse_model.py +1 -0
  96. udata/tests/site/test_site_rdf.py +3 -1
  97. udata/tests/test_cors.py +3 -0
  98. udata/tests/test_owned.py +4 -4
  99. udata/tests/test_routing.py +1 -1
  100. udata/tests/test_tags.py +1 -1
  101. udata/tests/test_transfer.py +2 -1
  102. udata/tests/workers/test_jobs_commands.py +1 -1
  103. udata/utils.py +0 -12
  104. {udata-9.1.4.dist-info → udata-9.1.4.dev30965.dist-info}/METADATA +4 -16
  105. {udata-9.1.4.dist-info → udata-9.1.4.dev30965.dist-info}/RECORD +109 -109
  106. udata/static/chunks/12.576e63b7a990f8eab784.js.map +0 -1
  107. udata/static/chunks/28.1ef31a46255dc2bf56d1.js.map +0 -1
  108. udata/tests/api/test_activities_api.py +0 -69
  109. {udata-9.1.4.dist-info → udata-9.1.4.dev30965.dist-info}/LICENSE +0 -0
  110. {udata-9.1.4.dist-info → udata-9.1.4.dev30965.dist-info}/WHEEL +0 -0
  111. {udata-9.1.4.dist-info → udata-9.1.4.dev30965.dist-info}/entry_points.txt +0 -0
  112. {udata-9.1.4.dist-info → udata-9.1.4.dev30965.dist-info}/top_level.txt +0 -0
udata/core/reuse/api.py CHANGED
@@ -1,19 +1,15 @@
1
1
  from datetime import datetime
2
2
 
3
- import mongoengine
4
3
  from bson.objectid import ObjectId
5
4
  from flask import request
6
- from flask_login import current_user
7
5
 
8
6
  from udata.api import API, api, errors
9
7
  from udata.api.parsers import ModelApiParser
10
- from udata.api_fields import patch, patch_and_save
11
8
  from udata.auth import admin_permission
12
9
  from udata.core.badges import api as badges_api
13
10
  from udata.core.badges.fields import badge_fields
14
11
  from udata.core.dataset.api_fields import dataset_ref_fields
15
12
  from udata.core.followers.api import FollowAPI
16
- from udata.core.reuse.constants import REUSE_TOPICS, REUSE_TYPES
17
13
  from udata.core.storages.api import (
18
14
  image_parser,
19
15
  parse_uploaded_image,
@@ -23,10 +19,14 @@ from udata.models import Dataset
23
19
  from udata.utils import id_or_404
24
20
 
25
21
  from .api_fields import (
22
+ reuse_fields,
23
+ reuse_page_fields,
26
24
  reuse_suggestion_fields,
27
25
  reuse_topic_fields,
28
26
  reuse_type_fields,
29
27
  )
28
+ from .constants import REUSE_TOPICS, REUSE_TYPES
29
+ from .forms import ReuseForm
30
30
  from .models import Reuse
31
31
  from .permissions import ReuseEditPermission
32
32
 
@@ -96,30 +96,24 @@ reuse_parser = ReuseApiParser()
96
96
  @ns.route("/", endpoint="reuses")
97
97
  class ReuseListAPI(API):
98
98
  @api.doc("list_reuses")
99
- @api.expect(Reuse.__index_parser__)
100
- @api.marshal_with(Reuse.__page_fields__)
99
+ @api.expect(reuse_parser.parser)
100
+ @api.marshal_with(reuse_page_fields)
101
101
  def get(self):
102
- query = Reuse.objects(deleted=None, private__ne=True)
103
-
104
- return Reuse.apply_sort_filters_and_pagination(query)
102
+ args = reuse_parser.parse()
103
+ reuses = Reuse.objects(deleted=None, private__ne=True)
104
+ reuses = reuse_parser.parse_filters(reuses, args)
105
+ sort = args["sort"] or ("$text_score" if args["q"] else None) or DEFAULT_SORTING
106
+ return reuses.order_by(sort).paginate(args["page"], args["page_size"])
105
107
 
106
108
  @api.secure
107
109
  @api.doc("create_reuse")
108
- @api.expect(Reuse.__write_fields__)
110
+ @api.expect(reuse_fields)
109
111
  @api.response(400, "Validation error")
110
- @api.marshal_with(Reuse.__read_fields__, code=201)
112
+ @api.marshal_with(reuse_fields)
111
113
  def post(self):
112
- reuse = patch(Reuse(), request)
113
-
114
- if not reuse.owner and not reuse.organization:
115
- reuse.owner = current_user._get_current_object()
116
-
117
- try:
118
- reuse.save()
119
- except mongoengine.errors.ValidationError as e:
120
- api.abort(400, e.message)
121
-
122
- return patch_and_save(reuse, request), 201
114
+ """Create a new object"""
115
+ form = api.validate(ReuseForm)
116
+ return form.save(), 201
123
117
 
124
118
 
125
119
  @ns.route("/<reuse:reuse>/", endpoint="reuse", doc=common_doc)
@@ -127,7 +121,7 @@ class ReuseListAPI(API):
127
121
  @api.response(410, "Reuse has been deleted")
128
122
  class ReuseAPI(API):
129
123
  @api.doc("get_reuse")
130
- @api.marshal_with(Reuse.__read_fields__)
124
+ @api.marshal_with(reuse_fields)
131
125
  def get(self, reuse):
132
126
  """Fetch a given reuse"""
133
127
  if reuse.deleted and not ReuseEditPermission(reuse).can():
@@ -136,8 +130,8 @@ class ReuseAPI(API):
136
130
 
137
131
  @api.secure
138
132
  @api.doc("update_reuse")
139
- @api.expect(Reuse.__write_fields__)
140
- @api.marshal_with(Reuse.__read_fields__)
133
+ @api.expect(reuse_fields)
134
+ @api.marshal_with(reuse_fields)
141
135
  @api.response(400, errors.VALIDATION_ERROR)
142
136
  def put(self, reuse):
143
137
  """Update a given reuse"""
@@ -145,9 +139,8 @@ class ReuseAPI(API):
145
139
  if reuse.deleted and request_deleted is not None:
146
140
  api.abort(410, "This reuse has been deleted")
147
141
  ReuseEditPermission(reuse).test()
148
-
149
- # This is a patch but old API acted like PATCH on PUT requests.
150
- return patch_and_save(reuse, request)
142
+ form = api.validate(ReuseForm, reuse)
143
+ return form.save()
151
144
 
152
145
  @api.secure
153
146
  @api.doc("delete_reuse")
@@ -167,8 +160,8 @@ class ReuseDatasetsAPI(API):
167
160
  @api.secure
168
161
  @api.doc("reuse_add_dataset", **common_doc)
169
162
  @api.expect(dataset_ref_fields)
170
- @api.response(200, "The dataset is already present", Reuse.__read_fields__)
171
- @api.marshal_with(Reuse.__read_fields__, code=201)
163
+ @api.response(200, "The dataset is already present", reuse_fields)
164
+ @api.marshal_with(reuse_fields, code=201)
172
165
  def post(self, reuse):
173
166
  """Add a dataset to a given reuse"""
174
167
  if "id" not in request.json:
@@ -218,7 +211,7 @@ class ReuseBadgeAPI(API):
218
211
  class ReuseFeaturedAPI(API):
219
212
  @api.doc("feature_reuse")
220
213
  @api.secure(admin_permission)
221
- @api.marshal_with(Reuse.__read_fields__)
214
+ @api.marshal_with(reuse_fields)
222
215
  def post(self, reuse):
223
216
  """Mark a reuse as featured"""
224
217
  reuse.featured = True
@@ -227,7 +220,7 @@ class ReuseFeaturedAPI(API):
227
220
 
228
221
  @api.doc("unfeature_reuse")
229
222
  @api.secure(admin_permission)
230
- @api.marshal_with(Reuse.__read_fields__)
223
+ @api.marshal_with(reuse_fields)
231
224
  def delete(self, reuse):
232
225
  """Unmark a reuse as featured"""
233
226
  reuse.featured = False
@@ -1,9 +1,108 @@
1
- from udata.api import api, fields
1
+ from udata.api import api, base_reference, fields
2
+ from udata.core.badges.fields import badge_fields
3
+ from udata.core.dataset.api_fields import dataset_fields
4
+ from udata.core.organization.api_fields import org_ref_fields
5
+ from udata.core.user.api_fields import user_ref_fields
2
6
 
3
- from .constants import IMAGE_SIZES
7
+ from .constants import IMAGE_SIZES, REUSE_TOPICS, REUSE_TYPES
4
8
 
5
9
  BIGGEST_IMAGE_SIZE = IMAGE_SIZES[0]
6
10
 
11
+
12
+ reuse_fields = api.model(
13
+ "Reuse",
14
+ {
15
+ "id": fields.String(description="The reuse identifier", readonly=True),
16
+ "title": fields.String(description="The reuse title", required=True),
17
+ "slug": fields.String(description="The reuse permalink string", readonly=True),
18
+ "type": fields.String(description="The reuse type", required=True, enum=list(REUSE_TYPES)),
19
+ "url": fields.String(description="The reuse remote URL (website)", required=True),
20
+ "description": fields.Markdown(
21
+ description="The reuse description in Markdown", required=True
22
+ ),
23
+ "tags": fields.List(fields.String, description="Some keywords to help in search"),
24
+ "badges": fields.List(
25
+ fields.Nested(badge_fields), description="The reuse badges", readonly=True
26
+ ),
27
+ "topic": fields.String(
28
+ description="The reuse topic", required=True, enum=list(REUSE_TOPICS)
29
+ ),
30
+ "featured": fields.Boolean(description="Is the reuse featured", readonly=True),
31
+ "private": fields.Boolean(
32
+ description="Is the reuse private to the owner or the organization"
33
+ ),
34
+ "image": fields.ImageField(description="The reuse thumbnail thumbnail (cropped) URL"),
35
+ "image_thumbnail": fields.ImageField(
36
+ attribute="image",
37
+ size=BIGGEST_IMAGE_SIZE,
38
+ description="The reuse thumbnail thumbnail URL. This is the square "
39
+ "({0}x{0}) and cropped version.".format(BIGGEST_IMAGE_SIZE),
40
+ ),
41
+ "created_at": fields.ISODateTime(description="The reuse creation date", readonly=True),
42
+ "last_modified": fields.ISODateTime(
43
+ description="The reuse last modification date", readonly=True
44
+ ),
45
+ "deleted": fields.ISODateTime(description="The deletion date if deleted", readonly=True),
46
+ "archived": fields.ISODateTime(
47
+ description="The archivation date if archived", allow_null=True
48
+ ),
49
+ "datasets": fields.List(fields.Nested(dataset_fields), description="The reused datasets"),
50
+ "organization": fields.Nested(
51
+ org_ref_fields,
52
+ allow_null=True,
53
+ description="The publishing organization",
54
+ readonly=True,
55
+ ),
56
+ "owner": fields.Nested(
57
+ user_ref_fields, description="The owner user", readonly=True, allow_null=True
58
+ ),
59
+ "metrics": fields.Raw(
60
+ attribute=lambda o: o.get_metrics(), description="The reuse metrics", readonly=True
61
+ ),
62
+ "uri": fields.UrlFor(
63
+ "api.reuse", lambda o: {"reuse": o}, description="The reuse API URI", readonly=True
64
+ ),
65
+ "page": fields.UrlFor(
66
+ "reuses.show",
67
+ lambda o: {"reuse": o},
68
+ description="The reuse page URL",
69
+ readonly=True,
70
+ fallback_endpoint="api.reuse",
71
+ ),
72
+ "extras": fields.Raw(description="Extras attributes as key-value pairs"),
73
+ },
74
+ mask="*,datasets{title,uri,page}",
75
+ )
76
+
77
+ reuse_page_fields = api.model("ReusePage", fields.pager(reuse_fields))
78
+
79
+
80
+ reuse_ref_fields = api.inherit(
81
+ "ReuseReference",
82
+ base_reference,
83
+ {
84
+ "title": fields.String(description="The reuse title", readonly=True),
85
+ "image": fields.ImageField(description="The reuse thumbnail thumbnail (cropped) URL"),
86
+ "image_thumbnail": fields.ImageField(
87
+ attribute="image",
88
+ size=BIGGEST_IMAGE_SIZE,
89
+ description="The reuse thumbnail thumbnail URL. This is the square "
90
+ "({0}x{0}) and cropped version.".format(BIGGEST_IMAGE_SIZE),
91
+ ),
92
+ "uri": fields.UrlFor(
93
+ "api.reuse", lambda o: {"reuse": o}, description="The reuse API URI", readonly=True
94
+ ),
95
+ "page": fields.UrlFor(
96
+ "reuses.show",
97
+ lambda o: {"reuse": o},
98
+ description="The reuse page URL",
99
+ readonly=True,
100
+ fallback_endpoint="api.reuse",
101
+ ),
102
+ },
103
+ )
104
+
105
+
7
106
  reuse_type_fields = api.model(
8
107
  "ReuseType",
9
108
  {
udata/core/reuse/apiv2.py CHANGED
@@ -2,13 +2,13 @@ from flask import request
2
2
 
3
3
  from udata import search
4
4
  from udata.api import API, apiv2
5
- from udata.core.reuse.models import Reuse
6
5
  from udata.utils import multi_to_dict
7
6
 
7
+ from .api_fields import reuse_fields, reuse_page_fields
8
8
  from .search import ReuseSearch
9
9
 
10
- apiv2.inherit("ReusePage", Reuse.__page_fields__)
11
- apiv2.inherit("Reuse (read)", Reuse.__read_fields__)
10
+ apiv2.inherit("ReusePage", reuse_page_fields)
11
+ apiv2.inherit("Reuse", reuse_fields)
12
12
 
13
13
  ns = apiv2.namespace("reuses", "Reuse related operations")
14
14
 
@@ -23,7 +23,7 @@ class ReuseSearchAPI(API):
23
23
 
24
24
  @apiv2.doc("search_reuses")
25
25
  @apiv2.expect(search_parser)
26
- @apiv2.marshal_with(Reuse.__page_fields__)
26
+ @apiv2.marshal_with(reuse_page_fields)
27
27
  def get(self):
28
28
  """Search all reuses"""
29
29
  search_parser.parse_args()
@@ -0,0 +1,45 @@
1
+ from udata.core.reuse.constants import REUSE_TOPICS, REUSE_TYPES
2
+ from udata.forms import ModelForm, fields, validators
3
+ from udata.i18n import lazy_gettext as _
4
+ from udata.models import Reuse
5
+
6
+ from .constants import DESCRIPTION_SIZE_LIMIT, IMAGE_SIZES, TITLE_SIZE_LIMIT
7
+
8
+ __all__ = ("ReuseForm",)
9
+
10
+
11
+ def check_url_does_not_exists(form, field):
12
+ """Ensure a reuse URL is not yet registered"""
13
+ if field.data != field.object_data and Reuse.url_exists(field.data):
14
+ raise validators.ValidationError(_("This URL is already registered"))
15
+
16
+
17
+ class ReuseForm(ModelForm):
18
+ model_class = Reuse
19
+
20
+ title = fields.StringField(
21
+ _("Title"), [validators.DataRequired(), validators.Length(max=TITLE_SIZE_LIMIT)]
22
+ )
23
+ description = fields.MarkdownField(
24
+ _("Description"),
25
+ [validators.DataRequired(), validators.Length(max=DESCRIPTION_SIZE_LIMIT)],
26
+ description=_(
27
+ "The details about the reuse (build process, specifics, " "self-critics...)."
28
+ ),
29
+ )
30
+ type = fields.SelectField(_("Type"), choices=list(REUSE_TYPES.items()))
31
+ url = fields.URLField(_("URL"), [validators.DataRequired(), check_url_does_not_exists])
32
+ image = fields.ImageField(_("Image"), sizes=IMAGE_SIZES, placeholder="reuse")
33
+ tags = fields.TagField(_("Tags"), description=_("Some taxonomy keywords"))
34
+ datasets = fields.DatasetListField(_("Used datasets"))
35
+ private = fields.BooleanField(
36
+ _("Private"),
37
+ description=_("Restrict the dataset visibility to you or " "your organization only."),
38
+ )
39
+ topic = fields.SelectField(_("Topic"), choices=list(REUSE_TOPICS.items()))
40
+
41
+ owner = fields.CurrentUserField()
42
+ organization = fields.PublishAsField(_("Publish as"))
43
+ deleted = fields.DateTimeField()
44
+ archived = fields.DateTimeField()
45
+ extras = fields.ExtrasField()
@@ -2,15 +2,11 @@ from blinker import Signal
2
2
  from mongoengine.signals import post_save, pre_save
3
3
  from werkzeug.utils import cached_property
4
4
 
5
- from udata.api_fields import field, function_field, generate_fields
6
- from udata.core.dataset.api_fields import dataset_fields
7
5
  from udata.core.owned import Owned, OwnedQuerySet
8
- from udata.core.reuse.api_fields import BIGGEST_IMAGE_SIZE
9
6
  from udata.core.storages import default_image_basename, images
10
7
  from udata.frontend.markdown import mdstrip
11
8
  from udata.i18n import lazy_gettext as _
12
9
  from udata.models import BadgeMixin, WithMetrics, db
13
- from udata.mongo.errors import FieldValidationError
14
10
  from udata.uris import endpoint_for
15
11
  from udata.utils import hash_url
16
12
 
@@ -27,100 +23,32 @@ class ReuseQuerySet(OwnedQuerySet):
27
23
  return self(db.Q(private=True) | db.Q(datasets__0__exists=False) | db.Q(deleted__ne=None))
28
24
 
29
25
 
30
- def check_url_does_not_exists(url):
31
- """Ensure a reuse URL is not yet registered"""
32
- if url and Reuse.url_exists(url):
33
- raise FieldValidationError(_("This URL is already registered"), field="url")
34
-
35
-
36
- @generate_fields(
37
- searchable=True,
38
- additionalSorts=[
39
- {"key": "datasets", "value": "metrics.datasets"},
40
- {"key": "followers", "value": "metrics.followers"},
41
- {"key": "views", "value": "metrics.views"},
42
- ],
43
- )
44
26
  class Reuse(db.Datetimed, WithMetrics, BadgeMixin, Owned, db.Document):
45
- title = field(
46
- db.StringField(required=True),
47
- sortable=True,
48
- show_as_ref=True,
49
- )
50
- slug = field(
51
- db.SlugField(
52
- max_length=255, required=True, populate_from="title", update=True, follow=True
53
- ),
54
- readonly=True,
55
- )
56
- description = field(
57
- db.StringField(required=True),
58
- markdown=True,
59
- )
60
- type = field(
61
- db.StringField(required=True, choices=list(REUSE_TYPES)),
62
- filterable={},
63
- )
64
- url = field(
65
- db.StringField(required=True),
66
- description="The remote URL (website)",
67
- check=check_url_does_not_exists,
27
+ title = db.StringField(required=True)
28
+ slug = db.SlugField(
29
+ max_length=255, required=True, populate_from="title", update=True, follow=True
68
30
  )
31
+ description = db.StringField(required=True)
32
+ type = db.StringField(required=True, choices=list(REUSE_TYPES))
33
+ url = db.StringField(required=True)
69
34
  urlhash = db.StringField(required=True, unique=True)
70
35
  image_url = db.StringField()
71
- image = field(
72
- db.ImageField(
73
- fs=images,
74
- basename=default_image_basename,
75
- max_size=IMAGE_MAX_SIZE,
76
- thumbnails=IMAGE_SIZES,
77
- ),
78
- readonly=True,
79
- show_as_ref=True,
80
- thumbnail_info={
81
- "size": BIGGEST_IMAGE_SIZE,
82
- },
83
- )
84
- datasets = field(
85
- db.ListField(
86
- field(
87
- db.ReferenceField("Dataset", reverse_delete_rule=db.PULL),
88
- nested_fields=dataset_fields,
89
- ),
90
- ),
91
- filterable={
92
- "key": "dataset",
93
- },
94
- )
95
- tags = field(
96
- db.TagListField(),
97
- filterable={
98
- "key": "tag",
99
- },
100
- )
101
- topic = field(
102
- db.StringField(required=True, choices=list(REUSE_TOPICS)),
103
- filterable={},
36
+ image = db.ImageField(
37
+ fs=images, basename=default_image_basename, max_size=IMAGE_MAX_SIZE, thumbnails=IMAGE_SIZES
104
38
  )
39
+ datasets = db.ListField(db.ReferenceField("Dataset", reverse_delete_rule=db.PULL))
40
+ tags = db.TagListField()
41
+ topic = db.StringField(required=True, choices=list(REUSE_TOPICS))
105
42
  # badges = db.ListField(db.EmbeddedDocumentField(ReuseBadge))
106
43
 
107
- private = field(db.BooleanField(default=False))
44
+ private = db.BooleanField(default=False)
108
45
 
109
46
  ext = db.MapField(db.GenericEmbeddedDocumentField())
110
- extras = field(db.ExtrasField())
47
+ extras = db.ExtrasField()
111
48
 
112
- featured = field(
113
- db.BooleanField(),
114
- filterable={},
115
- readonly=True,
116
- )
117
- deleted = field(
118
- db.DateTimeField(),
119
- readonly=True,
120
- )
121
- archived = field(
122
- db.DateTimeField(),
123
- )
49
+ featured = db.BooleanField()
50
+ deleted = db.DateTimeField()
51
+ archived = db.DateTimeField()
124
52
 
125
53
  def __str__(self):
126
54
  return self.title or ""
@@ -182,16 +110,6 @@ class Reuse(db.Datetimed, WithMetrics, BadgeMixin, Owned, db.Document):
182
110
 
183
111
  display_url = property(url_for)
184
112
 
185
- @function_field(description="Link to the API endpoint for this reuse", show_as_ref=True)
186
- def uri(self):
187
- return endpoint_for("api.reuse", reuse=self, _external=True)
188
-
189
- @function_field(description="Link to the udata web page for this reuse", show_as_ref=True)
190
- def page(self):
191
- return endpoint_for(
192
- "reuses.show", reuse=self, _external=True, fallback_endpoint="api.reuse"
193
- )
194
-
195
113
  @property
196
114
  def is_visible(self):
197
115
  return not self.is_hidden
udata/core/site/api.py CHANGED
@@ -1,10 +1,12 @@
1
1
  from bson import ObjectId
2
2
  from flask import json, make_response, redirect, request, url_for
3
+ from mongoengine import Q
3
4
 
4
5
  from udata.api import API, api, fields
5
6
  from udata.auth import admin_permission
6
7
  from udata.core.dataservices.models import Dataservice
7
8
  from udata.core.dataset.api_fields import dataset_fields
9
+ from udata.core.reuse.api_fields import reuse_fields
8
10
  from udata.models import Dataset, Reuse
9
11
  from udata.rdf import CONTEXT, RDF_EXTENSIONS, graph_response, negociate_content
10
12
  from udata.utils import multi_to_dict
@@ -60,7 +62,7 @@ class SiteHomeDatasetsAPI(API):
60
62
  class SiteHomeReusesAPI(API):
61
63
  @api.doc("get_home_reuses")
62
64
  @api.secure(admin_permission)
63
- @api.marshal_list_with(Reuse.__read_fields__)
65
+ @api.marshal_list_with(reuse_fields)
64
66
  def get(self):
65
67
  """List homepage featured reuses"""
66
68
  return current_site.settings.home_reuses
@@ -68,7 +70,7 @@ class SiteHomeReusesAPI(API):
68
70
  @api.secure(admin_permission)
69
71
  @api.doc("set_home_reuses")
70
72
  @api.expect(([str], "Reuse IDs to put in homepage"))
71
- @api.marshal_list_with(Reuse.__read_fields__)
73
+ @api.marshal_list_with(reuse_fields)
72
74
  def put(self):
73
75
  """Set the homepage reuses editorial selection"""
74
76
  if not isinstance(request.json, list):
@@ -106,7 +108,31 @@ class SiteRdfCatalogFormat(API):
106
108
  if "tag" in params:
107
109
  datasets = datasets.filter(tags=params.get("tag", ""))
108
110
  datasets = datasets.paginate(page, page_size)
109
- dataservices = Dataservice.objects.visible().filter_by_dataset_pagination(datasets, page)
111
+
112
+ # We need to add Dataservice to the catalog.
113
+ # In the best world, we want:
114
+ # - Keep the correct number of datasets on the page (if the requested page size is 100, we should have 100 datasets)
115
+ # - Have simple MongoDB queries
116
+ # - Do not duplicate the datasets (each dataset is present once in the catalog)
117
+ # - Do not duplicate the dataservices (each dataservice is present once in the catalog)
118
+ # - Every referenced dataset for one dataservices present on the page (hard to do)
119
+ #
120
+ # Multiple solutions are possible but none check all the constraints.
121
+ # The selected one is to put all the dataservices referencing at least one of the dataset on
122
+ # the page at the end of it. It means dataservices could be duplicated (present on multiple pages)
123
+ # and these dataservices may referenced some datasets not present in the current page. It's working
124
+ # if somebody is doing the same thing as us (keeping the list of all the datasets IDs for the entire catalog then
125
+ # listing all dataservices in a second pass)
126
+ # Another option is to do some tricky Mongo requests to order/group datasets by their presence in some dataservices but
127
+ # it could be really hard to do with a n..n relation.
128
+ # Let's keep this solution simple right now and iterate on it in the future.
129
+ dataservices_filter = Q(datasets__in=[d.id for d in datasets])
130
+
131
+ # On the first page, add all dataservices without datasets
132
+ if page == 1:
133
+ dataservices_filter = dataservices_filter | Q(datasets__size=0)
134
+
135
+ dataservices = Dataservice.objects.visible().filter(dataservices_filter)
110
136
 
111
137
  catalog = build_catalog(current_site, datasets, dataservices=dataservices, format=format)
112
138
  # bypass flask-restplus make_response, since graph_response
@@ -177,8 +177,8 @@ def migrate():
177
177
 
178
178
  level_summary = "\n".join(
179
179
  [
180
- " - {0}: {1}".format(geolevel.id, counter[geolevel.id])
181
- for geolevel in GeoLevel.objects.order_by("admin_level")
180
+ " - {0}: {1}".format(l.id, counter[l.id])
181
+ for l in GeoLevel.objects.order_by("admin_level")
182
182
  ]
183
183
  )
184
184
  summary = "\n".join(
@@ -188,7 +188,7 @@ def migrate():
188
188
  Summary
189
189
  =======
190
190
  Processed {zones} zones in {datasets} datasets:\
191
- """.format(**counter)
191
+ """.format(level_summary, **counter)
192
192
  ),
193
193
  level_summary,
194
194
  ]
@@ -2,7 +2,7 @@ import factory
2
2
  from faker.providers import BaseProvider
3
3
  from geojson.utils import generate_random
4
4
 
5
- from udata.factories import ModelFactory
5
+ from udata.factories import DateRangeFactory, ModelFactory
6
6
  from udata.utils import faker_provider
7
7
 
8
8
  from . import geoids
@@ -62,7 +62,7 @@ class GeomField(Field):
62
62
  self.data = geojson.loads(value)
63
63
  else:
64
64
  self.data = geojson.GeoJSON.to_instance(value)
65
- except Exception:
65
+ except:
66
66
  self.data = None
67
67
  log.exception("Unable to parse GeoJSON")
68
68
  raise ValueError(self.gettext("Not a valid GeoJSON"))
@@ -120,7 +120,7 @@ class GeoZone(WithMetrics, db.Document):
120
120
  @cache.memoize()
121
121
  def get_spatial_granularities(lang):
122
122
  with language(lang):
123
- return [(geolevel.id, _(geolevel.name)) for geolevel in GeoLevel.objects] + [
123
+ return [(l.id, _(l.name)) for l in GeoLevel.objects] + [
124
124
  (id, str(label)) for id, label in BASE_GRANULARITIES
125
125
  ]
126
126
 
@@ -130,7 +130,7 @@ spatial_granularities = LocalProxy(lambda: get_spatial_granularities(get_locale(
130
130
 
131
131
  @cache.cached(timeout=50, key_prefix="admin_levels")
132
132
  def get_spatial_admin_levels():
133
- return dict((geolevel.id, geolevel.admin_level) for geolevel in GeoLevel.objects)
133
+ return dict((l.id, l.admin_level) for l in GeoLevel.objects)
134
134
 
135
135
 
136
136
  admin_levels = LocalProxy(get_spatial_admin_levels)
@@ -2,7 +2,7 @@ from datetime import timedelta
2
2
 
3
3
  from udata.tests import DBTestMixin, TestCase
4
4
 
5
- from ..factories import GeoZoneFactory
5
+ from ..factories import GeoLevelFactory, GeoZoneFactory
6
6
  from ..models import GeoZone, SpatialCoverage
7
7
 
8
8
  A_YEAR = timedelta(days=365)
@@ -1,7 +1,5 @@
1
1
  # Here to force translations by gettext
2
- def _(s):
3
- return s # noqa: force translations
4
-
2
+ _ = lambda s: s # noqa: force translations
5
3
 
6
4
  TRANSLATIONS = (
7
5
  _("French region"),
udata/core/topic/api.py CHANGED
@@ -2,7 +2,7 @@ from udata.api import API, api, fields
2
2
  from udata.core.dataset.api_fields import dataset_fields
3
3
  from udata.core.discussions.models import Discussion
4
4
  from udata.core.organization.api_fields import org_ref_fields
5
- from udata.core.reuse.models import Reuse
5
+ from udata.core.reuse.api_fields import reuse_fields
6
6
  from udata.core.spatial.api_fields import spatial_coverage_fields
7
7
  from udata.core.topic.parsers import TopicApiParser
8
8
  from udata.core.topic.permissions import TopicEditPermission
@@ -33,7 +33,7 @@ topic_fields = api.model(
33
33
  attribute=lambda o: [d.fetch() for d in o.datasets],
34
34
  ),
35
35
  "reuses": fields.List(
36
- fields.Nested(Reuse.__read_fields__),
36
+ fields.Nested(reuse_fields),
37
37
  description="The topic reuses",
38
38
  attribute=lambda o: [r.fetch() for r in o.reuses],
39
39
  ),
udata/core/topic/apiv2.py CHANGED
@@ -10,6 +10,7 @@ from udata.core.dataset.apiv2 import dataset_page_fields
10
10
  from udata.core.dataset.models import Dataset
11
11
  from udata.core.organization.api_fields import org_ref_fields
12
12
  from udata.core.reuse.api import ReuseApiParser
13
+ from udata.core.reuse.apiv2 import reuse_page_fields
13
14
  from udata.core.reuse.models import Reuse
14
15
  from udata.core.spatial.api_fields import spatial_coverage_fields
15
16
  from udata.core.topic.models import Topic
@@ -215,7 +216,7 @@ class TopicDatasetAPI(API):
215
216
  class TopicReusesAPI(API):
216
217
  @apiv2.doc("topic_reuses")
217
218
  @apiv2.expect(reuse_parser.parser)
218
- @apiv2.marshal_with(Reuse.__page_fields__)
219
+ @apiv2.marshal_with(reuse_page_fields)
219
220
  def get(self, topic):
220
221
  """Get a given topic reuses, with filters"""
221
222
  args = reuse_parser.parse()