udata 10.8.3.dev37152__py2.py3-none-any.whl → 10.8.3.dev37185__py2.py3-none-any.whl

udata/core/dataset/api_fields.py

@@ -266,6 +266,7 @@ DEFAULT_MASK = ",".join(
         "acronym",
         "slug",
         "description",
+        "description_short",
         "created_at",
         "last_modified",
         "deleted",
@@ -327,6 +328,7 @@ dataset_fields = api.model(
         "description": fields.Markdown(
             description="The dataset description in markdown", required=True
         ),
+        "description_short": fields.String(description="The dataset short description"),
         "created_at": fields.ISODateTime(
             description="This date is computed between harvested creation date if any and site's internal creation date",
             required=True,

udata/core/dataset/apiv2.py

@@ -44,6 +44,7 @@ DEFAULT_MASK_APIV2 = ",".join(
         "acronym",
         "slug",
         "description",
+        "description_short",
         "created_at",
         "last_modified",
         "deleted",
@@ -105,6 +106,9 @@ dataset_fields = apiv2.model(
         "description": fields.Markdown(
             description="The dataset description in markdown", required=True
         ),
+        "description_short": fields.String(
+            description="The dataset short description", required=False
+        ),
         "created_at": fields.ISODateTime(
             description="The dataset creation date", required=True, readonly=True
         ),

udata/core/dataset/constants.py

@@ -89,5 +89,6 @@ SCHEMA_CACHE_DURATION = 60 * 5  # In seconds
 
 TITLE_SIZE_LIMIT = 350
 DESCRIPTION_SIZE_LIMIT = 100000
+DESCRIPTION_SHORT_SIZE_LIMIT = 200
 
 FULL_OBJECTS_HEADER = "X-Get-Datasets-Full-Objects"

udata/core/dataset/csv.py

@@ -26,6 +26,7 @@ class DatasetCsvAdapter(csv.Adapter):
         ("owner_id", "owner.id"),
         # 'contact_point', #  ?
         "description",
+        "description_short",
         "frequency",
         "license",
         "temporal_coverage.start",

udata/core/dataset/forms.py

@@ -7,6 +7,7 @@ from udata.mongo.errors import FieldValidationError
 from .constants import (
     CHECKSUM_TYPES,
     DEFAULT_FREQUENCY,
+    DESCRIPTION_SHORT_SIZE_LIMIT,
     DESCRIPTION_SIZE_LIMIT,
     LEGACY_FREQUENCIES,
     RESOURCE_FILETYPES,
@@ -151,6 +152,11 @@ class DatasetForm(ModelForm):
         [validators.DataRequired(), validators.Length(max=DESCRIPTION_SIZE_LIMIT)],
         description=_("The details about the dataset (collection process, specifics...)."),
     )
+    description_short = fields.StringField(
+        _("Short description"),
+        [validators.Length(max=DESCRIPTION_SHORT_SIZE_LIMIT)],
+        description=_("A short description of the dataset."),
+    )
     license = fields.ModelSelectField(_("License"), model=License, allow_blank=True)
     frequency = fields.SelectField(
         _("Update frequency"),

udata/core/dataset/models.py

@@ -35,6 +35,7 @@ from .constants import (
     CHECKSUM_TYPES,
     CLOSED_FORMATS,
     DEFAULT_LICENSE,
+    DESCRIPTION_SHORT_SIZE_LIMIT,
     LEGACY_FREQUENCIES,
     MAX_DISTANCE,
     PIVOTAL_DATA,
@@ -560,6 +561,7 @@ class Dataset(Auditable, WithMetrics, DatasetBadgeMixin, Owned, Linkable, db.Doc
         auditable=False,
     )
     description = field(db.StringField(required=True, default=""))
+    description_short = field(db.StringField(max_length=DESCRIPTION_SHORT_SIZE_LIMIT))
     license = field(db.ReferenceField("License"))
 
     tags = field(db.TagListField())

udata/harvest/backends/dcat.py

@@ -72,16 +72,16 @@ class DcatBackend(BaseBackend):
         fmt = self.get_format()
         self.job.data = {"format": fmt}
 
-        serialized_graphs = []
+        pages = []
 
         for page_number, page in self.walk_graph(self.source.url, fmt):
             self.process_one_datasets_page(page_number, page)
-            serialized_graphs.append(page.serialize(format=fmt, indent=None))
+            pages.append((page_number, page))
 
         # We do a second pass to have all datasets in memory and attach datasets
         # to dataservices. It could be better to be one pass of graph walking and
         # then one pass of attaching datasets to dataservices.
-        for page_number, page in self.walk_graph(self.source.url, fmt):
+        for page_number, page in pages:
             self.process_one_dataservices_page(page_number, page)
 
         if not self.dryrun and self.has_reached_max_items():
@@ -100,6 +100,8 @@ class DcatBackend(BaseBackend):
 
         bucket = current_app.config.get("HARVEST_GRAPHS_S3_BUCKET")
 
+        serialized_graphs = [p.serialize(format=fmt, indent=None) for _, p in pages]
+
         if (
             bucket is not None
             and sum([len(g.encode("utf-8")) for g in serialized_graphs])