udata 9.2.2.dev31819__py2.py3-none-any.whl → 9.2.3__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 """
 
-__version__ = "9.2.2.dev"
+__version__ = "9.2.3"
 __description__ = "Open data portal"

udata/core/discussions/api.py

@@ -77,7 +77,7 @@ comment_discussion_fields = api.model(
 discussion_page_fields = api.model("DiscussionPage", fields.pager(discussion_fields))
 
 parser = api.parser()
-sorting_keys: list[str] = ["created", "title", "closed"]
+sorting_keys: list[str] = ["created", "title", "closed", "discussion.posted_on"]
 sorting_choices: list[str] = sorting_keys + ["-" + k for k in sorting_keys]
 parser.add_argument(
     "sort",

udata/migrations/2024-10-01-remove-ods-duplicates.py

@@ -0,0 +1,97 @@
+"""
+Remove duplicate OpenDataSoft resources
+The duplicate are due to ODS modifying the URL of their CSV and XLSX exports,
+the URL being the identifier of the resources in DCAT harvesting.
+The default /export/csv had been existing since harvesting ODS with DCAT.
+2024-08-07 : `use_labels=false` appended at the end of the URL -> first duplicates
+2024-08-09 : replaced by `use_labels=true` -> second set of duplicates
+"""
+
+import logging
+import traceback
+
+from mongoengine.errors import ValidationError
+
+from udata.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    count = 0
+    errors = 0
+
+    original = "/exports/{0}"
+    first_duplication = "?use_labels=false"
+    second_duplication = "?use_labels=true"
+
+    # Datasets with use_labels=false are the one that were most surely impacted by the URL modifications
+    datasets = Dataset.objects(resources__url__contains=first_duplication)
+
+    log.info("Starting")
+    log.info(f"{datasets.count()} datasets to process...")
+    for dat in datasets:
+        to_save = False
+        for res_format in ["csv", "xlsx"]:
+            resources = [res for res in dat.resources if original.format(res_format) in res.url]
+            if not resources:
+                # This dataset doesn't have csv or xlsx resources
+                continue
+            if len(resources) not in [3, 4]:
+                log.info(
+                    f"Skipping, {len(resources)} {res_format} duplicate resources found for {dat.id}. We're expecting 3 or 4"
+                )
+                continue
+            try:
+                res_original = next(
+                    res
+                    for res in resources
+                    if res.url.endswith(original.format(res_format))
+                    and res.title.endswith(f".{res_format}")
+                )
+                res_first_duplication = next(
+                    res for res in resources if res.url.endswith(first_duplication)
+                )
+                res_second_duplication = next(
+                    res for res in resources if res.url.endswith(second_duplication)
+                )
+
+                # Resource that exists prior to ODS to DCAT migration and somehow did not get migrated properly
+                # cf https://github.com/opendatateam/udata-ods/pull/247
+                res_migration_duplicate = next(
+                    (
+                        res
+                        for res in resources
+                        if res.title == f"Export au format {res_format.upper()}"
+                    ),
+                    None,
+                )
+
+            except StopIteration:
+                log.info(
+                    f"Could not find expected params on {res_format} for {dat.id} : {set(res.url.split('/')[-1] for res in resources)}"
+                )
+                errors += 1
+                continue
+
+            # Keep the original resource and update its URL
+            res_original.url += second_duplication
+            # Remove all the other duplicates
+            dat.remove_resource(res_first_duplication)
+            dat.remove_resource(res_second_duplication)
+            if res_migration_duplicate:
+                dat.remove_resource(res_migration_duplicate)
+
+            to_save = True
+
+        if to_save:
+            try:
+                dat.save()
+                count += 1
+            except ValidationError:
+                log.info(f"Could not save dataset {dat.id}")
+                log.info(traceback.format_exc())
+                errors += 1
+
+    log.info("Done !")
+    log.info(f"Updated {count} datasets. Failed on {errors} objects.")

{udata-9.2.2.dev31819.dist-info → udata-9.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 9.2.2.dev31819
+Version: 9.2.3
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -138,7 +138,12 @@ It is collectively taken care of by members of the
 
 # Changelog
 
-## Current (in progress)
+## 9.2.3 (2024-10-14)
+
+- Add migration to delete duplicate resources due to ODS harvesting [#3158](https://github.com/opendatateam/udata/pull/3158)
+- Add discussion.posted_on in discussion sort choices [3168](https://github.com/opendatateam/udata/pull/3168)
+
+## 9.2.2 (2024-10-08)
 
 - Add a filter on organization and document sort parameters in the `/discussions` endpoint [#3147](https://github.com/opendatateam/udata/pull/3147)
 - Move discussion catalog creation and add fields [#3152](https://github.com/opendatateam/udata/pull/3152) and [#3154](https://github.com/opendatateam/udata/pull/3154)

{udata-9.2.2.dev31819.dist-info → udata-9.2.3.dist-info}/RECORD

@@ -1,6 +1,6 @@
 tasks/__init__.py,sha256=nubUI6ljumym4uv6NvAJEkWHtsdurFpEGSq-AxDWYDM,8153
 tasks/helpers.py,sha256=70fS9tI_m0DTWmKx9Zl5-LG-nxdz_ZaPyvvsFkN2r48,1091
-udata/__init__.py,sha256=iy3SaE2OS6d8cVQIyTPY4Cbr-q1XUGkY4wN2YMkJJzs,101
+udata/__init__.py,sha256=pwft13dMLFAWF3NT-MhqgjA9nRu8KfeOF-x0rqj4s4Y,97
 udata/api_fields.py,sha256=Oq0WTuI2b-TZXcxr_RExxKHT1wQYlqmDo1gKJsOfXp8,19369
 udata/app.py,sha256=xjk2D3EgboYBpTwBwdIxd2klt2yMoWMyCrkry5fz0LA,7292
 udata/assets.py,sha256=H5Hrc2vnKM0IFLyWfLXmJ2Kj35w1i8W1D8Cgy8_cUj4,657
@@ -108,7 +108,7 @@ udata/core/dataset/signals.py,sha256=WN4sV-lJlNsRkhcnhoy0SYJvCoYmK_5QFYZd1u-h4gs
 udata/core/dataset/tasks.py,sha256=yDPK2oKSzTXfST8Up7vMd13XPEK5r6iapq7hUIow6BI,8493
 udata/core/discussions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/core/discussions/actions.py,sha256=seKxsKut6xLkobfC2Yy2wpD_-flsT8HcFPcN6SDpz6k,789
-udata/core/discussions/api.py,sha256=YsVz7JB7RazAeiydxGe1q1t8sIpxCM_w1b71oT9oN_8,9422
+udata/core/discussions/api.py,sha256=5yMdKeWp0c9H2ipy5bXoMf82r3ULMIARhvuIdM-XJp0,9446
 udata/core/discussions/constants.py,sha256=nbZgXESpg0TykIGPxW8xUtHtk7TwQoyOL0Ky4U4B7c8,27
 udata/core/discussions/csv.py,sha256=c7ZHMicIi8R0kvMjnsUooO0WlH0t90hJ9kO2mmFnUM0,672
 udata/core/discussions/factories.py,sha256=CgQaUmmyDu90XtyBsqXVa-TmZMrN7Dwuu1Cnr5wNrVc,350
@@ -347,6 +347,7 @@ udata/migrations/2023-02-08-rename-internal-dates.py,sha256=jE1oR16Zc98450mATKpK
 udata/migrations/2024-01-29-fix-reuse-and-dataset-with-private-None.py,sha256=9LMwAdv7527qsrQQikZAskFsheqwZpEP_VewjwROm6o,592
 udata/migrations/2024-03-22-migrate-activity-kwargs-to-extras.py,sha256=ucoXSqZlGToMCNiba0pRw_WoYb-lSBKvnbBLe-YVVrs,402
 udata/migrations/2024-06-11-fix-reuse-datasets-references.py,sha256=p0ksSutxk93rY9AMPyN7w_9lU5Y3sRU75iWOLR8MmsY,768
+udata/migrations/2024-10-01-remove-ods-duplicates.py,sha256=e7u0oGUxFgp4_cmxJlEpWZtKp2gn6T1tVYrQgb6V2wo,3621
 udata/migrations/__init__.py,sha256=RBCBDaTlLjuMs_Qzwji6Z6T4r7FCGXhESKoxQbT5qAA,11221
 udata/models/__init__.py,sha256=txbZwa-lRG3mq99eQ9E5YcFWiNUdjDVSyJJvlqUMFfs,1413
 udata/mongo/__init__.py,sha256=y4Rv-kq3o_kcEulcNpePLzocXPBNpx3Jd82G-VZPaMc,1421
@@ -698,9 +699,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=3ICJrrfuylqcmgkWApGylKGYLKKmcC
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=AnLimXvln78bzu8GHq6rj9Xp0LjewKi-yJrrtHzi7d4,44645
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=SqzVRRevggMrUyGUxFyvf8Shf3GHDDeCpLFzr7oHI9o,28163
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=Bfj0FURHtLPxBHrXpUDmJHgM9vslS_VTyGUgXVjfuM8,51137
-udata-9.2.2.dev31819.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-9.2.2.dev31819.dist-info/METADATA,sha256=IKkLtM9Rq0IRFeX-LEyksYilrghHuLV-zPNPteOFp9U,131519
-udata-9.2.2.dev31819.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-udata-9.2.2.dev31819.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
-udata-9.2.2.dev31819.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-9.2.2.dev31819.dist-info/RECORD,,
+udata-9.2.3.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-9.2.3.dist-info/METADATA,sha256=o04kwCcLHDez5LZGH9kOsWRhViWU0T97-Yl_8wZxL4g,131766
+udata-9.2.3.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+udata-9.2.3.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
+udata-9.2.3.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-9.2.3.dist-info/RECORD,,