udata 10.0.6.dev32964__py2.py3-none-any.whl → 10.0.6.dev32994__py2.py3-none-any.whl

udata/commands/dcat.py

@@ -2,7 +2,8 @@ import logging
 
 import click
 import mongoengine
-from rdflib import Graph
+from rdflib import Graph, URIRef
+from rdflib.namespace import RDF
 
 from udata.commands import cli, cyan, echo, green, magenta, yellow
 from udata.core.dataset.factories import DatasetFactory
@@ -12,7 +13,8 @@ from udata.harvest.backends.dcat import (
     CswIso19139DcatBackend,
     DcatBackend,
 )
-from udata.rdf import namespace_manager
+from udata.harvest.models import HarvestItem
+from udata.rdf import DCAT, DCT, namespace_manager
 
 log = logging.getLogger(__name__)
 
@@ -61,15 +63,22 @@ def parse_url(url, csw, iso, quiet=False, rid=""):
     backend.job = MockJob()
     format = backend.get_format()
     echo(yellow("Detected format: {}".format(format)))
-    graphs = backend.parse_graph(url, format)
+    graphs = backend.walk_graph(url, format)
 
     # serialize/unserialize graph like in the job mechanism
     graph = Graph(namespace_manager=namespace_manager)
-    for subgraph in graphs:
+    for page_number, subgraph in graphs:
         serialized = subgraph.serialize(format=format, indent=None)
         _subgraph = Graph(namespace_manager=namespace_manager)
         graph += _subgraph.parse(data=serialized, format=format)
 
+        for node in subgraph.subjects(RDF.type, DCAT.Dataset):
+            identifier = subgraph.value(node, DCT.identifier)
+            kwargs = {"nid": str(node), "page": page_number}
+            kwargs["type"] = "uriref" if isinstance(node, URIRef) else "blank"
+            item = HarvestItem(remote_id=str(identifier), kwargs=kwargs)
+            backend.job.items.append(item)
+
     for item in backend.job.items:
         if not rid or rid in item.remote_id:
             echo(magenta("Processing item {}".format(item.remote_id)))

udata/tests/test_dcat_commands.py

@@ -0,0 +1,26 @@
+import pytest
+
+
+@pytest.mark.usefixtures("clean_db")
+class ParseUrlCommandTest:
+    def test_parse_url(self, cli, requests_mock, caplog) -> None:
+        logs = []
+
+        def mock_echo(message: str) -> None:
+            logs.append(message)
+
+        local_file: str = "./udata/harvest/tests/dcat/sig.oreme.rdf"
+        # This mock_url is to make requests' url parsing happy, but the mock will return the local file content.
+        mock_url = "https://example.com/sig.oreme.rdf"
+        with open(local_file, "r") as test_rdf_file:
+            requests_mock.get(mock_url, text=test_rdf_file.read())
+            requests_mock.head(mock_url, text="sig.oreme.rdf")
+        dataset_id = "0437a976-cff1-4fa6-807a-c23006df2f8f"
+        result = cli(
+            "dcat",
+            "parse-url",
+            mock_url,
+            "--rid",
+            dataset_id,
+        )
+        assert "Dataset is valid" in result.stdout

{udata-10.0.6.dev32964.dist-info → udata-10.0.6.dev32994.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 10.0.6.dev32964
+Version: 10.0.6.dev32994
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -140,7 +140,7 @@ It is collectively taken care of by members of the
 
 ## Current (in progress)
 
-- Nothing yet
+- Fix the `parse-url` command [#3225](https://github.com/opendatateam/udata/pull/3225)
 
 ## 10.0.5 (2024-12-09)
 

{udata-10.0.6.dev32964.dist-info → udata-10.0.6.dev32994.dist-info}/RECORD

@@ -41,7 +41,7 @@ udata/auth/views.py,sha256=PX4bRHATNzPHWu01_KC3mgcZpAEahdcDAJVBhy4oL4Y,6110
 udata/commands/__init__.py,sha256=Won_rW_hIU9TA3o4oNe6kI46l1fnDBM_oW0Hc1XS9F8,7711
 udata/commands/cache.py,sha256=bLdrf_fCWFYX9ULlL2ADsZRwijkI4pArsJxfx24OivM,341
 udata/commands/db.py,sha256=x5ui77hoT-JcECneBhhXZtpH4g65gaj7SyeM0bnORhA,14694
-udata/commands/dcat.py,sha256=L1J1AxQCmMDmwdxzugQNfy4VZQYf1rI1wPhSdUaMB5E,3419
+udata/commands/dcat.py,sha256=f6jT2AGZem-w1CaRH_ahfWB9A4oCDvjG13tPmBpeCqw,3910
 udata/commands/fixtures.py,sha256=oZkZPqpBNTjYQwNwODfi-bZyRZIjBIG-gBhpNaaAiNc,9221
 udata/commands/images.py,sha256=0rVojLik5DYgJ6W4uNEfMP2g2QUU2V761tj3z6lo8no,2050
 udata/commands/info.py,sha256=A5WMo3_N_rlt3cySVJrZqKWrbIowX97ZLKMIFQE5178,1545
@@ -590,6 +590,7 @@ udata/tests/schemas.json,sha256=szM1jDpkogfOG4xWbjIGjLgG8l9-ZyE3JKQtecJyD1E,4990
 udata/tests/test_activity.py,sha256=x-pDK6VW9wAG0uxYRZQ3DWTRjfCU729iaMGMJb1rWYU,3195
 udata/tests/test_api_fields.py,sha256=4sDs40CtdarVQHzAdNIeRCOYughYsTlPehTY18jaq58,11293
 udata/tests/test_cors.py,sha256=b_pyxKeIyqhnsXxXryPf4d0V0QxaLQ1P_VjY89Q_j3g,3233
+udata/tests/test_dcat_commands.py,sha256=fDAnAjkja8AXw_qzaAWnVTgglkBAvK2mjPMHUCtqrrU,919
 udata/tests/test_discussions.py,sha256=Qo7nhIUY7YAATZfWl5Mgo0XhngGwFULgqEoMhsHZtFE,33587
 udata/tests/test_i18n.py,sha256=u60344JNRG_8s0t89ghXtQ1FbF4TayEHBzuBFxqnQ_Y,3152
 udata/tests/test_linkchecker.py,sha256=W8jrwKYXM8wWXZFjiaBwpWGRBhZ8bsSHGHzL9voDN7U,10218
@@ -709,9 +710,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=18Y5YtzVKInDejw-R-45HNzsB3OVwJ
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=6IQvFk0NTDV5Jq-kLkkzpioWfrMaLDa1oQSevKFbxKQ,44943
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=LWzCNS68hS2lbeo1bncZw4ZKqV2Dk_JUC-4SiF3o994,28163
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=X1ieIsJDmk_A_YiblpmUBzWWrVHta9F8yzCZLrjeW-Y,51435
-udata-10.0.6.dev32964.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-10.0.6.dev32964.dist-info/METADATA,sha256=NNf-ZhhambKXxPpKpghn7i6zy7asPYm_tDuGR_EcFeA,137244
-udata-10.0.6.dev32964.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-udata-10.0.6.dev32964.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
-udata-10.0.6.dev32964.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-10.0.6.dev32964.dist-info/RECORD,,
+udata-10.0.6.dev32994.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-10.0.6.dev32994.dist-info/METADATA,sha256=K82JxoPltu5jwPmaLfNeI-9hhFUXFeCxMl-1upmOxe4,137317
+udata-10.0.6.dev32994.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+udata-10.0.6.dev32994.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
+udata-10.0.6.dev32994.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-10.0.6.dev32994.dist-info/RECORD,,