udata 10.0.6__py2.py3-none-any.whl → 10.0.6.dev32964__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 """
 
-__version__ = "10.0.6"
+__version__ = "10.0.6.dev"
 __description__ = "Open data portal"

udata/commands/dcat.py

@@ -2,8 +2,7 @@ import logging
 
 import click
 import mongoengine
-from rdflib import Graph, URIRef
-from rdflib.namespace import RDF
+from rdflib import Graph
 
 from udata.commands import cli, cyan, echo, green, magenta, yellow
 from udata.core.dataset.factories import DatasetFactory
@@ -13,8 +12,7 @@ from udata.harvest.backends.dcat import (
     CswIso19139DcatBackend,
     DcatBackend,
 )
-from udata.harvest.models import HarvestItem
-from udata.rdf import DCAT, DCT, namespace_manager
+from udata.rdf import namespace_manager
 
 log = logging.getLogger(__name__)
 
@@ -63,22 +61,15 @@ def parse_url(url, csw, iso, quiet=False, rid=""):
     backend.job = MockJob()
     format = backend.get_format()
     echo(yellow("Detected format: {}".format(format)))
-    graphs = backend.walk_graph(url, format)
+    graphs = backend.parse_graph(url, format)
 
     # serialize/unserialize graph like in the job mechanism
     graph = Graph(namespace_manager=namespace_manager)
-    for page_number, subgraph in graphs:
+    for subgraph in graphs:
         serialized = subgraph.serialize(format=format, indent=None)
         _subgraph = Graph(namespace_manager=namespace_manager)
         graph += _subgraph.parse(data=serialized, format=format)
 
-        for node in subgraph.subjects(RDF.type, DCAT.Dataset):
-            identifier = subgraph.value(node, DCT.identifier)
-            kwargs = {"nid": str(node), "page": page_number}
-            kwargs["type"] = "uriref" if isinstance(node, URIRef) else "blank"
-            item = HarvestItem(remote_id=str(identifier), kwargs=kwargs)
-            backend.job.items.append(item)
-
     for item in backend.job.items:
         if not rid or rid in item.remote_id:
             echo(magenta("Processing item {}".format(item.remote_id)))

udata/core/dataservices/rdf.py

@@ -156,10 +156,12 @@ def dataservice_to_rdf(dataservice: Dataservice, graph=None):
 
     if is_hvd:
         # We also want to automatically add any HVD category tags of the dataservice's datasets.
-        dataset_ids = [dat.id for dat in dataservice.datasets]
-        for tag in TAG_TO_EU_HVD_CATEGORIES:
-            if Dataset.objects(id__in=dataset_ids, tags="hvd").filter(tags=tag).first():
-                hvd_category_tags.add(tag)
+        for dataset in dataservice.datasets:
+            if "hvd" not in dataset.tags:  # Only check HVD datasets for their categories.
+                continue
+            for tag in dataset.tags:
+                if tag in TAG_TO_EU_HVD_CATEGORIES:
+                    hvd_category_tags.add(tag)
     for tag in hvd_category_tags:
         d.add(DCATAP.hvdCategory, URIRef(TAG_TO_EU_HVD_CATEGORIES[tag]))
 

udata/core/dataservices/tasks.py

@@ -1,7 +1,8 @@
 from celery.utils.log import get_task_logger
 
 from udata.core.dataservices.models import Dataservice
-from udata.harvest.models import HarvestJob
+
+# from udata.harvest.models import HarvestJob
 from udata.models import Discussion, Follow, Transfer
 from udata.tasks import job
 
@@ -17,7 +18,8 @@ def purge_dataservices(self):
         # Remove discussions
         Discussion.objects(subject=dataservice).delete()
         # Remove HarvestItem references
-        HarvestJob.objects(items__dataservice=dataservice).update(set__items__S__dataservice=None)
+        # TODO: uncomment when adding dataservice harvest
+        # HarvestJob.objects(items__dataservice=dataservice).update(set__items__S__dataservice=None)
         # Remove associated Transfers
         Transfer.objects(subject=dataservice).delete()
         # Remove dataservice

udata/core/dataset/tasks.py

@@ -48,11 +48,6 @@ def purge_datasets(self):
             datasets = topic.datasets
             datasets.remove(dataset)
             topic.update(datasets=datasets)
-        # Remove dataservices related dataset
-        for dataservice in Dataservice.objects(datasets=dataset):
-            datasets = dataservice.datasets
-            datasets.remove(dataset)
-            dataservice.update(datasets=datasets)
         # Remove HarvestItem references
         HarvestJob.objects(items__dataset=dataset).update(set__items__S__dataset=None)
         # Remove associated Transfers
@@ -128,11 +123,7 @@ def send_frequency_reminder(self):
 def update_datasets_reuses_metrics(self):
     all_datasets = Dataset.objects.visible().timeout(False)
     for dataset in all_datasets:
-        try:
-            dataset.count_reuses()
-        except Exception as e:
-            log.error(f"Error for dataset {dataset} during reuses metrics update: {e}")
-            continue
+        dataset.count_reuses()
 
 
 def get_queryset(model_cls):

udata/core/organization/models.py

@@ -117,7 +117,7 @@ class Organization(WithMetrics, OrganizationBadgeMixin, db.Datetimed, db.Documen
         max_length=255, required=True, populate_from="name", update=True, follow=True
     )
     description = db.StringField(required=True)
-    url = db.URLField()
+    url = db.StringField()
     image_url = db.StringField()
     logo = db.ImageField(
         fs=avatars, basename=default_image_basename, max_size=LOGO_MAX_SIZE, thumbnails=LOGO_SIZES

udata/core/post/api.py

@@ -1,7 +1,6 @@
 from datetime import datetime
 
 from udata.api import API, api, fields
-from udata.auth import Permission as AdminPermission
 from udata.auth import admin_permission
 from udata.core.dataset.api_fields import dataset_fields
 from udata.core.reuse.models import Reuse
@@ -61,13 +60,6 @@ parser = api.page_parser()
 parser.add_argument(
     "sort", type=str, default="-created_at", location="args", help="The sorting attribute"
 )
-parser.add_argument(
-    "with_drafts",
-    type=bool,
-    default=False,
-    location="args",
-    help="`True` also returns the unpublished posts (only for super-admins)",
-)
 
 
 @ns.route("/", endpoint="posts")
@@ -78,13 +70,11 @@ class PostsAPI(API):
     def get(self):
         """List all posts"""
         args = parser.parse_args()
-
-        posts = Post.objects()
-
-        if not (AdminPermission().can() and args["with_drafts"]):
-            posts = posts.published()
-
-        return posts.order_by(args["sort"]).paginate(args["page"], args["page_size"])
+        return (
+            Post.objects.published()
+            .order_by(args["sort"])
+            .paginate(args["page"], args["page_size"])
+        )
 
     @api.doc("create_post")
     @api.secure(admin_permission)

udata/core/post/tests/test_api.py

@@ -15,28 +15,14 @@ class PostsAPITest:
 
     def test_post_api_list(self, api):
         """It should fetch a post list from the API"""
-        PostFactory.create_batch(3)
-        draft = PostFactory(published=None)
+        posts = PostFactory.create_batch(3)
+        posts.append(PostFactory(published=None))
 
         response = api.get(url_for("api.posts"))
         assert200(response)
         # Response should not contain the unpublished post
         assert len(response.json["data"]) == 3
 
-        api.login(AdminFactory())
-
-        response = api.get(url_for("api.posts"))
-        assert200(response)
-
-        assert len(response.json["data"]) == 3
-        assert str(draft.id) not in [post["id"] for post in response.json["data"]]
-
-        response = api.get(url_for("api.posts", with_drafts=True))
-        assert200(response)
-
-        assert len(response.json["data"]) == 4
-        assert str(draft.id) in [post["id"] for post in response.json["data"]]
-
     def test_post_api_get(self, api):
         """It should fetch a post from the API"""
         post = PostFactory()

udata/harvest/tasks.py

@@ -13,9 +13,8 @@ def harvest(self, ident):
     log.info('Launching harvest job for source "%s"', ident)
 
     source = HarvestSource.get(ident)
-    if source.deleted or not source.active:
-        log.info('Ignoring inactive or deleted source "%s"', ident)
-        return  # Ignore deleted and inactive sources
+    if source.deleted:
+        return  # Ignore deleted sources
     Backend = backends.get(current_app, source.backend)
     backend = Backend(source)