udata 12.0.2.dev9__py3-none-any.whl → 12.0.2.dev11__py3-none-any.whl

udata/commands/__init__.py

@@ -186,7 +186,6 @@ MODULES_WITH_COMMANDS = [
     "core.spatial",
     "core.user",
     "harvest",
-    "linkchecker",
     "search",
 ]
 

udata/core/dataset/api.py

@@ -46,7 +46,6 @@ from udata.core.storages.api import handle_upload, upload_parser
 from udata.core.topic.models import Topic
 from udata.frontend.markdown import md
 from udata.i18n import gettext as _
-from udata.linkchecker.checker import check_resource
 from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
 from udata.utils import get_by
 
@@ -902,20 +901,6 @@ class AllowedExtensionsAPI(API):
         return sorted(current_app.config["ALLOWED_RESOURCES_EXTENSIONS"])
 
 
-@ns.route(
-    "/<dataset:dataset>/resources/<uuid:rid>/check/",
-    endpoint="check_dataset_resource",
-    doc=common_doc,
-)
-@api.param("rid", "The resource unique identifier")
-class CheckDatasetResource(API, ResourceMixin):
-    @api.doc("check_dataset_resource")
-    def get(self, dataset, rid):
-        """Checks that a resource's URL exists and returns metadata."""
-        resource = self.get_resource_or_404(dataset, rid)
-        return check_resource(resource)
-
-
 @ns.route("/resource_types/", endpoint="resource_types")
 class ResourceTypesAPI(API):
     @api.doc("resource_types")

udata/core/dataset/models.py

@@ -1,6 +1,6 @@
 import logging
 import re
-from datetime import datetime, timedelta
+from datetime import datetime
 from pydoc import locate
 from typing import Self
 from urllib.parse import urlparse
@@ -8,7 +8,6 @@ from urllib.parse import urlparse
 import Levenshtein
 import requests
 from blinker import signal
-from dateutil.parser import parse as parse_dt
 from flask import current_app, url_for
 from mongoengine import ValidationError as MongoEngineValidationError
 from mongoengine.fields import DateTimeField
@@ -369,7 +368,13 @@ class ResourceMixin(object):
     mime = db.StringField()
     filesize = db.IntField()  # `size` is a reserved keyword for mongoengine.
     fs_filename = db.StringField()
-    extras = db.ExtrasField()
+    extras = db.ExtrasField(
+        {
+            "check:available": db.BooleanField,
+            "check:status": db.IntField,
+            "check:date": db.DateTimeField,
+        }
+    )
     harvest = db.EmbeddedDocumentField(HarvestResourceMetadata)
     schema = db.EmbeddedDocumentField(Schema)
 
@@ -428,41 +433,6 @@ class ResourceMixin(object):
         """
         return self.extras.get("check:available", "unknown")
 
-    def need_check(self):
-        """Does the resource needs to be checked against its linkchecker?
-
-        We check unavailable resources often, unless they go over the
-        threshold. Available resources are checked less and less frequently
-        based on their historical availability.
-        """
-        min_cache_duration, max_cache_duration, ko_threshold = [
-            current_app.config.get(k)
-            for k in (
-                "LINKCHECKING_MIN_CACHE_DURATION",
-                "LINKCHECKING_MAX_CACHE_DURATION",
-                "LINKCHECKING_UNAVAILABLE_THRESHOLD",
-            )
-        ]
-        count_availability = self.extras.get("check:count-availability", 1)
-        is_available = self.check_availability()
-        if is_available == "unknown":
-            return True
-        elif is_available or count_availability > ko_threshold:
-            delta = min(min_cache_duration * count_availability, max_cache_duration)
-        else:
-            delta = min_cache_duration
-        if self.extras.get("check:date"):
-            limit_date = datetime.utcnow() - timedelta(minutes=delta)
-            check_date = self.extras["check:date"]
-            if not isinstance(check_date, datetime):
-                try:
-                    check_date = parse_dt(check_date)
-                except (ValueError, TypeError):
-                    return True
-            if check_date >= limit_date:
-                return False
-        return True
-
     @property
     def latest(self):
         """

udata/entrypoints.py

@@ -4,10 +4,8 @@ import pkg_resources
 ENTRYPOINTS = {
     "udata.avatars": "Avatar rendering backends",
     "udata.harvesters": "Harvest backends",
-    "udata.linkcheckers": "Link checker backends",
     "udata.metrics": "Extra metrics",
     "udata.models": "Models and migrations",
-    "udata.preview": "Displays preview for resources",
     "udata.plugins": "Generic plugin",
     "udata.tasks": "Tasks and jobs",
     "udata.themes": "Themes",

udata/models/__init__.py

@@ -31,8 +31,6 @@ from udata.features.territories.models import *  # noqa
 # Load HarvestSource model as harvest for catalog
 from udata.harvest.models import HarvestSource as Harvest  # noqa
 
-import udata.linkchecker.models  # noqa
-
 
 def init_app(app):
     entrypoints.get_enabled("udata.models", app)

udata/mongo/extras_fields.py

@@ -11,15 +11,16 @@ ALLOWED_TYPES = (str, int, float, bool, datetime, date, list, dict)
 
 
 class ExtrasField(DictField):
-    def __init__(self, **kwargs):
+    def __init__(self, keys_types={}, **kwargs):
         self.registered = {}
+        for key, dbtype in keys_types.items():
+            self.register(key, dbtype)
         super(ExtrasField, self).__init__()
 
     def register(self, key, dbtype):
         """Register a DB type to add constraint on a given extra key"""
         if not issubclass(dbtype, (BaseField, EmbeddedDocument)):
-            msg = "ExtrasField can only register MongoEngine fields"
-            raise TypeError(msg)
+            raise TypeError("ExtrasField can only register MongoEngine fields")
         self.registered[key] = dbtype
 
     def validate(self, values):

udata/settings.py

@@ -331,16 +331,6 @@ class Defaults(object):
     # The order is important to compute parents/children, smaller first.
     HANDLED_LEVELS = tuple()
 
-    LINKCHECKING_ENABLED = True
-    # Resource types ignored by linkchecker
-    LINKCHECKING_UNCHECKED_TYPES = ("api",)
-    LINKCHECKING_IGNORE_DOMAINS = []
-    LINKCHECKING_IGNORE_PATTERNS = ["format=shp"]
-    LINKCHECKING_MIN_CACHE_DURATION = 60  # in minutes
-    LINKCHECKING_MAX_CACHE_DURATION = 1080  # in minutes (1 week)
-    LINKCHECKING_UNAVAILABLE_THRESHOLD = 100
-    LINKCHECKING_DEFAULT_LINKCHECKER = "no_check"
-
     # Ignore some endpoint from API tracking
     # By default ignore the 3 most called APIs
     TRACKING_BLACKLIST = [

{udata-12.0.2.dev9.dist-info → udata-12.0.2.dev11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: udata
-Version: 12.0.2.dev9
+Version: 12.0.2.dev11
 Summary: Open data portal
 Author-email: Opendata Team <opendatateam@data.gouv.fr>
 Maintainer-email: Opendata Team <opendatateam@data.gouv.fr>

{udata-12.0.2.dev9.dist-info → udata-12.0.2.dev11.dist-info}/RECORD

@@ -2,7 +2,7 @@ udata/__init__.py,sha256=U0HEYqKCLOY43O1UCVeuAb3b3SSX1pPhsJGpHJmK67k,75
 udata/api_fields.py,sha256=XI0XoM1fxO4DEzxGptOAB5SL_fJr-u58-bfQVAvzgBg,36549
 udata/app.py,sha256=By-eZvSVSCNtWeKm_lA8TF81qaHdzScvggvbgHCDHYI,8992
 udata/cors.py,sha256=JttAogsNVSFWEV9-1L2kdbwwsNewn3KjgBErXReNwfc,3801
-udata/entrypoints.py,sha256=mbAAUVT8ZenzSYdang2PbAwZcK1pENtA3axBmPRiWCw,2717
+udata/entrypoints.py,sha256=90n21thjq-r7EEyWf6IJZ6DS1q404OoP_VDkpRjZMvs,2611
 udata/errors.py,sha256=E8W7b4PH7c5B85g_nsUMt8fHqMVpDFOZFkO6wMPl6bA,117
 udata/factories.py,sha256=MoklZnU8iwNL25dm3JsoXhoQs1PQWSVYL1WvcUBtJqM,492
 udata/i18n.py,sha256=bC9ajf66YgcYoJffvresLZLa32rb6NsY-JGMtFiVsG4,8163
@@ -10,7 +10,7 @@ udata/mail.py,sha256=Huhx_1QthJkLvuRUuP6jqb5Qq5R4iSmqeEpLVO9ZkQ4,2671
 udata/rdf.py,sha256=aJKmnE1r6YyMKXLo-VRlUvOXoZSJuvNeNbMCqEl0kdY,19370
 udata/routing.py,sha256=Hnc1ktmKVS-RUHNKw2zYTft2HJ903FhjtlcenQ9igwI,8044
 udata/sentry.py,sha256=ekcxqUSqxfM98TtvCsPaOoX5i2l6PEcYt7kb4l3od-Q,3223
-udata/settings.py,sha256=o6sV0gAXgZY4if2pU017yR0pKwDTEFBFbnEWYDyBxZg,22110
+udata/settings.py,sha256=1gDu1fnorsgzRzkMIEzFjWiVZ_7UPe1kRW-GQulb7O0,21686
 udata/sitemap.py,sha256=oRRWoPI7ZsFFnUAOqGT1YuXFFKHBe8EcRnUCNHD7xjM,979
 udata/tags.py,sha256=ydq4uokd6bzdeGVSpEXASVtGvDfO2LfQs9mptvvKJCM,631
 udata/tasks.py,sha256=Sv01dhvATtq_oHOBp3J1j1VT1HQe0Pab7zxwIeIdKoo,5122
@@ -34,7 +34,7 @@ udata/auth/mails.py,sha256=ggGfgYEgNLtF-p5HocrmuQAk0b6fteWtN4UzK3ZvMlA,1759
 udata/auth/password_validation.py,sha256=ODVdEsiXbtq_8ws4Yf3hs5Sq7jz-IDa1RxAm_WPIJnA,1806
 udata/auth/proconnect.py,sha256=hsvQ71Hqy42NvwgYtcMniRXWax3Q7LX1INcmaS7gaIQ,5073
 udata/auth/views.py,sha256=83nlLQiRlqLPILPOkKilNnrTJTs6c7XKqe-nt2I5l8s,7861
-udata/commands/__init__.py,sha256=Won_rW_hIU9TA3o4oNe6kI46l1fnDBM_oW0Hc1XS9F8,7711
+udata/commands/__init__.py,sha256=y0ncKCjOTuxm9Tn8A0WMvub1Mai0wLQ15dq9y8GRV7c,7692
 udata/commands/cache.py,sha256=bLdrf_fCWFYX9ULlL2ADsZRwijkI4pArsJxfx24OivM,341
 udata/commands/db.py,sha256=OyVBcuSIqYqNywlZAi19F2yRJCAIdFKKyQ9H9alqbfI,20426
 udata/commands/dcat.py,sha256=f6jT2AGZem-w1CaRH_ahfWB9A4oCDvjG13tPmBpeCqw,3910
@@ -92,7 +92,7 @@ udata/core/dataservices/tasks.py,sha256=fHG1r5ymfJRXJ_Lug6je3VKZoK30XKXE2rQ8x0R-
 udata/core/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/core/dataset/actions.py,sha256=mX6xox0PiMrbcAPZ3VZsI26rfM-ciYfEXxN6sqqImKA,1222
 udata/core/dataset/activities.py,sha256=eGxMUnC47YHxTgcls6igQ3qP7cYgwFtPfj0asCylGsI,3315
-udata/core/dataset/api.py,sha256=4KjNwgJ5PGV6WWyGaJdpiivWoNYs79BeHH5e509GV7s,35861
+udata/core/dataset/api.py,sha256=JBdYH2RXac1WVOF67ZzBSvImVxixMfN0rxEknnfthRU,35338
 udata/core/dataset/api_fields.py,sha256=p7ZnmGNImZ4sgZTpoyHpI0CgOukpEIx8QdGnxlmgl2I,18032
 udata/core/dataset/apiv2.py,sha256=1H4557ZMi6rwEyrwB1Ha20m0bf3Avhg_vDLiDQt5Fi0,21030
 udata/core/dataset/commands.py,sha256=3mKSdJ-M7ggdG29AVn77C4ouZanbYoqkTaGQoBKOp3s,3471
@@ -103,7 +103,7 @@ udata/core/dataset/exceptions.py,sha256=uKiayLSpSzsnLvClObS6hOO0qXEqvURKN7_w8eim
 udata/core/dataset/factories.py,sha256=tb18axsk8Tx5iUIqWM9IELdt-2Ryp2UN0-iY4fdea4U,9059
 udata/core/dataset/forms.py,sha256=gGXOqy3WXFNvWMXngDq3TEqMM18-9DBpy2V7msrOsTw,7084
 udata/core/dataset/metrics.py,sha256=s8Xs_rqRXfNWsErkiJTuRMG5o_cU5iSK8mUJFKVSc7w,1204
-udata/core/dataset/models.py,sha256=Ny79cFiD7vOJt0JIfOZ7AJQOBx-_EEWPxdnhBpUr_-E,42980
+udata/core/dataset/models.py,sha256=1iXf6l0wjNv04if6gAq-NXSPLub2Mx2KOyCg2RQtids,41617
 udata/core/dataset/permissions.py,sha256=zXQ6kU-Ni3Pl5tDtat-ZPupug9InsNeCN7xRLc2Vcrc,1097
 udata/core/dataset/preview.py,sha256=uFEpK-p5nIAlY8hVOMhd7mtkwFt6C_PQRMNxPvAyoo4,839
 udata/core/dataset/rdf.py,sha256=jHyHgay3g3Z04Ju5kLyMk3Vyb-kWJYtqqg8Smamv7WI,32970
@@ -356,12 +356,6 @@ udata/harvest/tests/dcat/partial-collection-1.jsonld,sha256=emPZGvpdaqgIVTgtmlWN
 udata/harvest/tests/dcat/partial-collection-2.jsonld,sha256=zJ1ggcs2b4IJBDJ6zKJn8w5arjJBU_EHr6qFd8tu0I8,3691
 udata/harvest/tests/dcat/sig.oreme.rdf,sha256=6F1P-hPyE8bZC-5uQTskTawRd19U0opIy9LCDwp0sd4,6315
 udata/harvest/tests/dcat/udata.xml,sha256=co7tLKinEdPOwEScHUXSqSAKAFSgLElYOCI3uu50Sgo,14532
-udata/linkchecker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-udata/linkchecker/backends.py,sha256=Xe_nWwjKsdMv7kgs3mbQDppVO4lqaJlboUxa_0nk1XM,1018
-udata/linkchecker/checker.py,sha256=LB6WSMp5yvdmfdlq2ygp7uHoYEVsK0IPk6qnmhrS4GI,2903
-udata/linkchecker/commands.py,sha256=14c78P0FTLncuJpAgPizoCY62bUIknKumev1FPZ4JQU,454
-udata/linkchecker/models.py,sha256=V5PBSOGnCXZrZLoFQwKb91enpgVTbX4Q3_oWOPihJic,423
-udata/linkchecker/tasks.py,sha256=jETwfn6v-mP4jh_ovz4EwlxBUcRij_wHJVK1AmC0qo4,1918
 udata/migrations/2020-07-24-remove-s-from-scope-oauth.py,sha256=bCfBRcd4azHSV-fpZQaiHkWu7likHoWNpq9PeJvi5S0,665
 udata/migrations/2020-08-24-add-fs-filename.py,sha256=7mz5ufFvY67NDKEmQn0HS3u1SdR9uH077ngJfd2uot4,1819
 udata/migrations/2020-09-28-update-reuses-datasets-metrics.py,sha256=d7zizz1fwy-jCOJWmfveBpBZJSHJu46HSIBf3Xnd0eI,407
@@ -404,13 +398,13 @@ udata/migrations/2025-07-30-purge-old-harvest-dynamic-fields.py,sha256=ijeu6WvX6
 udata/migrations/2025-09-04-update-legacy-frequencies.py,sha256=8YAROAHhytf6Kses_54aFscmPNes2aHRYqTwNpawdVk,1693
 udata/migrations/2025-10-01-delete-orphaned-topic-elements.py,sha256=Mhx5ANOihZL4botxtjvfsll-xKBtkVQBkPcSq0BJ-Ec,788
 udata/migrations/__init__.py,sha256=RBCBDaTlLjuMs_Qzwji6Z6T4r7FCGXhESKoxQbT5qAA,11221
-udata/models/__init__.py,sha256=77OriDFm4dJzQDIb-7ADKSkf9GAxkXbMHeWXoYYnTsk,1459
+udata/models/__init__.py,sha256=RT0WaPKm2linlYnTmD0xck7M2XuIoIJ94E0ErzNGzaw,1418
 udata/mongo/__init__.py,sha256=y4Rv-kq3o_kcEulcNpePLzocXPBNpx3Jd82G-VZPaMc,1421
 udata/mongo/datetime_fields.py,sha256=xACagQZu1OKPvpcznI-bMC1tJfAvo-VBUe7OOadnBdg,2089
 udata/mongo/document.py,sha256=yJl4rzE0L69SvNbtmnmyCALTGhXwBPrj7nvM-J6sDpE,1792
 udata/mongo/engine.py,sha256=JF9N55j7joDIn9NrItMtlIrA5CwVLhS_jlB2ptX94oA,2408
 udata/mongo/errors.py,sha256=SpTMAc_aNIfGkqyXGCbTlIAmYxU86rGM_NtIYaB642c,472
-udata/mongo/extras_fields.py,sha256=FfyVvRkpLn4pUeCqwI33NBJblHOywGlnA05RCEZ-ugs,4139
+udata/mongo/extras_fields.py,sha256=knb0fwt8eIEOA0jjeUAs9Gmn_cfUNVPuRdGEVcJzE2Y,4218
 udata/mongo/queryset.py,sha256=fXfYkUHsCWAUoub3OR7v825USPv-PQQIHkv4U5FnjYg,3954
 udata/mongo/slug_fields.py,sha256=tEUlwozrdQfF42KR5dxk5PUNSX7zISTIXsSgHxR4YMg,7522
 udata/mongo/taglist_field.py,sha256=RPi8DlgMEMK1wk8hbQDLAyH2GnzZCfNpWXQsllxPB6g,1371
@@ -504,7 +498,6 @@ udata/tests/test_api_fields.py,sha256=NCUTtOMEaTM5-tK-YUxhjEud2IPIDOHR3vbZWAQdEC
 udata/tests/test_cors.py,sha256=i1SQS91lm-i3YEUqoHKUFpOI7TCpDx89MzHoWd3r2uk,2932
 udata/tests/test_dcat_commands.py,sha256=fDAnAjkja8AXw_qzaAWnVTgglkBAvK2mjPMHUCtqrrU,919
 udata/tests/test_discussions.py,sha256=a2yBGfOSm93w8zP2s2gXy51LGniiZ0lbrej0uuXNd2E,47685
-udata/tests/test_linkchecker.py,sha256=W8jrwKYXM8wWXZFjiaBwpWGRBhZ8bsSHGHzL9voDN7U,10218
 udata/tests/test_mail.py,sha256=f-8meP9r1Xrz0eOTsvmdynoV9OFHLwwMr7XM5WWv_gk,4182
 udata/tests/test_migrations.py,sha256=Iq0gt3hjK6ohliIvEpqyt0tYJz0MKIzBg_yfw7dOcHo,15698
 udata/tests/test_model.py,sha256=mwWf8tbh4iFVzqIpTvpobjfCQOdo8EJ5x3t8mXHmd7g,21999
@@ -632,9 +625,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=nv80xZLfIfUsSOMBcr29L268FDc_Gt
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=bUp-7Ray8t8ALgJk3Icw1jmiGIc9_pEJQHiGw_2EU2o,50989
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=Y_XpUxD074wXc63oJTnoVOyOQ2lmBxl-MrgluZ0Qdw4,27961
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=qh8mrz9AFuVQtXYSSP4QWsXLM_Lv3EHVifHT1NflWXY,57529
-udata-12.0.2.dev9.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-12.0.2.dev9.dist-info/METADATA,sha256=BBbAgWGTEx3bRQ_tAsh5fbWIao80z-0khr74HXfnQso,5272
-udata-12.0.2.dev9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-udata-12.0.2.dev9.dist-info/entry_points.txt,sha256=v2u12qO11i2lyLNIp136WmLJ-NHT-Kew3Duu8J-AXPM,614
-udata-12.0.2.dev9.dist-info/top_level.txt,sha256=EF6CE6YSHd_og-8LCEA4q25ALUpWVe8D0okOLdMAE3A,6
-udata-12.0.2.dev9.dist-info/RECORD,,
+udata-12.0.2.dev11.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-12.0.2.dev11.dist-info/METADATA,sha256=Yno-kuZXIaNIlsxQaFAEjEfwiWeFtm-i4tzMXULFLYs,5273
+udata-12.0.2.dev11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+udata-12.0.2.dev11.dist-info/entry_points.txt,sha256=v2u12qO11i2lyLNIp136WmLJ-NHT-Kew3Duu8J-AXPM,614
+udata-12.0.2.dev11.dist-info/top_level.txt,sha256=EF6CE6YSHd_og-8LCEA4q25ALUpWVe8D0okOLdMAE3A,6
+udata-12.0.2.dev11.dist-info/RECORD,,

udata/linkchecker/backends.py

@@ -1,31 +0,0 @@
-import logging
-from datetime import datetime
-
-from flask import current_app
-
-from udata.entrypoints import get_enabled
-
-log = logging.getLogger(__name__)
-
-
-ENTRYPOINT = "udata.linkcheckers"
-
-
-class NoCheckLinkchecker(object):
-    """Dummy linkchecker for resources that need no check"""
-
-    def check(self, _):
-        return {"check:status": 204, "check:available": True, "check:date": datetime.utcnow()}
-
-
-def get(name):
-    """Get a linkchecker given its name or fallback on default"""
-    linkcheckers = get_enabled(ENTRYPOINT, current_app)
-    linkcheckers.update(no_check=NoCheckLinkchecker)  # no_check always enabled
-    selected_linkchecker = linkcheckers.get(name)
-    if not selected_linkchecker:
-        default_linkchecker = current_app.config.get("LINKCHECKING_DEFAULT_LINKCHECKER")
-        selected_linkchecker = linkcheckers.get(default_linkchecker)
-    if not selected_linkchecker:
-        log.error("No linkchecker found ({} requested and no fallback)".format(name))
-    return selected_linkchecker

udata/linkchecker/checker.py

@@ -1,75 +0,0 @@
-from urllib.parse import urlparse
-
-from flask import current_app
-
-from .backends import NoCheckLinkchecker
-from .backends import get as get_linkchecker
-
-
-def _get_check_keys(the_dict, resource, previous_status):
-    check_keys = {k: v for k, v in the_dict.items() if k.startswith("check:")}
-    check_keys["check:count-availability"] = _compute_count_availability(
-        resource, check_keys.get("check:available"), previous_status
-    )
-    return check_keys
-
-
-def _compute_count_availability(resource, status, previous_status):
-    """Compute the `check:count-availability` extra value"""
-    count_availability = resource.extras.get("check:count-availability", 1)
-    return count_availability + 1 if status == previous_status else 1
-
-
-def is_ignored(resource):
-    """Check if the resource's URL is to be ignored"""
-    ignored_domains = current_app.config["LINKCHECKING_IGNORE_DOMAINS"]
-    ignored_patterns = current_app.config["LINKCHECKING_IGNORE_PATTERNS"]
-    url = resource.url
-    if not url:
-        return True
-    parsed_url = urlparse(url)
-    ignored_domains_match = parsed_url.netloc in ignored_domains
-    ignored_patterns_match = any([p in url for p in ignored_patterns])
-    return ignored_domains_match or ignored_patterns_match
-
-
-def dummy_check_response():
-    """Trigger a dummy check"""
-    return NoCheckLinkchecker().check(None)
-
-
-def check_resource(resource):
-    """
-    Check a resource availability against a linkchecker backend
-
-    The linkchecker used can be configured on a resource basis by setting
-    the `resource.extras['check:checker']` attribute with a key that points
-    to a valid `udata.linkcheckers` entrypoint. If not set, it will
-    fallback on the default linkchecker defined by the configuration variable
-    `LINKCHECKING_DEFAULT_LINKCHECKER`.
-
-    Returns
-    -------
-    dict or (dict, int)
-        Check results dict and status code (if error).
-    """
-    linkchecker_type = resource.extras.get("check:checker")
-    LinkChecker = get_linkchecker(linkchecker_type)
-    if not LinkChecker:
-        return {"error": "No linkchecker configured."}, 503
-    if is_ignored(resource):
-        return dummy_check_response()
-    result = LinkChecker().check(resource)
-    if not result:
-        return {"error": "No response from linkchecker"}, 503
-    elif result.get("check:error"):
-        return {"error": result["check:error"]}, 500
-    elif not result.get("check:status"):
-        return {"error": "No status in response from linkchecker"}, 503
-    # store the check result in the resource's extras
-    # XXX maybe this logic should be in the `Resource` model?
-    previous_status = resource.extras.get("check:available")
-    check_keys = _get_check_keys(result, resource, previous_status)
-    resource.extras.update(check_keys)
-    resource.save(signal_kwargs={"ignores": ["post_save"]})  # Prevent signal triggering on dataset
-    return result

udata/linkchecker/commands.py

@@ -1,21 +0,0 @@
-import logging
-
-import click
-
-from udata.commands import cli
-from udata.linkchecker.tasks import check_resources
-
-log = logging.getLogger(__name__)
-
-
-@cli.group("linkchecker")
-def grp():
-    """Link checking operations"""
-    pass
-
-
-@grp.command()
-@click.option("-n", "--number", type=int, default=5000, help="Number of URLs to check")
-def check(number):
-    """Check <number> of URLs that have not been (recently) checked"""
-    check_resources(number)

udata/linkchecker/models.py

@@ -1,9 +0,0 @@
-from udata.core.dataset.models import ResourceMixin
-from udata.mongo import db
-
-# Register harvest extras
-ResourceMixin.extras.register("check:available", db.BooleanField)
-ResourceMixin.extras.register("check:count-availability", db.IntField)
-ResourceMixin.extras.register("check:status", db.IntField)
-ResourceMixin.extras.register("check:url", db.StringField)
-ResourceMixin.extras.register("check:date", db.DateTimeField)

udata/linkchecker/tasks.py

@@ -1,55 +0,0 @@
-import logging
-import uuid
-
-from flask import current_app
-
-from udata.models import Dataset
-from udata.tasks import job
-from udata.utils import get_by
-
-from .checker import check_resource
-
-log = logging.getLogger(__name__)
-
-
-@job("check_resources")
-def check_resources(self, number):
-    """Check <number> of URLs that have not been (recently) checked"""
-    if not current_app.config.get("LINKCHECKING_ENABLED"):
-        log.error("Link checking is disabled.")
-        return
-
-    base_pipeline = [
-        {"$match": {"resources": {"$gt": []}}},
-        {"$project": {"resources._id": True, "resources.extras.check:date": True}},
-        {"$unwind": "$resources"},
-    ]
-    # unchecked resources
-    pipeline = base_pipeline + [
-        {"$match": {"resources.extras.check:date": {"$eq": None}}},
-        {"$limit": number},
-    ]
-    resources = list(Dataset.objects.aggregate(*pipeline))
-    # not recently checked resources
-    slots_left = number - len(resources)
-    if slots_left:
-        pipeline = base_pipeline + [
-            {"$match": {"resources.extras.check:date": {"$ne": None}}},
-            {"$sort": {"resources.extras.check:date": 1}},
-            {"$limit": slots_left},
-        ]
-        resources += list(Dataset.objects.aggregate(*pipeline))
-
-    nb_resources = len(resources)
-    log.info("Checking %s resources...", nb_resources)
-    for idx, dataset_resource in enumerate(resources):
-        dataset_obj = Dataset.objects.get(id=dataset_resource["_id"])
-        resource_id = dataset_resource["resources"]["_id"]
-        rid = uuid.UUID(resource_id)
-        resource_obj = get_by(dataset_obj.resources, "id", rid)
-        log.info("Checking resource %s (%s/%s)", resource_id, idx + 1, nb_resources)
-        if resource_obj.need_check():
-            check_resource(resource_obj)
-        else:
-            log.info("--> Skipping this resource, cache is fresh enough.")
-    log.info("Done.")

udata/tests/test_linkchecker.py

@@ -1,277 +0,0 @@
-from datetime import datetime, timedelta
-
-import mock
-import pytest
-
-from udata.auth import login_user
-from udata.core.activity import init_app as init_activity
-from udata.core.activity.models import Activity
-from udata.core.dataset.factories import DatasetFactory, ResourceFactory
-from udata.core.user.factories import UserFactory
-from udata.linkchecker.checker import check_resource
-from udata.settings import Testing
-from udata.tests import TestCase
-
-
-class LinkcheckerTestSettings(Testing):
-    LINKCHECKING_ENABLED = True
-    LINKCHECKING_IGNORE_DOMAINS = ["example-ignore.com"]
-    LINKCHECKING_IGNORE_PATTERNS = ["format=shp"]
-    LINKCHECKING_MIN_CACHE_DURATION = 0.5
-    LINKCHECKING_UNAVAILABLE_THRESHOLD = 100
-    LINKCHECKING_MAX_CACHE_DURATION = 100
-
-
-@pytest.fixture
-def activity_app(app):
-    init_activity(app)
-    yield app
-
-
-def test_check_resource_creates_no_activity(activity_app, mocker):
-    resource = ResourceFactory()
-    dataset = DatasetFactory(resources=[resource])
-    user = UserFactory()
-    login_user(user)
-    check_res = {"check:status": 200, "check:available": True, "check:date": datetime.utcnow()}
-
-    class DummyLinkchecker:
-        def check(self, _):
-            return check_res
-
-    mocker.patch("udata.linkchecker.checker.get_linkchecker", return_value=DummyLinkchecker)
-
-    check_resource(resource)
-
-    activities = Activity.objects.filter(related_to=dataset)
-    assert len(activities) == 0
-
-
-class LinkcheckerTest(TestCase):
-    settings = LinkcheckerTestSettings
-
-    def setUp(self):
-        self.resource = ResourceFactory()
-        self.dataset = DatasetFactory(resources=[self.resource])
-
-    @mock.patch("udata.linkchecker.checker.get_linkchecker")
-    def test_check_resource_no_linkchecker(self, mock_fn):
-        mock_fn.return_value = None
-        res = check_resource(self.resource)
-        self.assertEqual(res, ({"error": "No linkchecker configured."}, 503))
-
-    @mock.patch("udata.linkchecker.checker.get_linkchecker")
-    def test_check_resource_linkchecker_ok(self, mock_fn):
-        check_res = {"check:status": 200, "check:available": True, "check:date": datetime.utcnow()}
-
-        class DummyLinkchecker:
-            def check(self, _):
-                return check_res
-
-        mock_fn.return_value = DummyLinkchecker
-
-        res = check_resource(self.resource)
-        self.assertEqual(res, check_res)
-        check_res.update({"check:count-availability": 1})
-        self.assertEqual(self.resource.extras, check_res)
-
-    @mock.patch("udata.linkchecker.checker.get_linkchecker")
-    def test_check_resource_filter_result(self, mock_fn):
-        check_res = {"check:status": 200, "dummy": "dummy"}
-
-        class DummyLinkchecker:
-            def check(self, _):
-                return check_res
-
-        mock_fn.return_value = DummyLinkchecker
-
-        res = check_resource(self.resource)
-        self.assertEqual(res, check_res)
-        self.assertNotIn("dummy", self.resource.extras)
-
-    @mock.patch("udata.linkchecker.checker.get_linkchecker")
-    def test_check_resource_linkchecker_no_status(self, mock_fn):
-        class DummyLinkchecker:
-            def check(self, _):
-                return {"check:available": True}
-
-        mock_fn.return_value = DummyLinkchecker
-        res = check_resource(self.resource)
-        self.assertEqual(res, ({"error": "No status in response from linkchecker"}, 503))
-
-    @mock.patch("udata.linkchecker.checker.get_linkchecker")
-    def test_check_resource_linkchecker_check_error(self, mock_fn):
-        class DummyLinkchecker:
-            def check(self, _):
-                return {"check:error": "ERROR"}
-
-        mock_fn.return_value = DummyLinkchecker
-        res = check_resource(self.resource)
-        self.assertEqual(res, ({"error": "ERROR"}, 500))
-
-    @mock.patch("udata.linkchecker.checker.get_linkchecker")
-    def test_check_resource_linkchecker_in_resource(self, mock_fn):
-        self.resource.extras["check:checker"] = "another_linkchecker"
-        self.resource.save()
-        check_resource(self.resource)
-        args, kwargs = mock_fn.call_args
-        self.assertEqual(args, ("another_linkchecker",))
-
-    def test_check_resource_linkchecker_no_check(self):
-        self.resource.extras["check:checker"] = "no_check"
-        self.resource.save()
-        res = check_resource(self.resource)
-        self.assertEqual(res.get("check:status"), 204)
-        self.assertEqual(res.get("check:available"), True)
-
-    def test_check_resource_ignored_domain(self):
-        self.resource.extras = {}
-        self.resource.url = "http://example-ignore.com/url"
-        self.resource.save()
-        res = check_resource(self.resource)
-        self.assertEqual(res.get("check:status"), 204)
-        self.assertEqual(res.get("check:available"), True)
-
-    def test_check_resource_ignored_pattern(self):
-        self.resource.extras = {}
-        self.resource.url = "http://example.com/url?format=shp"
-        self.resource.save()
-        res = check_resource(self.resource)
-        self.assertEqual(res.get("check:status"), 204)
-        self.assertEqual(res.get("check:available"), True)
-
-    def test_is_need_check(self):
-        self.resource.extras = {
-            "check:available": True,
-            "check:date": datetime.utcnow(),
-            "check:status": 42,
-        }
-        self.assertFalse(self.resource.need_check())
-
-    def test_is_need_check_unknown_status(self):
-        self.resource.extras = {}
-        self.assertTrue(self.resource.need_check())
-
-    def test_is_need_check_cache_expired(self):
-        self.resource.extras = {
-            "check:available": True,
-            "check:date": datetime.utcnow() - timedelta(seconds=3600),
-            "check:status": 42,
-        }
-        self.assertTrue(self.resource.need_check())
-
-    def test_is_need_check_date_string(self):
-        check_date = (datetime.utcnow() - timedelta(seconds=3600)).isoformat()
-        self.resource.extras = {
-            "check:available": True,
-            "check:date": check_date,
-            "check:status": 42,
-        }
-        self.assertTrue(self.resource.need_check())
-
-    def test_is_need_check_wrong_check_date(self):
-        check_date = "123azerty"
-        self.resource.extras = {
-            "check:available": True,
-            "check:date": check_date,
-            "check:status": 42,
-        }
-        self.assertTrue(self.resource.need_check())
-
-    def test_is_need_check_wrong_check_date_int(self):
-        check_date = 42
-        self.resource.extras = {
-            "check:available": True,
-            "check:date": check_date,
-            "check:status": 42,
-        }
-        self.assertTrue(self.resource.need_check())
-
-    def test_is_need_check_count_availability(self):
-        self.resource.extras = {
-            # should need a new check after 100 * 30s = 3000s < 3600s
-            "check:count-availability": 100,
-            "check:available": True,
-            "check:date": datetime.utcnow() - timedelta(seconds=3600),
-            "check:status": 42,
-        }
-        self.assertTrue(self.resource.need_check())
-
-    def test_is_need_check_count_availability_expired(self):
-        self.resource.extras = {
-            # should need a new check after 150 * 30s = 4500s > 3600s
-            "check:count-availability": 150,
-            "check:available": True,
-            "check:date": datetime.utcnow() - timedelta(seconds=3600),
-            "check:status": 42,
-        }
-        self.assertFalse(self.resource.need_check())
-
-    def test_is_need_check_count_availability_unavailable(self):
-        self.resource.extras = {
-            # should need a new check after 30s < 3600S
-            # count-availability is below threshold
-            "check:count-availability": 95,
-            "check:available": False,
-            "check:date": datetime.utcnow() - timedelta(seconds=3600),
-            "check:status": 42,
-        }
-        self.assertTrue(self.resource.need_check())
-
-    @mock.patch("udata.linkchecker.checker.get_linkchecker")
-    def test_count_availability_increment(self, mock_fn):
-        check_res = {"check:status": 200, "check:available": True, "check:date": datetime.utcnow()}
-
-        class DummyLinkchecker:
-            def check(self, _):
-                return check_res
-
-        mock_fn.return_value = DummyLinkchecker
-
-        check_resource(self.resource)
-        self.assertEqual(self.resource.extras["check:count-availability"], 1)
-
-        check_resource(self.resource)
-        self.assertEqual(self.resource.extras["check:count-availability"], 2)
-
-    @mock.patch("udata.linkchecker.checker.get_linkchecker")
-    def test_count_availability_reset(self, mock_fn):
-        self.resource.extras = {
-            "check:status": 200,
-            "check:available": True,
-            "check:date": datetime.utcnow(),
-            "check:count-availability": 2,
-        }
-        check_res = {"check:status": 200, "check:available": False, "check:date": datetime.utcnow()}
-
-        class DummyLinkchecker:
-            def check(self, _):
-                return check_res
-
-        mock_fn.return_value = DummyLinkchecker
-
-        check_resource(self.resource)
-        self.assertEqual(self.resource.extras["check:count-availability"], 1)
-
-    def test_count_availability_threshold(self):
-        self.resource.extras = {
-            "check:status": 404,
-            "check:available": False,
-            # if it weren't above threshold, should need check (>30s)
-            # and we're still below max_cache 101 * 0.5 < 100
-            "check:date": datetime.utcnow() - timedelta(seconds=60),
-            "check:count-availability": 101,
-        }
-        self.assertFalse(self.resource.need_check())
-
-    def test_count_availability_max_cache_duration(self):
-        self.resource.extras = {
-            "check:status": 200,
-            "check:available": True,
-            # next check should be at 300 * 0.5 = 150min
-            # but we are above max cache duration 150min > 100min
-            # and 120m > 100 min so we should need a new check
-            "check:date": datetime.utcnow() - timedelta(minutes=120),
-            "check:count-availability": 300,
-        }
-        self.assertTrue(self.resource.need_check())