udata 12.0.2.dev10__py3-none-any.whl → 13.0.1.dev21__py3-none-any.whl

udata/linkchecker/backends.py

@@ -1,31 +0,0 @@
-import logging
-from datetime import datetime
-
-from flask import current_app
-
-from udata.entrypoints import get_enabled
-
-log = logging.getLogger(__name__)
-
-
-ENTRYPOINT = "udata.linkcheckers"
-
-
-class NoCheckLinkchecker(object):
-    """Dummy linkchecker for resources that need no check"""
-
-    def check(self, _):
-        return {"check:status": 204, "check:available": True, "check:date": datetime.utcnow()}
-
-
-def get(name):
-    """Get a linkchecker given its name or fallback on default"""
-    linkcheckers = get_enabled(ENTRYPOINT, current_app)
-    linkcheckers.update(no_check=NoCheckLinkchecker)  # no_check always enabled
-    selected_linkchecker = linkcheckers.get(name)
-    if not selected_linkchecker:
-        default_linkchecker = current_app.config.get("LINKCHECKING_DEFAULT_LINKCHECKER")
-        selected_linkchecker = linkcheckers.get(default_linkchecker)
-    if not selected_linkchecker:
-        log.error("No linkchecker found ({} requested and no fallback)".format(name))
-    return selected_linkchecker

udata/linkchecker/checker.py

@@ -1,75 +0,0 @@
-from urllib.parse import urlparse
-
-from flask import current_app
-
-from .backends import NoCheckLinkchecker
-from .backends import get as get_linkchecker
-
-
-def _get_check_keys(the_dict, resource, previous_status):
-    check_keys = {k: v for k, v in the_dict.items() if k.startswith("check:")}
-    check_keys["check:count-availability"] = _compute_count_availability(
-        resource, check_keys.get("check:available"), previous_status
-    )
-    return check_keys
-
-
-def _compute_count_availability(resource, status, previous_status):
-    """Compute the `check:count-availability` extra value"""
-    count_availability = resource.extras.get("check:count-availability", 1)
-    return count_availability + 1 if status == previous_status else 1
-
-
-def is_ignored(resource):
-    """Check if the resource's URL is to be ignored"""
-    ignored_domains = current_app.config["LINKCHECKING_IGNORE_DOMAINS"]
-    ignored_patterns = current_app.config["LINKCHECKING_IGNORE_PATTERNS"]
-    url = resource.url
-    if not url:
-        return True
-    parsed_url = urlparse(url)
-    ignored_domains_match = parsed_url.netloc in ignored_domains
-    ignored_patterns_match = any([p in url for p in ignored_patterns])
-    return ignored_domains_match or ignored_patterns_match
-
-
-def dummy_check_response():
-    """Trigger a dummy check"""
-    return NoCheckLinkchecker().check(None)
-
-
-def check_resource(resource):
-    """
-    Check a resource availability against a linkchecker backend
-
-    The linkchecker used can be configured on a resource basis by setting
-    the `resource.extras['check:checker']` attribute with a key that points
-    to a valid `udata.linkcheckers` entrypoint. If not set, it will
-    fallback on the default linkchecker defined by the configuration variable
-    `LINKCHECKING_DEFAULT_LINKCHECKER`.
-
-    Returns
-    -------
-    dict or (dict, int)
-        Check results dict and status code (if error).
-    """
-    linkchecker_type = resource.extras.get("check:checker")
-    LinkChecker = get_linkchecker(linkchecker_type)
-    if not LinkChecker:
-        return {"error": "No linkchecker configured."}, 503
-    if is_ignored(resource):
-        return dummy_check_response()
-    result = LinkChecker().check(resource)
-    if not result:
-        return {"error": "No response from linkchecker"}, 503
-    elif result.get("check:error"):
-        return {"error": result["check:error"]}, 500
-    elif not result.get("check:status"):
-        return {"error": "No status in response from linkchecker"}, 503
-    # store the check result in the resource's extras
-    # XXX maybe this logic should be in the `Resource` model?
-    previous_status = resource.extras.get("check:available")
-    check_keys = _get_check_keys(result, resource, previous_status)
-    resource.extras.update(check_keys)
-    resource.save(signal_kwargs={"ignores": ["post_save"]})  # Prevent signal triggering on dataset
-    return result

udata/linkchecker/commands.py

@@ -1,21 +0,0 @@
-import logging
-
-import click
-
-from udata.commands import cli
-from udata.linkchecker.tasks import check_resources
-
-log = logging.getLogger(__name__)
-
-
-@cli.group("linkchecker")
-def grp():
-    """Link checking operations"""
-    pass
-
-
-@grp.command()
-@click.option("-n", "--number", type=int, default=5000, help="Number of URLs to check")
-def check(number):
-    """Check <number> of URLs that have not been (recently) checked"""
-    check_resources(number)

udata/linkchecker/models.py

@@ -1,9 +0,0 @@
-from udata.core.dataset.models import ResourceMixin
-from udata.mongo import db
-
-# Register harvest extras
-ResourceMixin.extras.register("check:available", db.BooleanField)
-ResourceMixin.extras.register("check:count-availability", db.IntField)
-ResourceMixin.extras.register("check:status", db.IntField)
-ResourceMixin.extras.register("check:url", db.StringField)
-ResourceMixin.extras.register("check:date", db.DateTimeField)

udata/linkchecker/tasks.py

@@ -1,55 +0,0 @@
-import logging
-import uuid
-
-from flask import current_app
-
-from udata.models import Dataset
-from udata.tasks import job
-from udata.utils import get_by
-
-from .checker import check_resource
-
-log = logging.getLogger(__name__)
-
-
-@job("check_resources")
-def check_resources(self, number):
-    """Check <number> of URLs that have not been (recently) checked"""
-    if not current_app.config.get("LINKCHECKING_ENABLED"):
-        log.error("Link checking is disabled.")
-        return
-
-    base_pipeline = [
-        {"$match": {"resources": {"$gt": []}}},
-        {"$project": {"resources._id": True, "resources.extras.check:date": True}},
-        {"$unwind": "$resources"},
-    ]
-    # unchecked resources
-    pipeline = base_pipeline + [
-        {"$match": {"resources.extras.check:date": {"$eq": None}}},
-        {"$limit": number},
-    ]
-    resources = list(Dataset.objects.aggregate(*pipeline))
-    # not recently checked resources
-    slots_left = number - len(resources)
-    if slots_left:
-        pipeline = base_pipeline + [
-            {"$match": {"resources.extras.check:date": {"$ne": None}}},
-            {"$sort": {"resources.extras.check:date": 1}},
-            {"$limit": slots_left},
-        ]
-        resources += list(Dataset.objects.aggregate(*pipeline))
-
-    nb_resources = len(resources)
-    log.info("Checking %s resources...", nb_resources)
-    for idx, dataset_resource in enumerate(resources):
-        dataset_obj = Dataset.objects.get(id=dataset_resource["_id"])
-        resource_id = dataset_resource["resources"]["_id"]
-        rid = uuid.UUID(resource_id)
-        resource_obj = get_by(dataset_obj.resources, "id", rid)
-        log.info("Checking resource %s (%s/%s)", resource_id, idx + 1, nb_resources)
-        if resource_obj.need_check():
-            check_resource(resource_obj)
-        else:
-            log.info("--> Skipping this resource, cache is fresh enough.")
-    log.info("Done.")

udata/templates/mail/account_deleted.html

@@ -1,5 +0,0 @@
-{% extends 'mail/base.html' %}
-
-{% block body %}
-<p style="margin: 0;padding: 0;">{{ _('Your account has now been deleted') }}</p>
-{% endblock %}

udata/templates/mail/account_deleted.txt

@@ -1,6 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _('Your account has now been deleted') }}.
-
-{% endblock %}

udata/templates/mail/account_inactivity.html

@@ -1,40 +0,0 @@
-{% extends 'mail/base.html' %}
-{% from 'mail/button.html' import mail_button %}
-
-{% block body %}
-<p style="margin: 0;padding: 0;">
-    {{ _(
-            'We have noticed that your account associated to (%(user_email)s) has been inactive for %(inactivity_years)d years or more'
-            ' on %(site)s, the open platform for public data.',
-            user_email=user.email,
-            inactivity_years=config.YEARS_OF_INACTIVITY_BEFORE_DELETION,
-            site=config.SITE_TITLE
-        )
-    }}
-</p>
-<br/>
-<p style="margin: 0;padding: 0;"><b>
-    {{ _(
-        'If you want to keep your account, please log in with your account on %(home)s.',
-        home=homepage_url()
-        )
-    }}
-</b></p>
-<br/>
-<p style="margin: 0;padding: 0;">
-    {{ _(
-        'Without logging in, your account will be deleted within %(notify_delay)d days.',
-        notify_delay=config.DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY
-        )
-    }}
-</p>
-<br/>
-<p style="margin: 0;padding: 0;">
-    {{ _(
-        'This account is not tied to your other administration accounts and '
-        'you can always re-create an account on the %(site)s platform if necessary.',
-        site=config.SITE_TITLE
-        )
-    }}
-</p>
-{% endblock %}

udata/templates/mail/account_inactivity.txt

@@ -1,31 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _(
-      'We have noticed that your account associated to (%(user_email)s) has been inactive for %(inactivity_years)d years or more'
-      ' on %(site)s, the open platform for public data.',
-      user_email=user.email,
-      inactivity_years=config.YEARS_OF_INACTIVITY_BEFORE_DELETION,
-      site=config.SITE_TITLE
-    )
-}}
-
-{{ _(
-      'If you want to keep your account, please log in with your account on %(home)s.',
-      home=homepage_url()
-    )
-}}
-
-{{ _(
-      'Without logging in, your account will be deleted within %(notify_delay)d days.',
-      notify_delay=config.DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY
-    )
-}}
-{{
-  _(
-      'This account is not tied to your other administration accounts and '
-      'you can always re-create an account on the %(site)s platform if necessary',
-      site=config.SITE_TITLE
-  )
-}}
-{% endblock %}

udata/templates/mail/badge_added_association.html

@@ -1,33 +0,0 @@
-{% extends 'mail/base.html' %}
-{% from 'mail/button.html' import mail_button %}
-
-{% block body %}
-<p style="margin: 0;padding: 0;">{{
-    _('%(user)s has identified your organization "%(name)s" as an association',
-    user=(
-        '<a href="'|safe
-        + badge.created_by.url_for(_mailCampaign=True)
-        + '">'|safe
-        + badge.created_by.fullname
-        + '</a>'|safe
-    ),
-    name=(
-        '<a href="'|safe
-        + organization.url_for(_mailCampaign=True)
-        + '">'|safe
-        + organization.name|string
-        + '</a>'|safe
-    )
-) }}.</p>
-<br/>
-<table width="100%" border="0" cellspacing="0" cellpadding="0">
-    <tr>
-        <td align="center">
-            {{ mail_button(
-                _('See the badge'),
-                organization.url_for(_mailCampaign=True)
-            ) }}
-        </td>
-    </tr>
-</table>
-{% endblock %}

udata/templates/mail/badge_added_association.txt

@@ -1,11 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _('%(user)s has identified your organization "%(name)s" as an association',
-  name=organization.name,
-  user=badge.created_by.fullname
-) }}.
-
-{{ _('You can see the badge on this page:') }}
-{{ organization.url_for(_mailCampaign=True) }}
-{% endblock %}

udata/templates/mail/badge_added_certified.html

@@ -1,33 +0,0 @@
-{% extends 'mail/base.html' %}
-{% from 'mail/button.html' import mail_button %}
-
-{% block body %}
-<p style="margin: 0;padding: 0;">{{
-    _('%(user)s has certified your organization "%(name)s"',
-    user=(
-        '<a href="'|safe
-        + badge.created_by.url_for(_mailCampaign=True)
-        + '">'|safe
-        + badge.created_by.fullname
-        + '</a>'|safe
-    ),
-    name=(
-        '<a href="'|safe
-        + organization.url_for(_mailCampaign=True)
-        + '">'|safe
-        + organization.name|string
-        + '</a>'|safe
-    )
-) }}.</p>
-<br/>
-<table width="100%" border="0" cellspacing="0" cellpadding="0">
-    <tr>
-        <td align="center">
-            {{ mail_button(
-                _('See the badge'),
-                organization.url_for(_mailCampaign=True)
-            ) }}
-        </td>
-    </tr>
-</table>
-{% endblock %}

udata/templates/mail/badge_added_certified.txt

@@ -1,11 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _('%(user)s has certified your organization "%(name)s"',
-        name=organization.name,
-        user=badge.created_by.fullname
-) }}.
-
-{{ _('You can see the badge on this page:') }}
-{{ organization.url_for(_mailCampaign=True) }}
-{% endblock %}

udata/templates/mail/badge_added_company.html

@@ -1,33 +0,0 @@
-{% extends 'mail/base.html' %}
-{% from 'mail/button.html' import mail_button %}
-
-{% block body %}
-<p style="margin: 0;padding: 0;">{{
-    _('%(user)s has identified your organization "%(name)s" as a company',
-    user=(
-        '<a href="'|safe
-        + badge.created_by.url_for(_mailCampaign=True)
-        + '">'|safe
-        + badge.created_by.fullname
-        + '</a>'|safe
-    ),
-    name=(
-        '<a href="'|safe
-        + organization.url_for(_mailCampaign=True)
-        + '">'|safe
-        + organization.name|string
-        + '</a>'|safe
-    )
-) }}.</p>
-<br/>
-<table width="100%" border="0" cellspacing="0" cellpadding="0">
-    <tr>
-        <td align="center">
-            {{ mail_button(
-                _('See the badge'),
-                organization.url_for(_mailCampaign=True)
-            ) }}
-        </td>
-    </tr>
-</table>
-{% endblock %}

udata/templates/mail/badge_added_company.txt

@@ -1,11 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _('%(user)s has identified your organization "%(name)s" as a company',
-  name=organization.name,
-  user=badge.created_by.fullname
-) }}.
-
-{{ _('You can see the badge on this page:') }}
-{{ organization.url_for(_mailCampaign=True) }}
-{% endblock %}

udata/templates/mail/badge_added_local_authority.html

@@ -1,33 +0,0 @@
-{% extends 'mail/base.html' %}
-{% from 'mail/button.html' import mail_button %}
-
-{% block body %}
-<p style="margin: 0;padding: 0;">{{
-    _('%(user)s has identified your organization "%(name)s" as a local authority',
-    user=(
-        '<a href="'|safe
-        + badge.created_by.url_for(_mailCampaign=True)
-        + '">'|safe
-        + badge.created_by.fullname
-        + '</a>'|safe
-    ),
-    name=(
-        '<a href="'|safe
-        + organization.url_for(_mailCampaign=True)
-        + '">'|safe
-        + organization.name|string
-        + '</a>'|safe
-    )
-) }}.</p>
-<br/>
-<table width="100%" border="0" cellspacing="0" cellpadding="0">
-    <tr>
-        <td align="center">
-            {{ mail_button(
-                _('See the badge'),
-                organization.url_for(_mailCampaign=True)
-            ) }}
-        </td>
-    </tr>
-</table>
-{% endblock %}

udata/templates/mail/badge_added_local_authority.txt

@@ -1,11 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _('%(user)s has identified your organization "%(name)s" as a local authority',
-  name=organization.name,
-  user=badge.created_by.fullname
-) }}.
-
-{{ _('You can see the badge on this page:') }}
-{{ organization.url_for(_mailCampaign=True) }}
-{% endblock %}

udata/templates/mail/badge_added_public_service.html

@@ -1,33 +0,0 @@
-{% extends 'mail/base.html' %}
-{% from 'mail/button.html' import mail_button %}
-
-{% block body %}
-<p style="margin: 0;padding: 0;">{{
-    _('%(user)s has identified your organization "%(name)s" as public service',
-    user=(
-        '<a href="'|safe
-        + badge.created_by.url_for(_mailCampaign=True)
-        + '">'|safe
-        + badge.created_by.fullname
-        + '</a>'|safe
-    ),
-    name=(
-        '<a href="'|safe
-        + organization.url_for(_mailCampaign=True)
-        + '">'|safe
-        + organization.name|string
-        + '</a>'|safe
-    )
-) }}.</p>
-<br/>
-<table width="100%" border="0" cellspacing="0" cellpadding="0">
-    <tr>
-        <td align="center">
-            {{ mail_button(
-                _('See the badge'),
-                organization.url_for(_mailCampaign=True)
-            ) }}
-        </td>
-    </tr>
-</table>
-{% endblock %}

udata/templates/mail/badge_added_public_service.txt

@@ -1,11 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _('%(user)s has identified your organization "%(name)s" as a public service',
-  name=organization.name,
-  user=badge.created_by.fullname
-) }}.
-
-{{ _('You can see the badge on this page:') }}
-{{ organization.url_for(_mailCampaign=True) }}
-{% endblock %}

udata/templates/mail/discussion_closed.html

@@ -1,47 +0,0 @@
-{% extends 'mail/base.html' %}
-{% from 'mail/button.html' import mail_button %}
-
-{% block body %}
-<p style="margin: 0;padding: 0;">{{
-    _('%(user)s closed an discussion on your %(type)s %(subject)s',
-        type=discussion.subject.verbose_name,
-        user=(
-            '<a href="'|safe
-            + discussion.closed_by_org_or_user.url_for(_mailCampaign=True)
-            + '">'|safe
-            + discussion.closed_by_name
-            + '</a>'|safe
-        ),
-        subject=(
-            '<a href="'|safe
-            + discussion.url_for(_mailCampaign=True)
-            + '">'|safe
-            + discussion.subject|string
-            + '</a>'|safe
-        )
-    )
-}}.</p>
-<br/>
-<p style="margin: 0;padding: 0;">
-    <b>{{ _('Title') }}:</b>
-    {{ discussion.title }}
-</p>
-
-{% if message %}
-<p style="margin: 0;padding: 0;">
-    <b>{{ _('Message') }}:</b>
-    {{ message.content | markdown }}
-</p>
-{% endif %}
-
-<table width="100%" border="0" cellspacing="0" cellpadding="0">
-    <tr>
-        <td align="center">
-            {{ mail_button(
-                _('See the discussion'),
-                discussion.url_for(_mailCampaign=True)
-            ) }}
-        </td>
-    </tr>
-</table>
-{% endblock %}

udata/templates/mail/discussion_closed.txt

@@ -1,16 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _('%(user)s closed an discussion on your %(type)s %(subject)s',
-    type=discussion.subject.verbose_name,
-    user=discussion.closed_by_name,
-    subject=discussion.subject|string
-) }}.
-
-
-{{ _('Title') }}: {{ discussion.title }}
-
-
-{{ _('You can see the discussion on this page:') }}
-{{ discussion.url_for(_mailCampaign=True) }}
-{% endblock %}

udata/templates/mail/inactive_account_deleted.html

@@ -1,5 +0,0 @@
-{% extends 'mail/base.html' %}
-
-{% block body %}
-<p style="margin: 0;padding: 0;">{{ _('Your account on %(site)s has been deleted due to inactivity', site=config.SITE_TITLE) }}</p>
-{% endblock %}

udata/templates/mail/inactive_account_deleted.txt

@@ -1,6 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _('Your account on %(site)s has been deleted due to inactivity', site=config.SITE_TITLE) }}.
-
-{% endblock %}

udata/templates/mail/membership_refused.html

@@ -1,20 +0,0 @@
-{% extends 'mail/base.html' %}
-{% from 'mail/button.html' import mail_button %}
-
-
-{% block body %}
-<p style="margin: 0;padding: 0;font-family: &quot;Helvetica Neue&quot;, &quot;Helvetica&quot;, Helvetica, Arial, sans-serif;">
-{{ _('Your membership for the organization "%(org)s" has been refused',
-    org=(
-        '<a href="'|safe
-        + org.url_for(_mailCampaign=True)
-        + '">'|safe
-        + org.name
-        + '</a>'|safe
-    )
-) }}.</p>
-
-<br/>
-<p style="margin: 0;padding: 0;"><b>{{ _('Refusal reason') }}:</b></p>
-<p style="margin: 0;padding: 0;">{{ request.refusal_comment }}</p>
-{% endblock %}

udata/templates/mail/membership_refused.txt

@@ -1,11 +0,0 @@
-{% extends 'mail/base.txt' %}
-
-{% block body %}
-{{ _('Your membership for the organization "%(org)s" has been refused',
-    org=org.name
-) }}.
-
-{{ _('Refusal reason') }}:
-
-{{ request.refusal_comment }}
-{% endblock %}