udata 9.2.5.dev32170__py2.py3-none-any.whl → 9.2.5.dev32222__py2.py3-none-any.whl

udata/core/dataservices/search.py

@@ -0,0 +1,118 @@
+import datetime
+
+from bson.objectid import ObjectId
+from flask_restx.inputs import boolean
+
+from udata.api import api
+from udata.api.parsers import ModelApiParser
+from udata.models import Dataservice, Organization, User
+from udata.search import (
+    BoolFilter,
+    Filter,
+    ModelSearchAdapter,
+    ModelTermsFilter,
+    register,
+)
+from udata.utils import to_iso_datetime
+
+__all__ = ("DataserviceSearch",)
+
+DEFAULT_SORTING = "-created_at"
+
+
+class DataserviceApiParser(ModelApiParser):
+    sorts = {
+        "created": "created_at",
+    }
+
+    def __init__(self):
+        super().__init__()
+        self.parser.add_argument("tag", type=str, location="args")
+        self.parser.add_argument("organization", type=str, location="args")
+        self.parser.add_argument("is_restricted", type=bool, location="args")
+
+    @staticmethod
+    def parse_filters(dataservices, args):
+        if args.get("q"):
+            # Following code splits the 'q' argument by spaces to surround
+            # every word in it with quotes before rebuild it.
+            # This allows the search_text method to tokenise with an AND
+            # between tokens whereas an OR is used without it.
+            phrase_query = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
+            dataservices = dataservices.search_text(phrase_query)
+        if args.get("tag"):
+            dataservices = dataservices.filter(tags=args["tag"])
+        if args.get("organization"):
+            if not ObjectId.is_valid(args["organization"]):
+                api.abort(400, "Organization arg must be an identifier")
+            dataservices = dataservices.filter(organization=args["organization"])
+        if "is_restricted" in args:
+            dataservices = dataservices.filter(is_restricted=boolean(args["is_restricted"]))
+        return dataservices
+
+
+@register
+class DataserviceSearch(ModelSearchAdapter):
+    model = Dataservice
+    search_url = "dataservices/"
+
+    sorts = {
+        "created": "created_at",
+    }
+
+    filters = {
+        "tag": Filter(),
+        "organization": ModelTermsFilter(model=Organization),
+        "archived": BoolFilter(),
+    }
+
+    @classmethod
+    def is_indexable(cls, dataservice: Dataservice) -> bool:
+        return dataservice.deleted_at is None and not dataservice.private
+
+    @classmethod
+    def mongo_search(cls, args):
+        dataservices = Dataservice.objects.visible()
+        dataservices = DataserviceApiParser.parse_filters(dataservices, args)
+
+        sort = (
+            cls.parse_sort(args["sort"])
+            or ("$text_score" if args["q"] else None)
+            or DEFAULT_SORTING
+        )
+        return dataservices.order_by(sort).paginate(args["page"], args["page_size"])
+
+    @classmethod
+    def serialize(cls, dataservice: Dataservice) -> dict:
+        organization = None
+        owner = None
+        if dataservice.organization:
+            org = Organization.objects(id=dataservice.organization.id).first()
+            organization = {
+                "id": str(org.id),
+                "name": org.name,
+                "public_service": 1 if org.public_service else 0,
+                "followers": org.metrics.get("followers", 0),
+            }
+        elif dataservice.owner:
+            owner = User.objects(id=dataservice.owner.id).first()
+        extras = {}
+        for key, value in dataservice.extras.items():
+            extras[key] = to_iso_datetime(value) if isinstance(value, datetime.datetime) else value
+
+        return {
+            "id": str(dataservice.id),
+            "title": dataservice.title,
+            "description": dataservice.description,
+            "base_api_url": dataservice.base_api_url,
+            "created_at": to_iso_datetime(dataservice.created_at),
+            "archived": to_iso_datetime(dataservice.archived_at)
+            if dataservice.archived_at
+            else None,
+            "organization": organization,
+            "owner": str(owner.id) if owner else None,
+            "tags": dataservice.tags,
+            "extras": extras,
+            "followers": dataservice.metrics.get("followers", 0),
+            "is_restricted": dataservice.is_restricted or False,
+        }

udata/core/dataset/api.py

@@ -36,6 +36,7 @@ from udata.core.badges.fields import badge_fields
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import CHECKSUM_TYPES
 from udata.core.followers.api import FollowAPI
+from udata.core.organization.models import Organization
 from udata.core.storages.api import handle_upload, upload_parser
 from udata.core.topic.models import Topic
 from udata.linkchecker.checker import check_resource
@@ -96,6 +97,12 @@ class DatasetApiParser(ModelApiParser):
         self.parser.add_argument("granularity", type=str, location="args")
         self.parser.add_argument("temporal_coverage", type=str, location="args")
         self.parser.add_argument("organization", type=str, location="args")
+        self.parser.add_argument(
+            "organization_badge",
+            type=str,
+            choices=list(Organization.__badges__),
+            location="args",
+        )
         self.parser.add_argument("owner", type=str, location="args")
         self.parser.add_argument("format", type=str, location="args")
         self.parser.add_argument("schema", type=str, location="args")
@@ -131,6 +138,9 @@ class DatasetApiParser(ModelApiParser):
             if not ObjectId.is_valid(args["organization"]):
                 api.abort(400, "Organization arg must be an identifier")
             datasets = datasets.filter(organization=args["organization"])
+        if args.get("organization_badge"):
+            orgs = Organization.objects.with_badge(args["organization_badge"]).only("id")
+            datasets = datasets.filter(organization__in=orgs)
         if args.get("owner"):
             if not ObjectId.is_valid(args["owner"]):
                 api.abort(400, "Owner arg must be an identifier")

udata/core/dataset/models.py

@@ -14,12 +14,13 @@ from mongoengine.signals import post_save, pre_save
 from stringdist import rdlevenshtein
 from werkzeug.utils import cached_property
 
+from udata.api_fields import field
 from udata.app import cache
 from udata.core import storages
 from udata.core.owned import Owned, OwnedQuerySet
 from udata.frontend.markdown import mdstrip
 from udata.i18n import lazy_gettext as _
-from udata.models import BadgeMixin, SpatialCoverage, WithMetrics, db
+from udata.models import Badge, BadgeMixin, BadgesList, SpatialCoverage, WithMetrics, db
 from udata.mongo.errors import FieldValidationError
 from udata.uris import ValidationError, endpoint_for
 from udata.uris import validate as validate_url
@@ -53,6 +54,10 @@ __all__ = (
     "ResourceSchema",
 )
 
+BADGES: dict[str, str] = {
+    PIVOTAL_DATA: _("Pivotal data"),
+}
+
 NON_ASSIGNABLE_SCHEMA_TYPES = ["datapackage"]
 
 log = logging.getLogger(__name__)
@@ -498,7 +503,16 @@ class Resource(ResourceMixin, WithMetrics, db.EmbeddedDocument):
         self.dataset.save(*args, **kwargs)
 
 
-class Dataset(WithMetrics, BadgeMixin, Owned, db.Document):
+class DatasetBadge(Badge):
+    kind = db.StringField(required=True, choices=list(BADGES.keys()))
+
+
+class DatasetBadgeMixin(BadgeMixin):
+    badges = field(BadgesList(DatasetBadge), **BadgeMixin.default_badges_list_params)
+    __badges__ = BADGES
+
+
+class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
     title = db.StringField(required=True)
     acronym = db.StringField(max_length=128)
     # /!\ do not set directly the slug when creating or updating a dataset
@@ -539,10 +553,6 @@ class Dataset(WithMetrics, BadgeMixin, Owned, db.Document):
     def __str__(self):
         return self.title or ""
 
-    __badges__ = {
-        PIVOTAL_DATA: _("Pivotal data"),
-    }
-
     __metrics_keys__ = [
         "discussions",
         "reuses",

udata/core/dataset/search.py

@@ -32,8 +32,9 @@ class DatasetSearch(ModelSearchAdapter):
 
     filters = {
         "tag": Filter(),
-        "badge": Filter(),
+        "badge": Filter(choices=list(Dataset.__badges__)),
         "organization": ModelTermsFilter(model=Organization),
+        "organization_badge": Filter(choices=list(Organization.__badges__)),
         "owner": ModelTermsFilter(model=User),
         "license": ModelTermsFilter(model=License),
         "geozone": ModelTermsFilter(model=GeoZone),
@@ -51,7 +52,7 @@ class DatasetSearch(ModelSearchAdapter):
 
     @classmethod
     def mongo_search(cls, args):
-        datasets = Dataset.objects(archived=None, deleted=None, private=False)
+        datasets = Dataset.objects.visible()
         datasets = DatasetApiParser.parse_filters(datasets, args)
 
         sort = (
@@ -59,8 +60,7 @@ class DatasetSearch(ModelSearchAdapter):
             or ("$text_score" if args["q"] else None)
             or DEFAULT_SORTING
         )
-        offset = (args["page"] - 1) * args["page_size"]
-        return datasets.order_by(sort).skip(offset).limit(args["page_size"]), datasets.count()
+        return datasets.order_by(sort).paginate(args["page"], args["page_size"])
 
     @classmethod
     def serialize(cls, dataset):
@@ -76,6 +76,7 @@ class DatasetSearch(ModelSearchAdapter):
                 "name": org.name,
                 "public_service": 1 if org.public_service else 0,
                 "followers": org.metrics.get("followers", 0),
+                "badges": [badge.kind for badge in org.badges],
             }
         elif dataset.owner:
             owner = User.objects(id=dataset.owner.id).first()

udata/core/organization/api.py

@@ -63,6 +63,15 @@ class OrgApiParser(ModelApiParser):
         "last_modified": "last_modified",
     }
 
+    def __init__(self):
+        super().__init__()
+        self.parser.add_argument(
+            "badge",
+            type=str,
+            choices=list(Organization.__badges__),
+            location="args",
+        )
+
     @staticmethod
     def parse_filters(organizations, args):
         if args.get("q"):
@@ -72,6 +81,8 @@ class OrgApiParser(ModelApiParser):
             # between tokens whereas an OR is used without it.
             phrase_query = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
             organizations = organizations.search_text(phrase_query)
+        if args.get("badge"):
+            organizations = organizations.with_badge(args["badge"])
         return organizations
 
 

udata/core/organization/models.py

@@ -5,7 +5,8 @@ from blinker import Signal
 from mongoengine.signals import post_save, pre_save
 from werkzeug.utils import cached_property
 
-from udata.core.badges.models import BadgeMixin
+from udata.api_fields import field
+from udata.core.badges.models import Badge, BadgeMixin, BadgesList
 from udata.core.metrics.models import WithMetrics
 from udata.core.storages import avatars, default_image_basename
 from udata.frontend.markdown import mdstrip
@@ -29,6 +30,14 @@ from .constants import (
 
 __all__ = ("Organization", "Team", "Member", "MembershipRequest")
 
+BADGES: dict[str, str] = {
+    PUBLIC_SERVICE: _("Public Service"),
+    CERTIFIED: _("Certified"),
+    ASSOCIATION: _("Association"),
+    COMPANY: _("Company"),
+    LOCAL_AUTHORITY: _("Local authority"),
+}
+
 
 class Team(db.EmbeddedDocument):
     name = db.StringField(required=True)
@@ -82,8 +91,20 @@ class OrganizationQuerySet(db.BaseQuerySet):
     def get_by_id_or_slug(self, id_or_slug):
         return self(slug=id_or_slug).first() or self(id=id_or_slug).first()
 
+    def with_badge(self, kind):
+        return self(badges__kind=kind)
+
+
+class OrganizationBadge(Badge):
+    kind = db.StringField(required=True, choices=list(BADGES.keys()))
+
 
-class Organization(WithMetrics, BadgeMixin, db.Datetimed, db.Document):
+class OrganizationBadgeMixin(BadgeMixin):
+    badges = field(BadgesList(OrganizationBadge), **BadgeMixin.default_badges_list_params)
+    __badges__ = BADGES
+
+
+class Organization(WithMetrics, OrganizationBadgeMixin, db.Datetimed, db.Document):
     name = db.StringField(required=True)
     acronym = db.StringField(max_length=128)
     slug = db.SlugField(
@@ -126,14 +147,6 @@ class Organization(WithMetrics, BadgeMixin, db.Datetimed, db.Document):
     def __str__(self):
         return self.name or ""
 
-    __badges__ = {
-        PUBLIC_SERVICE: _("Public Service"),
-        CERTIFIED: _("Certified"),
-        ASSOCIATION: _("Association"),
-        COMPANY: _("Company"),
-        LOCAL_AUTHORITY: _("Local authority"),
-    }
-
     __metrics_keys__ = [
         "datasets",
         "members",

udata/core/organization/search.py

@@ -3,7 +3,7 @@ import datetime
 from udata import search
 from udata.core.organization.api import DEFAULT_SORTING, OrgApiParser
 from udata.models import Organization
-from udata.search.fields import Filter
+from udata.search.fields import ModelTermsFilter
 from udata.utils import to_iso_datetime
 
 __all__ = ("OrganizationSearch",)
@@ -22,7 +22,11 @@ class OrganizationSearch(search.ModelSearchAdapter):
         "created": "created_at",
     }
 
-    filters = {"badge": Filter()}
+    filters = {
+        "badge": ModelTermsFilter(
+            model=Organization, field_name="badges", choices=list(Organization.__badges__)
+        ),
+    }
 
     @classmethod
     def is_indexable(cls, org):
@@ -30,7 +34,7 @@ class OrganizationSearch(search.ModelSearchAdapter):
 
     @classmethod
     def mongo_search(cls, args):
-        orgs = Organization.objects(deleted=None)
+        orgs = Organization.objects.visible()
         orgs = OrgApiParser.parse_filters(orgs, args)
 
         sort = (
@@ -38,8 +42,7 @@ class OrganizationSearch(search.ModelSearchAdapter):
             or ("$text_score" if args["q"] else None)
             or DEFAULT_SORTING
         )
-        offset = (args["page"] - 1) * args["page_size"]
-        return orgs.order_by(sort).skip(offset).limit(args["page_size"]), orgs.count()
+        return orgs.order_by(sort).paginate(args["page"], args["page_size"])
 
     @classmethod
     def serialize(cls, organization):

udata/core/reuse/api.py

@@ -13,6 +13,7 @@ from udata.core.badges import api as badges_api
 from udata.core.badges.fields import badge_fields
 from udata.core.dataset.api_fields import dataset_ref_fields
 from udata.core.followers.api import FollowAPI
+from udata.core.organization.models import Organization
 from udata.core.reuse.constants import REUSE_TOPICS, REUSE_TYPES
 from udata.core.storages.api import (
     image_parser,
@@ -49,6 +50,12 @@ class ReuseApiParser(ModelApiParser):
         self.parser.add_argument("dataset", type=str, location="args")
         self.parser.add_argument("tag", type=str, location="args")
         self.parser.add_argument("organization", type=str, location="args")
+        self.parser.add_argument(
+            "organization_badge",
+            type=str,
+            choices=list(Organization.__badges__),
+            location="args",
+        )
         self.parser.add_argument("owner", type=str, location="args")
         self.parser.add_argument("type", type=str, location="args")
         self.parser.add_argument("topic", type=str, location="args")
@@ -79,6 +86,9 @@ class ReuseApiParser(ModelApiParser):
             if not ObjectId.is_valid(args["organization"]):
                 api.abort(400, "Organization arg must be an identifier")
             reuses = reuses.filter(organization=args["organization"])
+        if args.get("organization_badge"):
+            orgs = Organization.objects.with_badge(args["organization_badge"])
+            reuses = reuses.filter(organization__in=orgs)
         if args.get("owner"):
             if not ObjectId.is_valid(args["owner"]):
                 api.abort(400, "Owner arg must be an identifier")

udata/core/reuse/models.py

@@ -9,7 +9,7 @@ from udata.core.reuse.api_fields import BIGGEST_IMAGE_SIZE
 from udata.core.storages import default_image_basename, images
 from udata.frontend.markdown import mdstrip
 from udata.i18n import lazy_gettext as _
-from udata.models import BadgeMixin, WithMetrics, db
+from udata.models import Badge, BadgeMixin, BadgesList, WithMetrics, db
 from udata.mongo.errors import FieldValidationError
 from udata.uris import endpoint_for
 from udata.utils import hash_url
@@ -18,6 +18,8 @@ from .constants import IMAGE_MAX_SIZE, IMAGE_SIZES, REUSE_TOPICS, REUSE_TYPES
 
 __all__ = ("Reuse",)
 
+BADGES: dict[str, str] = {}
+
 
 class ReuseQuerySet(OwnedQuerySet):
     def visible(self):
@@ -33,15 +35,25 @@ def check_url_does_not_exists(url):
         raise FieldValidationError(_("This URL is already registered"), field="url")
 
 
+class ReuseBadge(Badge):
+    kind = db.StringField(required=True, choices=list(BADGES.keys()))
+
+
+class ReuseBadgeMixin(BadgeMixin):
+    badges = field(BadgesList(ReuseBadge), **BadgeMixin.default_badges_list_params)
+    __badges__ = BADGES
+
+
 @generate_fields(
     searchable=True,
-    additionalSorts=[
+    additional_sorts=[
         {"key": "datasets", "value": "metrics.datasets"},
         {"key": "followers", "value": "metrics.followers"},
         {"key": "views", "value": "metrics.views"},
     ],
+    additional_filters={"organization_badge": "organization.badges"},
 )
-class Reuse(db.Datetimed, WithMetrics, BadgeMixin, Owned, db.Document):
+class Reuse(db.Datetimed, WithMetrics, ReuseBadgeMixin, Owned, db.Document):
     title = field(
         db.StringField(required=True),
         sortable=True,
@@ -124,8 +136,6 @@ class Reuse(db.Datetimed, WithMetrics, BadgeMixin, Owned, db.Document):
     def __str__(self):
         return self.title or ""
 
-    __badges__ = {}
-
     __metrics_keys__ = [
         "discussions",
         "datasets",

udata/core/reuse/search.py

@@ -29,9 +29,10 @@ class ReuseSearch(ModelSearchAdapter):
     filters = {
         "tag": Filter(),
         "organization": ModelTermsFilter(model=Organization),
+        "organization_badge": Filter(choices=list(Organization.__badges__)),
         "owner": ModelTermsFilter(model=User),
         "type": Filter(),
-        "badge": Filter(),
+        "badge": Filter(choices=list(Reuse.__badges__)),
         "featured": BoolFilter(),
         "topic": Filter(),
         "archived": BoolFilter(),
@@ -43,7 +44,7 @@ class ReuseSearch(ModelSearchAdapter):
 
     @classmethod
     def mongo_search(cls, args):
-        reuses = Reuse.objects(deleted=None, private__ne=True)
+        reuses = Reuse.objects.visible()
         reuses = ReuseApiParser.parse_filters(reuses, args)
 
         sort = (
@@ -51,8 +52,7 @@ class ReuseSearch(ModelSearchAdapter):
             or ("$text_score" if args["q"] else None)
             or DEFAULT_SORTING
         )
-        offset = (args["page"] - 1) * args["page_size"]
-        return reuses.order_by(sort).skip(offset).limit(args["page_size"]), reuses.count()
+        return reuses.order_by(sort).paginate(args["page"], args["page_size"])
 
     @classmethod
     def serialize(cls, reuse: Reuse) -> dict:
@@ -65,6 +65,7 @@ class ReuseSearch(ModelSearchAdapter):
                 "name": org.name,
                 "public_service": 1 if org.public_service else 0,
                 "followers": org.metrics.get("followers", 0),
+                "badges": [badge.kind for badge in org.badges],
             }
         elif reuse.owner:
             owner = User.objects(id=reuse.owner.id).first()

udata/search/fields.py

@@ -19,8 +19,12 @@ OR_SEPARATOR = "|"
 
 
 class Filter:
-    @staticmethod
-    def as_request_parser_kwargs():
+    def __init__(self, choices=None):
+        self.choices = choices
+
+    def as_request_parser_kwargs(self):
+        if self.choices:
+            return {"type": clean_string, "choices": self.choices}
         return {"type": clean_string}
 
 
@@ -31,9 +35,10 @@ class BoolFilter(Filter):
 
 
 class ModelTermsFilter(Filter):
-    def __init__(self, model, field_name="id"):
+    def __init__(self, model, field_name="id", choices=None):
         self.model = model
         self.field_name = field_name
+        super().__init__(choices=choices)
 
     @property
     def model_field(self):

udata/search/query.py

@@ -54,8 +54,10 @@ class SearchQuery:
                 "sort": self.sort,
             }
             query_args.update(self._filters)
-            result, total = self.adapter.mongo_search(query_args)
-            return SearchResult(query=self, mongo_objects=result, total=total, **query_args)
+            result = self.adapter.mongo_search(query_args)
+            return SearchResult(
+                query=self, mongo_objects=list(result), total=result.total, **query_args
+            )
 
     def to_url(self, url=None, replace=False, **kwargs):
         """Serialize the query into an URL"""