udata 9.1.4__py2.py3-none-any.whl → 9.1.4.dev30965__py2.py3-none-any.whl

tasks/__init__.py

@@ -194,10 +194,10 @@ def qa(ctx):
 
 
 @task
-def serve(ctx, host="localhost", port="7000"):
+def serve(ctx, host="localhost"):
     """Run a development server"""
     with ctx.cd(ROOT):
-        ctx.run(f"python manage.py serve -d -r -h {host} -p {port}")
+        ctx.run("python manage.py serve -d -r -h %s" % host)
 
 
 @task

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 """
 
-__version__ = "9.1.4"
+__version__ = "9.1.4.dev"
 __description__ = "Open data portal"

udata/api/__init__.py

@@ -1,4 +1,5 @@
 import inspect
+import itertools
 import logging
 import urllib.parse
 from functools import wraps
@@ -17,7 +18,7 @@ from flask import (
 from flask_restx import Api, Resource
 from flask_storage import UnauthorizedFileType
 
-from udata import entrypoints, tracking
+from udata import cors, entrypoints, tracking
 from udata.app import csrf
 from udata.auth import Permission, PermissionDenied, RoleNeed, current_user, login_user
 from udata.i18n import get_locale
@@ -324,7 +325,7 @@ def init_app(app):
     import udata.harvest.api  # noqa
 
     for module in entrypoints.get_enabled("udata.apis", app).values():
-        module if inspect.ismodule(module) else import_module(module)
+        api_module = module if inspect.ismodule(module) else import_module(module)
 
     # api.init_app(app)
     app.register_blueprint(apiv1_blueprint)

udata/api/commands.py

@@ -4,6 +4,7 @@ import os
 import click
 from flask import current_app, json
 from flask_restx import schemas
+from werkzeug.security import gen_salt
 
 from udata.api import api
 from udata.api.oauth2 import OAuth2Client

udata/api/fields.py

@@ -4,28 +4,7 @@ import logging
 import pytz
 from dateutil.parser import parse
 from flask import request, url_for
-
-# Explicitly import all of flask_restx fields so they're available throughout the codebase as api.fields
-from flask_restx.fields import Arbitrary as Arbitrary
-from flask_restx.fields import Boolean as Boolean
-from flask_restx.fields import ClassName as ClassName
-from flask_restx.fields import Date as Date
-from flask_restx.fields import DateTime as DateTime
-from flask_restx.fields import Fixed as Fixed
-from flask_restx.fields import Float as Float
-from flask_restx.fields import FormattedString as FormattedString
-from flask_restx.fields import Integer as Integer
-from flask_restx.fields import List as List
-from flask_restx.fields import MarshallingError as MarshallingError
-from flask_restx.fields import MinMaxMixin as MinMaxMixin
-from flask_restx.fields import Nested as Nested
-from flask_restx.fields import NumberMixin as NumberMixin
-from flask_restx.fields import Polymorph as Polymorph
-from flask_restx.fields import Raw as Raw
-from flask_restx.fields import String as String
-from flask_restx.fields import StringMixin as StringMixin
-from flask_restx.fields import Url as Url
-from flask_restx.fields import Wildcard as Wildcard
+from flask_restx.fields import *  # noqa
 
 from udata.uris import endpoint_for
 from udata.utils import multi_to_dict

udata/api_fields.py

@@ -2,10 +2,10 @@ import flask_restx.fields as restx_fields
 import mongoengine
 import mongoengine.fields as mongo_fields
 from bson import ObjectId
-from flask_storage.mongo import ImageField as FlaskStorageImageField
 
 import udata.api.fields as custom_restx_fields
-from udata.api import api, base_reference
+from udata.api import api
+from udata.mongo.engine import db
 from udata.mongo.errors import FieldValidationError
 
 lazy_reference = api.model(
@@ -17,7 +17,7 @@ lazy_reference = api.model(
 )
 
 
-def convert_db_to_field(key, field, info):
+def convert_db_to_field(key, field, info={}):
     """
     This function maps a Mongo field to a Flask RestX field.
     Most of the types are a simple 1-to-1 mapping except lists and references that requires
@@ -28,6 +28,8 @@ def convert_db_to_field(key, field, info):
     params. Since merging the params involve a litte bit of work (merging default params with read/write params and then with
     user-supplied overrides, setting the readonly flag…), it's easier to have do this one time at the end of the function.
     """
+    info = {**getattr(field, "__additional_field_info__", {}), **info}
+
     params = {}
     params["required"] = field.required
 
@@ -43,9 +45,7 @@ def convert_db_to_field(key, field, info):
         # is always good enough.
         return info.get("convert_to"), info.get("convert_to")
     elif isinstance(field, mongo_fields.StringField):
-        constructor = (
-            custom_restx_fields.Markdown if info.get("markdown", False) else restx_fields.String
-        )
+        constructor = restx_fields.String
         params["min_length"] = field.min_length
         params["max_length"] = field.max_length
         params["enum"] = field.choices
@@ -61,42 +61,18 @@ def convert_db_to_field(key, field, info):
         constructor = custom_restx_fields.ISODateTime
     elif isinstance(field, mongo_fields.DictField):
         constructor = restx_fields.Raw
-    elif isinstance(field, mongo_fields.ImageField) or isinstance(field, FlaskStorageImageField):
-        size = info.get("size", None)
-        if size:
-            params["description"] = f"URL of the cropped and squared image ({size}x{size})"
-        else:
-            params["description"] = "URL of the image"
-
-        if info.get("is_thumbnail", False):
-            constructor_read = custom_restx_fields.ImageField
-            write_params["read_only"] = True
-        else:
-            constructor = custom_restx_fields.ImageField
-
     elif isinstance(field, mongo_fields.ListField):
         # For lists, we convert the inner value from Mongo to RestX then we create
         # the `List` RestX type with this converted inner value.
-        # There is three level of information, from most important to least
-        #     1. `inner_field_info` inside `__additional_field_info__` on the parent
-        #     2. `__additional_field_info__` of the inner field
-        #     3. `__additional_field_info__` of the parent
-        inner_info = getattr(field.field, "__additional_field_info__", {})
         field_read, field_write = convert_db_to_field(
-            f"{key}.inner", field.field, {**info, **inner_info, **info.get("inner_field_info", {})}
+            f"{key}.inner", field.field, info.get("inner_field_info", {})
         )
-
-        def constructor_read(**kwargs):
-            return restx_fields.List(field_read, **kwargs)
-
-        def constructor_write(**kwargs):
-            return restx_fields.List(field_write, **kwargs)
+        constructor_read = lambda **kwargs: restx_fields.List(field_read, **kwargs)
+        constructor_write = lambda **kwargs: restx_fields.List(field_write, **kwargs)
     elif isinstance(
         field, (mongo_fields.GenericReferenceField, mongoengine.fields.GenericLazyReferenceField)
     ):
-
-        def constructor(**kwargs):
-            return restx_fields.Nested(lazy_reference, **kwargs)
+        constructor = lambda **kwargs: restx_fields.Nested(lazy_reference, **kwargs)
     elif isinstance(field, mongo_fields.ReferenceField):
         # For reference we accept while writing a String representing the ID of the referenced model.
         # For reading, if the user supplied a `nested_fields` (RestX model), we use it to convert
@@ -107,38 +83,34 @@ def convert_db_to_field(key, field, info):
             # If there is no `nested_fields` convert the object to the string representation.
             constructor_read = restx_fields.String
         else:
-
-            def constructor_read(**kwargs):
-                return restx_fields.Nested(nested_fields, **kwargs)
+            constructor_read = lambda **kwargs: restx_fields.Nested(nested_fields, **kwargs)
 
         write_params["description"] = "ID of the reference"
         constructor_write = restx_fields.String
     elif isinstance(field, mongo_fields.EmbeddedDocumentField):
         nested_fields = info.get("nested_fields")
         if nested_fields is not None:
-
-            def constructor(**kwargs):
-                return restx_fields.Nested(nested_fields, **kwargs)
+            constructor = lambda **kwargs: restx_fields.Nested(nested_fields, **kwargs)
         elif hasattr(field.document_type_obj, "__read_fields__"):
-
-            def constructor_read(**kwargs):
-                return restx_fields.Nested(field.document_type_obj.__read_fields__, **kwargs)
-
-            def constructor_write(**kwargs):
-                return restx_fields.Nested(field.document_type_obj.__write_fields__, **kwargs)
+            constructor_read = lambda **kwargs: restx_fields.Nested(
+                field.document_type_obj.__read_fields__, **kwargs
+            )
+            constructor_write = lambda **kwargs: restx_fields.Nested(
+                field.document_type_obj.__write_fields__, **kwargs
+            )
         else:
             raise ValueError(
                 f"EmbeddedDocumentField `{key}` requires a `nested_fields` param to serialize/deserialize or a `@generate_fields()` definition."
             )
 
     else:
-        raise ValueError(f"Unsupported MongoEngine field type {field.__class__}")
+        raise ValueError(f"Unsupported MongoEngine field type {field.__class__.__name__}")
 
     read_params = {**params, **read_params, **info}
     write_params = {**params, **write_params, **info}
 
     read = constructor_read(**read_params) if constructor_read else constructor(**read_params)
-    if write_params.get("readonly", False) or (constructor_write is None and constructor is None):
+    if write_params.get("readonly", False):
         write = None
     else:
         write = (
@@ -147,26 +119,6 @@ def convert_db_to_field(key, field, info):
     return read, write
 
 
-def get_fields(cls):
-    """
-    Returns all the exposed fields of the class (fields decorated with `field()`)
-    It also expends image fields to add thumbnail fields.
-    """
-    for key, field in cls._fields.items():
-        info: dict | None = getattr(field, "__additional_field_info__", None)
-        if info is None:
-            continue
-
-        yield key, field, info
-
-        if isinstance(field, mongo_fields.ImageField) or isinstance(field, FlaskStorageImageField):
-            yield (
-                f"{key}_thumbnail",
-                field,
-                {**info, **info.get("thumbnail_info", {}), "is_thumbnail": True, "attribute": key},
-            )
-
-
 def generate_fields(**kwargs):
     """
     This decorator will create two auto-generated attributes on the class `__read_fields__` and `__write_fields__`
@@ -176,18 +128,18 @@ def generate_fields(**kwargs):
     def wrapper(cls):
         read_fields = {}
         write_fields = {}
-        ref_fields = {}
-        sortables = kwargs.get("additionalSorts", [])
+        sortables = []
         filterables = []
 
-        read_fields["id"] = restx_fields.String(required=True, readonly=True)
+        read_fields["id"] = restx_fields.String(required=True)
 
-        for key, field, info in get_fields(cls):
-            sortable_key = info.get("sortable", False)
-            if sortable_key:
-                sortables.append(
-                    {"key": sortable_key if isinstance(sortable_key, str) else key, "value": key}
-                )
+        for key, field in cls._fields.items():
+            info = getattr(field, "__additional_field_info__", None)
+            if info is None:
+                continue
+
+            if info.get("sortable", False):
+                sortables.append(key)
 
             filterable = info.get("filterable", None)
             if filterable is not None:
@@ -204,26 +156,18 @@ def generate_fields(**kwargs):
                     ):
                         filterable["constraints"].append("objectid")
 
-                if "type" not in filterable:
-                    filterable["type"] = str
-                    if isinstance(field, mongo_fields.BooleanField):
-                        filterable["type"] = bool
-
                 # We may add more information later here:
                 # - type of mongo query to execute (right now only simple =)
 
                 filterables.append(filterable)
 
-            read, write = convert_db_to_field(key, field, info)
+            read, write = convert_db_to_field(key, field)
 
             if read:
                 read_fields[key] = read
             if write:
                 write_fields[key] = write
 
-            if read and info.get("show_as_ref", False):
-                ref_fields[key] = read
-
         # The goal of this loop is to fetch all functions (getters) of the class
         # If a function has an `__additional_field_info__` attribute it means
         # it has been decorated with `@function_field()` and should be included
@@ -255,12 +199,9 @@ def generate_fields(**kwargs):
             read_fields[method_name] = restx_fields.String(
                 attribute=make_lambda(method), **{"readonly": True, **info}
             )
-            if info.get("show_as_ref", False):
-                ref_fields[key] = read_fields[method_name]
 
         cls.__read_fields__ = api.model(f"{cls.__name__} (read)", read_fields, **kwargs)
         cls.__write_fields__ = api.model(f"{cls.__name__} (write)", write_fields, **kwargs)
-        cls.__ref_fields__ = api.inherit(f"{cls.__name__}Reference", base_reference, ref_fields)
 
         mask = kwargs.pop("mask", None)
         if mask is not None:
@@ -285,9 +226,7 @@ def generate_fields(**kwargs):
             )
 
         if sortables:
-            choices = [sortable["key"] for sortable in sortables] + [
-                "-" + sortable["key"] for sortable in sortables
-            ]
+            choices = sortables + ["-" + k for k in sortables]
             parser.add_argument(
                 "sort",
                 type=str,
@@ -296,12 +235,8 @@ def generate_fields(**kwargs):
                 help="The field (and direction) on which sorting apply",
             )
 
-        searchable = kwargs.pop("searchable", False)
-        if searchable:
-            parser.add_argument("q", type=str, location="args")
-
         for filterable in filterables:
-            parser.add_argument(filterable["key"], type=filterable["type"], location="args")
+            parser.add_argument(filterable["key"], type=str, location="args")
 
         cls.__index_parser__ = parser
 
@@ -309,23 +244,7 @@ def generate_fields(**kwargs):
             args = cls.__index_parser__.parse_args()
 
             if sortables and args["sort"]:
-                negate = args["sort"].startswith("-")
-                sort_key = args["sort"][1:] if negate else args["sort"]
-
-                sort_by = next(
-                    (sortable["value"] for sortable in sortables if sortable["key"] == sort_key),
-                    None,
-                )
-
-                if sort_by:
-                    if negate:
-                        sort_by = "-" + sort_by
-
-                    base_query = base_query.order_by(sort_by)
-
-            if searchable and args.get("q"):
-                phrase_query = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
-                base_query = base_query.search_text(phrase_query)
+                base_query = base_query.order_by(args["sort"])
 
             for filterable in filterables:
                 if args.get(filterable["key"]):
@@ -374,16 +293,11 @@ def patch(obj, request):
     Patch the object with the data from the request.
     Only fields decorated with the `field()` decorator will be read (and not readonly).
     """
-    from udata.mongo.engine import db
-
     for key, value in request.json.items():
         field = obj.__write_fields__.get(key)
         if field is not None and not field.readonly:
             model_attribute = getattr(obj.__class__, key)
-
-            if hasattr(model_attribute, "from_input"):
-                value = model_attribute.from_input(value)
-            elif isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
+            if isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
                 model_attribute.field, mongoengine.fields.ReferenceField
             ):
                 # TODO `wrap_primary_key` do Mongo request, do a first pass to fetch all documents before calling it (to avoid multiple queries).
@@ -409,7 +323,7 @@ def patch(obj, request):
             # `check` field attribute allows to do validation from the request before setting
             # the attribute
             check = info.get("check", None)
-            if check is not None and value != getattr(obj, key):
+            if check is not None:
                 check(**{key: value})  # TODO add other model attributes in function parameters
 
             setattr(obj, key, value)
@@ -417,21 +331,10 @@ def patch(obj, request):
     return obj
 
 
-def patch_and_save(obj, request):
-    obj = patch(obj, request)
-
-    try:
-        obj.save()
-    except mongoengine.errors.ValidationError as e:
-        api.abort(400, e.message)
-
-    return obj
-
-
 def wrap_primary_key(
     field_name: str,
     foreign_field: mongoengine.fields.ReferenceField | mongoengine.fields.GenericReferenceField,
-    value: str | None,
+    value: str,
     document_type=None,
 ):
     """
@@ -440,12 +343,6 @@ def wrap_primary_key(
 
     TODO: we only check the document reference if the ID is a `String` field (not in the case of a classic `ObjectId`).
     """
-    if value is None:
-        return value
-
-    if isinstance(value, dict) and "id" in value:
-        return wrap_primary_key(field_name, foreign_field, value["id"], document_type)
-
     document_type = document_type or foreign_field.document_type().__class__
     id_field_name = document_type._meta["id_field"]
 
@@ -468,7 +365,7 @@ def wrap_primary_key(
         return foreign_document
 
     if isinstance(id_field, mongoengine.fields.ObjectIdField):
-        return foreign_document.to_dbref()
+        return ObjectId(value)
     elif isinstance(id_field, mongoengine.fields.StringField):
         # Right now I didn't find a simpler way to make mongoengine happy.
         # For references, it expects `ObjectId`, `DBRef`, `LazyReference` or `document` but since

udata/app.py

@@ -206,7 +206,7 @@ def register_extensions(app):
         mail,
         models,
         mongo,
-        notifications,  # noqa
+        notifications,
         routing,
         search,
         sentry,

udata/auth/__init__.py

@@ -2,14 +2,18 @@ import logging
 
 from flask import current_app, render_template
 from flask_principal import Permission as BasePermission
-from flask_principal import PermissionDenied as PermissionDenied
-from flask_principal import RoleNeed as RoleNeed
-from flask_principal import UserNeed as UserNeed
-from flask_principal import identity_loaded as identity_loaded
-from flask_security import Security as Security
-from flask_security import current_user as current_user
-from flask_security import login_required as login_required
-from flask_security import login_user as login_user
+from flask_principal import (
+    PermissionDenied,  # noqa: facade pattern
+    RoleNeed,
+    UserNeed,  # noqa: facade pattern
+    identity_loaded,  # noqa: facade pattern
+)
+from flask_security import (  # noqa
+    Security,  # noqa
+    current_user,
+    login_required,
+    login_user,
+)
 from werkzeug.utils import import_string
 
 log = logging.getLogger(__name__)

udata/commands/db.py

@@ -159,10 +159,10 @@ def check_references(models_to_check):
     references = []
     for model in set(_models):
         if model.__name__ == "Activity":
-            print("Skipping Activity model, scheduled for deprecation")
+            print(f"Skipping Activity model, scheduled for deprecation")
             continue
         if model.__name__ == "GeoLevel":
-            print("Skipping GeoLevel model, scheduled for deprecation")
+            print(f"Skipping GeoLevel model, scheduled for deprecation")
             continue
 
         if models_to_check and model.__name__ not in models_to_check:
@@ -367,7 +367,7 @@ def check_references(models_to_check):
                                     )
                         else:
                             print_and_save(f'Unknown ref type {reference["type"]}')
-                    except mongoengine.errors.FieldDoesNotExist:
+                    except mongoengine.errors.FieldDoesNotExist as e:
                         print_and_save(
                             f"[ERROR for {model.__name__} {obj.id}] {traceback.format_exc()}"
                         )

udata/commands/dcat.py

@@ -33,7 +33,7 @@ def parse_url(url, csw, iso, quiet=False, rid=""):
     """Parse the datasets in a DCAT format located at URL (debug)"""
     if quiet:
         verbose_loggers = ["rdflib", "udata.core.dataset"]
-        [logging.getLogger(logger).setLevel(logging.ERROR) for logger in verbose_loggers]
+        [logging.getLogger(l).setLevel(logging.ERROR) for l in verbose_loggers]
 
     class MockSource:
         url = ""

udata/commands/fixtures.py

@@ -16,6 +16,7 @@ from udata.commands import cli
 from udata.core.contact_point.factories import ContactPointFactory
 from udata.core.contact_point.models import ContactPoint
 from udata.core.dataservices.factories import DataserviceFactory
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.factories import (
     CommunityResourceFactory,
     DatasetFactory,
@@ -26,7 +27,6 @@ from udata.core.organization.factories import OrganizationFactory
 from udata.core.organization.models import Member, Organization
 from udata.core.reuse.factories import ReuseFactory
 from udata.core.user.factories import UserFactory
-from udata.core.user.models import User
 
 log = logging.getLogger(__name__)
 
@@ -56,21 +56,22 @@ UNWANTED_KEYS: dict[str, list[str]] = {
         "quality",
     ],
     "resource": ["latest", "preview_url", "last_modified"],
-    "organization": ["class", "page", "uri", "logo_thumbnail"],
-    "reuse": ["datasets", "image_thumbnail", "page", "uri", "owner"],
+    "organization": ["members", "page", "uri", "logo_thumbnail"],
+    "reuse": ["datasets", "image_thumbnail", "page", "uri", "organization", "owner"],
     "community": [
         "dataset",
+        "organization",
         "owner",
         "latest",
         "last_modified",
         "preview_url",
     ],
-    "discussion": ["subject", "url", "class"],
-    "user": ["uri", "page", "class", "avatar_thumbnail", "email"],
-    "posted_by": ["uri", "page", "class", "avatar_thumbnail", "email"],
+    "discussion": ["subject", "user", "url", "class"],
+    "message": ["posted_by"],
     "dataservice": [
         "datasets",
         "license",
+        "organization",
         "owner",
         "self_api_url",
         "self_web_url",
@@ -80,8 +81,6 @@ UNWANTED_KEYS: dict[str, list[str]] = {
 
 def remove_unwanted_keys(obj: dict, filter_type: str) -> dict:
     """Remove UNWANTED_KEYS from a dict."""
-    if filter_type not in UNWANTED_KEYS:
-        return obj
     for unwanted_key in UNWANTED_KEYS[filter_type]:
         if unwanted_key in obj:
             del obj[unwanted_key]
@@ -152,29 +151,6 @@ def generate_fixtures_file(data_source: str, results_filename: str) -> None:
         print(f"Fixtures saved to file {results_filename}")
 
 
-def get_or_create(data, key, model, factory):
-    """Try getting the object. If it doesn't exist yet, create it with the provided factory."""
-    if key not in data or data[key] is None:
-        return
-    data[key] = remove_unwanted_keys(data[key], key)
-    obj = model.objects(id=data[key]["id"]).first()
-    if not obj:
-        obj = factory(**data[key])
-    return obj
-
-
-def get_or_create_organization(data):
-    return get_or_create(data, "organization", Organization, OrganizationFactory)
-
-
-def get_or_create_owner(data):
-    return get_or_create(data, "owner", User, UserFactory)
-
-
-def get_or_create_user(data):
-    return get_or_create(data, "user", User, UserFactory)
-
-
 @cli.command()
 @click.argument("source", default=DEFAULT_FIXTURE_FILE)
 def import_fixtures(source):
@@ -190,45 +166,49 @@ def import_fixtures(source):
             user = UserFactory()
             dataset = fixture["dataset"]
             dataset = remove_unwanted_keys(dataset, "dataset")
-            if fixture["organization"]:
-                organization = fixture["organization"]
-                organization["members"] = [
-                    Member(user=get_or_create_user(member), role=member["role"])
-                    for member in organization["members"]
-                ]
-                fixture["organization"] = organization
-                org = get_or_create_organization(fixture)
-                dataset = DatasetFactory(**dataset, organization=org)
-            else:
+            if not fixture["organization"]:
                 dataset = DatasetFactory(**dataset, owner=user)
+            else:
+                org = Organization.objects(id=fixture["organization"]["id"]).first()
+                if not org:
+                    organization = fixture["organization"]
+                    organization = remove_unwanted_keys(organization, "organization")
+                    org = OrganizationFactory(**organization, members=[Member(user=user)])
+                dataset = DatasetFactory(**dataset, organization=org)
             for resource in fixture["resources"]:
                 resource = remove_unwanted_keys(resource, "resource")
                 res = ResourceFactory(**resource)
                 dataset.add_resource(res)
             for reuse in fixture["reuses"]:
                 reuse = remove_unwanted_keys(reuse, "reuse")
-                reuse["owner"] = get_or_create_owner(reuse)
-                reuse["organization"] = get_or_create_organization(reuse)
-                ReuseFactory(**reuse, datasets=[dataset])
+                ReuseFactory(**reuse, datasets=[dataset], owner=user)
             for community in fixture["community_resources"]:
                 community = remove_unwanted_keys(community, "community")
-                community["owner"] = get_or_create_owner(community)
-                community["organization"] = get_or_create_organization(community)
-                CommunityResourceFactory(**community, dataset=dataset)
+                CommunityResourceFactory(**community, dataset=dataset, owner=user)
             for discussion in fixture["discussions"]:
                 discussion = remove_unwanted_keys(discussion, "discussion")
-                discussion["closed_by"] = get_or_create(discussion, "closed_by", User, UserFactory)
-                for message in discussion["discussion"]:
-                    message["posted_by"] = get_or_create(message, "posted_by", User, UserFactory)
-                discussion["discussion"] = [
-                    MessageDiscussionFactory(**message) for message in discussion["discussion"]
-                ]
-                discussion["user"] = get_or_create_user(discussion)
-                DiscussionFactory(**discussion, subject=dataset)
+                messages = discussion.pop("discussion")
+                for message in messages:
+                    message = remove_unwanted_keys(message, "message")
+                DiscussionFactory(
+                    **discussion,
+                    subject=dataset,
+                    user=user,
+                    discussion=[
+                        MessageDiscussionFactory(**message, posted_by=user) for message in messages
+                    ],
+                )
             for dataservice in fixture["dataservices"]:
                 dataservice = remove_unwanted_keys(dataservice, "dataservice")
-                dataservice["contact_point"] = get_or_create(
-                    dataservice, "contact_point", ContactPoint, ContactPointFactory
-                )
-                dataservice["organization"] = get_or_create_organization(dataservice)
-                DataserviceFactory(**dataservice, datasets=[dataset])
+                if not dataservice["contact_point"]:
+                    DataserviceFactory(**dataservice, datasets=[dataset])
+                else:
+                    contact_point = ContactPoint.objects(
+                        id=dataservice["contact_point"]["id"]
+                    ).first()
+                    if not contact_point:
+                        contact_point = ContactPointFactory(**dataservice["contact_point"])
+                    dataservice.pop("contact_point")
+                    DataserviceFactory(
+                        **dataservice, datasets=[dataset], contact_point=contact_point
+                    )