udata 9.1.4.dev31182__py2.py3-none-any.whl → 9.1.4.dev31221__py2.py3-none-any.whl

udata/commands/fixtures.py

@@ -26,6 +26,7 @@ from udata.core.organization.factories import OrganizationFactory
 from udata.core.organization.models import Member, Organization
 from udata.core.reuse.factories import ReuseFactory
 from udata.core.user.factories import UserFactory
+from udata.core.user.models import User
 
 log = logging.getLogger(__name__)
 
@@ -55,22 +56,21 @@ UNWANTED_KEYS: dict[str, list[str]] = {
         "quality",
     ],
     "resource": ["latest", "preview_url", "last_modified"],
-    "organization": ["members", "page", "uri", "logo_thumbnail"],
-    "reuse": ["datasets", "image_thumbnail", "page", "uri", "organization", "owner"],
+    "organization": ["class", "page", "uri", "logo_thumbnail"],
+    "reuse": ["datasets", "image_thumbnail", "page", "uri", "owner"],
     "community": [
         "dataset",
-        "organization",
         "owner",
         "latest",
         "last_modified",
         "preview_url",
     ],
-    "discussion": ["subject", "user", "url", "class"],
-    "message": ["posted_by"],
+    "discussion": ["subject", "url", "class"],
+    "user": ["uri", "page", "class", "avatar_thumbnail", "email"],
+    "posted_by": ["uri", "page", "class", "avatar_thumbnail", "email"],
     "dataservice": [
         "datasets",
         "license",
-        "organization",
         "owner",
         "self_api_url",
         "self_web_url",
@@ -80,6 +80,8 @@ UNWANTED_KEYS: dict[str, list[str]] = {
 
 def remove_unwanted_keys(obj: dict, filter_type: str) -> dict:
     """Remove UNWANTED_KEYS from a dict."""
+    if filter_type not in UNWANTED_KEYS:
+        return obj
     for unwanted_key in UNWANTED_KEYS[filter_type]:
         if unwanted_key in obj:
             del obj[unwanted_key]
@@ -150,6 +152,29 @@ def generate_fixtures_file(data_source: str, results_filename: str) -> None:
         print(f"Fixtures saved to file {results_filename}")
 
 
+def get_or_create(data, key, model, factory):
+    """Try getting the object. If it doesn't exist yet, create it with the provided factory."""
+    if key not in data or data[key] is None:
+        return
+    data[key] = remove_unwanted_keys(data[key], key)
+    obj = model.objects(id=data[key]["id"]).first()
+    if not obj:
+        obj = factory(**data[key])
+    return obj
+
+
+def get_or_create_organization(data):
+    return get_or_create(data, "organization", Organization, OrganizationFactory)
+
+
+def get_or_create_owner(data):
+    return get_or_create(data, "owner", User, UserFactory)
+
+
+def get_or_create_user(data):
+    return get_or_create(data, "user", User, UserFactory)
+
+
 @cli.command()
 @click.argument("source", default=DEFAULT_FIXTURE_FILE)
 def import_fixtures(source):
@@ -165,52 +190,45 @@ def import_fixtures(source):
             user = UserFactory()
             dataset = fixture["dataset"]
             dataset = remove_unwanted_keys(dataset, "dataset")
-            if not fixture["organization"]:
-                dataset = DatasetFactory(**dataset, owner=user)
-            else:
-                org = Organization.objects(id=fixture["organization"]["id"]).first()
-                if not org:
-                    organization = fixture["organization"]
-                    organization = remove_unwanted_keys(organization, "organization")
-                    org = OrganizationFactory(**organization, members=[Member(user=user)])
+            if fixture["organization"]:
+                organization = fixture["organization"]
+                organization["members"] = [
+                    Member(user=get_or_create_user(member), role=member["role"])
+                    for member in organization["members"]
+                ]
+                fixture["organization"] = organization
+                org = get_or_create_organization(fixture)
                 dataset = DatasetFactory(**dataset, organization=org)
+            else:
+                dataset = DatasetFactory(**dataset, owner=user)
             for resource in fixture["resources"]:
                 resource = remove_unwanted_keys(resource, "resource")
                 res = ResourceFactory(**resource)
                 dataset.add_resource(res)
             for reuse in fixture["reuses"]:
                 reuse = remove_unwanted_keys(reuse, "reuse")
-                ReuseFactory(**reuse, datasets=[dataset], owner=user)
+                reuse["owner"] = get_or_create_owner(reuse)
+                reuse["organization"] = get_or_create_organization(reuse)
+                ReuseFactory(**reuse, datasets=[dataset])
             for community in fixture["community_resources"]:
                 community = remove_unwanted_keys(community, "community")
-                CommunityResourceFactory(**community, dataset=dataset, owner=user)
+                community["owner"] = get_or_create_owner(community)
+                community["organization"] = get_or_create_organization(community)
+                CommunityResourceFactory(**community, dataset=dataset)
             for discussion in fixture["discussions"]:
                 discussion = remove_unwanted_keys(discussion, "discussion")
-                messages = discussion.pop("discussion")
-                for message in messages:
-                    message = remove_unwanted_keys(message, "message")
-                DiscussionFactory(
-                    **discussion,
-                    subject=dataset,
-                    user=user,
-                    discussion=[
-                        MessageDiscussionFactory(**message, posted_by=user) for message in messages
-                    ],
-                )
+                discussion["closed_by"] = get_or_create(discussion, "closed_by", User, UserFactory)
+                for message in discussion["discussion"]:
+                    message["posted_by"] = get_or_create(message, "posted_by", User, UserFactory)
+                discussion["discussion"] = [
+                    MessageDiscussionFactory(**message) for message in discussion["discussion"]
+                ]
+                discussion["user"] = get_or_create_user(discussion)
+                DiscussionFactory(**discussion, subject=dataset)
             for dataservice in fixture["dataservices"]:
                 dataservice = remove_unwanted_keys(dataservice, "dataservice")
-                if not dataservice["contact_point"]:
-                    DataserviceFactory(**dataservice, datasets=[dataset], organization=org)
-                else:
-                    contact_point = ContactPoint.objects(
-                        id=dataservice["contact_point"]["id"]
-                    ).first()
-                    if not contact_point:
-                        contact_point = ContactPointFactory(**dataservice["contact_point"])
-                    dataservice.pop("contact_point")
-                    DataserviceFactory(
-                        **dataservice,
-                        datasets=[dataset],
-                        organization=org,
-                        contact_point=contact_point,
-                    )
+                dataservice["contact_point"] = get_or_create(
+                    dataservice, "contact_point", ContactPoint, ContactPointFactory
+                )
+                dataservice["organization"] = get_or_create_organization(dataservice)
+                DataserviceFactory(**dataservice, datasets=[dataset])

udata/commands/tests/test_fixtures.py

@@ -27,21 +27,28 @@ class FixturesTest:
         """Test generating fixtures from the current env, then importing them back."""
         assert models.Dataset.objects.count() == 0  # Start with a clean slate.
         user = UserFactory()
-        org = OrganizationFactory(**{}, members=[Member(user=user)])
+        admin = UserFactory()
+        org = OrganizationFactory(
+            members=[Member(user=user, role="editor"), Member(user=admin, role="admin")]
+        )
         # Set the same slug we're 'exporting' from the FIXTURE_DATASET_SLUG config, see the
         # @pytest.mark.options above.
-        dataset = DatasetFactory(**{}, slug="some-test-dataset-slug", organization=org)
-        res = ResourceFactory(**{})
+        dataset = DatasetFactory(slug="some-test-dataset-slug", organization=org)
+        res = ResourceFactory()
         dataset.add_resource(res)
-        ReuseFactory(**{}, datasets=[dataset], owner=user)
-        CommunityResourceFactory(**{}, dataset=dataset, owner=user)
+        ReuseFactory(datasets=[dataset], owner=user)
+        CommunityResourceFactory(dataset=dataset, owner=user)
         DiscussionFactory(
             **{},
             subject=dataset,
             user=user,
-            discussion=[MessageDiscussionFactory(**{}, posted_by=user)],
+            discussion=[
+                MessageDiscussionFactory(posted_by=user),
+                MessageDiscussionFactory(posted_by=admin),
+            ],
+            closed_by=admin,
         )
-        DataserviceFactory(**{}, datasets=[dataset])
+        DataserviceFactory(datasets=[dataset], organization=org)
 
         with NamedTemporaryFile(mode="w+", delete=True) as fixtures_fd:
             # Get the fixtures from the local instance.
@@ -69,13 +76,25 @@ class FixturesTest:
             # Then load them in the database to make sure they're correct.
             result = cli("import-fixtures", fixtures_fd.name)
         assert models.Organization.objects(slug=org.slug).count() > 0
+        result_org = models.Organization.objects.get(slug=org.slug)
+        assert result_org.members[0].user.id == user.id
+        assert result_org.members[0].role == "editor"
+        assert result_org.members[1].user.id == admin.id
+        assert result_org.members[1].role == "admin"
         assert models.Dataset.objects.count() > 0
         assert models.Discussion.objects.count() > 0
+        result_discussion = models.Discussion.objects.first()
+        assert result_discussion.user.id == user.id
+        assert result_discussion.closed_by.id == admin.id
+        assert len(result_discussion.discussion) == 2
+        assert result_discussion.discussion[0].posted_by.id == user.id
+        assert result_discussion.discussion[1].posted_by.id == admin.id
         assert models.CommunityResource.objects.count() > 0
         assert models.User.objects.count() > 0
         assert models.Dataservice.objects.count() > 0
         # Make sure we also import the dataservice organization
-        assert models.Dataservice.objects(organization__exists=True).count() > 0
+        result_dataservice = models.Dataservice.objects.first()
+        assert result_dataservice.organization == org
 
     def test_import_fixtures_from_default_file(self, cli):
         """Test importing fixtures from udata.commands.fixture.DEFAULT_FIXTURE_FILE."""

udata/core/dataset/events.py

@@ -49,7 +49,10 @@ def publish(url, document, resource_id, action):
         "dataset_id": str(document.id),
         "document": resource,
     }
-    r = requests.post(url, json=payload)
+    headers = {}
+    if current_app.config["RESOURCES_ANALYSER_API_KEY"]:
+        headers = {"Authorization": f"Bearer {current_app.config['RESOURCES_ANALYSER_API_KEY']}"}
+    r = requests.post(url, json=payload, headers=headers)
     r.raise_for_status()
 
 

udata/settings.py

@@ -522,6 +522,7 @@ class Defaults(object):
     FIXTURE_DATASET_SLUGS = []
     PUBLISH_ON_RESOURCE_EVENTS = False
     RESOURCES_ANALYSER_URI = "http://localhost:8000"
+    RESOURCES_ANALYSER_API_KEY = None
 
     # Datasets quality settings
     ###########################################################################