OneStop4All-Indexer 2.8.0.dev11__tar.gz → 2.8.0.dev13__tar.gz

{onestop4all_indexer-2.8.0.dev11 → onestop4all_indexer-2.8.0.dev13/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev11
+Version: 2.8.0.dev13
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.8.0.dev11 → onestop4all_indexer-2.8.0.dev13}/OneStop4All_Indexer.egg-info/SOURCES.txt

@@ -24,7 +24,7 @@ harvesters/harvester_metadatastandards.py
 harvesters/harvester_organization.py
 harvesters/harvester_repository.py
 harvesters/harvester_service.py
-harvesters/harvester_softwaresourcecode.py
+harvesters/harvester_software.py
 utils/__init__.py
 utils/cli.py
 utils/configs.py

{onestop4all_indexer-2.8.0.dev11/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev11
+Version: 2.8.0.dev13
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.8.0.dev11 → onestop4all_indexer-2.8.0.dev13}/harvesters/__init__.py

@@ -1,7 +1,7 @@
 from .harvester_repository import *
 from .harvester_organization import *
 from .harvester_article import *
-from .harvester_softwaresourcecode import *
+from .harvester_software import *
 from .harvester_learningresource import *
 from .harvester_metadatastandards import *
 from .harvester_document import *

{onestop4all_indexer-2.8.0.dev11 → onestop4all_indexer-2.8.0.dev13}/harvesters/harvester_organization.py

@@ -1,10 +1,10 @@
-from .harvester_base import Harvester
-from data_repositories.repository_person import RepositoryPerson
-from utils import sparql
-from utils import flatten_dict
-from utils import is_truthy
 import logging
 
+from data_repositories.repository_person import RepositoryPerson
+from utils import sparql, flatten_dict, is_truthy
+
+from .harvester_base import Harvester
+
 log = logging.getLogger(__name__)
 
 
@@ -20,32 +20,37 @@ class Organization_Harvester(Harvester):
         PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
         PREFIX org: <http://www.w3.org/ns/org#>
         PREFIX schema: <http://schema.org/>
+        PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
 
         SELECT ?subject ?predicate ?object ?geo_as_wkt ?isN4EMember
         WHERE {
-            ?subject rdf:type foaf:Organization .
+            {
+                SELECT DISTINCT ?subject
+                WHERE {
+                ?subject rdf:type foaf:Organization .
+                BIND(STRAFTER(STR(?subject), "/objects/") AS ?id)
+                FILTER(STRSTARTS(?id, "n4e/"))
+                FILTER(!CONTAINS(?id, "jsonPointer"))
+                FILTER EXISTS { ?subject n4e:sourceSystem ?source }
+                }
+                OFFSET %s
+                LIMIT %s
+            }
+
             ?subject ?predicate ?object .
-            # Exclude messy data that was harvested from remote source, contains duplicates, etc
-            FILTER EXISTS { ?subject n4e:sourceSystem ?source }
+
             OPTIONAL {
                 ?subject geo:hasGeometry ?geometry .
                 ?geometry geo:asWKT ?geo_as_wkt .
             }
+
             OPTIONAL {
-                ?subject n4e:sourceSystem ?source_system .
-                OPTIONAL { ?source_system dct:title ?sourceSystem_title . }
-                OPTIONAL { ?source_system foaf:homepage ?sourceSystem_homepage . }
-            }
-            OPTIONAL {
-                ?subject rdf:type foaf:Organization ;
-                        org:hasMembership ?membership .
+                ?subject org:hasMembership ?membership .
                 ?membership org:organization ?n4eproject .
                 ?n4eproject schema:url "https://nfdi4earth.de/"^^xsd:anyURI .
                 BIND(true AS ?isN4EMember)
             }
         }
-        OFFSET %d
-        LIMIT %d
     """
 
     def __init__(self, persons_repo: RepositoryPerson, **kw):
@@ -83,11 +88,34 @@ class Organization_Harvester(Harvester):
         )
 
         organization_list = []
-        for (
-            key,
-            value,
-        ) in organizations.items():  # transform orga dict to list for indexing
+        for key, value in organizations.items():  # transform orga dict to list for indexing
             organization = value
+
+            # clean geometry: if more than one geometry is present,
+            # keep only the first one (should not happen, but just in case)
+            geom = organization.get("geometry", [])
+            if len(geom) > 1:
+                organization["geometry"] = geom[:1]
+
+            # clean rorId: if more than one rorId is present,
+            # keep only unique ones (should not happen, but just in case)
+            rorid = organization.get("rorId", [])
+            if len(rorid) > 1:
+                organization["rorId"] = list(set(rorid))
+
+            # Transform locations
+            localities = organization.get("locality", [])
+            countries = organization.get("countryName", [])
+            complete_locations = []
+
+            for loc, country in zip(localities, countries):
+                complete_locations.append(f"{loc}, {country}")
+
+            # Filter duplication
+            organization["location"] = list(dict.fromkeys(complete_locations))
+            organization["locality"] = list(dict.fromkeys(localities))
+            organization["countryName"] = list(dict.fromkeys(countries))
+
             # ensure mainTitle
             if (
                 "mainTitle" not in organization
@@ -132,9 +160,23 @@ class Organization_Harvester(Harvester):
                     assignto_dict[subject[0]][attribute] = []
                 assignto_dict[subject[0]][attribute].extend(organization_name)
 
+    def does_object_exist(self, value, attribute: str, data: dict):
+        return attribute in data and value in data[attribute]
+
     def parse_response(
         self, hits, organizations, issuborganization, hasN4Econtact
     ):
+        PREDICATES = {
+            "http://xmlns.com/foaf/0.1/homepage": "homepage",
+            "http://www.w3.org/2002/07/owl#sameAs": "sameAs",
+            "http://w3id.org/nfdi4ing/metadata4ing#hasRorId": "rorId",
+            "http://www.w3.org/2004/02/skos/core#altLabel": "altLabel",
+            "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "type",
+            "http://nfdi4earth.de/ontology/sourceSystemURL": "sourceSystemURL",
+            "http://nfdi4earth.de/ontology/hasSignedCommitment": "hasSignedCommitment",
+            "http://nfdi4earth.de/ontology/sourceSystemID": "sourceSystem" + self.flatten_separator + "id"
+        }
+
         for hit in hits:
             subject = hit["subject"]["value"]
             predicate = hit["predicate"]["value"]
@@ -146,17 +188,22 @@ class Organization_Harvester(Harvester):
                 organizations[subject]["id"] = self.getID(
                     subject
                 )  # use ID from triple store also in Solr to ensure stable IDs
+                organizations[subject]["locality"] = []
+                organizations[subject]["countryName"] = []
 
             # set geometry if available and not already set
             if (
                 predicate == "http://www.opengis.net/ont/geosparql#hasGeometry"
-                and ("geometry" not in organizations[subject]
-                    or hit["geo_as_wkt"]["value"] not in organizations[subject]["geometry"])
+                and (
+                    "geometry" not in organizations[subject]
+                    or hit["geo_as_wkt"]["value"]
+                    not in organizations[subject]["geometry"]
+                )
             ):
                 self.addValue(
                     dict=organizations[subject],
                     attribute="geometry",
-                    value=hit["geo_as_wkt"]["value"]
+                    value=hit["geo_as_wkt"]["value"],
                 )
 
             # set membership in N4E project
@@ -171,8 +218,9 @@ class Organization_Harvester(Harvester):
 
             if predicate == "http://schema.org/name":  # name
                 if (
-                    "xml:lang" not in hit["object"]
-                    or hit["object"]["xml:lang"] == "en"
+                    ("xml:lang" not in hit["object"]
+                    or hit["object"]["xml:lang"] == "en")
+                    and not self.does_object_exist(object, "name", organizations[subject])
                 ):  # use international name for orga name
                     self.addValue(
                         dict=organizations[subject],
@@ -181,60 +229,30 @@ class Organization_Harvester(Harvester):
                     )
                     organizations[subject]["mainTitle"] = object  # mainTitle
                 if (
-                    "name_alt" not in organizations[subject]
-                    or object not in organizations[subject]["name_alt"]
+                    not self.does_object_exist(object, "name_alt", organizations[subject])
                 ):  # prevent duplicates
                     self.addValue(
                         dict=organizations[subject],
                         attribute="name_alt",
                         value=object,
                     )
-            elif predicate == "http://xmlns.com/foaf/0.1/homepage":  # homepage
-                self.addValue(
-                    dict=organizations[subject],
-                    attribute="homepage",
-                    value=object,
-                )
-            elif (
-                predicate == "http://www.w3.org/2006/vcard/ns#locality"
-            ):  # locality
-                self.addValue(
-                    dict=organizations[subject],
-                    attribute="locality",
-                    value=object,
-                )
-            elif (
-                predicate == "http://www.w3.org/2006/vcard/ns#country-name"
-            ):  # countryName
-                self.addValue(
-                    dict=organizations[subject],
-                    attribute="countryName",
-                    value=object,
-                )
-            elif predicate == "http://www.w3.org/2002/07/owl#sameAs":  # sameAs
-                self.addValue(
-                    dict=organizations[subject],
-                    attribute="sameAs",
-                    value=object,
-                )
-            elif (
-                predicate == "http://w3id.org/nfdi4ing/metadata4ing#hasRorId"
-            ):  # rorId
-                self.addValue(
-                    dict=organizations[subject],
-                    attribute="rorId",
-                    value=object,
-                )
-            elif (
-                predicate == "http://www.w3.org/2004/02/skos/core#altLabel"
-            ):  # altLabel
-                self.addValue(
-                    dict=organizations[subject],
-                    attribute="altLabel",
-                    value=object,
-                )
+
+            elif predicate == "http://www.w3.org/2006/vcard/ns#locality":
+                organizations[subject]["locality"].append(object)
+
+            elif predicate == "http://www.w3.org/2006/vcard/ns#country-name":
+                organizations[subject]["countryName"].append(object)
+
+            elif predicate in PREDICATES:
+                attribute = PREDICATES[predicate]
+                if not self.does_object_exist(object, attribute, organizations[subject]):
+                    self.addValue(
+                        dict=organizations[subject],
+                        attribute=attribute,
+                        value=object)
             elif (
                 predicate == "http://www.w3.org/ns/org#subOrganizationOf"
+                and (subject, object) not in issuborganization
             ):  # subOrganizationOf
                 issuborganization.append(
                     (subject, object)
@@ -242,60 +260,8 @@ class Organization_Harvester(Harvester):
             elif (
                 predicate
                 == "http://nfdi4earth.de/ontology/hasNFDI4EarthContactPerson"
+                and (subject, object) not in hasN4Econtact
             ):  # NFDI4EarthContactPerson
                 hasN4Econtact.append(
                     (subject, object)
                 )  # store, resolve contact info later
-            elif (
-                predicate == "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
-            ):  # type
-                self.addValue(
-                    dict=organizations[subject], attribute="type", value=object
-                )
-            elif (
-                predicate == "http://nfdi4earth.de/ontology/sourceSystem"
-            ):  # sourceSystem
-                if "sourceSystem_homepage" in hit:
-                    self.addValue(
-                        organizations[subject],
-                        "sourceSystem" + self.flatten_separator + "homepage",
-                        hit["sourceSystem_homepage"]["value"],
-                    )
-                if "sourceSystem_title" in hit:
-                    self.addValue(
-                        organizations[subject],
-                        "sourceSystem" + self.flatten_separator + "title",
-                        hit["sourceSystem_title"]["value"],
-                    )
-            elif (
-                predicate == "http://nfdi4earth.de/ontology/sourceSystemID"
-            ):  # sourceSystemID
-                if (
-                    "sourceSystem" + self.flatten_separator + "id"
-                    not in organizations[subject]
-                ):
-                    # only set if not already present
-                    self.addValue(
-                        dict=organizations[subject],
-                        attribute="sourceSystem"
-                        + self.flatten_separator
-                        + "id",
-                        value=object,
-                    )
-            elif (
-                predicate == "http://nfdi4earth.de/ontology/sourceSystemURL"
-            ):  # sourceSystemURL
-                self.addValue(
-                    dict=organizations[subject],
-                    attribute="sourceSystemURL",
-                    value=object,
-                )
-            elif (
-                predicate
-                == "http://nfdi4earth.de/ontology/hasSignedCommitment"
-            ):  # hasSignedCommitment
-                self.addValue(
-                    dict=organizations[subject],
-                    attribute="hasSignedCommitment",
-                    value=object,
-                )