OneStop4All-Indexer 2.8.0.dev11__tar.gz → 2.8.0.dev12__tar.gz

{onestop4all_indexer-2.8.0.dev11 → onestop4all_indexer-2.8.0.dev12/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev11
+Version: 2.8.0.dev12
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.8.0.dev11/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev11
+Version: 2.8.0.dev12
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.8.0.dev11 → onestop4all_indexer-2.8.0.dev12}/harvesters/harvester_organization.py

@@ -1,10 +1,10 @@
-from .harvester_base import Harvester
-from data_repositories.repository_person import RepositoryPerson
-from utils import sparql
-from utils import flatten_dict
-from utils import is_truthy
 import logging
 
+from data_repositories.repository_person import RepositoryPerson
+from utils import sparql, flatten_dict, is_truthy
+
+from .harvester_base import Harvester
+
 log = logging.getLogger(__name__)
 
 
@@ -20,32 +20,37 @@ class Organization_Harvester(Harvester):
         PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
         PREFIX org: <http://www.w3.org/ns/org#>
         PREFIX schema: <http://schema.org/>
+        PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
 
         SELECT ?subject ?predicate ?object ?geo_as_wkt ?isN4EMember
         WHERE {
-            ?subject rdf:type foaf:Organization .
+            {
+                SELECT DISTINCT ?subject
+                WHERE {
+                ?subject rdf:type foaf:Organization .
+                BIND(STRAFTER(STR(?subject), "/objects/") AS ?id)
+                FILTER(STRSTARTS(?id, "n4e/"))
+                FILTER(!CONTAINS(?id, "jsonPointer"))
+                FILTER EXISTS { ?subject n4e:sourceSystem ?source }
+                }
+                OFFSET %s
+                LIMIT %s
+            }
+
             ?subject ?predicate ?object .
-            # Exclude messy data that was harvested from remote source, contains duplicates, etc
-            FILTER EXISTS { ?subject n4e:sourceSystem ?source }
+
             OPTIONAL {
                 ?subject geo:hasGeometry ?geometry .
                 ?geometry geo:asWKT ?geo_as_wkt .
             }
+
             OPTIONAL {
-                ?subject n4e:sourceSystem ?source_system .
-                OPTIONAL { ?source_system dct:title ?sourceSystem_title . }
-                OPTIONAL { ?source_system foaf:homepage ?sourceSystem_homepage . }
-            }
-            OPTIONAL {
-                ?subject rdf:type foaf:Organization ;
-                        org:hasMembership ?membership .
+                ?subject org:hasMembership ?membership .
                 ?membership org:organization ?n4eproject .
                 ?n4eproject schema:url "https://nfdi4earth.de/"^^xsd:anyURI .
                 BIND(true AS ?isN4EMember)
             }
         }
-        OFFSET %d
-        LIMIT %d
     """
 
     def __init__(self, persons_repo: RepositoryPerson, **kw):
@@ -150,13 +155,16 @@ class Organization_Harvester(Harvester):
             # set geometry if available and not already set
             if (
                 predicate == "http://www.opengis.net/ont/geosparql#hasGeometry"
-                and ("geometry" not in organizations[subject]
-                    or hit["geo_as_wkt"]["value"] not in organizations[subject]["geometry"])
+                and (
+                    "geometry" not in organizations[subject]
+                    or hit["geo_as_wkt"]["value"]
+                    not in organizations[subject]["geometry"]
+                )
             ):
                 self.addValue(
                     dict=organizations[subject],
                     attribute="geometry",
-                    value=hit["geo_as_wkt"]["value"]
+                    value=hit["geo_as_wkt"]["value"],
                 )
 
             # set membership in N4E project

{onestop4all_indexer-2.8.0.dev11 → onestop4all_indexer-2.8.0.dev12}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="OneStop4All-Indexer",
-    version="2.8.0.dev11",
+    version="2.8.0.dev12",
     description="Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index",
     author="Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer",
     author_email="m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de",