OneStop4All-Indexer 2.8.0.dev10__tar.gz → 2.8.0.dev12__tar.gz

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev12/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev10
+Version: 2.8.0.dev12
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
@@ -13,6 +13,7 @@ Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
 Provides-Extra: airflow
 Requires-Dist: apache-airflow==3.1.7; extra == "airflow"
+Requires-Dist: apache-airflow-providers-amazon[s3fs]; extra == "airflow"
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev12}/OneStop4All_Indexer.egg-info/requires.txt

@@ -7,3 +7,4 @@ shapely~=2.0.5
 
 [airflow]
 apache-airflow==3.1.7
+apache-airflow-providers-amazon[s3fs]

{onestop4all_indexer-2.8.0.dev10/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev10
+Version: 2.8.0.dev12
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
@@ -13,6 +13,7 @@ Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
 Provides-Extra: airflow
 Requires-Dist: apache-airflow==3.1.7; extra == "airflow"
+Requires-Dist: apache-airflow-providers-amazon[s3fs]; extra == "airflow"
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev12}/harvesters/harvester_dataset.py

@@ -299,7 +299,6 @@ class Dataset_Harvester(HarvesterCordra):
                                         value=val,
                                     )
                         if "downloadURL" in distribution:
-                            print(distribution["downloadURL"])
                             for download_url in distribution["downloadURL"]:
                                 val = self.get_string_from_jsonld(
                                     download_url, subject

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev12}/harvesters/harvester_organization.py

@@ -1,10 +1,10 @@
-from .harvester_base import Harvester
-from data_repositories.repository_person import RepositoryPerson
-from utils import sparql
-from utils import flatten_dict
-from utils import is_truthy
 import logging
 
+from data_repositories.repository_person import RepositoryPerson
+from utils import sparql, flatten_dict, is_truthy
+
+from .harvester_base import Harvester
+
 log = logging.getLogger(__name__)
 
 
@@ -20,32 +20,37 @@ class Organization_Harvester(Harvester):
         PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
         PREFIX org: <http://www.w3.org/ns/org#>
         PREFIX schema: <http://schema.org/>
+        PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
 
         SELECT ?subject ?predicate ?object ?geo_as_wkt ?isN4EMember
         WHERE {
-            ?subject rdf:type foaf:Organization .
+            {
+                SELECT DISTINCT ?subject
+                WHERE {
+                ?subject rdf:type foaf:Organization .
+                BIND(STRAFTER(STR(?subject), "/objects/") AS ?id)
+                FILTER(STRSTARTS(?id, "n4e/"))
+                FILTER(!CONTAINS(?id, "jsonPointer"))
+                FILTER EXISTS { ?subject n4e:sourceSystem ?source }
+                }
+                OFFSET %s
+                LIMIT %s
+            }
+
             ?subject ?predicate ?object .
-            # Exclude messy data that was harvested from remote source, contains duplicates, etc
-            FILTER EXISTS { ?subject n4e:sourceSystem ?source }
+
             OPTIONAL {
                 ?subject geo:hasGeometry ?geometry .
                 ?geometry geo:asWKT ?geo_as_wkt .
             }
+
             OPTIONAL {
-                ?subject n4e:sourceSystem ?source_system .
-                OPTIONAL { ?source_system dct:title ?sourceSystem_title . }
-                OPTIONAL { ?source_system foaf:homepage ?sourceSystem_homepage . }
-            }
-            OPTIONAL {
-                ?subject rdf:type foaf:Organization ;
-                        org:hasMembership ?membership .
+                ?subject org:hasMembership ?membership .
                 ?membership org:organization ?n4eproject .
                 ?n4eproject schema:url "https://nfdi4earth.de/"^^xsd:anyURI .
                 BIND(true AS ?isN4EMember)
             }
         }
-        OFFSET %d
-        LIMIT %d
     """
 
     def __init__(self, persons_repo: RepositoryPerson, **kw):
@@ -150,13 +155,16 @@ class Organization_Harvester(Harvester):
             # set geometry if available and not already set
             if (
                 predicate == "http://www.opengis.net/ont/geosparql#hasGeometry"
-                and ("geometry" not in organizations[subject]
-                    or hit["geo_as_wkt"]["value"] not in organizations[subject]["geometry"])
+                and (
+                    "geometry" not in organizations[subject]
+                    or hit["geo_as_wkt"]["value"]
+                    not in organizations[subject]["geometry"]
+                )
             ):
                 self.addValue(
                     dict=organizations[subject],
                     attribute="geometry",
-                    value=hit["geo_as_wkt"]["value"]
+                    value=hit["geo_as_wkt"]["value"],
                 )
 
             # set membership in N4E project

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev12}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="OneStop4All-Indexer",
-    version="2.8.0.dev10",
+    version="2.8.0.dev12",
     description="Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index",
     author="Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer",
     author_email="m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de",
@@ -16,7 +16,10 @@ setup(
         "shapely ~= 2.0.5",
     ],
     extras_require={
-        "airflow": ["apache-airflow==3.1.7"],
+        "airflow": [
+            "apache-airflow==3.1.7",
+            "apache-airflow-providers-amazon[s3fs]",
+        ],
     },
     include_package_data=True,
     entry_points={

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev12}/utils/solr.py

@@ -57,7 +57,9 @@ class Solr(object):
             )
             return HTTPBasicAuth(username, password)
 
-    def index_documents(self, documents: List[Dict]) -> None:
+    def index_documents(
+        self, documents: List[Dict], commit=True, ping=True
+    ) -> None:
         # solr_endpoint = coreurl(solr_url, solr_core)
         log.info(f"start indexing {len(documents)} documents")
         batch_size = 50000
@@ -87,13 +89,19 @@ class Solr(object):
             if len(batch) < batch_size:
                 break
 
-        log.info("commit changes to index")
-        self.client.ping()
-        log.info("solr healtcheck successful")
-        self.client.commit()
-        log.info("sucessfully commited changes to index")
+        if ping is True:
+            self.client.ping()
+            log.info("solr healtcheck successful")
+
+        if commit is True:
+            log.info("commit changes to index")
+            self.client.commit()
+            log.info("sucessfully commited changes to index")
         log.info("finished indexing")
 
+    def commit(self):
+        self.client.commit()
+
     def reset_index(self):
         self.client.delete(q="*:*")
         self.client.commit()