OneStop4All-Indexer 2.7.1__tar.gz → 2.8.0.dev2__tar.gz

{onestop4all_indexer-2.7.1 → onestop4all_indexer-2.8.0.dev2/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.7.1
+Version: 2.8.0.dev2
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.7.1/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.7.1
+Version: 2.8.0.dev2
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.7.1 → onestop4all_indexer-2.8.0.dev2}/harvesters/harvester_dataset.py

@@ -24,9 +24,9 @@ class Dataset_Harvester(HarvesterCordra):
         iteration_end=None,
         page_size=50000,
         query='type:"Dataset"',
-        #query='id:"n4e/dthb-oai-pangaea.de-doi-10.1594-PANGAEA.981078"', #downloadURL: https://cordra.knowledgehub.test.n4e.geo.tu-dresden.de/objects/n4e/dthb-oai-pangaea.de-doi-10.1594-PANGAEA.981078
-        #query='id:"n4e/dthb-GB_NERC_BAS_PDC_01994"', #accessURL: https://cordra.knowledgehub.test.n4e.geo.tu-dresden.de/objects/n4e/dthb-GB_NERC_BAS_PDC_01994
-        #query='id:"n4e/dthb-6A0D8B9D-1BBD-441B-BA5C-6159EE41EE71"', #multiple accessURLs: https://cordra.knowledgehub.nfdi4earth.de/objects/n4e/dthb-6A0D8B9D-1BBD-441B-BA5C-6159EE41EE71,
+        # query='id:"n4e/dthb-oai-pangaea.de-doi-10.1594-PANGAEA.981078"', #downloadURL: https://cordra.knowledgehub.test.n4e.geo.tu-dresden.de/objects/n4e/dthb-oai-pangaea.de-doi-10.1594-PANGAEA.981078
+        # query='id:"n4e/dthb-GB_NERC_BAS_PDC_01994"', #accessURL: https://cordra.knowledgehub.test.n4e.geo.tu-dresden.de/objects/n4e/dthb-GB_NERC_BAS_PDC_01994
+        # query='id:"n4e/dthb-6A0D8B9D-1BBD-441B-BA5C-6159EE41EE71"', #multiple accessURLs: https://cordra.knowledgehub.nfdi4earth.de/objects/n4e/dthb-6A0D8B9D-1BBD-441B-BA5C-6159EE41EE71,
         solr_validation=True,
         **kw,
     ):
@@ -35,9 +35,15 @@ class Dataset_Harvester(HarvesterCordra):
         self.links_repo = links_repo
         self.removed_geometries = []
         self.solr_validator = SolrValidator()
-        self.iteration_start = int(iteration_start)
-        self.iteration_end = int(iteration_end)
-        self.page_size = int(page_size)
+        self.iteration_start = (
+            int(iteration_start)
+            if iteration_start is not None
+            else iteration_start
+        )
+        self.iteration_end = (
+            int(iteration_end) if iteration_end is not None else iteration_end
+        )
+        self.page_size = int(page_size) if page_size is not None else page_size
         self.query = query
         self.solr_validation = solr_validation
         log.info("#" * 20)
@@ -277,7 +283,9 @@ class Dataset_Harvester(HarvesterCordra):
                                 if val:
                                     self.addValue(
                                         dict=datasets[subject],
-                                        attribute="distribution" + self.flatten_separator + "accessURL",
+                                        attribute="distribution"
+                                        + self.flatten_separator
+                                        + "accessURL",
                                         value=val,
                                     )
                         if "title" in distribution:
@@ -288,16 +296,23 @@ class Dataset_Harvester(HarvesterCordra):
                                 if val:
                                     self.addValue(
                                         dict=datasets[subject],
-                                        attribute="distribution" + self.flatten_separator + "title",
+                                        attribute="distribution"
+                                        + self.flatten_separator
+                                        + "title",
                                         value=val,
                                     )
                         if "downloadURL" in distribution:
+                            print(distribution["downloadURL"])
                             for download_url in distribution["downloadURL"]:
-                                val = self.get_string_from_jsonld(download_url, subject)
+                                val = self.get_string_from_jsonld(
+                                    download_url, subject
+                                )
                                 if val:
                                     self.addValue(
                                         dict=datasets[subject],
-                                        attribute="distribution" + self.flatten_separator + "downloadURL",
+                                        attribute="distribution"
+                                        + self.flatten_separator
+                                        + "downloadURL",
                                         value=val,
                                     )
                 elif key == "spatialCoverage":

{onestop4all_indexer-2.7.1 → onestop4all_indexer-2.8.0.dev2}/harvesters/harvester_repository.py

@@ -27,7 +27,7 @@ class Repository_Harvester(Harvester):
         {
             ?subject rdf:type ?type.
             ?subject ?predicate ?object.
-            FILTER (?predicate NOT IN (dct:publisher, dcat:distribution, dcat:contactPoint))
+            FILTER (?predicate NOT IN (dct:publisher, dcat:distribution, dcat:contactPoint, n4e:hasCertificate))
         }
         UNION{
             VALUES  ?predicate { dct:publisher }
@@ -44,6 +44,13 @@ class Repository_Harvester(Harvester):
             ?metadataStandard dct:title ?object.
             optional {?metadataStandard n4e:hasWebsite ?metadataStandard_homepage.}
         }
+        UNION {
+            VALUES ?predicate { n4e:hasCertificate }
+            ?subject rdf:type ?type;
+                        n4e:hasCertificate ?certificate.
+            optional { ?certificate n4e:certificateName ?certificate_name }
+            bind(coalesce(?certificate_name, ?certificate) as ?object)
+        }
         UNION {
             VALUES ?predicate { dcat:contactPoint }
             ?subject rdf:type ?type;
@@ -157,9 +164,8 @@ class Repository_Harvester(Harvester):
 
             #execute additional sparql query to determine if (some) datasets in this repository are harvested by n4e
             #key = subject
-            has_harvested_datasets = self.request_has_harvested_datasets(repo_subject=key)
-            repo["isHarvestedByNFDI4Earth"] = has_harvested_datasets 
-
+            has_harvested_datasets = self.request_has_harvested_datasets(repo_subject=key, repo=repo)
+            repo["isHarvestedByNFDI4Earth"] = has_harvested_datasets
 
         return repos_list
 
@@ -536,7 +542,7 @@ class Repository_Harvester(Harvester):
                         r["object"]["value"],
                     )
 
-    def request_has_harvested_datasets(self, repo_subject): 
+    def request_has_harvested_datasets(self, repo_subject, repo=None):
         sparql_query = """
         PREFIX n4e: <http://nfdi4earth.de/ontology/>
         select  COUNT(?s) AS ?datasetCount 
@@ -552,7 +558,13 @@ class Repository_Harvester(Harvester):
 
         if not hits or not hits[0]:
             return False
-        else:
-            count = int(hits[0]["datasetCount"]["value"])
-            has_harvested_datasets = count > 0
-            return has_harvested_datasets
+
+        # Check for Earth Data Portal repo
+        if repo is not None and (
+            repo["id"] == "n4e-service0044"
+            or "Earth Data Portal" in repo["title"]):
+            return True
+
+        count = int(hits[0]["datasetCount"]["value"])
+        has_harvested_datasets = count > 0
+        return has_harvested_datasets

{onestop4all_indexer-2.7.1 → onestop4all_indexer-2.8.0.dev2}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="OneStop4All-Indexer",
-    version="2.7.1",
+    version="2.8.0.dev2",
     description="Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index",
     author="Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer",
     author_email="m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de",

onestop4all_indexer-2.8.0.dev2/utils/configs.py

@@ -0,0 +1,89 @@
+import logging
+import os
+from jproperties import Properties
+
+log = logging.getLogger(__name__)
+
+
+app_properties = Properties()
+
+try:
+    with open("./application.properties", "rb") as config_file:
+        app_properties.load(config_file)
+except FileNotFoundError:
+    app_properties = Properties()
+    log.warning(
+        "Warnung: application.properties not found! "
+        "Using default values and environment variables only."
+    )
+
+
+def parse_bool_config(value):
+    if value.lower() in ["false", "0", "f", "no", "n"]:
+        return False
+    elif value.lower() in ["true", "1", "t", "yes", "y"]:
+        return True
+
+
+# merge env variables and application.properties
+# if env varible is set override application.properties
+config = {
+    "cordra_endpoint": os.getenv(
+        "CORDRA_ENDPOINT",
+        default=(
+            app_properties.get("cordra_endpoint").data
+            if app_properties.get("cordra_endpoint")
+            else None
+        ),
+    ),
+    "sparql_endpoint": os.getenv(
+        "SPARQL_ENDPOINT",
+        default=(
+            app_properties.get("sparql_endpoint").data
+            if app_properties.get("sparql_endpoint")
+            else None
+        ),
+    ),
+    "solr_auth": os.getenv(
+        "SOLR_AUTH",
+        default=(
+            app_properties.get("solr_auth").data
+            if app_properties.get("solr_auth")
+            else None
+        ),
+    ),
+    "solr_core": os.getenv(
+        "SOLR_CORE",
+        default=(
+            app_properties.get("solr_core").data
+            if app_properties.get("solr_core")
+            else None
+        ),
+    ),
+    "solr_url": os.getenv(
+        "SOLR_URL",
+        default=(
+            app_properties.get("solr_url").data
+            if app_properties.get("solr_url")
+            else None
+        ),
+    ),
+    "mailserver_url": os.getenv(
+        "MAILSERVER_URL",
+        default=(
+            app_properties.get("mailserver_url").data
+            if app_properties.get("mailserver_url")
+            else None
+        ),
+    ),
+    "reset_index": parse_bool_config(
+        os.getenv(
+            "RESET_INDEX",
+            default=(
+                app_properties.get("reset_index").data
+                if app_properties.get("reset_index")
+                else None
+            ),
+        )
+    ),
+}

onestop4all_indexer-2.7.1/utils/configs.py

@@ -1,44 +0,0 @@
-import os
-from jproperties import Properties
-
-app_properties = Properties()
-
-with open("./application.properties", "rb") as config_file:
-    app_properties.load(config_file)
-
-
-def parse_bool_config(value):
-    if value.lower() in ["false", "0", "f", "no", "n"]:
-        return False
-    elif value.lower() in ["true", "1", "t", "yes", "y"]:
-        return True
-
-
-# merge env variables and application.properties
-# if env varible is set override application.properties
-config = {
-    "cordra_endpoint": os.getenv(
-        "CORDRA_ENDPOINT", default=app_properties.get("cordra_endpoint").data
-    ),
-    "sparql_endpoint": os.getenv(
-        "SPARQL_ENDPOINT", default=app_properties.get("sparql_endpoint").data
-    ),
-    "solr_auth": os.getenv(
-        "SOLR_AUTH", default=app_properties.get("solr_auth").data
-    ),
-    "solr_core": os.getenv(
-        "SOLR_CORE", default=app_properties.get("solr_core").data
-    ),
-    "solr_url": os.getenv(
-        "SOLR_URL", default=app_properties.get("solr_url").data
-    ),
-    "mailserver_url": os.getenv(
-        "MAILSERVER_URL", default=app_properties.get("mailserver_url").data
-    ),
-    "reset_index": parse_bool_config(
-        os.getenv(
-            "RESET_INDEX",
-            default=app_properties.get("reset_index").data,
-        )
-    ),
-}