OneStop4All-Indexer 2.8.0.dev9__tar.gz → 2.8.0.dev11__tar.gz

{onestop4all_indexer-2.8.0.dev9 → onestop4all_indexer-2.8.0.dev11/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev9
+Version: 2.8.0.dev11
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
@@ -13,6 +13,7 @@ Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
 Provides-Extra: airflow
 Requires-Dist: apache-airflow==3.1.7; extra == "airflow"
+Requires-Dist: apache-airflow-providers-amazon[s3fs]; extra == "airflow"
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev9 → onestop4all_indexer-2.8.0.dev11}/OneStop4All_Indexer.egg-info/requires.txt

@@ -7,3 +7,4 @@ shapely~=2.0.5
 
 [airflow]
 apache-airflow==3.1.7
+apache-airflow-providers-amazon[s3fs]

{onestop4all_indexer-2.8.0.dev9/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev9
+Version: 2.8.0.dev11
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
@@ -13,6 +13,7 @@ Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
 Provides-Extra: airflow
 Requires-Dist: apache-airflow==3.1.7; extra == "airflow"
+Requires-Dist: apache-airflow-providers-amazon[s3fs]; extra == "airflow"
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev9 → onestop4all_indexer-2.8.0.dev11}/harvesters/harvester_dataset.py

@@ -299,7 +299,6 @@ class Dataset_Harvester(HarvesterCordra):
                                         value=val,
                                     )
                         if "downloadURL" in distribution:
-                            print(distribution["downloadURL"])
                             for download_url in distribution["downloadURL"]:
                                 val = self.get_string_from_jsonld(
                                     download_url, subject

{onestop4all_indexer-2.8.0.dev9 → onestop4all_indexer-2.8.0.dev11}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="OneStop4All-Indexer",
-    version="2.8.0.dev9",
+    version="2.8.0.dev11",
     description="Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index",
     author="Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer",
     author_email="m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de",
@@ -16,7 +16,10 @@ setup(
         "shapely ~= 2.0.5",
     ],
     extras_require={
-        "airflow": ["apache-airflow==3.1.7"],
+        "airflow": [
+            "apache-airflow==3.1.7",
+            "apache-airflow-providers-amazon[s3fs]",
+        ],
     },
     include_package_data=True,
     entry_points={

{onestop4all_indexer-2.8.0.dev9 → onestop4all_indexer-2.8.0.dev11}/utils/solr.py

@@ -1,8 +1,9 @@
 import logging
 
-from pysolr import Solr as SolrBase
+from pysolr import Solr as SolrClient
 from requests.auth import HTTPBasicAuth
 from typing import List, Dict, Optional, Literal
+from urllib.parse import urljoin
 
 from utils import config
 
@@ -21,7 +22,7 @@ class Solr(object):
         self.solr_url = solr_url
         self.solr_core = solr_core
         self.auth = solr_auth
-        self.solr = SolrBase(
+        self.client = SolrClient(
             self.endpoint,
             auth=self.authentication,
             always_commit=always_commit,
@@ -34,8 +35,17 @@ class Solr(object):
         # overwrite with initially given values
         # TODO: check if endpoint is reachable, if not raise error
         solr_url = self.solr_url if self.solr_url else config["solr_url"]
+        log.debug(f"configured solr url: {solr_url}")
+        if solr_url.startswith("http://"):
+            raise ValueError(
+                "Insecure solr url configured. "
+                "Please check your configuration and use https."
+            )
         solr_core = self.solr_core if self.solr_core else config["solr_core"]
-        return f"{solr_url.rstrip('/')}/{solr_core}"
+        log.debug(f"configured solr core: {solr_core}")
+        _endpoint = urljoin(solr_url, solr_core)
+        log.info(f"initialized solr client with endpoint: {_endpoint}")
+        return _endpoint
 
     @property
     def authentication(self):
@@ -47,12 +57,11 @@ class Solr(object):
             )
             return HTTPBasicAuth(username, password)
 
-    def index_documents(self, documents: List[Dict]) -> None:
+    def index_documents(
+        self, documents: List[Dict], commit=True, ping=True
+    ) -> None:
         # solr_endpoint = coreurl(solr_url, solr_core)
-        log.info(
-            f"start indexing {len(documents)} documents to {self.endpoint}"
-        )
-
+        log.info(f"start indexing {len(documents)} documents")
         batch_size = 50000
         offset = 0
         iteration = 0
@@ -65,11 +74,11 @@ class Solr(object):
             if len(batch) == 0:
                 break
 
-            self.solr.ping()
+            self.client.ping()
             log.info("solr healtcheck successful")
             try:
                 log.info(f"start adding batch of {len(batch)} documents")
-                self.solr.add(batch)
+                self.client.add(batch)
                 log.info(f"finished adding batch of {len(batch)} documents")
             except Exception as e:
                 log.error(e)
@@ -80,17 +89,33 @@ class Solr(object):
             if len(batch) < batch_size:
                 break
 
-        log.info("commit changes to index")
-        self.solr.ping()
-        log.info("solr healtcheck successful")
-        self.solr.commit()
-        log.info("sucessfully commited changes to index")
+        if ping is True:
+            self.client.ping()
+            log.info("solr healtcheck successful")
+
+        if commit is True:
+            log.info("commit changes to index")
+            self.client.commit()
+            log.info("sucessfully commited changes to index")
         log.info("finished indexing")
 
+    def commit(self):
+        self.client.commit()
+
     def reset_index(self):
-        log.info(f"reset index for: {self.endpoint}")
-        self.solr.delete(q="*:*")
-        self.solr.commit()
+        self.client.delete(q="*:*")
+        self.client.commit()
+
+    def exists(self, document_id: str) -> bool:
+        result = self.search(f"id:{document_id}")
+        return result.hits > 0
+
+    def search(self, *args, **kwargs):
+        return self.client.search(*args, **kwargs)
+
+    def delete(self, *args, **kwargs):
+        self.client.delete(*args, **kwargs)
+        self.client.commit()
 
 
 class SolrValidator(Solr):
@@ -105,7 +130,7 @@ class SolrValidator(Solr):
         }
 
         try:
-            self.solr.add(dataset)
+            self.client.add(dataset)
             return True
         except Exception as e:
             log.error(e)
@@ -114,5 +139,5 @@ class SolrValidator(Solr):
     def close(self):
         # technically not necessary to check existence of test document,
         # but when urls are misconfigured, this will throw an error
-        if self.solr.search("id:geomValidationTest").hits:
-            self.solr.delete(q="id:geomValidationTest")
+        if self.exists("geomValidationTest"):
+            self.delete(id="geomValidationTest")