OneStop4All-Indexer 2.8.0.dev9__tar.gz → 2.8.0.dev10__tar.gz

{onestop4all_indexer-2.8.0.dev9 → onestop4all_indexer-2.8.0.dev10/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev9
+Version: 2.8.0.dev10
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.8.0.dev9/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev9
+Version: 2.8.0.dev10
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.8.0.dev9 → onestop4all_indexer-2.8.0.dev10}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="OneStop4All-Indexer",
-    version="2.8.0.dev9",
+    version="2.8.0.dev10",
     description="Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index",
     author="Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer",
     author_email="m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de",

{onestop4all_indexer-2.8.0.dev9 → onestop4all_indexer-2.8.0.dev10}/utils/solr.py

@@ -1,8 +1,9 @@
 import logging
 
-from pysolr import Solr as SolrBase
+from pysolr import Solr as SolrClient
 from requests.auth import HTTPBasicAuth
 from typing import List, Dict, Optional, Literal
+from urllib.parse import urljoin
 
 from utils import config
 
@@ -21,7 +22,7 @@ class Solr(object):
         self.solr_url = solr_url
         self.solr_core = solr_core
         self.auth = solr_auth
-        self.solr = SolrBase(
+        self.client = SolrClient(
             self.endpoint,
             auth=self.authentication,
             always_commit=always_commit,
@@ -34,8 +35,17 @@ class Solr(object):
         # overwrite with initially given values
         # TODO: check if endpoint is reachable, if not raise error
         solr_url = self.solr_url if self.solr_url else config["solr_url"]
+        log.debug(f"configured solr url: {solr_url}")
+        if solr_url.startswith("http://"):
+            raise ValueError(
+                "Insecure solr url configured. "
+                "Please check your configuration and use https."
+            )
         solr_core = self.solr_core if self.solr_core else config["solr_core"]
-        return f"{solr_url.rstrip('/')}/{solr_core}"
+        log.debug(f"configured solr core: {solr_core}")
+        _endpoint = urljoin(solr_url, solr_core)
+        log.info(f"initialized solr client with endpoint: {_endpoint}")
+        return _endpoint
 
     @property
     def authentication(self):
@@ -49,10 +59,7 @@ class Solr(object):
 
     def index_documents(self, documents: List[Dict]) -> None:
         # solr_endpoint = coreurl(solr_url, solr_core)
-        log.info(
-            f"start indexing {len(documents)} documents to {self.endpoint}"
-        )
-
+        log.info(f"start indexing {len(documents)} documents")
         batch_size = 50000
         offset = 0
         iteration = 0
@@ -65,11 +72,11 @@ class Solr(object):
             if len(batch) == 0:
                 break
 
-            self.solr.ping()
+            self.client.ping()
             log.info("solr healtcheck successful")
             try:
                 log.info(f"start adding batch of {len(batch)} documents")
-                self.solr.add(batch)
+                self.client.add(batch)
                 log.info(f"finished adding batch of {len(batch)} documents")
             except Exception as e:
                 log.error(e)
@@ -81,16 +88,26 @@ class Solr(object):
                 break
 
         log.info("commit changes to index")
-        self.solr.ping()
+        self.client.ping()
         log.info("solr healtcheck successful")
-        self.solr.commit()
+        self.client.commit()
         log.info("sucessfully commited changes to index")
         log.info("finished indexing")
 
     def reset_index(self):
-        log.info(f"reset index for: {self.endpoint}")
-        self.solr.delete(q="*:*")
-        self.solr.commit()
+        self.client.delete(q="*:*")
+        self.client.commit()
+
+    def exists(self, document_id: str) -> bool:
+        result = self.search(f"id:{document_id}")
+        return result.hits > 0
+
+    def search(self, *args, **kwargs):
+        return self.client.search(*args, **kwargs)
+
+    def delete(self, *args, **kwargs):
+        self.client.delete(*args, **kwargs)
+        self.client.commit()
 
 
 class SolrValidator(Solr):
@@ -105,7 +122,7 @@ class SolrValidator(Solr):
         }
 
         try:
-            self.solr.add(dataset)
+            self.client.add(dataset)
             return True
         except Exception as e:
             log.error(e)
@@ -114,5 +131,5 @@ class SolrValidator(Solr):
     def close(self):
         # technically not necessary to check existence of test document,
         # but when urls are misconfigured, this will throw an error
-        if self.solr.search("id:geomValidationTest").hits:
-            self.solr.delete(q="id:geomValidationTest")
+        if self.exists("geomValidationTest"):
+            self.delete(id="geomValidationTest")