OneStop4All-Indexer 2.8.0.dev6__tar.gz → 2.8.0.dev8__tar.gz

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,19 +1,16 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev6
+Version: 2.8.0.dev8
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
 License-File: LICENSE
 Requires-Dist: click
 Requires-Dist: sparqlwrapper~=2.0.0
-Requires-Dist: pysolr>=3.11.0
+Requires-Dist: pysolr~=3.11.0
 Requires-Dist: jproperties~=2.1.1
 Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
-Requires-Dist: sentence-transformers~=5.1.0
-Requires-Dist: qdrant-client~=1.15.1
-Requires-Dist: xformers
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8}/OneStop4All_Indexer.egg-info/SOURCES.txt

@@ -28,9 +28,7 @@ harvesters/harvester_softwaresourcecode.py
 utils/__init__.py
 utils/cli.py
 utils/configs.py
-utils/embeddings.py
 utils/harvest.py
-utils/qdrant.py
 utils/solr.py
 utils/sparql.py
 utils/util.py

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8}/OneStop4All_Indexer.egg-info/requires.txt

@@ -1,9 +1,6 @@
 click
 sparqlwrapper~=2.0.0
-pysolr>=3.11.0
+pysolr~=3.11.0
 jproperties~=2.1.1
 geomet~=1.1.0
 shapely~=2.0.5
-sentence-transformers~=5.1.0
-qdrant-client~=1.15.1
-xformers

{onestop4all_indexer-2.8.0.dev6/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev8}/PKG-INFO

@@ -1,19 +1,16 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev6
+Version: 2.8.0.dev8
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
 License-File: LICENSE
 Requires-Dist: click
 Requires-Dist: sparqlwrapper~=2.0.0
-Requires-Dist: pysolr>=3.11.0
+Requires-Dist: pysolr~=3.11.0
 Requires-Dist: jproperties~=2.1.1
 Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
-Requires-Dist: sentence-transformers~=5.1.0
-Requires-Dist: qdrant-client~=1.15.1
-Requires-Dist: xformers
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8}/harvesters/harvester_dataservice.py

@@ -1,6 +1,4 @@
 import logging
-import json
-import os
 from .harvester_base import Harvester
 from data_repositories.repository_theme import RepositoryTheme
 from data_repositories import RepositoryPerson
@@ -78,8 +76,6 @@ class DataService_Harvester(Harvester):
         self.solr_validator = SolrValidator()
         self.solr_validation = solr_validation
         self.removed_geometries = []
-        self.load_from_file = os.getenv("DS_LOAD_FROM_PATH", None)
-        self.save_to_file = os.getenv("DS_SAVE_TO_PATH", None)
         self.iteration_start = (
             int(iteration_start) if iteration_start is not None else 0
         )
@@ -96,13 +92,6 @@ iter_end={self.iteration_end}"""
         )
 
     def harvest(self):
-
-        if self.load_from_file is not None:
-            log.info(f"Loading DataServices from file {self.load_from_file}")
-            with open(self.load_from_file, "r", encoding="utf-8") as f:
-                services_list = json.load(f)
-                return services_list
-
         limit = self.page_size
         # convert to list of repo documents for indexing
         services = {}  # repos dict
@@ -111,7 +100,7 @@ iter_end={self.iteration_end}"""
         hits = {}
         # split sparql query
         while True:
-            if self.iteration_end is not None and i >= self.iteration_end:
+            if self.iteration_end is not None and i > self.iteration_end:
                 break
 
             query_splitted = self.sparql_query % (limit * i, limit)
@@ -140,11 +129,6 @@ iter_end={self.iteration_end}"""
             service["mainTitle"] = service["mainTitle"].strip()
             services_list.append(service)
 
-        if self.save_to_file is not None:
-            log.info(f"Saving DataServices to file {self.save_to_file}")
-            with open(self.save_to_file, "w", encoding="utf-8") as f:
-                json.dump(services_list, f, ensure_ascii=False, indent=2)
-
         return services_list
 
     def parse_response(self, hits, services):

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8}/harvesters/harvester_dataset.py

@@ -24,9 +24,6 @@ class Dataset_Harvester(HarvesterCordra):
         iteration_end=None,
         page_size=50000,
         query='type:"Dataset"',
-        # query='id:"n4e/dthb-oai-pangaea.de-doi-10.1594-PANGAEA.981078"', #downloadURL: https://cordra.knowledgehub.test.n4e.geo.tu-dresden.de/objects/n4e/dthb-oai-pangaea.de-doi-10.1594-PANGAEA.981078
-        # query='id:"n4e/dthb-GB_NERC_BAS_PDC_01994"', #accessURL: https://cordra.knowledgehub.test.n4e.geo.tu-dresden.de/objects/n4e/dthb-GB_NERC_BAS_PDC_01994
-        # query='id:"n4e/dthb-6A0D8B9D-1BBD-441B-BA5C-6159EE41EE71"', #multiple accessURLs: https://cordra.knowledgehub.nfdi4earth.de/objects/n4e/dthb-6A0D8B9D-1BBD-441B-BA5C-6159EE41EE71,
         solr_validation=True,
         **kw,
     ):
@@ -302,6 +299,7 @@ class Dataset_Harvester(HarvesterCordra):
                                         value=val,
                                     )
                         if "downloadURL" in distribution:
+                            print(distribution["downloadURL"])
                             for download_url in distribution["downloadURL"]:
                                 val = self.get_string_from_jsonld(
                                     download_url, subject

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8}/harvesters/harvester_organization.py

@@ -149,11 +149,15 @@ class Organization_Harvester(Harvester):
 
             # set geometry if available and not already set
             if (
-                "geo_as_wkt" in hit
-                and hit["geo_as_wkt"]["value"]
-                and "geometry" not in organizations[subject]
+                predicate == "http://www.opengis.net/ont/geosparql#hasGeometry"
+                and ("geometry" not in organizations[subject]
+                    or hit["geo_as_wkt"]["value"] not in organizations[subject]["geometry"])
             ):
-                organizations[subject]["geometry"] = hit["geo_as_wkt"]["value"]
+                self.addValue(
+                    dict=organizations[subject],
+                    attribute="geometry",
+                    value=hit["geo_as_wkt"]["value"]
+                )
 
             # set membership in N4E project
             is_n4e_member = hit.get("isN4EMember", {}).get("value", None)

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="OneStop4All-Indexer",
-    version="2.8.0.dev6",
+    version="2.8.0.dev8",
     description="Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index",
     author="Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer",
     author_email="m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de",
@@ -10,13 +10,10 @@ setup(
     install_requires=[
         "click",
         "sparqlwrapper~= 2.0.0",
-        "pysolr>= 3.11.0",
+        "pysolr~= 3.11.0",
         "jproperties~= 2.1.1 ",
         "geomet ~= 1.1.0",
         "shapely ~= 2.0.5",
-        "sentence-transformers ~= 5.1.0",
-        "qdrant-client ~= 1.15.1",
-        "xformers",
     ],
     include_package_data=True,
     entry_points={

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8}/utils/configs.py

@@ -88,20 +88,4 @@ config = {
             ),
         )
     ),
-    "qdrant_url": os.getenv(
-        "QDRANT_URL",
-        default=(
-            app_properties.get("qdrant_url").data
-            if app_properties.get("qdrant_url")
-            else None
-        ),
-    ),
-    "qdrant_api_key": os.getenv(
-        "QDRANT_API_KEY",
-        default=(
-            app_properties.get("qdrant_api_key").data
-            if app_properties.get("qdrant_api_key")
-            else None
-        ),
-    ),
 }

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8}/utils/harvest.py

@@ -25,8 +25,6 @@ from harvesters import (
     Service_Harvester,
 )
 from utils import config, Solr
-from utils import embeddings
-from utils.qdrant import Qdrant
 
 log = logging.getLogger(__name__)
 
@@ -121,7 +119,6 @@ def run(
     **further_options,
 ):
     solr = Solr()
-    qdrant = Qdrant()  # vector db
 
     if reset_index:
         solr.reset_index()
@@ -141,29 +138,12 @@ def run(
             documents = harvester.harvest()
             elapsed_time = timeit.default_timer() - start_time
 
-            try:
-                generate_embeddings(documents)
-            except Exception as e:
-                log.error(e)
-
-            log.info(f"index harvested {len(documents)} documents (vector db)")
-            qdrant.index_documents(documents)
-            log.info(
-                f"finished indexing for harvester {i} of {len(harvesters)}) (vector db)"
-            )
-
-            # remove embeddings from document, otherwise embeddings would be indexed in solr
-            # comment loop if embeddings should be indexed in solr as well.
-            for document in documents:
-                document.pop("embedding_384", None)
-
             log.info(f"index harvested {len(documents)} documents")
             solr.index_documents(documents)
             document_count += len(documents)
             log.info(
                 f"finished indexing for harvester {i} of {len(harvesters)})"
             )
-
             stats[harvester.get_type()] = {
                 "document_count": len(documents),
                 "processing_time": round(elapsed_time, 4),
@@ -190,12 +170,5 @@ def run(
         )
 
 
-def generate_embeddings(documents):
-    log.info("create embeddings for " + str(len(documents)) + " documents")
-    for document in documents:
-        embeddings.add_embedding(document)
-    log.info("created embeddings for batch")
-
-
 if __name__ == "__main__":
     run()

{onestop4all_indexer-2.8.0.dev6 → onestop4all_indexer-2.8.0.dev8}/utils/solr.py

@@ -3,7 +3,6 @@ import logging
 from pysolr import Solr as SolrBase
 from requests.auth import HTTPBasicAuth
 from typing import List, Dict, Optional, Literal
-from urllib.parse import urljoin
 
 from utils import config
 
@@ -33,9 +32,10 @@ class Solr(object):
     def endpoint(self):
         # using config-values (by default) OR
         # overwrite with initially given values
+        # TODO: check if endpoint is reachable, if not raise error
         solr_url = self.solr_url if self.solr_url else config["solr_url"]
         solr_core = self.solr_core if self.solr_core else config["solr_core"]
-        return urljoin(solr_url, solr_core)
+        return f"{solr_url.rstrip('/')}/{solr_core}"
 
     @property
     def authentication(self):
@@ -112,4 +112,7 @@ class SolrValidator(Solr):
             return False
 
     def close(self):
-        self.solr.delete({"id": "geomValidationTest"})
+        # technically not necessary to check existence of test document,
+        # but when urls are misconfigured, this will throw an error
+        if self.solr.search("id:geomValidationTest").hits:
+            self.solr.delete(q="id:geomValidationTest")

onestop4all_indexer-2.8.0.dev6/utils/embeddings.py

@@ -1,67 +0,0 @@
-from sentence_transformers import SentenceTransformer
-import logging
-
-log = logging.getLogger(__name__)
-
-model_384 = SentenceTransformer(
-    "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
-)
-# model_768 = SentenceTransformer("sentence-transformers/paraphrase-multilingual-mpnet-base-v2")
-# model_1024 = SentenceTransformer("Qwen/Qwen3-Embedding-0.6B")
-
-
-def add_embedding(document):
-    if document["type"] == "person_nested" or "person_nested" in ",".join(
-        document
-    ):  # no embeddings for nested author documents
-        return
-
-    try:
-        emb_str = get_entity_card(document)
-    except Exception as e:
-        log.error(e)
-        log.error(document)
-    log.info(emb_str if emb_str.isprintable() else document["id"])
-    try:
-        if emb_str is not None:
-            document["embedding_384"] = model_384.encode(emb_str).tolist()
-            # document["embedding_768"] = model_768.encode(emb_str).tolist()
-            # document["embedding_1024"] = model_1024.encode(emb_str).tolist()
-    except Exception as e:
-        log.error(e)
-
-
-def get_entity_card(document):
-    text = document["mainTitle"]
-    if "description" in document and len(document["description"]) > 0:
-        text += "\n" + "\n".join(document["description"])
-    if "keyword" in document and len(document["keyword"]) > 0:
-        text += "\n" + ",".join(document["keyword"])
-    elif "keywords" in document and len(document["keywords"]) > 0:
-        text += "\n" + ",".join(document["keywords"])
-
-    if (
-        "http://xmlns.com/foaf/0.1/Organization" in document["type"]
-    ):  # special attributes for organizations
-        if "altLabel" in document:
-            text += "\n" + ",".join(document["altLabel"])
-        if "countryName" in document:
-            text += "\n" + "country: " + ",".join(document["countryName"])
-        if "locality" in document:
-            text += "\n" + "location: " + ",".join(document["locality"])
-
-    return text
-
-
-def get_type_str(
-    document,
-):  # might need improvement with proper mapping (aligned with OS4A)
-    type_uri = document["type"]
-    if isinstance(type_uri, str):
-        type_val = type_uri.strip().lstrip("/").split("/")[-1]
-    elif isinstance(type_uri, list):
-        uris = []
-        for uri in type_uri:
-            uris.append(uri.strip().lstrip("/").split("/")[-1])
-        type_val = ",".join(uris)
-    return type_val

onestop4all_indexer-2.8.0.dev6/utils/qdrant.py

@@ -1,97 +0,0 @@
-import datetime
-import logging
-import uuid
-
-from qdrant_client import QdrantClient, models
-from typing import List, Dict
-
-from utils import config
-
-log = logging.getLogger(__name__)
-
-
-class Qdrant:
-
-    def __init__(self):
-        self.endpoint = config["qdrant_url"]
-        self.client = QdrantClient(self.endpoint, api_key=config["qdrant_url"])
-        log.debug(f"Initialized Qdrant client with endpoint {self.endpoint}")
-        self.collections = [
-            {"dim": 384, "name": "embedding_384"}
-            # {"dim": 768, "name": "embedding_768"},
-            # {"dim": 1024, "name": "embedding_1024"}
-        ]
-
-        self.init_collections()
-
-    def init_collections(self):
-        for collection in self.collections:
-            self.init_collection(collection["dim"], collection["name"])
-
-    def init_collection(self, vector_dim, collection_name):
-        collections = self.client.get_collections()  # existing collections
-        collection_names = [col.name for col in collections.collections]
-
-        try:
-            if (
-                collection_name not in collection_names
-            ):  # only create if not existing
-                self.client.create_collection(
-                    collection_name=collection_name,
-                    vectors_config=models.VectorParams(
-                        size=vector_dim, distance=models.Distance.COSINE
-                    ),
-                )
-                self.client.create_payload_index(
-                    collection_name=collection_name,
-                    field_name="id",
-                    field_schema="keyword",
-                )
-        except Exception as e:
-            log.error(e)
-
-    def index_documents(self, documents):
-        for collection in self.collections:
-            self.index_documents_dim(
-                documents,
-                collection_name=collection["name"],
-                embedding_key=collection["name"],
-            )
-
-    def index_documents_dim(
-        self, documents: List[Dict], collection_name, embedding_key
-    ) -> None:
-        log.info(
-            f"start indexing {len(documents)} documents to {self.endpoint}"
-        )
-
-        embeddings = []
-        payloads = []
-        ids = []
-
-        try:
-            for document in documents:
-                ids.append(
-                    str(uuid.uuid5(uuid.NAMESPACE_DNS, document["id"]))
-                )  # generates always the same uuid for the same document id
-                embeddings.append(document[str(embedding_key)])
-                payload = {
-                    "id": document["id"],
-                    "mainTitle": document["mainTitle"],
-                    "type": document["type"],
-                    "indexedAt": datetime.datetime.now(datetime.timezone.utc),
-                }
-                if "description" in document:
-                    payload["description"] = document["description"]
-                if "keywords" in document:
-                    payload["keyword"] = document["keyword"]
-                payloads.append(payload)
-
-            self.client.upload_collection(
-                collection_name=collection_name,
-                vectors=embeddings,
-                ids=ids,
-                payload=payloads,
-            )
-        except Exception as e:
-            log.error(e)