OneStop4All-Indexer 2.8.0.dev3__tar.gz → 2.8.0.dev5__tar.gz

{onestop4all_indexer-2.8.0.dev3 → onestop4all_indexer-2.8.0.dev5/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,16 +1,19 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev3
+Version: 2.8.0.dev5
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
 License-File: LICENSE
 Requires-Dist: click
 Requires-Dist: sparqlwrapper~=2.0.0
-Requires-Dist: pysolr~=3.9.0
+Requires-Dist: pysolr>=3.11.0
 Requires-Dist: jproperties~=2.1.1
 Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
+Requires-Dist: sentence-transformers~=5.1.0
+Requires-Dist: qdrant-client~=1.15.1
+Requires-Dist: xformers
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev3 → onestop4all_indexer-2.8.0.dev5}/OneStop4All_Indexer.egg-info/SOURCES.txt

@@ -28,7 +28,9 @@ harvesters/harvester_softwaresourcecode.py
 utils/__init__.py
 utils/cli.py
 utils/configs.py
+utils/embeddings.py
 utils/harvest.py
+utils/qdrant.py
 utils/solr.py
 utils/sparql.py
 utils/util.py

{onestop4all_indexer-2.8.0.dev3 → onestop4all_indexer-2.8.0.dev5}/OneStop4All_Indexer.egg-info/requires.txt

@@ -1,6 +1,9 @@
 click
 sparqlwrapper~=2.0.0
-pysolr~=3.9.0
+pysolr>=3.11.0
 jproperties~=2.1.1
 geomet~=1.1.0
 shapely~=2.0.5
+sentence-transformers~=5.1.0
+qdrant-client~=1.15.1
+xformers

{onestop4all_indexer-2.8.0.dev3/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev5}/PKG-INFO

@@ -1,16 +1,19 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev3
+Version: 2.8.0.dev5
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
 License-File: LICENSE
 Requires-Dist: click
 Requires-Dist: sparqlwrapper~=2.0.0
-Requires-Dist: pysolr~=3.9.0
+Requires-Dist: pysolr>=3.11.0
 Requires-Dist: jproperties~=2.1.1
 Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
+Requires-Dist: sentence-transformers~=5.1.0
+Requires-Dist: qdrant-client~=1.15.1
+Requires-Dist: xformers
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev3 → onestop4all_indexer-2.8.0.dev5}/harvesters/harvester_dataservice.py

@@ -1,4 +1,5 @@
 import logging
+import json
 from .harvester_base import Harvester
 from data_repositories.repository_theme import RepositoryTheme
 from data_repositories import RepositoryPerson
@@ -68,6 +69,10 @@ class DataService_Harvester(Harvester):
         iteration_start=0,
         iteration_end=None,
         page_size=10000,
+        load_from_file="/tmp/dataservices.json",
+        # load_from_file=None,
+        # save_to_file="/tmp/dataservices.json",
+        save_to_file=None,
         **kw,
     ):
         super().__init__(**kw)
@@ -76,6 +81,8 @@ class DataService_Harvester(Harvester):
         self.solr_validator = SolrValidator()
         self.solr_validation = solr_validation
         self.removed_geometries = []
+        self.load_from_file = load_from_file
+        self.save_to_file = save_to_file
         self.iteration_start = (
             int(iteration_start) if iteration_start is not None else 0
         )
@@ -92,6 +99,13 @@ iter_end={self.iteration_end}"""
         )
 
     def harvest(self):
+
+        if self.load_from_file is not None:
+            log.info(f"Loading DataServices from file {self.load_from_file}")
+            with open(self.load_from_file, "r", encoding="utf-8") as f:
+                services_list = json.load(f)
+                return services_list
+
         limit = self.page_size
         # convert to list of repo documents for indexing
         services = {}  # repos dict
@@ -129,6 +143,11 @@ iter_end={self.iteration_end}"""
             service["mainTitle"] = service["mainTitle"].strip()
             services_list.append(service)
 
+        if self.save_to_file is not None:
+            log.info(f"Saving DataServices to file {self.save_to_file}")
+            with open(self.save_to_file, "w", encoding="utf-8") as f:
+                json.dump(services_list, f, ensure_ascii=False, indent=2)
+
         return services_list
 
     def parse_response(self, hits, services):

{onestop4all_indexer-2.8.0.dev3 → onestop4all_indexer-2.8.0.dev5}/harvesters/harvester_dataset.py

@@ -302,7 +302,6 @@ class Dataset_Harvester(HarvesterCordra):
                                         value=val,
                                     )
                         if "downloadURL" in distribution:
-                            print(distribution["downloadURL"])
                             for download_url in distribution["downloadURL"]:
                                 val = self.get_string_from_jsonld(
                                     download_url, subject

{onestop4all_indexer-2.8.0.dev3 → onestop4all_indexer-2.8.0.dev5}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="OneStop4All-Indexer",
-    version="2.8.0.dev3",
+    version="2.8.0.dev5",
     description="Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index",
     author="Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer",
     author_email="m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de",
@@ -10,10 +10,13 @@ setup(
     install_requires=[
         "click",
         "sparqlwrapper~= 2.0.0",
-        "pysolr~= 3.9.0",
+        "pysolr>= 3.11.0",
         "jproperties~= 2.1.1 ",
         "geomet ~= 1.1.0",
         "shapely ~= 2.0.5",
+        "sentence-transformers ~= 5.1.0",
+        "qdrant-client ~= 1.15.1",
+        "xformers",
     ],
     include_package_data=True,
     entry_points={

{onestop4all_indexer-2.8.0.dev3 → onestop4all_indexer-2.8.0.dev5}/utils/configs.py

@@ -88,4 +88,20 @@ config = {
             ),
         )
     ),
+    "qdrant_url": os.getenv(
+        "QDRANT_URL",
+        default=(
+            app_properties.get("qdrant_url").data
+            if app_properties.get("qdrant_url")
+            else None
+        ),
+    ),
+    "qdrant_api_key": os.getenv(
+        "QDRANT_API_KEY",
+        default=(
+            app_properties.get("qdrant_api_key").data
+            if app_properties.get("qdrant_api_key")
+            else None
+        ),
+    ),
 }

onestop4all_indexer-2.8.0.dev5/utils/embeddings.py

@@ -0,0 +1,67 @@
+from sentence_transformers import SentenceTransformer
+import logging
+
+log = logging.getLogger(__name__)
+
+model_384 = SentenceTransformer(
+    "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+)
+# model_768 = SentenceTransformer("sentence-transformers/paraphrase-multilingual-mpnet-base-v2")
+# model_1024 = SentenceTransformer("Qwen/Qwen3-Embedding-0.6B")
+
+
+def add_embedding(document):
+    if document["type"] == "person_nested" or "person_nested" in ",".join(
+        document
+    ):  # no embeddings for nested author documents
+        return
+
+    try:
+        emb_str = get_entity_card(document)
+    except Exception as e:
+        log.error(e)
+        log.error(document)
+    log.info(emb_str if emb_str.isprintable() else document["id"])
+    try:
+        if emb_str is not None:
+            document["embedding_384"] = model_384.encode(emb_str).tolist()
+            # document["embedding_768"] = model_768.encode(emb_str).tolist()
+            # document["embedding_1024"] = model_1024.encode(emb_str).tolist()
+    except Exception as e:
+        log.error(e)
+
+
+def get_entity_card(document):
+    text = document["mainTitle"]
+    if "description" in document and len(document["description"]) > 0:
+        text += "\n" + "\n".join(document["description"])
+    if "keyword" in document and len(document["keyword"]) > 0:
+        text += "\n" + ",".join(document["keyword"])
+    elif "keywords" in document and len(document["keywords"]) > 0:
+        text += "\n" + ",".join(document["keywords"])
+
+    if (
+        "http://xmlns.com/foaf/0.1/Organization" in document["type"]
+    ):  # special attributes for organizations
+        if "altLabel" in document:
+            text += "\n" + ",".join(document["altLabel"])
+        if "countryName" in document:
+            text += "\n" + "country: " + ",".join(document["countryName"])
+        if "locality" in document:
+            text += "\n" + "location: " + ",".join(document["locality"])
+
+    return text
+
+
+def get_type_str(
+    document,
+):  # might need improvement with proper mapping (aligned with OS4A)
+    type_uri = document["type"]
+    if isinstance(type_uri, str):
+        type_val = type_uri.strip().lstrip("/").split("/")[-1]
+    elif isinstance(type_uri, list):
+        uris = []
+        for uri in type_uri:
+            uris.append(uri.strip().lstrip("/").split("/")[-1])
+        type_val = ",".join(uris)
+    return type_val

onestop4all_indexer-2.8.0.dev5/utils/harvest.py

@@ -0,0 +1,201 @@
+import logging
+import requests
+import timeit
+import traceback
+
+from socket import gethostname
+from typing import Optional
+
+from data_repositories import (
+    RepositoryTheme,
+    RepositoryPerson,
+    RepositoryResourceLinks,
+    RepositoryN4EOrganization,
+)
+from harvesters import (
+    Repository_Harvester,
+    Organization_Harvester,
+    Article_Harvester,
+    Softwaresourcecode_Harvester,
+    Learningresource_Harvester,
+    Metadatastandard_Harvester,
+    Document_Harvester,
+    Dataset_Harvester,
+    DataService_Harvester,
+    Service_Harvester,
+)
+from utils import config, Solr
+from utils import embeddings
+from utils.qdrant import Qdrant
+
+log = logging.getLogger(__name__)
+
+
+def get_harvester(
+    harvester_name=None,
+    dataset_options={},
+    dataservice_options={},
+):
+
+    # Lazy initialization of repository singletons
+    # Only one instance per repository type will be created and reused
+    repo_singletons = {}
+    repo_classes = {
+        "themes_repo": RepositoryTheme,
+        "persons_repo": RepositoryPerson,
+        "links_repo": RepositoryResourceLinks,
+        "n4e_orgas_repo": RepositoryN4EOrganization,
+    }
+
+    def get_repo(repo_name):
+        # Returns a singleton instance of the requested repository.
+        # Instantiates the repository only on first access.
+        if repo_name not in repo_singletons:
+            repo_singletons[repo_name] = repo_classes[repo_name]()
+        return repo_singletons[repo_name]
+
+    # Mapping from harvester name to factory function
+    # Each factory uses get_repo to ensure repositories are only created once
+    harvester_factories = {
+        "Service": lambda: Service_Harvester(get_repo("n4e_orgas_repo")),
+        "DataService": lambda: DataService_Harvester(
+            get_repo("themes_repo"),
+            get_repo("persons_repo"),
+            **dataservice_options,
+        ),
+        "Repository": lambda: Repository_Harvester(
+            get_repo("themes_repo"),
+            get_repo("n4e_orgas_repo"),
+        ),
+        "Organization": lambda: Organization_Harvester(
+            get_repo("persons_repo")
+        ),
+        "Article": lambda: Article_Harvester(
+            get_repo("persons_repo"),
+            get_repo("links_repo"),
+            get_repo("themes_repo"),
+        ),
+        "Softwaresourcecode": lambda: Softwaresourcecode_Harvester(
+            get_repo("persons_repo"),
+            get_repo("links_repo"),
+        ),
+        "Learningresource": lambda: Learningresource_Harvester(
+            get_repo("persons_repo"),
+            get_repo("themes_repo"),
+        ),
+        "Metadatastandard": lambda: Metadatastandard_Harvester(
+            get_repo("themes_repo")
+        ),
+        "Document": lambda: Document_Harvester(
+            get_repo("persons_repo"),
+            get_repo("links_repo"),
+        ),
+        "Dataset": lambda: Dataset_Harvester(
+            get_repo("persons_repo"),
+            get_repo("links_repo"),
+            **dataset_options,
+        ),
+    }
+
+    # If no harvester_name is given, instantiate all harvesters
+    if not harvester_name:
+        return [factory() for factory in harvester_factories.values()]
+
+    # If a name or list/tuple of names is given, instantiate only those
+    # This keeps compatibility with previous behavior
+    if isinstance(harvester_name, (list, tuple)):
+        names = harvester_name
+    else:
+        names = [harvester_name]
+
+    return [
+        harvester_factories[name]()
+        for name in harvester_factories
+        if name in names
+    ]
+
+
+def run(
+    requested_harvester: Optional[tuple] = None,
+    reset_index: Optional[bool] = None,
+    **further_options,
+):
+    solr = Solr()
+    qdrant = Qdrant()  # vector db
+
+    if reset_index:
+        solr.reset_index()
+
+    stats = {}
+
+    i = 1
+    harvesters = get_harvester(
+        harvester_name=requested_harvester, **further_options
+    )
+    log.info(f"Initialized {len(harvesters)} harvesters.")
+    document_count = 0
+    try:
+        for harvester in harvesters:
+            log.info(f"start harvester {harvester} ({i} of {len(harvesters)}")
+            start_time = timeit.default_timer()
+            documents = harvester.harvest()
+            elapsed_time = timeit.default_timer() - start_time
+
+            try:
+                generate_embeddings(documents)
+            except Exception as e:
+                log.error(e)
+
+            log.info(f"index harvested {len(documents)} documents (vector db)")
+            qdrant.index_documents(documents)
+            log.info(
+                f"finished indexing for harvester {i} of {len(harvesters)}) (vector db)"
+            )
+
+            # remove embeddings from document, otherwise embeddings would be indexed in solr
+            # comment loop if embeddings should be indexed in solr as well.
+            for document in documents:
+                document.pop("embedding_384", None)
+
+            log.info(f"index harvested {len(documents)} documents")
+            solr.index_documents(documents)
+            document_count += len(documents)
+            log.info(
+                f"finished indexing for harvester {i} of {len(harvesters)})"
+            )
+
+            stats[harvester.get_type()] = {
+                "document_count": len(documents),
+                "processing_time": round(elapsed_time, 4),
+            }
+            if harvester.get_notes():
+                stats[harvester.get_type()]["notes"] = harvester.get_notes()
+            i += 1
+    except Exception as e:
+        data = {
+            "exception": str(e),
+            "traceback": traceback.format_exc(),
+            "hostname": gethostname(),
+        }
+        if config["mailserver_url"]:
+            requests.post(config["mailserver_url"], json=data, verify=False)
+    else:
+        log.info(
+            "harvesting completed, indexed {} documents".format(document_count)
+        )
+        requests.post(
+            config["mailserver_url"],
+            json={"stats": stats, "hostname": gethostname()},
+            verify=False,
+        )
+
+
+def generate_embeddings(documents):
+    log.info("create embeddings for " + str(len(documents)) + " documents")
+    for document in documents:
+        embeddings.add_embedding(document)
+    log.info("created embeddings for batch")
+
+
+if __name__ == "__main__":
+    run()

onestop4all_indexer-2.8.0.dev5/utils/qdrant.py

@@ -0,0 +1,97 @@
+import datetime
+import logging
+import uuid
+
+from qdrant_client import QdrantClient, models
+from typing import List, Dict
+
+from utils import config
+
+log = logging.getLogger(__name__)
+
+
+class Qdrant:
+
+    def __init__(self):
+        self.endpoint = config["qdrant_url"]
+        self.client = QdrantClient(self.endpoint, api_key=config["qdrant_url"])
+        log.debug(f"Initialized Qdrant client with endpoint {self.endpoint}")
+        self.collections = [
+            {"dim": 384, "name": "embedding_384"}
+            # {"dim": 768, "name": "embedding_768"},
+            # {"dim": 1024, "name": "embedding_1024"}
+        ]
+
+        self.init_collections()
+
+    def init_collections(self):
+        for collection in self.collections:
+            self.init_collection(collection["dim"], collection["name"])
+
+    def init_collection(self, vector_dim, collection_name):
+        collections = self.client.get_collections()  # existing collections
+        collection_names = [col.name for col in collections.collections]
+
+        try:
+            if (
+                collection_name not in collection_names
+            ):  # only create if not existing
+                self.client.create_collection(
+                    collection_name=collection_name,
+                    vectors_config=models.VectorParams(
+                        size=vector_dim, distance=models.Distance.COSINE
+                    ),
+                )
+                self.client.create_payload_index(
+                    collection_name=collection_name,
+                    field_name="id",
+                    field_schema="keyword",
+                )
+        except Exception as e:
+            log.error(e)
+
+    def index_documents(self, documents):
+        for collection in self.collections:
+            self.index_documents_dim(
+                documents,
+                collection_name=collection["name"],
+                embedding_key=collection["name"],
+            )
+
+    def index_documents_dim(
+        self, documents: List[Dict], collection_name, embedding_key
+    ) -> None:
+        log.info(
+            f"start indexing {len(documents)} documents to {self.endpoint}"
+        )
+
+        embeddings = []
+        payloads = []
+        ids = []
+
+        try:
+            for document in documents:
+                ids.append(
+                    str(uuid.uuid5(uuid.NAMESPACE_DNS, document["id"]))
+                )  # generates always the same uuid for the same document id
+                embeddings.append(document[str(embedding_key)])
+                payload = {
+                    "id": document["id"],
+                    "mainTitle": document["mainTitle"],
+                    "type": document["type"],
+                    "indexedAt": datetime.datetime.now(datetime.timezone.utc),
+                }
+                if "description" in document:
+                    payload["description"] = document["description"]
+                if "keywords" in document:
+                    payload["keyword"] = document["keyword"]
+                payloads.append(payload)
+
+            self.client.upload_collection(
+                collection_name=collection_name,
+                vectors=embeddings,
+                ids=ids,
+                payload=payloads,
+            )
+        except Exception as e:
+            log.error(e)

onestop4all_indexer-2.8.0.dev3/utils/harvest.py

@@ -1,123 +0,0 @@
-import logging
-import requests
-import timeit
-import traceback
-
-from socket import gethostname
-from typing import Optional
-
-from data_repositories import (
-    RepositoryTheme,
-    RepositoryPerson,
-    RepositoryResourceLinks,
-    RepositoryN4EOrganization,
-)
-from harvesters import (
-    Repository_Harvester,
-    Organization_Harvester,
-    Article_Harvester,
-    Softwaresourcecode_Harvester,
-    Learningresource_Harvester,
-    Metadatastandard_Harvester,
-    Document_Harvester,
-    Dataset_Harvester,
-    DataService_Harvester,
-    Service_Harvester,
-)
-from utils import config, Solr
-
-log = logging.getLogger(__name__)
-
-
-def run(
-    requested_harvester: Optional[tuple] = None,
-    reset_index: Optional[bool] = None,
-    dataset_options: Optional[dict] = {},
-    dataservice_options: Optional[dict] = {},
-):
-    # init in memory data repos
-    themes_repo = RepositoryTheme()
-    persons_repo = RepositoryPerson()
-    links_repo = RepositoryResourceLinks()
-    n4e_orgas_repo = RepositoryN4EOrganization()
-
-    solr = Solr()
-
-    if reset_index:
-        solr.reset_index()
-
-    stats = {}
-
-    _harvesters = {
-        "Service": Service_Harvester(n4e_organizations_repo=n4e_orgas_repo),
-        "DataService": DataService_Harvester(
-            themes_repo, persons_repo, **dataservice_options
-        ),
-        "Repository": Repository_Harvester(
-            themes_repo, n4e_organizations_repo=n4e_orgas_repo
-        ),
-        "Organization": Organization_Harvester(persons_repo),
-        "Article": Article_Harvester(persons_repo, links_repo, themes_repo),
-        "Softwaresourcecode": Softwaresourcecode_Harvester(
-            persons_repo, links_repo
-        ),
-        "Learningresource": Learningresource_Harvester(
-            persons_repo, themes_repo
-        ),
-        "Metadatastandard": Metadatastandard_Harvester(themes_repo),
-        "Document": Document_Harvester(persons_repo, links_repo),
-        "Dataset": Dataset_Harvester(
-            persons_repo, links_repo, **dataset_options
-        ),
-    }
-    i = 1
-    document_count = 0
-    try:
-        for harvester_name in _harvesters:
-            if (
-                requested_harvester
-                and harvester_name not in requested_harvester
-            ):
-                continue
-            log.info(
-                f"start harvester {harvester_name} ({i} of {len(_harvesters)}"
-            )
-            harvester = _harvesters[harvester_name]
-            start_time = timeit.default_timer()
-            documents = harvester.harvest()
-            elapsed_time = timeit.default_timer() - start_time
-
-            log.info(f"index harvested {len(documents)} documents")
-            solr.index_documents(documents)
-            document_count += len(documents)
-            log.info(
-                f"finished indexing for harvester {i} of {len(_harvesters)})"
-            )
-            stats[harvester.get_type()] = {
-                "document_count": len(documents),
-                "processing_time": round(elapsed_time, 4),
-            }
-            if harvester.get_notes():
-                stats[harvester.get_type()]["notes"] = harvester.get_notes()
-            i += 1
-    except Exception as e:
-        data = {
-            "exception": str(e),
-            "traceback": traceback.format_exc(),
-            "hostname": gethostname(),
-        }
-        if config["mailserver_url"]:
-            requests.post(config["mailserver_url"], json=data, verify=False)
-    else:
-        log.info(
-            "harvesting completed, indexed {} documents".format(document_count)
-        )
-        requests.post(
-            config["mailserver_url"],
-            json={"stats": stats, "hostname": gethostname()},
-            verify=False,
-        )
-
-
-if __name__ == "__main__":
-    run()