OneStop4All-Indexer 2.8.0.dev14__tar.gz → 2.8.0.dev20__tar.gz

{onestop4all_indexer-2.8.0.dev14 → onestop4all_indexer-2.8.0.dev20/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev14
+Version: 2.8.0.dev20
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.8.0.dev14/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev14
+Version: 2.8.0.dev20
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de

{onestop4all_indexer-2.8.0.dev14 → onestop4all_indexer-2.8.0.dev20}/harvesters/harvester_dataservice.py

@@ -427,3 +427,120 @@ iter_end={self.iteration_end}"""
                 return geom_wkt
         else:
             return geom_wkt
+
+
+# Batched Harvester für DataServices
+class DataService_HarvesterBatched(DataService_Harvester):
+
+    sparql_query_template = """
+        PREFIX fo: <http://www.w3.org/1999/XSL/Format#>
+        PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+        PREFIX dc: <http://purl.org/dc/elements/1.1/>
+        PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
+        prefix dcat: <http://www.w3.org/ns/dcat#>
+        prefix dct: <http://purl.org/dc/terms/>
+        prefix n4e: <http://nfdi4earth.de/ontology/>
+        prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+
+        SELECT  ?subject ?predicate ?object ?contactpoint_email ?contactpoint_url ?sourceSystem_title ?sourceSystem_homepage
+        {
+            VALUES ?subject { %s }
+            {
+                ?subject rdf:type dcat:DataService.
+                ?subject ?predicate ?object.
+                FILTER (?predicate NOT IN (<http://purl.org/dc/terms/publisher>, dct:spatial, dcat:contactPoint, n4e:sourceSystem))
+            }
+            UNION{
+                VALUES  ?predicate { <http://purl.org/dc/terms/publisher> }
+                ?subject rdf:type dcat:DataService;
+                        <http://purl.org/dc/terms/publisher> ?publisher.
+                ?publisher <http://schema.org/name> ?object.
+            }
+            UNION{
+                VALUES  ?predicate { dct:spatial }
+                ?subject rdf:type dcat:DataService;
+                dct:spatial ?spatial.
+                ?spatial dcat:bbox ?object
+            }
+            UNION {
+                VALUES ?predicate { dcat:contactPoint }
+                ?subject rdf:type dcat:DataService;
+                        dcat:contactPoint ?object.
+                optional { ?object vcard:hasEmail ?contactpoint_email. }
+                optional { ?object vcard:hasURL ?contactpoint_url. }
+            }
+            UNION{
+                VALUES  ?predicate { n4e:sourceSystem }
+                ?subject rdf:type dcat:DataService;
+                        n4e:sourceSystem ?object.
+                optional {?object dct:title ?sourceSystem_title.}
+                optional {?object  foaf:homepage ?sourceSystem_homepage.}
+            }
+        }
+    """
+
+    def get_all_ids(self):
+        """
+        Holt alle DataService-URIs aus dem Triple-Store, auch wenn der Endpoint ein Limit hat.
+        """
+        subjects = set()
+        offset = 0
+        limit = 10000  # typisches Virtuoso-Limit, ggf. anpassen
+        while True:
+            query = f"""
+            PREFIX dcat: <http://www.w3.org/ns/dcat#>
+            PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+            SELECT DISTINCT ?subject WHERE {{ ?subject rdf:type dcat:DataService. }} OFFSET {offset} LIMIT {limit}
+            """
+            hits = sparql.execute_query(self.sparql_endpoint, query)
+            batch_subjects = [hit["subject"]["value"] for hit in hits]
+            if not batch_subjects:
+                break
+            subjects.update(batch_subjects)
+            if len(batch_subjects) < limit:
+                break
+            offset += limit
+        return list(subjects)
+
+    def get_triples_for_subjects(self, subject_uris):
+        """
+        Holt alle Tripel für eine Liste von Subject-URIs.
+        """
+        values_str = " ".join(f"<{uri}>" for uri in subject_uris)
+        query = self.sparql_query_template % values_str
+        hits = sparql.execute_query(self.sparql_endpoint, query)
+        return hits
+
+    def harvest(self, records, batch_size=10):
+        services = {}
+        # dsh_batched.parse_response(triples, services)
+        while True:
+            # process in smaller sub-batches to avoid too long query string
+            records_sliced_sublist = records[:batch_size]
+            del records[:batch_size]
+            log.info(f"Remaining records in batch: {len(records)}")
+            if not records_sliced_sublist:
+                break
+            triples = self.get_triples_for_subjects(
+                records_sliced_sublist,
+            )
+            self.parse_response(triples, services)
+
+        self.solr_validator.close()
+
+        services_list = []
+        for (
+            key,
+            value,
+        ) in services.items():  # transform repos dict to list for indexing
+            service = value
+            if "mainTitle" not in service:
+                if len(service.get("title", [])) > 0:
+                    service["mainTitle"] = service["title"][0]
+                else:
+                    log.warning(f"Service {service['id']} has no mainTitle")
+                    continue  # skip services without mainTitle
+            service["mainTitle"] = service["mainTitle"].strip()
+            services_list.append(service)
+
+        return services_list

{onestop4all_indexer-2.8.0.dev14 → onestop4all_indexer-2.8.0.dev20}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="OneStop4All-Indexer",
-    version="2.8.0.dev14",
+    version="2.8.0.dev20",
     description="Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index",
     author="Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer",
     author_email="m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de",

{onestop4all_indexer-2.8.0.dev14 → onestop4all_indexer-2.8.0.dev20}/utils/cli.py

@@ -127,6 +127,31 @@ def dataservices(**options):
     )
 
 
+@debug.command()
+def dataservices_batched():
+    from utils import config
+    from harvesters.harvester_dataservice import DataService_HarvesterBatched
+    from data_repositories import RepositoryPerson, RepositoryTheme
+
+    print("und los")
+    print(config)
+    persons_repo = RepositoryPerson()
+    themes_repo = RepositoryTheme()
+    dsh_batched = DataService_HarvesterBatched(
+        persons_repo=persons_repo,
+        themes_repo=themes_repo,
+        solr_validation=False,
+    )
+    ids = dsh_batched.get_all_ids()
+    batch_size = 10
+    for i in range(0, len(ids), batch_size):
+        # print(f"Batch {i//batch_size}: IDs {i} to {i+batch_size}")
+        ids_batched = ids[i : i + batch_size]
+        print(f"Batch {i//batch_size}: IDs {ids_batched}")
+        triples = dsh_batched.get_triples_for_subjects(ids_batched)
+        print(f"Batch {i//batch_size}: Triples {len(triples)}")
+
+
 @main.command()
 @click.pass_context
 def reset(ctx):

{onestop4all_indexer-2.8.0.dev14 → onestop4all_indexer-2.8.0.dev20}/utils/solr.py

@@ -21,8 +21,8 @@ class Solr(object):
         timeout: int = 5 * 60,
     ) -> None:
         self.solr_url = solr_url if solr_url else config["solr_url"]
-        self.solr_core = solr_core if solr_url else config["solr_url"]
-        self.auth = solr_auth if solr_url else config["solr_url"]
+        self.solr_core = solr_core if solr_core else config["solr_core"]
+        self.auth = solr_auth if solr_auth else config["solr_auth"]
         self.client = SolrClient(
             self.endpoint,
             auth=self.authentication,