OneStop4All-Indexer 2.8.0.dev10__tar.gz → 2.8.0.dev11__tar.gz

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev11/OneStop4All_Indexer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev10
+Version: 2.8.0.dev11
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
@@ -13,6 +13,7 @@ Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
 Provides-Extra: airflow
 Requires-Dist: apache-airflow==3.1.7; extra == "airflow"
+Requires-Dist: apache-airflow-providers-amazon[s3fs]; extra == "airflow"
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev11}/OneStop4All_Indexer.egg-info/requires.txt

@@ -7,3 +7,4 @@ shapely~=2.0.5
 
 [airflow]
 apache-airflow==3.1.7
+apache-airflow-providers-amazon[s3fs]

{onestop4all_indexer-2.8.0.dev10/OneStop4All_Indexer.egg-info → onestop4all_indexer-2.8.0.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OneStop4All-Indexer
-Version: 2.8.0.dev10
+Version: 2.8.0.dev11
 Summary: Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index
 Author: Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer
 Author-email: m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de
@@ -13,6 +13,7 @@ Requires-Dist: geomet~=1.1.0
 Requires-Dist: shapely~=2.0.5
 Provides-Extra: airflow
 Requires-Dist: apache-airflow==3.1.7; extra == "airflow"
+Requires-Dist: apache-airflow-providers-amazon[s3fs]; extra == "airflow"
 Dynamic: author
 Dynamic: author-email
 Dynamic: license-file

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev11}/harvesters/harvester_dataset.py

@@ -299,7 +299,6 @@ class Dataset_Harvester(HarvesterCordra):
                                         value=val,
                                     )
                         if "downloadURL" in distribution:
-                            print(distribution["downloadURL"])
                             for download_url in distribution["downloadURL"]:
                                 val = self.get_string_from_jsonld(
                                     download_url, subject

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev11}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="OneStop4All-Indexer",
-    version="2.8.0.dev10",
+    version="2.8.0.dev11",
     description="Library to harvest data from NFDI4Earth-KnowledgeHub to OneStop4All-Index",
     author="Markus Konkol, Arne Vogt, Tom Niers, Ralf Klammer",
     author_email="m.konkol@52north.org, a.vogt@52north.org, tom.niers@tu-dresden.de, ralf.klammer@tu-dresden.de",
@@ -16,7 +16,10 @@ setup(
         "shapely ~= 2.0.5",
     ],
     extras_require={
-        "airflow": ["apache-airflow==3.1.7"],
+        "airflow": [
+            "apache-airflow==3.1.7",
+            "apache-airflow-providers-amazon[s3fs]",
+        ],
     },
     include_package_data=True,
     entry_points={

{onestop4all_indexer-2.8.0.dev10 → onestop4all_indexer-2.8.0.dev11}/utils/solr.py

@@ -57,7 +57,9 @@ class Solr(object):
             )
             return HTTPBasicAuth(username, password)
 
-    def index_documents(self, documents: List[Dict]) -> None:
+    def index_documents(
+        self, documents: List[Dict], commit=True, ping=True
+    ) -> None:
         # solr_endpoint = coreurl(solr_url, solr_core)
         log.info(f"start indexing {len(documents)} documents")
         batch_size = 50000
@@ -87,13 +89,19 @@ class Solr(object):
             if len(batch) < batch_size:
                 break
 
-        log.info("commit changes to index")
-        self.client.ping()
-        log.info("solr healtcheck successful")
-        self.client.commit()
-        log.info("sucessfully commited changes to index")
+        if ping is True:
+            self.client.ping()
+            log.info("solr healtcheck successful")
+
+        if commit is True:
+            log.info("commit changes to index")
+            self.client.commit()
+            log.info("sucessfully commited changes to index")
         log.info("finished indexing")
 
+    def commit(self):
+        self.client.commit()
+
     def reset_index(self):
         self.client.delete(q="*:*")
         self.client.commit()