rara-tools 0.6.14__tar.gz → 0.6.16__tar.gz

{rara_tools-0.6.14/rara_tools.egg-info → rara_tools-0.6.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.6.14
+Version: 0.6.16
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

rara_tools-0.6.16/VERSION

@@ -0,0 +1 @@
+0.6.16

{rara_tools-0.6.14 → rara_tools-0.6.16}/rara_tools/elastic.py

@@ -4,7 +4,7 @@ import elasticsearch_dsl
 from elastic_transport import ObjectApiResponse
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk
-from elasticsearch_dsl import Index
+from elasticsearch_dsl import Index, Search, Q
 from elasticsearch_dsl.response import Response
 
 from .decorators import _elastic_connection
@@ -84,20 +84,30 @@ class KataElastic:
     def add_mapping(self, index_name: str, schema: dict):
         index = Index(name=index_name)
         return index.put_mapping(body=schema, using=self.elasticsearch)
-    
-    
+
+    @_elastic_connection
+    def delete_by_query(self, index: str, query_kwargs: dict, query_type: str = "term", wait_for_completion=True):
+        query = Q(query_type, **query_kwargs)
+        s = Search(using=self.elasticsearch, index=index).query(query)
+        response = self.elasticsearch.delete_by_query(
+            index=index,
+            body={"query": s.to_dict()["query"]},
+            wait_for_completion=True
+        )
+        return response
+
     @_elastic_connection
     def add_vector_mapping(
-            self, 
-            index_name: str, 
-            field: str, 
-            schema: Optional[dict] = None, 
+            self,
+            index_name: str,
+            field: str,
+            schema: Optional[dict] = None,
             dims: int = 1024
     ) -> dict:
         vector_mapping = {
             "properties": {
                 field: {
-                    "type": "dense_vector", 
+                    "type": "dense_vector",
                     "dims": dims
                 }
             }
@@ -105,22 +115,21 @@ class KataElastic:
         mapping = schema or vector_mapping
         index = Index(name=index_name)
         return index.put_mapping(body=mapping, using=self.elasticsearch)
-    
-    
+
     @_elastic_connection
     def add_ann_vector_mapping(
-            self, 
+            self,
             index_name: str,
             field: str,
-            schema: Optional[dict] = None, 
+            schema: Optional[dict] = None,
             dims: int = 1024
     ) -> dict:
         vector_mapping = {
             "properties": {
                 field: {
-                    "type": "dense_vector", 
-                    "dims": dims, 
-                    "similarity": "cosine", 
+                    "type": "dense_vector",
+                    "dims": dims,
+                    "similarity": "cosine",
                     "index": True
                 }
             }
@@ -131,15 +140,19 @@ class KataElastic:
 
     @_elastic_connection
     def add_vector(
-            self, 
+            self,
             index_name: str,
-            document_id: str, 
-            vector: List[float], 
-            field: str
+            document_id: str,
+            vector: List[float],
+            field: str,
+            refresh: str = "wait_for"
     ) -> dict:
         schema = {"doc": {field: vector}}
         return self.elasticsearch.update(
-            index=index_name, id=document_id, body=schema, refresh="wait_for"
+            index=index_name,
+            id=document_id,
+            body=schema,
+            refresh=refresh
         )
 
     @_elastic_connection
@@ -204,7 +217,7 @@ class KataElastic:
         actions = [{"_index": last_index_name, "_source": document} for document in documents]
         successful_count, error_count = bulk(actions=actions, client=self.elasticsearch, max_retries=3, refresh=refresh)
         return successful_count, error_count
-    
+
     @_elastic_connection
     def bulk_index_without_rollver(
             self,
@@ -240,16 +253,15 @@ class KataElastic:
             s.scan(), key=lambda doc: [getattr(doc, field) for field in sort_fields]
         )
         return documents
-    
-    
+
     @_elastic_connection
     def execute_fuzzy_search(
-            self, 
+            self,
             index: str,
             field: str,
-            entity: str, 
-            fuzziness: int = 2, 
-            prefix_length: int = 1, 
+            entity: str,
+            fuzziness: int = 2,
+            prefix_length: int = 1,
             max_expansions: int = 50
     ) -> Response:
         """Executes a fuzzy search.
@@ -261,7 +273,7 @@ class KataElastic:
         :param: max_expansion int: maximum number of terms the fuzzy query 
             will match before halting the search
         :return: Dict on search results.
-        """  
+        """
         query_params = {
             f"{field}.keyword": {
                 "value": entity,
@@ -274,7 +286,7 @@ class KataElastic:
         s = s.query("fuzzy", **query_params)
         response = s.execute()
         return response
-    
+
     def execute_ann_vector_search(
             self,
             index: str,
@@ -303,8 +315,8 @@ class KataElastic:
         s = s.extra(
             knn={
                 "field": field,
-                "query_vector": query_vector, 
-                "k": k, 
+                "query_vector": query_vector,
+                "k": k,
                 "num_candidates": num_candidates
             }
         )
@@ -314,15 +326,14 @@ class KataElastic:
             s = s.query(
                 elasticsearch_dsl.Q("terms", _id=elastic_ids)
             )
-            
+
         # Sort by score and return `n_docs` best-matching documents
         s = s.extra(size=n_docs)
 
         # Execute the search
         response = s.execute()
         return response
-    
-        
+
     def execute_script_score_vector_search(
             self,
             index: str,
@@ -341,7 +352,7 @@ class KataElastic:
         :param: elastic_ids: List[str]: Elastic ID-s for restricting the search.
         """
         s = elasticsearch_dsl.Search(using=self.elasticsearch, index=index)
-        
+
         if elastic_ids:
             query = elasticsearch_dsl.Q("terms", _id=elastic_ids)
         else:
@@ -364,6 +375,5 @@ class KataElastic:
         response = s.execute()
         return response
 
-
     def __str__(self) -> str:
         return self.elasticsearch_url

{rara_tools-0.6.14 → rara_tools-0.6.16}/rara_tools/normalizers/base.py

@@ -280,8 +280,15 @@ class RecordNormalizer:
 
     @property
     def data(self) -> List[dict]:
-        """ Shorthand to get all normalized records as dict. """
-        return [record.as_dict() for record in self]
+        """Shorthand to get all normalized records as dict, skipping failures."""
+        result = []
+        for record in self:
+            try:
+                result.append(record.as_dict())
+            except Exception as e:
+                logger.error(f"Failed to normalize record: {e}")
+                continue
+        return result
 
     def __iter__(self) -> Iterator:
         viaf_id_path = "viaf.queryResult.records.record.0.recordData.VIAFCluster.viafID"

{rara_tools-0.6.14 → rara_tools-0.6.16/rara_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.6.14
+Version: 0.6.16
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.6.14 → rara_tools-0.6.16}/tests/test_elastic.py

@@ -4,7 +4,9 @@ import time
 import uuid
 from time import sleep
 
+import elasticsearch_dsl
 import pytest
+
 from rara_tools.elastic import KataElastic
 
 with open("./tests/test_data/elastic_docs.json") as fh:
@@ -28,6 +30,7 @@ def test_index_creation():
     assert created["acknowledged"] is True
     time.sleep(2)
 
+
 @pytest.mark.order(2)
 def test_check():
     """Tests health check method.
@@ -119,6 +122,25 @@ def test_document_deleting():
     result = ELASTIC.get_documents_by_key(TEST_INDEX_NAME, "bar")
     assert len(result) == 0
 
+    unique_id = uuid.uuid4().hex
+    document_amount = 10
+    documents = [{"doc_id": unique_id, "page": x} for x in range(document_amount)]
+    ELASTIC.bulk_index(documents, TEST_INDEX_NAME, rollover_limit=100, refresh="wait_for")
+    query = elasticsearch_dsl.Q("term", **{"doc_id.keyword": unique_id})
+    search = elasticsearch_dsl.Search(using=ELASTIC.elasticsearch, index=TEST_DOCUMENT_INDEX).query(query)
+    count = [hit.to_dict() for hit in search.scan()]
+    assert len(count) == document_amount
+    response = ELASTIC.delete_by_query(TEST_DOCUMENT_INDEX, {"doc_id.keyword": unique_id})
+
+    attempt = 0
+    while attempt < 3 and count != 0:
+        search = elasticsearch_dsl.Search(using=ELASTIC.elasticsearch, index=TEST_DOCUMENT_INDEX).query(query)
+        count = [hit.to_dict() for hit in search.scan()]
+        time.sleep(3)
+        attempt += 1
+
+    assert len(count) == 0
+
 
 @pytest.mark.order(8)
 def test_index_deleting():

rara_tools-0.6.14/VERSION

@@ -1 +0,0 @@
-0.6.14