rara-tools 0.0.11__tar.gz → 0.0.13__tar.gz

{rara_tools-0.0.11/rara_tools.egg-info → rara_tools-0.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rara-tools
-Version: 0.0.11
+Version: 0.0.13
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

rara_tools-0.0.13/VERSION

@@ -0,0 +1 @@
+0.0.13

rara_tools-0.0.13/rara_tools/converters.py

@@ -0,0 +1,81 @@
+from .exceptions import SierraResponseConverterException
+
+
+class SierraResponseConverter:
+    """Converts a JSON response from the Sierra API to MARC-in-JSON format."""
+    
+    def __init__(self, response: dict):
+        if not isinstance(response, dict):
+            raise SierraResponseConverterException("Please provide a valid JSON response.")
+        self.response = response
+    
+    def _map_control_fields(self, field: dict) -> dict:
+        # for tags < 010, no subfields, instead one str value in "value" 
+        return {field["tag"]: field["value"]}
+    
+    def _map_data_fields(self, field: dict) -> dict:
+        """ Maps marc fields > 010. 
+
+        Args:
+            field (dict): Contains the marc tag and list with indicators and subfields.
+
+        Returns:
+            dict: standardised marc-in-json format.
+        """
+        
+        data = field["data"]
+        
+        # Order matters ind1, in2, subfields
+        field_data = {
+            "ind1": data.get("ind1", " "),
+            "ind2": data.get("ind2", " "),
+            "subfields": data.get("subfields", [])
+        }
+        
+        return {field["tag"]: field_data}
+    
+    def _is_marc21structured(self, field: dict) -> bool:
+        """Checks if the field is already structured according to MARC21 in JSON"""
+        return any(key.isdigit() for key in field.keys())
+        
+    
+    def _handle_field_type(self, field: dict) -> dict:
+        
+        if self._is_marc21structured(field):
+            return field
+        
+        if field.get("data"):
+            return self._map_data_fields(field)
+        
+        tag = field.get("tag")
+        
+        if not tag:
+            raise SierraResponseConverterException("Field is missing MARC21 tag.")
+        
+        if tag < "010":
+            return self._map_control_fields(field)
+        else:
+            return self._map_data_fields(field)
+        
+    def _convert_response(self) -> list:
+        entries = self.response.get("entries")
+        if not entries:
+            raise SierraResponseConverterException("No entries found in the response.")
+        
+        try:
+            return {"fields": [
+                {e["id"]: [
+                    self._handle_field_type(f) for f in e["marc"]["fields"]
+                    ]}
+                for e in entries
+            ]}
+            
+        except KeyError as e:
+            raise SierraResponseConverterException(f"Malformed response: missing key {e}")
+        
+    
+    def convert(self) -> list:
+        try:
+            return self._convert_response()
+        except Exception as e:
+            raise SierraResponseConverterException(f"An unexpected error occurred: {e}")

{rara_tools-0.0.11 → rara_tools-0.0.13}/rara_tools/elastic.py

@@ -263,7 +263,7 @@ class KataElastic:
         response = s.execute()
         return response
     
-    def execute_vector_search(
+    def execute_ann_vector_search(
             self,
             index: str,
             field: str,
@@ -281,6 +281,7 @@ class KataElastic:
         :param: query vector List[float]: Vector to search matches for.
         :param: k int: Number of nearest neighbors to return.
         :param: num_candidates int: Number of candidates considered before selecting k results.
+        :param: n_docs: int: Number of documents to return.
         :param: elastic_ids: List[str]: Elastic ID-s for restricting the search.
         """
 
@@ -308,6 +309,48 @@ class KataElastic:
         # Execute the search
         response = s.execute()
         return response
+    
+        
+    def execute_script_score_vector_search(
+            self,
+            index: str,
+            field: str,
+            query_vector: List[float],
+            n_docs: int = 10,
+            elastic_ids: List[str] = []
+    ) -> Response:
+        """ Execute a vector search.
+        NB! Requires different mapping than ANN!
+        
+        :param: index str: Index to search from.
+        :param: field str: Field containing vectorized data.
+        :param: query vector List[float]: Vector to search matches for.
+        :param: n_docs: int: Number of documents to return.
+        :param: elastic_ids: List[str]: Elastic ID-s for restricting the search.
+        """
+        s = elasticsearch_dsl.Search(using=self.elasticsearch, index=index)
+        
+        if elastic_ids:
+            query = elasticsearch_dsl.Q("terms", _id=elastic_ids)
+        else:
+            query = elasticsearch_dsl.Q("match_all")
+        # Apply script_score query
+        s = s.query(
+            "script_score",
+            query=query,
+            script={
+                "source": f"1.0 + cosineSimilarity(params.query_vector, '{field}')",
+                "params": {
+                    "query_vector": query_vector
+                }
+            }
+        )
+        # Set min_score and limit number of documents
+        s = s.extra(size=n_docs)
+
+        # Execute search
+        response = s.execute()
+        return response
 
 
     def __str__(self) -> str:

{rara_tools-0.0.11 → rara_tools-0.0.13/rara_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rara-tools
-Version: 0.0.11
+Version: 0.0.13
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.0.11 → rara_tools-0.0.13}/tests/test_converters.py

@@ -5,12 +5,22 @@ import pytest
 from rara_tools.converters import SierraResponseConverter
 from rara_tools.exceptions import SierraResponseConverterException
 
+import json
+
 root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 SIERRA_TEST_DATA_DIR = os.path.join(root, "tests", "test_data", "sierra")
 INPUT_DIR = os.path.join(SIERRA_TEST_DATA_DIR, "input")
 OUTPUT_DIR = os.path.join(SIERRA_TEST_DATA_DIR, "output")
 
+def compare_results(expected, converted):
+    return json.dumps(expected) == json.dumps(converted)
+    
+def read_json_file(file_path):
+    with open(file_path, "r") as f:
+        data = f.read()
+        return json.loads(data)
+
 example_res =  {
             "total": 100,
             "start": 50000,
@@ -27,6 +37,8 @@ example_res =  {
                             {
                                 # "tag": "100",
                                 "data": {
+                                    "ind1": "1",
+                                    "ind2": " ",
                                     "subfields": [
                                         {
                                             "code": "a",
@@ -36,18 +48,13 @@ example_res =  {
                                             "code": "d",
                                             "data": "1975-"
                                         }
-                                    ],
-                                    "ind1": "1",
-                                    "ind2": " "
+                                    ]
                                 }
                             },
             ]}}]}
 
 
-def read_json_file(file_path):
-    with open(file_path, "r") as f:
-        data = f.read()
-        return json.loads(data)
+
         
 def test_convert_bibs_response():
     response = read_json_file(os.path.join(INPUT_DIR, "bibs.json"))	
@@ -55,8 +62,9 @@ def test_convert_bibs_response():
     converter = SierraResponseConverter(response)
     data = converter.convert()
     
-    expected = read_json_file(os.path.join(OUTPUT_DIR, "bibs.json"))    
-    assert data == expected 
+    expected = read_json_file(os.path.join(OUTPUT_DIR, "bibs.json"))
+    
+    assert compare_results(expected, data)
     
     
 def test_convert_keywords_response():
@@ -67,9 +75,10 @@ def test_convert_keywords_response():
     converter = SierraResponseConverter(response)
     data = converter.convert()
     
+
     expected = read_json_file(os.path.join(OUTPUT_DIR, "keywords.json"))
     
-    assert data == expected
+    assert compare_results(expected, data)
     
     
 def test_convert_authorities_response():
@@ -82,7 +91,20 @@ def test_convert_authorities_response():
     
     expected = read_json_file(os.path.join(OUTPUT_DIR, "authorities.json"))
     
-    assert data == expected
+    assert compare_results(expected, data)
+    
+def test_converter_handles_marc_in_json_response():
+    """ Gracefully handle entries already in MARC-in-JSON format """
+    with open(os.path.join(INPUT_DIR, "bibsmarc.json"), "r") as f:
+        response = f.read()
+        response = json.loads(response)
+        
+    converter = SierraResponseConverter(response)
+    data = converter.convert()
+    
+    expected = read_json_file(os.path.join(OUTPUT_DIR, "bibsmarc.json"))
+    
+    assert compare_results(expected, data)
     
 def test_convert_with_wrong_format():
     with pytest.raises(SierraResponseConverterException):

{rara_tools-0.0.11 → rara_tools-0.0.13}/tests/test_elastic_vector_and_search_operations.py

@@ -99,8 +99,8 @@ def test_fuzzy_search():
     
     
 @pytest.mark.order(6)
-def test_vector_search():
-    """ Tests vector search.
+def test_ann_vector_search():
+    """ Tests ANN vector search.
     """
     # Execut fuzzy search to get ID restrictions
     response = ELASTIC.execute_fuzzy_search(
@@ -113,7 +113,7 @@ def test_vector_search():
     assert total_hits == 3    
     elastic_ids = [hit.meta.id for hit in response]
     
-    response = ELASTIC.execute_vector_search(
+    response = ELASTIC.execute_ann_vector_search(
         index=TEST_ANN_INDEX_NAME,
         field="vector",
         query_vector=TEST_VECTOR,
@@ -126,8 +126,35 @@ def test_vector_search():
     assert len(descriptions) == 1
     assert descriptions[0] == "Eesti maletaja ja maleteoreetik"
 
-
+    
 @pytest.mark.order(7)
+def test_script_score_vector_search():
+    """ Tests ANN vector search.
+    """
+    # Execut fuzzy search to get ID restrictions
+    response = ELASTIC.execute_fuzzy_search(
+        index=TEST_KNN_INDEX_NAME,
+        field="variations",
+        entity="Paul Keres",
+        fuzziness=2
+    )
+    total_hits = response.hits.total.value
+    assert total_hits == 3    
+    elastic_ids = [hit.meta.id for hit in response]
+    
+    response = ELASTIC.execute_script_score_vector_search(
+        index=TEST_KNN_INDEX_NAME,
+        field="vector",
+        query_vector=TEST_VECTOR,
+        n_docs=1,
+        elastic_ids=elastic_ids
+    )
+    descriptions = [hit.description for hit in response]
+    assert len(descriptions) == 1
+    assert descriptions[0] == "Eesti maletaja ja maleteoreetik"
+
+    
+@pytest.mark.order(8)
 def test_index_deleting():
     """
     Tests deleting index. We delete the test index now.

rara_tools-0.0.11/VERSION

@@ -1 +0,0 @@
-0.0.11

rara_tools-0.0.11/rara_tools/converters.py

@@ -1,41 +0,0 @@
-from .exceptions import SierraResponseConverterException
-
-
-class SierraResponseConverter:
-    """  Takes a JSON response from the Sierra API (https://tester.ester.ee/iii/sierra-api/swagger/index.html)
-    and converts it to MARC-in-JSON format.
-    
-    """
-    
-    def __init__(self, response: dict):
-        if not isinstance(response, dict):
-            raise SierraResponseConverterException("Please provide a valid JSON response.")
-        self.response = response
-        
-    def _map_field_data(self, field):
-        tag = field.get("tag")
-        if not tag:
-            raise SierraResponseConverterException("Field is missing a valid 'tag'.")
-        data = field.get("data", {})
-        return {tag: data}
-        
-    def _convert_response(self):
-        response = self.response
-        
-        entries = response.get("entries")
-        if not entries:
-            raise SierraResponseConverterException("No entries found in the response.")
-        
-        try:
-            fields = [self._map_field_data(f) for e in entries for f in e["marc"]["fields"]]
-        except KeyError as e:
-            raise SierraResponseConverterException(f"Missing expected MARC fields in the response: {e}")
-        
-        return {"fields": fields}
-            
-    def convert(self):
-        """Runner method, converts the response to MARC-in-JSON format with error handling."""
-        try:
-            return self._convert_response()
-        except Exception as e:
-            raise SierraResponseConverterException(f"An unexpected error occurred during conversion: {e}")