rara-tools 0.0.9__tar.gz → 0.0.11__tar.gz

{rara_tools-0.0.9/rara_tools.egg-info → rara_tools-0.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rara-tools
-Version: 0.0.9
+Version: 0.0.11
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -14,6 +14,7 @@ Requires-Dist: elasticsearch==8.*
 Requires-Dist: elasticsearch_dsl==8.*
 Requires-Dist: minio==7.*
 Requires-Dist: requests
+Requires-Dist: iso639-lang
 Provides-Extra: testing
 Requires-Dist: pytest>=8.0; extra == "testing"
 Requires-Dist: pytest-order; extra == "testing"

rara_tools-0.0.11/VERSION

@@ -0,0 +1 @@
+0.0.11

rara_tools-0.0.11/rara_tools/digar_schema_converter.py

@@ -0,0 +1,416 @@
+from collections import defaultdict
+from typing import List, NoReturn
+
+from rara_tools.utils import lang_to_iso639_2, ratio_to_percentage
+
+
+UNDEFINED_LANGUAGE_VALUE = "unk"
+QUALITY_RATIO_TYPE = "Float"
+
+
+class ImagePageSchema:
+    def __init__(self, image: dict) -> NoReturn:
+        self.__image = image
+        self.__schema: dict = {}
+
+    @property
+    def schema(self) -> dict:
+        if not self.__schema:
+            self.__schema = {
+                "@type": "VisualArtwork",
+                "@id": "",
+                "value": self.__image.get("label"),
+                "description": "",
+                "schema:position": self.__image.get("page")
+            }
+        return self.__schema
+
+
+class TextPageSchema:
+    def __init__(self, page: dict) -> NoReturn:
+        self.__page: dict = page
+        self.__schema: dict = {}
+
+    @property
+    def schema(self) -> dict:
+        if not self.__schema:
+            self.__schema = {
+                "@type": "Text",  # CONSTANT
+                "@id": "",  # Will be added in a later stage
+                "value": "Textblock",  # CONSTANT
+                "content": self.__page.get("text"),
+                "schema:position": self.__page.get("start_page")  # start_page ?
+            }
+        return self.__schema
+
+
+class PageSchema:
+    def __init__(
+            self,
+            page_texts: List[dict],
+            page_images: List[dict],
+            page_number: int,
+            doc_id: str
+    ) -> NoReturn:
+        self.__page_texts: List[dict] = page_texts
+        self.__page_images: List[dict] = page_images
+        self.__page_nr: int = page_number
+        self.__page_id: str = ""
+        self.__doc_id: str = doc_id
+        self.__schema: dict = {}
+
+    def _add_segment_ids(self, segments: List[dict]) -> List[dict]:
+        for i, segment in enumerate(segments):
+            segment_id = f"{self.page_id}/{i + 1}"
+            segment["@id"] = segment_id
+        return segments
+
+    @property
+    def page_id(self) -> str:
+        if not self.__page_id:
+            self.__page_id = f"{self.__doc_id}/{self.__page_nr}"
+        return self.__page_id
+
+    @property
+    def schema(self) -> dict:
+        if not self.__schema:
+            self.__schema = {
+                "@type": "CreativeWork",  # CONSTANT for pages
+                "@id": self.page_id,
+                "hasPart": []
+            }
+            text_schemas = [
+                TextPageSchema(page).schema
+                for page in self.__page_texts
+            ]
+            image_schemas = [
+                ImagePageSchema(image).schema
+                for image in self.__page_images
+            ]
+
+            page_schemas = text_schemas + image_schemas
+            page_schemas_with_ids = self._add_segment_ids(page_schemas)
+
+            self.__schema["hasPart"].extend(page_schemas_with_ids)
+
+        return self.__schema
+
+
+class DocSchemas:
+    def __init__(
+            self,
+            doc_meta: dict,
+            sierra_id: str = "",
+            generated_id: str = "",
+            permalink: str = "",
+            min_language_ratio: float = 0.2,
+            convert_ratio: bool = True,
+            generated_id_type: str = "CustomID"
+    ) -> NoReturn:
+        self.__convert_ratio = convert_ratio
+        self.__min_language_ratio = min_language_ratio
+        self.__sierra_id = sierra_id
+        self.__generated_id = generated_id
+        self.__permalink = permalink
+        self.__generated_id_type = generated_id_type
+        self.__doc_meta = doc_meta
+        self.__ocr_accuracy_schema: dict = {}
+        self.__text_quality_schema: dict = {}
+        self.__language_schema: List[dict] = []
+        self.__identifier_schema: List[dict] = []
+        self.__origin_schema: dict = {}
+        self.__origin: str = ""
+
+    @property
+    def origin(self) -> str:
+        if not self.__origin:
+            if self.__doc_meta["ocr_applied"]:
+                self.__origin = "Reformatted digital"
+            else:
+                self.__origin = "Born digital"
+        return self.__origin
+
+    @property
+    def ocr_accuracy_schema(self) -> dict:
+        if not self.__ocr_accuracy_schema:
+            ocr_quality = self.__doc_meta.get("alto_text_quality")
+            if ocr_quality:
+                self.__ocr_accuracy_schema = {
+                    "comment": "Estimated OCR accuracy"
+                }
+                if self.__convert_ratio:
+                    type_and_value = {
+                        "@type": QUALITY_RATIO_TYPE,
+                        "value": ocr_quality
+                    }
+                else:
+                    type_and_value = {
+                        "@type": "Text",
+                        "value": ratio_to_percentage(ocr_quality)
+                    }
+                self.__ocr_accuracy_schema.update(type_and_value)
+        return self.__ocr_accuracy_schema
+
+    @property
+    def text_quality_schema(self) -> dict:
+        if not self.__text_quality_schema:
+            text_quality = self.__doc_meta.get("text_quality")
+            self.__text_quality_schema = {
+                "comment": "Estimated n-gram-based text quality"
+            }
+            if self.__convert_ratio:
+                type_and_value = {
+                    "@type": QUALITY_RATIO_TYPE,
+                    "value": text_quality
+                }
+            else:
+                type_and_value = {
+                    "@type": "Text",
+                    "value": ratio_to_percentage(text_quality)
+                }
+            self.__text_quality_schema.update(type_and_value)
+        return self.__text_quality_schema
+
+    @property
+    def language_schema(self) -> List[dict]:
+        if not self.__language_schema:
+            self.__language_schema = [
+                {
+                    "@type": "ISO 639-2",
+                    "value": lang_to_iso639_2(
+                        lang["language"],
+                        unk_code=UNDEFINED_LANGUAGE_VALUE
+                    )
+                }
+                for lang in self.__doc_meta["languages"]
+                if lang["ratio"] >= self.__min_language_ratio
+            ]
+        return self.__language_schema
+
+    @property
+    def identifier_schema(self) -> List[dict]:
+        if not self.__identifier_schema:
+            identifiers = []
+            if self.__sierra_id:
+                identifiers.append(
+                    {
+                        "@type": "Identifier",
+                        "qualifier": "OPAC",
+                        "value": self.__sierra_id
+                    }
+                )
+            if self.__permalink:
+                identifiers.append(
+                    {
+                        "@type": "Identifier",
+                        "qualifier": "Permalink",
+                        "value": self.__permalink
+                    }
+                )
+            if self.__generated_id:
+                identifiers.append(
+                    {
+                        "@type": "Identifier",
+                        "qualifier": self.__generated_id_type,
+                        "value": self.__generated_id
+                    }
+                )
+            self.__identifier_schema = identifiers
+
+        return self.__identifier_schema
+
+    @property
+    def origin_schema(self) -> dict:
+        if not self.__origin_schema:
+            self.__origin_schema = {
+                "@type": "Text",
+                "value": self.origin,
+                "comment": "Origin"
+            }
+        return self.__origin_schema
+
+
+class DIGARSchemaConverter:
+    def __init__(
+            self,
+            digitizer_output: dict,
+            generated_id: str,
+            sierra_id: str = "",
+            permalink: str = "",
+            generated_id_type: str = "CustomID",
+            min_language_ratio: float = 0.2,
+            convert_ratio: bool = False
+    ) -> NoReturn:
+        """ Initialize DIGARSchemaConverter object. 
+        
+        Parameters
+        ----------
+        digitizer_output: dict
+            Raw output of rara-digitizer (https://pypi.org/project/rara-digitizer/).
+        generated_id: str
+            Some non-standard/generated document identifier used in ID fields.
+        sierra_id: str
+            Document's corresponding Sierra ID.
+        permalink: str
+            Permanent link, where the document can be accessed.
+        generated_id_type: str
+            Method / type of generated ID (e.g. 'UUID')
+        min_language_ratio: float
+            Cutoff ratio for languages. If ratio for some language
+            does not exceed the set threshold, the language will not
+            be added to the final output.
+        convert_ratio: bool
+            If enabled, all ratios are converted into percentages.
+          
+        """
+        self.__digitizer_output: dict = digitizer_output
+        self.__min_language_ratio: float = min_language_ratio
+        self.__convert_ratio: bool = convert_ratio
+        self.__sierra_id: str = sierra_id
+        self.__generated_id: str = generated_id
+        self.__permalink: str = permalink.removesuffix("/")
+        self.__generated_id_type: str = generated_id_type
+        self.__texts: List[dict] = []
+        self.__images: List[dict] = []
+        self.__doc_meta: dict = {}
+        self.__page_mappings: List[dict] = []
+        self.__dcterms_haspart: dict = {}
+        self.__dcterms_conforms_to: dict = {}
+        self.__dc_language: dict = {}
+        self.__dc_origin: dict = {}
+        self.__dc_identifier: List[dict] = []
+        self.__doc_id: str = ""
+
+        self.__doc_schemas = DocSchemas(
+            doc_meta=self.doc_meta,
+            sierra_id=self.__sierra_id,
+            generated_id=self.__generated_id,
+            permalink=self.__permalink,
+            min_language_ratio=self.__min_language_ratio,
+            convert_ratio=self.__convert_ratio,
+            generated_id_type=self.__generated_id_type
+        )
+        self.__digar_schema: dict = {}
+
+    def _get_page_number(self, page_content: dict) -> int:
+        """ Retrieves page number from image or text object.
+        """
+        _segments = page_content["texts"] + page_content["images"]
+        _first_segment = _segments[0]
+        if "start_page" in _first_segment:
+            page_number = _first_segment.get("start_page")
+        elif "page" in _first_segment:
+            page_number = _first_segment.get("page")
+        return page_number
+
+    @property
+    def doc_id(self) -> str:
+        """ Retrieves document ID to use for generating
+        page and segment ids. Preference order:
+        1. permalink; 2. sierra_id; 3. generated document id
+        """
+        if not self.__doc_id:
+            if self.__permalink:
+                self.__doc_id = self.__permalink
+            elif self.__sierra_id:
+                self.__doc_id = self.__sierra_id
+            else:
+                self.__doc_id = self.__generated_id
+        return self.__doc_id
+
+    @property
+    def texts(self) -> List[dict]:
+        if not self.__texts:
+            self.__texts = self.__digitizer_output.get("texts")
+        return self.__texts
+
+    @property
+    def images(self) -> List[dict]:
+        if not self.__images:
+            self.__images = self.__digitizer_output.get("images")
+        return self.__images
+
+    @property
+    def doc_meta(self) -> dict:
+        if not self.__doc_meta:
+            self.__doc_meta = self.__digitizer_output.get("doc_meta")
+        return self.__doc_meta
+
+    @property
+    def page_mappings(self) -> List[dict]:
+        if not self.__page_mappings:
+            mapped = defaultdict(lambda: defaultdict(list))
+            for text in self.texts:
+                mapped[text["start_page"]]["texts"].append(text)
+            for img in self.images:
+                mapped[img["page"]]["images"].append(img)
+
+            self.__page_mappings = [
+                v for k, v in sorted(list(mapped.items()), key=lambda x: x[0])
+            ]
+        return self.__page_mappings
+
+    @property
+    def dcterms_haspart(self) -> dict:
+        if not self.__dcterms_haspart:
+            self.__dcterms_haspart = {
+                "dcterms:hasPart": [
+                    PageSchema(
+                        page_texts=page["texts"],
+                        page_images=page["images"],
+                        page_number=self._get_page_number(page),
+                        doc_id=self.doc_id
+                    ).schema
+                    for page in self.page_mappings
+                ]
+            }
+        return self.__dcterms_haspart
+
+    @property
+    def dcterms_conforms_to(self) -> dict:
+        if not self.__dcterms_conforms_to:
+            schema_content = [
+                self.__doc_schemas.text_quality_schema,
+            ]
+            # Add OCR Accuracy only when it is not empty:
+            if self.__doc_schemas.ocr_accuracy_schema:
+                schema_content.append(self.__doc_schemas.ocr_accuracy_schema)
+            self.__dcterms_conforms_to = {
+                "dcterms:conformsTo": schema_content
+            }
+        return self.__dcterms_conforms_to
+
+    @property
+    def dc_language(self) -> dict:
+        if not self.__dc_language:
+            self.__dc_language = {
+                "dc:language": self.__doc_schemas.language_schema
+            }
+        return self.__dc_language
+
+    @property
+    def dc_origin(self) -> dict:
+        if not self.__dc_origin:
+            self.__dc_origin = {
+                "dcterms:provenance": self.__doc_schemas.origin_schema
+            }
+        return self.__dc_origin
+
+    @property
+    def dc_identifier(self) -> List[dict]:
+        if not self.__dc_identifier:
+            self.__dc_identifier = {
+                "dc:identifier": self.__doc_schemas.identifier_schema
+            }
+        return self.__dc_identifier
+
+    @property
+    def digar_schema(self) -> dict:
+        if not self.__digar_schema:
+            self.__digar_schema = {}
+            self.__digar_schema.update(self.dcterms_conforms_to)
+            self.__digar_schema.update(self.dcterms_haspart)
+            self.__digar_schema.update(self.dc_language)
+            self.__digar_schema.update(self.dc_origin)
+            self.__digar_schema.update(self.dc_identifier)
+        return self.__digar_schema

{rara_tools-0.0.9 → rara_tools-0.0.11}/rara_tools/elastic.py

@@ -1,10 +1,11 @@
-from typing import Any, Dict, Iterator, Optional
+from typing import Any, Dict, Iterator, Optional, List
 
 import elasticsearch_dsl
 from elastic_transport import ObjectApiResponse
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk
 from elasticsearch_dsl import Index
+from elasticsearch_dsl.response import Response
 
 from .decorators import _elastic_connection
 
@@ -82,6 +83,63 @@ class KataElastic:
     def add_mapping(self, index_name: str, schema: dict):
         index = Index(name=index_name)
         return index.put_mapping(body=schema, using=self.elasticsearch)
+    
+    
+    @_elastic_connection
+    def add_vector_mapping(
+            self, 
+            index_name: str, 
+            field: str, 
+            schema: Optional[dict] = None, 
+            dims: int = 1024
+    ) -> dict:
+        vector_mapping = {
+            "properties": {
+                field: {
+                    "type": "dense_vector", 
+                    "dims": dims
+                }
+            }
+        }
+        mapping = schema or vector_mapping
+        index = Index(name=index_name)
+        return index.put_mapping(body=mapping, using=self.elasticsearch)
+    
+    
+    @_elastic_connection
+    def add_ann_vector_mapping(
+            self, 
+            index_name: str,
+            field: str,
+            schema: Optional[dict] = None, 
+            dims: int = 1024
+    ) -> dict:
+        vector_mapping = {
+            "properties": {
+                field: {
+                    "type": "dense_vector", 
+                    "dims": dims, 
+                    "similarity": "cosine", 
+                    "index": True
+                }
+            }
+        }
+        mapping = schema or vector_mapping
+        index = Index(name=index_name)
+        return index.put_mapping(body=mapping, using=self.elasticsearch)
+
+    @_elastic_connection
+    def add_vector(
+            self, 
+            index_name: str,
+            document_id: str, 
+            vector: List[float], 
+            field: str
+    ) -> dict:
+        schema = {"doc": {field: vector}}
+        return self.elasticsearch.update(
+            index=index_name, id=document_id, body=schema, refresh="wait_for"
+        )
 
     @_elastic_connection
     def create_index(
@@ -170,6 +228,87 @@ class KataElastic:
             s.scan(), key=lambda doc: [getattr(doc, field) for field in sort_fields]
         )
         return documents
+    
+    
+    @_elastic_connection
+    def execute_fuzzy_search(
+            self, 
+            index: str,
+            field: str,
+            entity: str, 
+            fuzziness: int = 2, 
+            prefix_length: int = 1, 
+            max_expansions: int = 50
+    ) -> Response:
+        """Executes a fuzzy search.
+        :param: index str: Index to search from.
+        :param: entity str: Entity to search matches for.
+        :param: fuzziness int: Maximum edit distance for a match.
+        :param: prefix_length int: Number of characters in the prefix that 
+            should overlap with the original entity's prefix.
+        :param: max_expansion int: maximum number of terms the fuzzy query 
+            will match before halting the search
+        :return: Dict on search results.
+        """  
+        query_params = {
+            f"{field}.keyword": {
+                "value": entity,
+                "fuzziness": fuzziness,
+                "max_expansions": max_expansions,
+                "prefix_length": prefix_length
+            }
+        }
+        s = elasticsearch_dsl.Search(using=self.elasticsearch, index=index)
+        s = s.query("fuzzy", **query_params)
+        response = s.execute()
+        return response
+    
+    def execute_vector_search(
+            self,
+            index: str,
+            field: str,
+            query_vector: List[float],
+            k: int = 10,
+            num_candidates: int = 100,
+            n_docs: int = 10,
+            elastic_ids: List[str] = []
+    ) -> Response:
+        """ Execute a vector search.
+        NB! Works only with ANN mapping!
+        
+        :param: index str: Index to search from.
+        :param: field str: Field containing vectorized data.
+        :param: query vector List[float]: Vector to search matches for.
+        :param: k int: Number of nearest neighbors to return.
+        :param: num_candidates int: Number of candidates considered before selecting k results.
+        :param: elastic_ids: List[str]: Elastic ID-s for restricting the search.
+        """
+
+        s = elasticsearch_dsl.Search(using=self.elasticsearch, index=index)
+
+        # Add kNN vector search
+        s = s.extra(
+            knn={
+                "field": field,
+                "query_vector": query_vector, 
+                "k": k, 
+                "num_candidates": num_candidates
+            }
+        )
+
+        # Add ID filtering, if elastic_ids are specified
+        if elastic_ids:
+            s = s.query(
+                elasticsearch_dsl.Q("terms", _id=elastic_ids)
+            )
+            
+        # Sort by score and return `n_docs` best-matching documents
+        s = s.extra(size=n_docs)
+
+        # Execute the search
+        response = s.execute()
+        return response
+
 
     def __str__(self) -> str:
         return self.elasticsearch_url

rara_tools-0.0.11/rara_tools/utils.py

@@ -0,0 +1,104 @@
+from iso639 import Lang
+
+
+def lang_to_iso639_1(lang: str, unk_code: str = "unk") -> str:
+    """ Converts language into ISO-639-1 standard. 
+    Input can be any language code in a valid ISO-639 
+    standard or even a full name of the language, 
+    e.g. "Estonian".
+    
+    Parameters
+    -----------
+    lang: str
+        Language code in any valid ISO-639 standard.
+        
+    unk_code: str
+        Code to return incase of invalid/unsupported
+        input language.
+        
+    Returns
+    -------
+    Language code in ISO-639-1 standard.
+    """
+    try:
+        lg = Lang(lang)
+        iso_639_1_lang = lg.pt1
+    except:
+        iso_639_1_lang = unk_code
+    return iso_639_1_lang
+
+
+def lang_to_iso639_2(lang: str, unk_code: str = "unk") -> str:
+    """ Converts language into ISO-639-2 standard. 
+    Input can be any language code in a valid ISO-639 
+    standard or even a full name of the language, 
+    e.g. "Estonian".
+    
+    Parameters
+    -----------
+    lang: str
+        Language code in any valid ISO-639 standard.
+        
+    unk_code: str
+        Code to return incase of invalid/unsupported
+        input language.
+        
+    Returns
+    -------
+    Language code in ISO-639-2 standard.
+    """
+    try:
+        lg = Lang(lang)
+        # NB! uses bibliographic identifier (e.g. "de" -> "ger")
+        # opposed to terminological identifier ("de" -> "deu").
+        # This can be changed by replaving lg.pt2b -> lg.pt2t
+        iso_639_2_lang = lg.pt2b
+    except:
+        iso_639_2_lang = unk_code
+    return iso_639_2_lang
+
+
+def lang_to_iso639_3(lang: str, unk_code: str = "unk") -> str:
+    """ Converts language into ISO-639-3 standard. 
+    Input can be any language code in a valid ISO-639 
+    standard or even a full name of the language, 
+    e.g. "Estonian".
+    
+    Parameters
+    -----------
+    lang: str
+        Language code in any valid ISO-639 standard.
+        unk_code: str
+        
+    Code to return incase of invalid/unsupported
+        input language.
+        
+    Returns
+    -------
+    str
+        Language code in ISO-639-3 standard.
+    """
+    try:
+        lg = Lang(lang)
+        iso_639_3_lang = lg.pt3
+    except:
+        iso_639_3_lang = unk_code
+    return iso_639_3_lang
+
+
+def ratio_to_percentage(ratio: float) -> str:
+    """ Converts ratio to corresponding percentage.
+    
+    Parameters
+    -----------
+    ratio: float
+        Float in range [0,1]
+    
+    Returns
+    --------
+    str
+        Percentage corresponding to the float.
+    
+    """
+    percentage = f"{int(ratio*100)}%"
+    return percentage

{rara_tools-0.0.9 → rara_tools-0.0.11/rara_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rara-tools
-Version: 0.0.9
+Version: 0.0.11
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -14,6 +14,7 @@ Requires-Dist: elasticsearch==8.*
 Requires-Dist: elasticsearch_dsl==8.*
 Requires-Dist: minio==7.*
 Requires-Dist: requests
+Requires-Dist: iso639-lang
 Provides-Extra: testing
 Requires-Dist: pytest>=8.0; extra == "testing"
 Requires-Dist: pytest-order; extra == "testing"

{rara_tools-0.0.9 → rara_tools-0.0.11}/rara_tools.egg-info/SOURCES.txt

@@ -5,10 +5,12 @@ pyproject.toml
 requirements.txt
 rara_tools/converters.py
 rara_tools/decorators.py
+rara_tools/digar_schema_converter.py
 rara_tools/elastic.py
 rara_tools/exceptions.py
 rara_tools/s3.py
 rara_tools/task_reporter.py
+rara_tools/utils.py
 rara_tools.egg-info/PKG-INFO
 rara_tools.egg-info/SOURCES.txt
 rara_tools.egg-info/dependency_links.txt
@@ -18,7 +20,9 @@ rara_tools/constants/__init__.py
 rara_tools/constants/digitizer.py
 rara_tools/constants/general.py
 tests/test_converters.py
+tests/test_digar_schema_converter.py
 tests/test_elastic.py
+tests/test_elastic_vector_and_search_operations.py
 tests/test_s3_exceptions.py
 tests/test_s3_file_operations.py
 tests/test_task_reporter.py

{rara_tools-0.0.9 → rara_tools-0.0.11}/rara_tools.egg-info/requires.txt

@@ -2,6 +2,7 @@ elasticsearch==8.*
 elasticsearch_dsl==8.*
 minio==7.*
 requests
+iso639-lang
 
 [testing]
 pytest>=8.0

{rara_tools-0.0.9 → rara_tools-0.0.11}/requirements.txt

@@ -2,3 +2,4 @@ elasticsearch==8.*
 elasticsearch_dsl==8.*
 minio==7.*
 requests
+iso639-lang

rara_tools-0.0.11/tests/test_digar_schema_converter.py

@@ -0,0 +1,133 @@
+import json
+import pytest
+import os
+import sys
+
+from rara_tools.digar_schema_converter import DIGARSchemaConverter
+
+def load_json(file_path: str):
+    with open(file_path, "r") as f:
+        data = json.load(f)
+    return data
+
+
+TEST_DIGITIZER_OUTPUT_FILE = os.path.join(".", "tests", "test_data", "b1267058_test_digitizer_output.json")
+TEST_DIGITIZER_OUTPUT = load_json(TEST_DIGITIZER_OUTPUT_FILE)
+TEST_SIERRA_ID = "b1267058"
+TEST_GENERATED_ID = "hsasaHSAHHGDhb"
+TEST_PERMALINK = "https://www.digar.ee/b1267058"
+
+def test_digar_schema_converstion_default():
+    converter = DIGARSchemaConverter(
+        digitizer_output=TEST_DIGITIZER_OUTPUT,
+        sierra_id=TEST_SIERRA_ID,
+        generated_id=TEST_GENERATED_ID
+    )
+    digar_schema = converter.digar_schema
+    
+    # check that all neseccary fields are present
+    assert "dc:language" in digar_schema
+    assert "dcterms:provenance" in digar_schema
+    assert "dc:identifier" in digar_schema
+    assert "dcterms:hasPart" in digar_schema
+    assert "dcterms:conformsTo" in digar_schema
+    
+    languages = [lang.get("value") for lang in digar_schema.get("dc:language")]
+    # check that languages are converted into ISO-693-2
+    for lang in languages:
+        assert len(lang) == 3
+        
+        
+    # check that ratio is converted into percentage
+    text_quality = digar_schema.get("dcterms:conformsTo")[0].get("value")
+    assert isinstance(text_quality, str)
+    
+    
+def test_digar_schema_id_generation():
+    """ Tests ID generation logic.
+    """
+    converter = DIGARSchemaConverter(
+        digitizer_output=TEST_DIGITIZER_OUTPUT,
+        sierra_id=TEST_SIERRA_ID,
+        generated_id=TEST_GENERATED_ID,
+        permalink=TEST_PERMALINK
+        
+    )
+    
+    #If permalink is given, this should be used as base ID
+    digar_schema = converter.digar_schema
+    first_segment_id = digar_schema["dcterms:hasPart"][0]["hasPart"][0]["@id"]
+    
+    assert first_segment_id.startswith(TEST_PERMALINK)
+  
+    converter = DIGARSchemaConverter(
+        digitizer_output=TEST_DIGITIZER_OUTPUT,
+        sierra_id=TEST_SIERRA_ID,
+        generated_id=TEST_GENERATED_ID
+    )
+    
+    #If permalink is NOT given, Sierra ID should be used as base ID
+    digar_schema = converter.digar_schema
+    first_segment_id = digar_schema["dcterms:hasPart"][0]["hasPart"][0]["@id"]
+    assert first_segment_id.startswith(TEST_SIERRA_ID)
+    
+    
+    converter = DIGARSchemaConverter(
+        digitizer_output=TEST_DIGITIZER_OUTPUT,
+        generated_id=TEST_GENERATED_ID
+    )
+    
+    #If neiter permalink nor Sierra ID is given, generated ID should be used as base ID
+    digar_schema = converter.digar_schema
+    first_segment_id = digar_schema["dcterms:hasPart"][0]["hasPart"][0]["@id"]
+    assert first_segment_id.startswith(TEST_GENERATED_ID)
+    
+    
+def test_restricting_languages_with_ratio():
+    """ Checks that param `min_language_ratio` influences
+    the number of output languages.
+    """
+    converter = DIGARSchemaConverter(
+        digitizer_output=TEST_DIGITIZER_OUTPUT,
+        sierra_id=TEST_SIERRA_ID,
+        generated_id=TEST_GENERATED_ID,
+        permalink=TEST_PERMALINK,
+        min_language_ratio=0
+        
+    )
+    
+    #If permalink is given, this should be used as base ID
+    digar_schema = converter.digar_schema
+    languages = [lang.get("value") for lang in digar_schema.get("dc:language")]
+    assert len(languages) == 7
+    
+    converter = DIGARSchemaConverter(
+        digitizer_output=TEST_DIGITIZER_OUTPUT,
+        sierra_id=TEST_SIERRA_ID,
+        generated_id=TEST_GENERATED_ID,
+        permalink=TEST_PERMALINK,
+        min_language_ratio=0.02
+        
+    )
+    
+    #If permalink is given, this should be used as base ID
+    digar_schema = converter.digar_schema
+    languages = [lang.get("value") for lang in digar_schema.get("dc:language")]
+    assert len(languages) == 2
+    
+    converter = DIGARSchemaConverter(
+        digitizer_output=TEST_DIGITIZER_OUTPUT,
+        sierra_id=TEST_SIERRA_ID,
+        generated_id=TEST_GENERATED_ID,
+        permalink=TEST_PERMALINK,
+        min_language_ratio=0.5
+        
+    )
+    
+    #If permalink is given, this should be used as base ID
+    digar_schema = converter.digar_schema
+    languages = [lang.get("value") for lang in digar_schema.get("dc:language")]
+    assert len(languages) == 1
+    
+
+

rara_tools-0.0.11/tests/test_elastic_vector_and_search_operations.py

@@ -0,0 +1,140 @@
+import json
+import os
+import pytest
+
+from time import sleep
+from rara_tools.elastic import KataElastic
+
+
+def load_json(file_path: str):
+    with open(file_path, "r") as fh:
+        data = json.load(fh)
+    return data
+
+TEST_DOCUMENTS = load_json("./tests/test_data/elastic_vectorized_docs.json")
+TEST_VECTOR_DATA = load_json("./tests/test_data/test_vector_data.json")
+TEST_VECTOR = TEST_VECTOR_DATA.get("vector")
+    
+es_url = os.getenv("ELASTIC_TEST_URL", "http://localhost:9200")
+ELASTIC = KataElastic(es_url)
+
+TEST_KNN_INDEX_NAME = "tools_knn_testing_index"
+TEST_ANN_INDEX_NAME = "tools_ann_testing_index"
+
+TEST_VECTOR_FIELD = "vector"
+
+
+
+@pytest.mark.order(1)
+def test_index_creation_with_knn_vector_mapping():
+    """ Tests if index created and documents indexed.
+    """
+    # Create test index
+    created = ELASTIC.create_index(TEST_KNN_INDEX_NAME)
+    assert created["acknowledged"] is True
+    result = ELASTIC.add_vector_mapping(
+        index_name=TEST_KNN_INDEX_NAME, 
+        field=TEST_VECTOR_FIELD
+    )
+    assert result["acknowledged"] is True
+    
+    
+@pytest.mark.order(2)
+def test_index_creation_with_ann_vector_mapping():
+    """ Tests if index created and documents indexed.
+    """
+    # Create test index
+    created = ELASTIC.create_index(TEST_ANN_INDEX_NAME)
+    assert created["acknowledged"] is True
+    result = ELASTIC.add_ann_vector_mapping(
+        index_name=TEST_ANN_INDEX_NAME, 
+        field=TEST_VECTOR_FIELD
+    )
+    assert result["acknowledged"] is True
+    
+
+@pytest.mark.order(3)
+def test_vectorized_document_addition_knn_index():
+    """ Tests indexing vectorized documents.
+    """
+    # Add test documents
+    for document in TEST_DOCUMENTS:
+        indexed = ELASTIC.index_document(TEST_KNN_INDEX_NAME, document)
+        assert indexed["result"] == "created"
+    # let it index
+    sleep(1)
+    
+@pytest.mark.order(4)
+def test_vectorized_document_addition_ann_index():
+    """ Tests indexing vectorized documents.
+    """
+    # Add test documents
+    for document in TEST_DOCUMENTS:
+        indexed = ELASTIC.index_document(TEST_ANN_INDEX_NAME, document)
+        assert indexed["result"] == "created"
+    # let it index
+    sleep(1)
+    
+@pytest.mark.order(5)
+def test_fuzzy_search():
+    """ Tests fuzzy search.
+    """
+    response = ELASTIC.execute_fuzzy_search(
+        index=TEST_ANN_INDEX_NAME,
+        field="variations",
+        entity="Paul Keres",
+        fuzziness=0
+    )
+    total_hits = response.hits.total.value
+    assert total_hits == 2
+    
+    response = ELASTIC.execute_fuzzy_search(
+        index=TEST_ANN_INDEX_NAME,
+        field="variations",
+        entity="Paul Keres",
+        fuzziness=2
+    )
+    total_hits = response.hits.total.value
+    assert total_hits == 3
+    
+    
+@pytest.mark.order(6)
+def test_vector_search():
+    """ Tests vector search.
+    """
+    # Execut fuzzy search to get ID restrictions
+    response = ELASTIC.execute_fuzzy_search(
+        index=TEST_ANN_INDEX_NAME,
+        field="variations",
+        entity="Paul Keres",
+        fuzziness=2
+    )
+    total_hits = response.hits.total.value
+    assert total_hits == 3    
+    elastic_ids = [hit.meta.id for hit in response]
+    
+    response = ELASTIC.execute_vector_search(
+        index=TEST_ANN_INDEX_NAME,
+        field="vector",
+        query_vector=TEST_VECTOR,
+        k=1,
+        n_docs=1,
+        num_candidates=10,
+        elastic_ids=elastic_ids
+    )
+    descriptions = [hit.description for hit in response]
+    assert len(descriptions) == 1
+    assert descriptions[0] == "Eesti maletaja ja maleteoreetik"
+
+
+@pytest.mark.order(7)
+def test_index_deleting():
+    """
+    Tests deleting index. We delete the test index now.
+    """
+    indices = [TEST_KNN_INDEX_NAME, TEST_ANN_INDEX_NAME]
+    for index in indices:
+        deleted = ELASTIC.delete_index(index)
+        sleep(1)
+        assert deleted["acknowledged"] is True
+        

rara_tools-0.0.9/VERSION

@@ -1 +0,0 @@
-0.0.9