rara-tools 0.0.9__py3-none-any.whl → 0.0.10__py3-none-any.whl

rara_tools/digar_schema_converter.py

@@ -0,0 +1,409 @@
+from collections import defaultdict
+from typing import List, NoReturn
+
+from rara_tools.utils import lang_to_iso639_2, ratio_to_percentage
+
+GENERAL_DOC_IDENTIFIER = "Filepath"
+UNDEFINED_LANGUAGE_VALUE = "unk"
+QUALITY_RATIO_TYPE = "Float"
+
+
+class ImagePageSchema:
+    def __init__(self, image: dict) -> NoReturn:
+        self.__image = image
+        self.__schema: dict = {}
+
+    @property
+    def schema(self) -> dict:
+        if not self.__schema:
+            self.__schema = {
+                "@type": "VisualArtwork",
+                "@id": "",
+                "value": self.__image.get("label"),
+                "description": "",
+                "schema:position": self.__image.get("page")
+            }
+        return self.__schema
+
+
+class TextPageSchema:
+    def __init__(self, page: dict) -> NoReturn:
+        self.__page: dict = page
+        self.__schema: dict = {}
+
+    @property
+    def schema(self) -> dict:
+        if not self.__schema:
+            self.__schema = {
+                "@type": "Text",  # CONSTANT
+                "@id": "",  # Will be added in a later stage
+                "value": "Textblock",  # CONSTANT
+                "content": self.__page.get("text"),
+                "schema:position": self.__page.get("start_page")  # start_page ?
+            }
+        return self.__schema
+
+
+class PageSchema:
+    def __init__(
+            self,
+            page_texts: List[dict],
+            page_images: List[dict],
+            page_number: int,
+            doc_id: str
+    ) -> NoReturn:
+        self.__page_texts: List[dict] = page_texts
+        self.__page_images: List[dict] = page_images
+        self.__page_nr: int = page_number
+        self.__page_id: str = ""
+        self.__doc_id: str = doc_id
+        self.__schema: dict = {}
+
+    def _add_segment_ids(self, segments: List[dict]) -> List[dict]:
+        for i, segment in enumerate(segments):
+            segment_id = f"{self.page_id}/{i + 1}"
+            segment["@id"] = segment_id
+        return segments
+
+    @property
+    def page_id(self) -> str:
+        if not self.__page_id:
+            self.__page_id = f"{self.__doc_id}/{self.__page_nr}"
+        return self.__page_id
+
+    @property
+    def schema(self) -> dict:
+        if not self.__schema:
+            self.__schema = {
+                "@type": "CreativeWork",  # CONSTANT for pages
+                "@id": self.page_id,
+                "hasPart": []
+            }
+            text_schemas = [
+                TextPageSchema(page).schema
+                for page in self.__page_texts
+            ]
+            image_schemas = [
+                ImagePageSchema(image).schema
+                for image in self.__page_images
+            ]
+
+            page_schemas = text_schemas + image_schemas
+            page_schemas_with_ids = self._add_segment_ids(page_schemas)
+
+            self.__schema["hasPart"].extend(page_schemas_with_ids)
+
+        return self.__schema
+
+
+class DocSchemas:
+    def __init__(
+            self,
+            doc_meta: dict,
+            sierra_id: str = "",
+            generated_id: str = "",
+            permalink: str = "",
+            min_language_ratio: float = 0.2,
+            convert_ratio: bool = True
+    ) -> NoReturn:
+        self.__convert_ratio = convert_ratio
+        self.__min_language_ratio = min_language_ratio
+        self.__sierra_id = sierra_id
+        self.__generated_id = generated_id
+        self.__permalink = permalink
+        self.__doc_meta = doc_meta
+        self.__ocr_accuracy_schema: dict = {}
+        self.__text_quality_schema: dict = {}
+        self.__language_schema: List[dict] = []
+        self.__identifier_schema: List[dict] = []
+        self.__origin_schema: dict = {}
+        self.__origin: str = ""
+
+    @property
+    def origin(self) -> str:
+        if not self.__origin:
+            if self.__doc_meta["ocr_applied"]:
+                self.__origin = "Reformatted digital"
+            else:
+                self.__origin = "Born digital"
+        return self.__origin
+
+    @property
+    def ocr_accuracy_schema(self) -> dict:
+        if not self.__ocr_accuracy_schema:
+            ocr_quality = self.__doc_meta.get("alto_text_quality")
+            if ocr_quality:
+                self.__ocr_accuracy_schema = {
+                    "comment": "Estimated OCR accuracy"
+                }
+                if self.__convert_ratio:
+                    type_and_value = {
+                        "@type": QUALITY_RATIO_TYPE,
+                        "value": ocr_quality
+                    }
+                else:
+                    type_and_value = {
+                        "@type": "Text",
+                        "value": ratio_to_percentage(ocr_quality)
+                    }
+                self.__ocr_accuracy_schema.update(type_and_value)
+        return self.__ocr_accuracy_schema
+
+    @property
+    def text_quality_schema(self) -> dict:
+        if not self.__text_quality_schema:
+            text_quality = self.__doc_meta.get("text_quality")
+            self.__text_quality_schema = {
+                "comment": "Estimated n-gram-based text quality"
+            }
+            if self.__convert_ratio:
+                type_and_value = {
+                    "@type": QUALITY_RATIO_TYPE,
+                    "value": text_quality
+                }
+            else:
+                type_and_value = {
+                    "@type": "Text",
+                    "value": ratio_to_percentage(text_quality)
+                }
+            self.__text_quality_schema.update(type_and_value)
+        return self.__text_quality_schema
+
+    @property
+    def language_schema(self) -> List[dict]:
+        if not self.__language_schema:
+            self.__language_schema = [
+                {
+                    "@type": "ISO 639-2",
+                    "value": lang_to_iso639_2(
+                        lang["language"],
+                        unk_code=UNDEFINED_LANGUAGE_VALUE
+                    )
+                }
+                for lang in self.__doc_meta["languages"]
+                if lang["ratio"] >= self.__min_language_ratio
+            ]
+        return self.__language_schema
+
+    @property
+    def identifier_schema(self) -> List[dict]:
+        if not self.__identifier_schema:
+            identifiers = []
+            if self.__sierra_id:
+                identifiers.append(
+                    {
+                        "@type": "Identifier",
+                        "qualifier": "OPAC",
+                        "value": self.__sierra_id
+                    }
+                )
+            if self.__permalink:
+                identifiers.append(
+                    {
+                        "@type": "Identifier",
+                        "qualifier": "Permalink",
+                        "value": self.__permalink
+                    }
+                )
+            if self.__generated_id:
+                identifiers.append(
+                    {
+                        "@type": "Identifier",
+                        "qualifier": GENERAL_DOC_IDENTIFIER,
+                        "value": self.__generated_id
+                    }
+                )
+            self.__identifier_schema = identifiers
+
+        return self.__identifier_schema
+
+    @property
+    def origin_schema(self) -> dict:
+        if not self.__origin_schema:
+            self.__origin_schema = {
+                "@type": "Text",
+                "value": self.origin,
+                "comment": "Origin"
+            }
+        return self.__origin_schema
+
+
+class DIGARSchemaConverter:
+    def __init__(
+            self,
+            digitizer_output: dict,
+            generated_id: str,
+            sierra_id: str = "",
+            permalink: str = "",
+            min_language_ratio: float = 0.2,
+            convert_ratio: bool = False
+    ) -> NoReturn:
+        """ Initialize DIGARSchemaConverter object. 
+        
+        Parameters
+        ----------
+        digitizer_output: dict
+            Raw output of rara-digitizer (https://pypi.org/project/rara-digitizer/).
+        generated_id: str
+            Some non-standard/generated document identifier used in ID fields.
+        sierra_id: str
+            Document's corresponding Sierra ID.
+        permalink: str
+            Permanent link, where the document can be accessed.
+        min_language_ratio: float
+            Cutoff ratio for languages. If ratio for some language
+            does not exceed the set threshold, the language will not
+            be added to the final output.
+        convert_ratio: bool
+            If enabled, all ratios are converted into percentages.
+          
+        """
+        self.__digitizer_output: dict = digitizer_output
+        self.__min_language_ratio: float = min_language_ratio
+        self.__convert_ratio: bool = convert_ratio
+        self.__sierra_id: str = sierra_id
+        self.__generated_id: str = generated_id
+        self.__permalink: str = permalink.removesuffix("/")
+        self.__texts: List[dict] = []
+        self.__images: List[dict] = []
+        self.__doc_meta: dict = {}
+        self.__page_mappings: List[dict] = []
+        self.__dcterms_haspart: dict = {}
+        self.__dcterms_conforms_to: dict = {}
+        self.__dc_language: dict = {}
+        self.__dc_origin: dict = {}
+        self.__dc_identifier: List[dict] = []
+        self.__doc_id: str = ""
+
+        self.__doc_schemas = DocSchemas(
+            doc_meta=self.doc_meta,
+            sierra_id=self.__sierra_id,
+            generated_id=self.__generated_id,
+            permalink=self.__permalink,
+            min_language_ratio=self.__min_language_ratio,
+            convert_ratio=self.__convert_ratio
+        )
+        self.__digar_schema: dict = {}
+
+    def _get_page_number(self, page_content: dict) -> int:
+        """ Retrieves page number from image or text object.
+        """
+        _segments = page_content["texts"] + page_content["images"]
+        _first_segment = _segments[0]
+        if "start_page" in _first_segment:
+            page_number = _first_segment.get("start_page")
+        elif "page" in _first_segment:
+            page_number = _first_segment.get("page")
+        return page_number
+
+    @property
+    def doc_id(self) -> str:
+        """ Retrieves document ID to use for generating
+        page and segment ids. Preference order:
+        1. permalink; 2. sierra_id; 3. generated document id
+        """
+        if not self.__doc_id:
+            if self.__permalink:
+                self.__doc_id = self.__permalink
+            elif self.__sierra_id:
+                self.__doc_id = self.__sierra_id
+            else:
+                self.__doc_id = self.__generated_id
+        return self.__doc_id
+
+    @property
+    def texts(self) -> List[dict]:
+        if not self.__texts:
+            self.__texts = self.__digitizer_output.get("texts")
+        return self.__texts
+
+    @property
+    def images(self) -> List[dict]:
+        if not self.__images:
+            self.__images = self.__digitizer_output.get("images")
+        return self.__images
+
+    @property
+    def doc_meta(self) -> dict:
+        if not self.__doc_meta:
+            self.__doc_meta = self.__digitizer_output.get("doc_meta")
+        return self.__doc_meta
+
+    @property
+    def page_mappings(self) -> List[dict]:
+        if not self.__page_mappings:
+            mapped = defaultdict(lambda: defaultdict(list))
+            for text in self.texts:
+                mapped[text["start_page"]]["texts"].append(text)
+            for img in self.images:
+                mapped[img["page"]]["images"].append(img)
+
+            self.__page_mappings = [
+                v for k, v in sorted(list(mapped.items()), key=lambda x: x[0])
+            ]
+        return self.__page_mappings
+
+    @property
+    def dcterms_haspart(self) -> dict:
+        if not self.__dcterms_haspart:
+            self.__dcterms_haspart = {
+                "dcterms:hasPart": [
+                    PageSchema(
+                        page_texts=page["texts"],
+                        page_images=page["images"],
+                        page_number=self._get_page_number(page),
+                        doc_id=self.doc_id
+                    ).schema
+                    for page in self.page_mappings
+                ]
+            }
+        return self.__dcterms_haspart
+
+    @property
+    def dcterms_conforms_to(self) -> dict:
+        if not self.__dcterms_conforms_to:
+            schema_content = [
+                self.__doc_schemas.text_quality_schema,
+            ]
+            # Add OCR Accuracy only when it is not empty:
+            if self.__doc_schemas.ocr_accuracy_schema:
+                schema_content.append(self.__doc_schemas.ocr_accuracy_schema)
+            self.__dcterms_conforms_to = {
+                "dcterms:conformsTo": schema_content
+            }
+        return self.__dcterms_conforms_to
+
+    @property
+    def dc_language(self) -> dict:
+        if not self.__dc_language:
+            self.__dc_language = {
+                "dc:language": self.__doc_schemas.language_schema
+            }
+        return self.__dc_language
+
+    @property
+    def dc_origin(self) -> dict:
+        if not self.__dc_origin:
+            self.__dc_origin = {
+                "dcterms:provenance": self.__doc_schemas.origin_schema
+            }
+        return self.__dc_origin
+
+    @property
+    def dc_identifier(self) -> List[dict]:
+        if not self.__dc_identifier:
+            self.__dc_identifier = {
+                "dc:identifier": self.__doc_schemas.identifier_schema
+            }
+        return self.__dc_identifier
+
+    @property
+    def digar_schema(self) -> dict:
+        if not self.__digar_schema:
+            self.__digar_schema = {}
+            self.__digar_schema.update(self.dcterms_conforms_to)
+            self.__digar_schema.update(self.dcterms_haspart)
+            self.__digar_schema.update(self.dc_language)
+            self.__digar_schema.update(self.dc_origin)
+            self.__digar_schema.update(self.dc_identifier)
+        return self.__digar_schema

rara_tools/utils.py

@@ -0,0 +1,104 @@
+from iso639 import Lang
+
+
+def lang_to_iso639_1(lang: str, unk_code: str = "unk") -> str:
+    """ Converts language into ISO-639-1 standard. 
+    Input can be any language code in a valid ISO-639 
+    standard or even a full name of the language, 
+    e.g. "Estonian".
+    
+    Parameters
+    -----------
+    lang: str
+        Language code in any valid ISO-639 standard.
+        
+    unk_code: str
+        Code to return incase of invalid/unsupported
+        input language.
+        
+    Returns
+    -------
+    Language code in ISO-639-1 standard.
+    """
+    try:
+        lg = Lang(lang)
+        iso_639_1_lang = lg.pt1
+    except:
+        iso_639_1_lang = unk_code
+    return iso_639_1_lang
+
+
+def lang_to_iso639_2(lang: str, unk_code: str = "unk") -> str:
+    """ Converts language into ISO-639-2 standard. 
+    Input can be any language code in a valid ISO-639 
+    standard or even a full name of the language, 
+    e.g. "Estonian".
+    
+    Parameters
+    -----------
+    lang: str
+        Language code in any valid ISO-639 standard.
+        
+    unk_code: str
+        Code to return incase of invalid/unsupported
+        input language.
+        
+    Returns
+    -------
+    Language code in ISO-639-2 standard.
+    """
+    try:
+        lg = Lang(lang)
+        # NB! uses bibliographic identifier (e.g. "de" -> "ger")
+        # opposed to terminological identifier ("de" -> "deu").
+        # This can be changed by replaving lg.pt2b -> lg.pt2t
+        iso_639_2_lang = lg.pt2b
+    except:
+        iso_639_2_lang = unk_code
+    return iso_639_2_lang
+
+
+def lang_to_iso639_3(lang: str, unk_code: str = "unk") -> str:
+    """ Converts language into ISO-639-3 standard. 
+    Input can be any language code in a valid ISO-639 
+    standard or even a full name of the language, 
+    e.g. "Estonian".
+    
+    Parameters
+    -----------
+    lang: str
+        Language code in any valid ISO-639 standard.
+        unk_code: str
+        
+    Code to return incase of invalid/unsupported
+        input language.
+        
+    Returns
+    -------
+    str
+        Language code in ISO-639-3 standard.
+    """
+    try:
+        lg = Lang(lang)
+        iso_639_3_lang = lg.pt3
+    except:
+        iso_639_3_lang = unk_code
+    return iso_639_3_lang
+
+
+def ratio_to_percentage(ratio: float) -> str:
+    """ Converts ratio to corresponding percentage.
+    
+    Parameters
+    -----------
+    ratio: float
+        Float in range [0,1]
+    
+    Returns
+    --------
+    str
+        Percentage corresponding to the float.
+    
+    """
+    percentage = f"{int(ratio*100)}%"
+    return percentage

{rara_tools-0.0.9.dist-info → rara_tools-0.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rara-tools
-Version: 0.0.9
+Version: 0.0.10
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -14,6 +14,7 @@ Requires-Dist: elasticsearch==8.*
 Requires-Dist: elasticsearch_dsl==8.*
 Requires-Dist: minio==7.*
 Requires-Dist: requests
+Requires-Dist: iso639-lang
 Provides-Extra: testing
 Requires-Dist: pytest>=8.0; extra == "testing"
 Requires-Dist: pytest-order; extra == "testing"

{rara_tools-0.0.9.dist-info → rara_tools-0.0.10.dist-info}/RECORD

@@ -1,14 +1,16 @@
 rara_tools/converters.py,sha256=JcS74VzV6jm12l3C6aqMJBY9nuVW_aevQeCe32KmfrE,1576
 rara_tools/decorators.py,sha256=MjOyvZ5nTkwxwx2JLFEGpKKBysvecFw6EN6UDrSvZLU,2187
+rara_tools/digar_schema_converter.py,sha256=gGwhqdwxyTXODF0LP5Xi0u8uRoICfaIU3MRe1EVBnEc,13935
 rara_tools/elastic.py,sha256=vEvrbIPRtdqTdrNrPH2cewHLMfOTSf87a4JOiRQgYyA,7146
 rara_tools/exceptions.py,sha256=BwNh4qWxau_ylr9RqZoYwd1KnExI6oWWWDno3jkh8q4,474
 rara_tools/s3.py,sha256=uNDu2HzMYHAWh33RcHeyPFK7gdQfQPxsdfohyIKezEY,4467
 rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
+rara_tools/utils.py,sha256=9vSbmuWYU5ydr4lXBKlUKa0xzDccFsaJv4T-XwgUfuY,2578
 rara_tools/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rara_tools/constants/digitizer.py,sha256=gJ3jOMwuZfKcLqgOAxTyB266VYsskLabJiMUiSz3xX4,297
 rara_tools/constants/general.py,sha256=E9Jaw-YxocS_tOZw9QBoxO3e9KK5EMbLoM0R7D4Iflw,171
-rara_tools-0.0.9.dist-info/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
-rara_tools-0.0.9.dist-info/METADATA,sha256=HhxVd2e_lhAizmc9p88dOVuaCygVRH5tDv3xrPZXVmk,3867
-rara_tools-0.0.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-rara_tools-0.0.9.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
-rara_tools-0.0.9.dist-info/RECORD,,
+rara_tools-0.0.10.dist-info/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
+rara_tools-0.0.10.dist-info/METADATA,sha256=jV6nZKhjjwDL6TWt-fKWudWNUAViZTVDL0J39fefFtM,3895
+rara_tools-0.0.10.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+rara_tools-0.0.10.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
+rara_tools-0.0.10.dist-info/RECORD,,