rara-tools 0.7.3__tar.gz → 0.7.5__tar.gz

{rara_tools-0.7.3/rara_tools.egg-info → rara_tools-0.7.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.7.3
+Version: 0.7.5
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

rara_tools-0.7.5/VERSION

@@ -0,0 +1 @@
+0.7.5

{rara_tools-0.7.3 → rara_tools-0.7.5}/rara_tools/constants/digitizer.py

@@ -26,3 +26,15 @@ class Queue:
 class Tasks:
     START_DIGITIZER_PIPELINE = "start_digitizer_pipeline"
     PURGE_MODELS = "purge_unused_digitizer_models"
+
+
+class Error:
+    NO_SPACE = "Disk out of space!"
+    COULDNT_DOWNLOAD = "Unknown error when downloading model!"
+    UNKNOWN = "Unknown system error!"
+    S3_CONNECTION = "Failed to connect to S3!"
+    UNSUPPORTED_FILETYPE = "Unsupported file type!"
+    COULDNT_UPLOAD = "Could not upload documents to Elasticsearch!"
+    FILE_IS_PROTECTED = "File is password protected or encrypted!"
+    UNKNOWN_OCR = "Unknown error when applying ocr!"
+    CUSTOM_MODEL_ERROR = "Couldn't download custom image classification model!"

{rara_tools-0.7.3 → rara_tools-0.7.5}/rara_tools/constants/linker.py

@@ -1,5 +1,6 @@
 import logging
-from rara_tools.constants.normalizers import EntityType, VIAF_ENTITY_MAP
+
+from rara_tools.constants.normalizers import EntityType
 
 COMPONENT_KEY = "linker"
 
@@ -24,11 +25,13 @@ class StatusKeys:
     VECTORIZE_CONTEXT = "vectorize_context"
     LINK_KEYWORDS = "link_keywords"
 
+
 class URLSource:
     VIAF = "VIAF"
     SIERRA = "Sierra"
     EMS = "EMS"
 
+
 class KeywordType:
     LOC = "Kohamärksõnad"
     TIME = "Ajamärksõnad"
@@ -54,17 +57,24 @@ class KeywordMARC:
     TITLE = 630
     TITLE_LINKED = 600
 
+
 class KeywordSource:
     EMS = "EMS"
     SIERRA = "SIERRA"
     VIAF = "VIAF"
     AI = "AI"
 
+
 class Filters:
     AUTHOR = "author"
     YEAR = "year"
 
 
+class Error:
+    VECTORIZATION = "Failed to vectorize text!"
+    LINKING_KEYWORDS = "Failed to link keywords!"
+    LINKING_META = "Failed to link meta!"
+
 UNLINKED_KEYWORD_MARC_FIELD = 693
 
 ALLOWED_FILTERS_MAP = {
@@ -110,7 +120,6 @@ ALLOWED_ENTITY_TYPES = [
     EntityType.UNK,
 ]
 
-
 KEYWORD_TYPE_MAP = {
     KeywordType.TIME: EntityType.KEYWORD,
     KeywordType.GENRE: EntityType.KEYWORD,

rara_tools-0.7.5/rara_tools/constants/meta_extractor.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass
+
+COMPONENT_KEY = "meta_extractor"
+
+
+class Tasks:
+    SINGLE = "extract_meta_from_text"
+    PIPELINE = "run_meta_extractor_with_core_logic"
+
+
+class Queue:
+    MAIN = "meta_extractor"
+
+
+class StatusKeys:
+    EXTRACT_METADATA = "extract_metadata"
+
+class Error:
+    UNKNOWN = "Failed to extract meta information from digitizer output!"
+
+
+@dataclass(frozen=True)
+class TitleType:
+    AUTHOR_WITHOUT_TITLE: str = "pealkirjata autor"
+    NORMALIZED_TITLE: str = "normitud eelispealkiri"
+    TITLE: str = "väljaandes esitatud kujul põhipealkiri"
+    PARALLEL_TITLE: str = "rööppealkiri"
+    ADDITIONAL_TITLE: str = "alampealkiri"
+    METS_TITLE: str = "väljaandes esitatud kujul põhipealkiri"
+    ANON: str = "anonüümne väljaanne"
+
+
+TITLE_TYPES_MAP = {
+    TitleType.AUTHOR_WITHOUT_TITLE: 130,
+    TitleType.NORMALIZED_TITLE: 240,
+    TitleType.TITLE: 245,
+    TitleType.PARALLEL_TITLE: 246,
+    TitleType.ADDITIONAL_TITLE: 245,
+    TitleType.METS_TITLE: 245,
+    TitleType.ANON: 130
+}
+
+
+PUBLISHER_KEY = "Väljaandja"

{rara_tools-0.7.3 → rara_tools-0.7.5}/rara_tools/constants/subject_indexer.py

@@ -18,6 +18,10 @@ class StatusKeys:
     EXTRACT_KEYWORDS = "extract_keywords"
 
 
+class Error:
+    UNKNOWN = "Could not extract keywords from text!"
+
+
 class URLSource:
     VIAF = "VIAF"
     SIERRA = "Sierra"

{rara_tools-0.7.3 → rara_tools-0.7.5}/rara_tools/core_formatters/core_formatter.py

@@ -2,6 +2,10 @@ from typing import List, Tuple, Any
 from rara_tools.core_formatters.formatted_keyword import FormattedKeyword
 from rara_tools.core_formatters.formatted_meta import FormattedAuthor
 from rara_tools.constants.linker import MAIN_TAXONOMY_LANG, KEYWORD_TYPES_TO_IGNORE, EntityType
+from rara_tools.constants.meta_extractor import TitleType, TITLE_TYPES_MAP, PUBLISHER_KEY
+from rara_tools.constants.subject_indexer import KeywordType
+
+import regex as re
 
 def get_primary_author(authors: List[dict]) -> str:
     primary_author = ""
@@ -10,8 +14,19 @@ def get_primary_author(authors: List[dict]) -> str:
             primary_author = author.get("name", "")
     return primary_author
 
-def format_authors(authors: List[dict]) -> List[dict]:
+def is_valid_keyword(keyword: str) -> bool:
+    # If keywords contains ONLY punctuation
+    # characters, we assume it`s not valid
+    if re.search(r"^(\W|_)+$", keyword):
+        return False
+    return True
+
+def format_series_info(series: str):
+    pass
+
+def format_authors(authors: List[dict]) -> Tuple[List[dict], dict]:
     formatted_authors = []
+    publisher = {}
     for author in authors:
         entity_type = author.get("type", EntityType.UNK)
 
@@ -20,8 +35,14 @@ def format_authors(authors: List[dict]) -> List[dict]:
             linked_doc=None,
             entity_type=entity_type
         ).to_dict()
+
+        # If author role == publisher, do not add it as an author
+        if formatted_author.get("author_role", "") == PUBLISHER_KEY:
+            publisher = formatted_author
+            continue
+
         formatted_authors.append(formatted_author)
-    return formatted_authors
+    return (formatted_authors, publisher)
 
 def format_sections(sections: List[dict]) -> List[dict]:
     for section in sections:
@@ -31,9 +52,16 @@ def format_sections(sections: List[dict]) -> List[dict]:
         if primary_author:
             for title in titles:
                 title["author_from_title"] = primary_author
+        if not authors:
+            for title in titles:
+                title["title_type"] = TitleType.ANON
+                title["title_type_int"] = TITLE_TYPES_MAP.get(TitleType.ANON)
         section["titles"] = titles
 
-        formatted_authors = format_authors(authors)
+        # Extract publisher, but do nothing with it
+        # as it is unlikely for the publishing info to be
+        # in a METS/ALTO section. Can update it, if proven otherwise
+        formatted_authors, publisher = format_authors(authors)
         section["authors"] = formatted_authors
 
     return sections
@@ -46,14 +74,25 @@ def format_meta(meta: dict) -> dict:
 
     authors = meta_to_format.pop("authors", [])
     sections = meta_to_format.pop("sections", [])
+    titles = meta_to_format.pop("titles", [])
 
-    formatted_authors = format_authors(authors)
+    formatted_authors, publisher = format_authors(authors)
     formatted_sections = format_sections(sections)
 
     if sections and formatted_sections:
         meta_to_format["sections"] = formatted_sections
     if authors and formatted_authors:
         meta_to_format["authors"] = formatted_authors
+    if titles and not authors:
+        for title in titles:
+            title["title_type"] = TitleType.ANON
+            title["title_type_int"] = TITLE_TYPES_MAP.get(TitleType.ANON)
+        meta_to_format["titles"] = titles
+
+    if publisher:
+        # Not sure, if it would be better to add original name or
+        # linked value. Currently adding original for safety
+        meta_to_format["publisher"] = publisher.get("original_name")
 
     meta["meta"] = meta_to_format
 
@@ -81,6 +120,30 @@ def format_keywords(flat_keywords: List[dict]) -> List[dict]:
             linked_doc=None,
             main_taxnomy_lang=MAIN_TAXONOMY_LANG
         ).to_dict()
-        formatted_keywords.append(formatted_keyword)
+        if is_valid_keyword(formatted_keyword.get("keyword")):
+            formatted_keywords.append(formatted_keyword)
 
     return formatted_keywords
+
+def get_udk072(flat_keywords: List[dict]) -> List[str]:
+    """ Filters out UDK from flat subject indexer output.
+    """
+    # keyword type: UDK
+    udk072 = [
+        keyword.get("keyword")
+        for keyword in flat_keywords
+        if keyword.get("entity_type") == KeywordType.UDK
+    ]
+    return udk072
+
+
+def get_udk080(flat_keywords: List[dict]) -> List[str]:
+    """ Filters out UDC from flat subject indexer output.
+    """
+    # keyword type: UDC
+    udk080 = [
+        keyword.get("keyword")
+        for keyword in flat_keywords
+        if keyword.get("entity_type") == KeywordType.UDC
+    ]
+    return udk080

{rara_tools-0.7.3 → rara_tools-0.7.5}/rara_tools/core_formatters/formatted_meta.py

@@ -11,7 +11,7 @@ class FormattedTitle(FormattedObject):
         super().__init__(
             object_dict=object_dict,
             linked_doc=linked_doc,
-            original_entity_key="name"
+            original_entity_key="title"
         )
 
 
@@ -41,17 +41,16 @@ class FormattedAuthor(FormattedObject):
         #self.__standardized_uri: str = ""
         self.__viaf_id: str = ""
 
+        self._default_author_type: str = EntityType.PER
+
 
     @property
     def primary_author_type(self) -> str:
         if self.__primary_author_type == None:
-            if self.is_primary:
-                if self.entity_type != EntityType.UNK:
+            self.__primary_author_type = self._default_author_type
+            if self.entity_type != EntityType.UNK:
+                if self.entity_type in [EntityType.ORG, EntityType.PER]:
                     self.__primary_author_type = self.entity_type
-                else:
-                    self.__primary_author_type = EntityType.PER
-            else:
-                self.__primary_author_type = ""
         return self.__primary_author_type
 
 

{rara_tools-0.7.3 → rara_tools-0.7.5/rara_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.7.3
+Version: 0.7.5
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

rara_tools-0.7.5/tests/test_formatters.py

@@ -0,0 +1,85 @@
+import pytest
+import os
+from pprint import pprint
+from rara_tools.core_formatters.core_formatter import format_keywords, format_meta, get_udk072, get_udk080
+from rara_tools.constants.meta_extractor import PUBLISHER_KEY
+from tests.test_utils import read_json_file
+
+ROOT_DIR = os.path.join("tests", "test_data", "formatter")
+INPUT_KEYWORDS_FILE_PATHS = [
+    os.path.join(ROOT_DIR, "keywords_1.json"),
+    os.path.join(ROOT_DIR, "keywords_2.json"),
+    os.path.join(ROOT_DIR, "keywords_3.json")
+]
+INPUT_META_FILE_PATHS = [
+    os.path.join(ROOT_DIR, "epub_meta.json"),
+    os.path.join(ROOT_DIR, "mets_alto_meta.json"),
+    os.path.join(ROOT_DIR, "pdf_meta_2.json"),
+    os.path.join(ROOT_DIR, "pdf_meta.json")
+]
+
+INPUT_KEYWORDS = [
+    read_json_file(keyword_file_path)
+    for keyword_file_path in INPUT_KEYWORDS_FILE_PATHS
+]
+
+INPUT_META_DICTS = [
+    read_json_file(meta_file_path)
+    for meta_file_path in INPUT_META_FILE_PATHS
+]
+
+def test_formatting_keywords_for_core():
+    for keyword_dict_list in INPUT_KEYWORDS:
+        formatted_keywords = format_keywords(keyword_dict_list)
+        #pprint(formatted_keywords)
+        assert formatted_keywords
+        assert isinstance(formatted_keywords, list)
+
+
+def test_formatting_meta_for_core():
+    for meta_dict in INPUT_META_DICTS:
+        formatted_meta = format_meta(meta_dict)
+        #pprint(formatted_meta)
+        assert formatted_meta
+        assert isinstance(formatted_meta, dict)
+
+
+def test_validating_keywords():
+    keyword_dict_list = INPUT_KEYWORDS[-1]
+    assert len(keyword_dict_list) == 6
+    formatted_keywords = format_keywords(keyword_dict_list)
+    assert len(formatted_keywords) == 2
+
+def test_removing_publisher_from_authors():
+    meta_dict = INPUT_META_DICTS[-1]
+    formatted_meta = format_meta(meta_dict)
+    assert formatted_meta["meta"]["publisher"] == "s.n"
+    for author in formatted_meta["meta"]["authors"]:
+        assert author.get("author_role") != PUBLISHER_KEY
+    assert isinstance(formatted_meta, dict)
+
+def test_title_key_without_authors():
+    meta_dict = INPUT_META_DICTS[2]
+    formatted_meta = format_meta(meta_dict)
+    for title in formatted_meta["meta"]["titles"]:
+        assert title.get("title_type_int") == 130
+    assert isinstance(formatted_meta, dict)
+
+def test_all_authors_have_types():
+    for meta_dict in INPUT_META_DICTS:
+        formatted_meta = format_meta(meta_dict)
+        authors = formatted_meta.get("authors", [])
+        for author in authors:
+            assert author.get("primary_author_type")
+
+def test_getting_udk072():
+    keyword_dict_list = INPUT_KEYWORDS[-1]
+    assert len(keyword_dict_list) == 6
+    udk072_list = get_udk072(keyword_dict_list)
+    assert len(udk072_list) == 1
+
+def test_getting_udk080():
+    keyword_dict_list = INPUT_KEYWORDS[-1]
+    assert len(keyword_dict_list) == 6
+    udk080_list = get_udk080(keyword_dict_list)
+    assert len(udk080_list) == 1

rara_tools-0.7.3/VERSION

@@ -1 +0,0 @@
-0.7.3

rara_tools-0.7.3/rara_tools/constants/meta_extractor.py

@@ -1,14 +0,0 @@
-COMPONENT_KEY = "meta_extractor"
-
-
-class Tasks:
-    SINGLE = "extract_meta_from_text"
-    PIPELINE = "run_meta_extractor_with_core_logic"
-
-
-class Queue:
-    MAIN = "meta_extractor"
-
-
-class StatusKeys:
-    EXTRACT_METADATA = "extract_metadata"

rara_tools-0.7.3/tests/test_formatters.py

@@ -1,41 +0,0 @@
-import pytest
-import os
-from pprint import pprint
-from rara_tools.core_formatters.core_formatter import format_keywords, format_meta
-from tests.test_utils import read_json_file
-
-ROOT_DIR = os.path.join("tests", "test_data", "formatter")
-INPUT_KEYWORDS_FILE_PATHS = [
-    os.path.join(ROOT_DIR, "keywords_1.json"),
-    os.path.join(ROOT_DIR, "keywords_2.json")
-]
-INPUT_META_FILE_PATHS = [
-    os.path.join(ROOT_DIR, "epub_meta.json"),
-    os.path.join(ROOT_DIR, "mets_alto_meta.json"),
-    os.path.join(ROOT_DIR, "pdf_meta.json")
-]
-
-INPUT_KEYWORDS = [
-    read_json_file(keyword_file_path)
-    for keyword_file_path in INPUT_KEYWORDS_FILE_PATHS
-]
-
-INPUT_META_DICTS = [
-    read_json_file(meta_file_path)
-    for meta_file_path in INPUT_META_FILE_PATHS
-]
-
-def test_formatting_keywords_for_core():
-    for keyword_dict in INPUT_KEYWORDS:
-        formatted_keywords = format_keywords(keyword_dict)
-        #pprint(formatted_keywords)
-        assert formatted_keywords
-        assert isinstance(formatted_keywords, list)
-
-
-def test_formatting_meta_for_core():
-    for meta_dict in INPUT_META_DICTS:
-        formatted_meta = format_meta(meta_dict)
-        #pprint(formatted_meta)
-        assert formatted_meta
-        assert isinstance(formatted_meta, dict)