rara-tools 0.6.13__tar.gz → 0.6.14__tar.gz

{rara_tools-0.6.13/rara_tools.egg-info → rara_tools-0.6.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.6.13
+Version: 0.6.14
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

rara_tools-0.6.14/VERSION

@@ -0,0 +1 @@
+0.6.14

rara_tools-0.6.14/rara_tools/constants/subject_indexer.py

@@ -0,0 +1,85 @@
+from rara_tools.constants.normalizers import EntityType
+
+COMPONENT_KEY = "subject_indexer"
+
+
+class Tasks:
+    SINGLE = "run_subject_indexer_process"
+    PIPELINE = "run_subject_indexer_with_core_logic"
+
+
+class Queue:
+    MAIN = "subject-indexer"
+
+
+class StatusKeys:
+    EXTRACT_KEYWORDS = "extract_keywords"
+
+
+class URLSource:
+    VIAF = "VIAF"
+    SIERRA = "Sierra"
+    EMS = "EMS"
+
+class KeywordType:
+    LOC = "Kohamärksõnad"
+    TIME = "Ajamärksõnad"
+    TOPIC = "Teemamärksõnad"
+    GENRE = "Vormimärksõnad"
+    TITLE = "Teose pealkiri"
+    PER = "Isikunimi"
+    ORG = "Kollektiivi nimi"
+    EVENT = "Ajutine kollektiiv või sündmus"
+    CATEGORY = "Valdkonnamärksõnad"
+    UDC = "UDC Summary"
+    UDK = "UDK Rahvusbibliograafia"
+
+
+class KeywordMARC:
+    PER = 600
+    ORG = 610
+    TOPIC = 650
+    GENRE = 655
+    TIME = 648
+    LOC = 651
+    EVENT = 611
+    TITLE = 630
+
+class KeywordSource:
+    EMS = "EMS"
+    SIERRA = "SIERRA"
+    VIAF = "VIAF"
+    AI = "AI"
+
+
+KEYWORD_TYPE_MAP = {
+    KeywordType.TIME: EntityType.KEYWORD,
+    KeywordType.GENRE: EntityType.KEYWORD,
+    KeywordType.LOC: EntityType.LOC,
+    KeywordType.PER: EntityType.PER,
+    KeywordType.ORG: EntityType.ORG,
+    KeywordType.TOPIC: EntityType.KEYWORD,
+    KeywordType.TITLE: EntityType.TITLE,
+    KeywordType.EVENT: EntityType.ORG
+}
+
+KEYWORD_MARC_MAP = {
+    KeywordType.LOC: KeywordMARC.LOC,
+    KeywordType.TIME: KeywordMARC.TIME,
+    KeywordType.TOPIC: KeywordMARC.TOPIC,
+    KeywordType.GENRE: KeywordMARC.GENRE,
+    KeywordType.TITLE: KeywordMARC.TITLE,
+    KeywordType.ORG: KeywordMARC.ORG,
+    KeywordType.PER: KeywordMARC.PER,
+    KeywordType.EVENT: KeywordMARC.EVENT
+}
+
+KEYWORD_TYPES_TO_IGNORE = [
+    KeywordType.CATEGORY,
+    KeywordType.UDC,
+    KeywordType.UDK
+]
+
+EMS_ENTITY_TYPES = [EntityType.KEYWORD, EntityType.LOC]
+SIERRA_ENTITY_TYPES = [EntityType.PER, EntityType.ORG, EntityType.TITLE]
+VIAF_ENTITY_TYPES = [EntityType.PER, EntityType.ORG, EntityType.TITLE]

rara_tools-0.6.14/rara_tools/formatters.py

@@ -0,0 +1,106 @@
+from typing import List, Tuple, Any
+from rara_tools.constants.subject_indexer import (
+    EntityType, KeywordType, KeywordMARC, KeywordSource, URLSource,
+    KEYWORD_TYPE_MAP, KEYWORD_MARC_MAP, KEYWORD_TYPES_TO_IGNORE,
+    EMS_ENTITY_TYPES, SIERRA_ENTITY_TYPES, VIAF_ENTITY_TYPES
+)
+
+def _get_keyword_source(linked_doc: Any, entity_type: str, is_linked: bool
+) -> str:
+    """ Find keyword source.
+    """
+    if not is_linked:
+        source = KeywordSource.AI
+    elif entity_type in EMS_ENTITY_TYPES:
+        source = KeywordSource.EMS
+    elif entity_type in SIERRA_ENTITY_TYPES:
+        if linked_doc and linked_doc.elastic:
+            source = KeywordSource.SIERRA
+        elif linked_doc and linked_doc.viaf:
+            source = KeywordSource.VIAF
+        else:
+            source = KeywordSource.AI
+    else:
+        source = KeywordSource.AI
+    return source
+
+def _find_indicators(entity_type: str, entity: str,
+        is_linked: bool
+) -> Tuple[str, str]:
+    """ Find MARC indicators 1 and 2.
+    """
+    ind1 = " "
+    ind2 = " "
+    if entity_type in SIERRA_ENTITY_TYPES:
+        if entity_type == EntityType.PER:
+            if "," in entity:
+                ind1 = "1"
+            else:
+                ind1 = "0"
+        else:
+            # 1 märksõna esimeseks elemendiks võimupiirkonna nimi, nt:
+            #    (a) Eesti (b) Riigikogu - raske automaatselt määrata
+            # 2 märksõna esimeseks elemendiks nimi pärijärjestuses
+            ind1 = "2"
+        if not is_linked:
+            ind2 = "4"
+    elif entity_type in EMS_ENTITY_TYPES:
+        ind2 = "4"
+    return (ind1, ind2)
+
+
+def format_keywords(flat_keywords: List[dict]) -> dict:
+    """ Formats unlinked keywords for Kata CORE.
+    """
+    ignored_keywords = []
+    filtered_keywords = []
+
+    for keyword_dict in flat_keywords:
+        keyword_type = keyword_dict.get("entity_type")
+        if keyword_type in KEYWORD_TYPES_TO_IGNORE:
+            ignored_keywords.append(keyword_dict)
+        else:
+            filtered_keywords.append(keyword_dict)
+
+    formatted_keywords = {
+        "keywords": [],
+        "other": ignored_keywords
+    }
+
+    for keyword_dict in filtered_keywords:
+        original_keyword = keyword_dict.get("keyword")
+        keyword_type = keyword_dict.get("entity_type")
+        entity_type = KEYWORD_TYPE_MAP.get(keyword_type, "")
+        marc_field = KEYWORD_MARC_MAP.get(str(keyword_type), "")
+        lang = keyword_dict.get("language", "")
+
+        ind1, ind2 = _find_indicators(
+            entity_type=entity_type,
+            entity=original_keyword,
+            is_linked=False
+        )
+        keyword_source = _get_keyword_source(
+            linked_doc=None,
+            is_linked=False,
+            entity_type=entity_type
+        )
+        new_keyword_dict = {
+            "dates": "",
+            "indicator1": ind1,
+            "indicator2": ind2,
+            "is_linked": False,
+            "keyword_source": keyword_source,
+            "lang": lang,
+            "location": "",
+            "marc_field": marc_field,
+            "numeration": "",
+            "organisation_sub_unit": "",
+            "original_keyword": original_keyword,
+            "persons_title": "",
+            "url": "",
+            "url_source": ""
+        }
+        new_keyword_dict.update(keyword_dict)
+        formatted_keywords["keywords"].append(new_keyword_dict)
+
+    return formatted_keywords

{rara_tools-0.6.13 → rara_tools-0.6.14/rara_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.6.13
+Version: 0.6.14
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.6.13 → rara_tools-0.6.14}/rara_tools.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ rara_tools/decorators.py
 rara_tools/digar_schema_converter.py
 rara_tools/elastic.py
 rara_tools/exceptions.py
+rara_tools/formatters.py
 rara_tools/s3.py
 rara_tools/task_reporter.py
 rara_tools/utils.py
@@ -48,6 +49,7 @@ tests/test_digar_schema_converter.py
 tests/test_elastic.py
 tests/test_elastic_vector_and_search_operations.py
 tests/test_entity_normalizers.py
+tests/test_formatters.py
 tests/test_marc_parsers.py
 tests/test_normalization.py
 tests/test_s3_exceptions.py

rara_tools-0.6.14/tests/test_formatters.py

@@ -0,0 +1,15 @@
+import pytest
+import os
+from pprint import pprint
+from rara_tools.formatters import format_keywords
+from tests.test_utils import read_json_file
+
+ROOT_DIR = os.path.join("tests", "test_data", "formatter")
+INPUT_KEYWORDS_FILE_PATH = os.path.join(ROOT_DIR, "keywords.json")
+
+INPUT_KEYWORDS = read_json_file(INPUT_KEYWORDS_FILE_PATH)
+
+def test_formatting_keywords_for_core():
+    formatted_keywords = format_keywords(INPUT_KEYWORDS)
+    assert formatted_keywords
+    assert isinstance(formatted_keywords, dict)

rara_tools-0.6.13/VERSION

@@ -1 +0,0 @@
-0.6.13

rara_tools-0.6.13/rara_tools/constants/subject_indexer.py

@@ -1,14 +0,0 @@
-COMPONENT_KEY = "subject_indexer"
-
-
-class Tasks:
-    SINGLE = "run_subject_indexer_process"
-    PIPELINE = "run_subject_indexer_with_core_logic"
-
-
-class Queue:
-    MAIN = "subject-indexer"
-
-
-class StatusKeys:
-    EXTRACT_KEYWORDS = "extract_keywords"