rara-tools 0.6.12__tar.gz → 0.6.14__tar.gz

{rara_tools-0.6.12/rara_tools.egg-info → rara_tools-0.6.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.6.12
+Version: 0.6.14
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

rara_tools-0.6.14/VERSION

@@ -0,0 +1 @@
+0.6.14

rara_tools-0.6.14/rara_tools/constants/subject_indexer.py

@@ -0,0 +1,85 @@
+from rara_tools.constants.normalizers import EntityType
+
+COMPONENT_KEY = "subject_indexer"
+
+
+class Tasks:
+    SINGLE = "run_subject_indexer_process"
+    PIPELINE = "run_subject_indexer_with_core_logic"
+
+
+class Queue:
+    MAIN = "subject-indexer"
+
+
+class StatusKeys:
+    EXTRACT_KEYWORDS = "extract_keywords"
+
+
+class URLSource:
+    VIAF = "VIAF"
+    SIERRA = "Sierra"
+    EMS = "EMS"
+
+class KeywordType:
+    LOC = "Kohamärksõnad"
+    TIME = "Ajamärksõnad"
+    TOPIC = "Teemamärksõnad"
+    GENRE = "Vormimärksõnad"
+    TITLE = "Teose pealkiri"
+    PER = "Isikunimi"
+    ORG = "Kollektiivi nimi"
+    EVENT = "Ajutine kollektiiv või sündmus"
+    CATEGORY = "Valdkonnamärksõnad"
+    UDC = "UDC Summary"
+    UDK = "UDK Rahvusbibliograafia"
+
+
+class KeywordMARC:
+    PER = 600
+    ORG = 610
+    TOPIC = 650
+    GENRE = 655
+    TIME = 648
+    LOC = 651
+    EVENT = 611
+    TITLE = 630
+
+class KeywordSource:
+    EMS = "EMS"
+    SIERRA = "SIERRA"
+    VIAF = "VIAF"
+    AI = "AI"
+
+
+KEYWORD_TYPE_MAP = {
+    KeywordType.TIME: EntityType.KEYWORD,
+    KeywordType.GENRE: EntityType.KEYWORD,
+    KeywordType.LOC: EntityType.LOC,
+    KeywordType.PER: EntityType.PER,
+    KeywordType.ORG: EntityType.ORG,
+    KeywordType.TOPIC: EntityType.KEYWORD,
+    KeywordType.TITLE: EntityType.TITLE,
+    KeywordType.EVENT: EntityType.ORG
+}
+
+KEYWORD_MARC_MAP = {
+    KeywordType.LOC: KeywordMARC.LOC,
+    KeywordType.TIME: KeywordMARC.TIME,
+    KeywordType.TOPIC: KeywordMARC.TOPIC,
+    KeywordType.GENRE: KeywordMARC.GENRE,
+    KeywordType.TITLE: KeywordMARC.TITLE,
+    KeywordType.ORG: KeywordMARC.ORG,
+    KeywordType.PER: KeywordMARC.PER,
+    KeywordType.EVENT: KeywordMARC.EVENT
+}
+
+KEYWORD_TYPES_TO_IGNORE = [
+    KeywordType.CATEGORY,
+    KeywordType.UDC,
+    KeywordType.UDK
+]
+
+EMS_ENTITY_TYPES = [EntityType.KEYWORD, EntityType.LOC]
+SIERRA_ENTITY_TYPES = [EntityType.PER, EntityType.ORG, EntityType.TITLE]
+VIAF_ENTITY_TYPES = [EntityType.PER, EntityType.ORG, EntityType.TITLE]

rara_tools-0.6.14/rara_tools/formatters.py

@@ -0,0 +1,106 @@
+from typing import List, Tuple, Any
+from rara_tools.constants.subject_indexer import (
+    EntityType, KeywordType, KeywordMARC, KeywordSource, URLSource,
+    KEYWORD_TYPE_MAP, KEYWORD_MARC_MAP, KEYWORD_TYPES_TO_IGNORE,
+    EMS_ENTITY_TYPES, SIERRA_ENTITY_TYPES, VIAF_ENTITY_TYPES
+)
+
+def _get_keyword_source(linked_doc: Any, entity_type: str, is_linked: bool
+) -> str:
+    """ Find keyword source.
+    """
+    if not is_linked:
+        source = KeywordSource.AI
+    elif entity_type in EMS_ENTITY_TYPES:
+        source = KeywordSource.EMS
+    elif entity_type in SIERRA_ENTITY_TYPES:
+        if linked_doc and linked_doc.elastic:
+            source = KeywordSource.SIERRA
+        elif linked_doc and linked_doc.viaf:
+            source = KeywordSource.VIAF
+        else:
+            source = KeywordSource.AI
+    else:
+        source = KeywordSource.AI
+    return source
+
+def _find_indicators(entity_type: str, entity: str,
+        is_linked: bool
+) -> Tuple[str, str]:
+    """ Find MARC indicators 1 and 2.
+    """
+    ind1 = " "
+    ind2 = " "
+    if entity_type in SIERRA_ENTITY_TYPES:
+        if entity_type == EntityType.PER:
+            if "," in entity:
+                ind1 = "1"
+            else:
+                ind1 = "0"
+        else:
+            # 1 märksõna esimeseks elemendiks võimupiirkonna nimi, nt:
+            #    (a) Eesti (b) Riigikogu - raske automaatselt määrata
+            # 2 märksõna esimeseks elemendiks nimi pärijärjestuses
+            ind1 = "2"
+        if not is_linked:
+            ind2 = "4"
+    elif entity_type in EMS_ENTITY_TYPES:
+        ind2 = "4"
+    return (ind1, ind2)
+
+
+def format_keywords(flat_keywords: List[dict]) -> dict:
+    """ Formats unlinked keywords for Kata CORE.
+    """
+    ignored_keywords = []
+    filtered_keywords = []
+
+    for keyword_dict in flat_keywords:
+        keyword_type = keyword_dict.get("entity_type")
+        if keyword_type in KEYWORD_TYPES_TO_IGNORE:
+            ignored_keywords.append(keyword_dict)
+        else:
+            filtered_keywords.append(keyword_dict)
+
+    formatted_keywords = {
+        "keywords": [],
+        "other": ignored_keywords
+    }
+
+    for keyword_dict in filtered_keywords:
+        original_keyword = keyword_dict.get("keyword")
+        keyword_type = keyword_dict.get("entity_type")
+        entity_type = KEYWORD_TYPE_MAP.get(keyword_type, "")
+        marc_field = KEYWORD_MARC_MAP.get(str(keyword_type), "")
+        lang = keyword_dict.get("language", "")
+
+        ind1, ind2 = _find_indicators(
+            entity_type=entity_type,
+            entity=original_keyword,
+            is_linked=False
+        )
+        keyword_source = _get_keyword_source(
+            linked_doc=None,
+            is_linked=False,
+            entity_type=entity_type
+        )
+        new_keyword_dict = {
+            "dates": "",
+            "indicator1": ind1,
+            "indicator2": ind2,
+            "is_linked": False,
+            "keyword_source": keyword_source,
+            "lang": lang,
+            "location": "",
+            "marc_field": marc_field,
+            "numeration": "",
+            "organisation_sub_unit": "",
+            "original_keyword": original_keyword,
+            "persons_title": "",
+            "url": "",
+            "url_source": ""
+        }
+        new_keyword_dict.update(keyword_dict)
+        formatted_keywords["keywords"].append(new_keyword_dict)
+
+    return formatted_keywords

{rara_tools-0.6.12 → rara_tools-0.6.14}/rara_tools/parsers/marc_records/organization_record.py

@@ -6,13 +6,8 @@ import regex as re
 import json
 
 # TODO: indikaatorid ind1 väljadel 100 ja 400?
-"""
-|c asutuse konverentsi toimumise koht (MK)
 
-|d asutuse konverentsi toimumise aeg (K)
 
-|n asutuse konverentsi järjenumber (K)
-"""
 class OrganizationRecord(BaseRecord):
     """ Generates a simplified organization JSON record
     from a pymarc MARC record.
@@ -45,6 +40,7 @@ class OrganizationRecord(BaseRecord):
         self.__name_specification: str = ""
         self.__dates: str = ""
         self.__location: str = ""
+        self.__numeration: str = ""
         self.__name_variations: List[str] = []
         self.__source: str = ""
         self.__description: str = ""
@@ -61,7 +57,10 @@ class OrganizationRecord(BaseRecord):
 
 
     def _clean_value(self, value: str) -> str:
-        cleaned_value = value.strip("., ")
+        try:
+            cleaned_value = value.strip("., ")
+        except Exception as e:
+            cleaned_value = ""
         return cleaned_value
 
     def _merge_and_clean(self, value: dict, keys: List[str]) -> str:
@@ -103,10 +102,10 @@ class OrganizationRecord(BaseRecord):
         if not self.__dates:
             values = self.get_values(
                 marc_ids=self.__name_field_id,
-                subfield_id=["d"]
+                subfield_id="d"
             )
             if values:
-                self.__dates = self.__clean_value(values[0])
+                self.__dates = self._clean_value(values[0])
         return self.__dates
 
     @property
@@ -114,12 +113,23 @@ class OrganizationRecord(BaseRecord):
         if not self.__location:
             values = self.get_values(
                 marc_ids=self.__name_field_id,
-                subfield_id=["c"]
+                subfield_id="c"
             )
             if values:
-                self.__location = self.__clean_value(values[0])
+                self.__location = self._clean_value(values[0])
         return self.__location
 
+    @property
+    def numeration(self) -> str:
+        if not self.__numeration:
+            values = self.get_values(
+                marc_ids=self.__name_field_id,
+                subfield_id="n"
+            )
+            if values:
+                self.__numeration = self._clean_value(values[0])
+        return self.__numeration
+
     @property
     def acronyms(self) -> List[str]:
         if not self.__acronyms:

{rara_tools-0.6.12 → rara_tools-0.6.14/rara_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.6.12
+Version: 0.6.14
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.6.12 → rara_tools-0.6.14}/rara_tools.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ rara_tools/decorators.py
 rara_tools/digar_schema_converter.py
 rara_tools/elastic.py
 rara_tools/exceptions.py
+rara_tools/formatters.py
 rara_tools/s3.py
 rara_tools/task_reporter.py
 rara_tools/utils.py
@@ -48,6 +49,7 @@ tests/test_digar_schema_converter.py
 tests/test_elastic.py
 tests/test_elastic_vector_and_search_operations.py
 tests/test_entity_normalizers.py
+tests/test_formatters.py
 tests/test_marc_parsers.py
 tests/test_normalization.py
 tests/test_s3_exceptions.py

rara_tools-0.6.14/tests/test_formatters.py

@@ -0,0 +1,15 @@
+import pytest
+import os
+from pprint import pprint
+from rara_tools.formatters import format_keywords
+from tests.test_utils import read_json_file
+
+ROOT_DIR = os.path.join("tests", "test_data", "formatter")
+INPUT_KEYWORDS_FILE_PATH = os.path.join(ROOT_DIR, "keywords.json")
+
+INPUT_KEYWORDS = read_json_file(INPUT_KEYWORDS_FILE_PATH)
+
+def test_formatting_keywords_for_core():
+    formatted_keywords = format_keywords(INPUT_KEYWORDS)
+    assert formatted_keywords
+    assert isinstance(formatted_keywords, dict)

{rara_tools-0.6.12 → rara_tools-0.6.14}/tests/test_marc_parsers.py

@@ -6,6 +6,7 @@ from rara_tools.parsers.marc_parsers.organization_parser import OrganizationsMAR
 from rara_tools.parsers.marc_parsers.location_parser import LocationMARCParser
 from rara_tools.parsers.marc_parsers.title_parser import TitlesMARCParser
 from rara_tools.parsers.marc_records.person_record import PersonRecord
+from rara_tools.parsers.marc_records.organization_record import OrganizationRecord
 from tests.test_utils import read_json_file
 
 ROOT_DIR = os.path.join("tests", "test_data", "marc_records")
@@ -15,7 +16,8 @@ EMS_TEST_FILE = os.path.join(MARC_ROOT_DIR, "ems_test_subset.mrc")
 PER_TEST_FILE = os.path.join(MARC_ROOT_DIR, "per_test_subset.mrc")
 ORG_TEST_FILE = os.path.join(MARC_ROOT_DIR, "org_test_subset.mrc")
 TITLE_TEST_FILE = os.path.join(MARC_ROOT_DIR, "title_test_subset.mrc")
-JSON_TEST_FILE = os.path.join(JSON_ROOT_DIR, "marc_json_record.json")
+PER_JSON_TEST_FILE = os.path.join(JSON_ROOT_DIR, "per_marc_json_record.json")
+ORG_JSON_TEST_FILE = os.path.join(JSON_ROOT_DIR, "org_marc_json_record.json")
 
 def test_ems_parser_without_variations():
     ems_marc_parser = EMSMARCParser(EMS_TEST_FILE, add_variations=False)
@@ -61,7 +63,17 @@ def test_title_parser_with_variations():
         assert "link_variations" in record
         assert len(record["link_variations"]) > 0
 
-def test_creating_marc_record_with_json_input():
-    json_data = read_json_file(JSON_TEST_FILE)
+def test_creating_per_marc_record_with_json_input():
+    json_data = read_json_file(PER_JSON_TEST_FILE)
     record = PersonRecord(json_data)
     assert record.name == "Koidula, Lydia"
+
+
+def test_creating_org_marc_record_with_json_input():
+    json_data = read_json_file(ORG_JSON_TEST_FILE)
+    record = OrganizationRecord(json_data)
+    assert record.original_name.get("a") == "Eesti"
+    assert record.original_name.get("b") == "Riigikogu"
+    assert not record.location
+    assert not record.dates
+    assert not record.numeration

rara_tools-0.6.12/VERSION

@@ -1 +0,0 @@
-0.6.12

rara_tools-0.6.12/rara_tools/constants/subject_indexer.py

@@ -1,14 +0,0 @@
-COMPONENT_KEY = "subject_indexer"
-
-
-class Tasks:
-    SINGLE = "run_subject_indexer_process"
-    PIPELINE = "run_subject_indexer_with_core_logic"
-
-
-class Queue:
-    MAIN = "subject-indexer"
-
-
-class StatusKeys:
-    EXTRACT_KEYWORDS = "extract_keywords"