rara-tools 0.6.16__py3-none-any.whl → 0.7.0__py3-none-any.whl

rara_tools/constants/linker.py

@@ -1,3 +1,6 @@
+import logging
+from rara_tools.constants.normalizers import EntityType, VIAF_ENTITY_MAP
+
 COMPONENT_KEY = "linker"
 
 
@@ -20,3 +23,114 @@ class Queue:
 class StatusKeys:
     VECTORIZE_CONTEXT = "vectorize_context"
     LINK_KEYWORDS = "link_keywords"
+
+class URLSource:
+    VIAF = "VIAF"
+    SIERRA = "Sierra"
+    EMS = "EMS"
+
+class KeywordType:
+    LOC = "Kohamärksõnad"
+    TIME = "Ajamärksõnad"
+    TOPIC = "Teemamärksõnad"
+    GENRE = "Vormimärksõnad"
+    TITLE = "Teose pealkiri"
+    PER = "Isikunimi"
+    ORG = "Kollektiivi nimi"
+    EVENT = "Ajutine kollektiiv või sündmus"
+    CATEGORY = "Valdkonnamärksõnad"
+    UDC = "UDC Summary"
+    UDK = "UDK Rahvusbibliograafia"
+
+
+class KeywordMARC:
+    PER = 600
+    ORG = 610
+    TOPIC = 650
+    GENRE = 655
+    TIME = 648
+    LOC = 651
+    EVENT = 611
+    TITLE = 630
+    TITLE_LINKED = 600
+
+class KeywordSource:
+    EMS = "EMS"
+    SIERRA = "SIERRA"
+    VIAF = "VIAF"
+    AI = "AI"
+
+class Filters:
+    AUTHOR = "author"
+    YEAR = "year"
+
+
+UNLINKED_KEYWORD_MARC_FIELD = 693
+
+ALLOWED_FILTERS_MAP = {
+    EntityType.PER: [Filters.YEAR],
+    EntityType.ORG: [Filters.YEAR],
+    EntityType.TITLE: [Filters.YEAR, Filters.AUTHOR],
+    EntityType.KEYWORD: [],
+    EntityType.LOC: []
+}
+KEYWORD_MARC_MAP = {
+    KeywordType.LOC: KeywordMARC.LOC,
+    KeywordType.TIME: KeywordMARC.TIME,
+    KeywordType.TOPIC: KeywordMARC.TOPIC,
+    KeywordType.GENRE: KeywordMARC.GENRE,
+    KeywordType.TITLE: KeywordMARC.TITLE,
+    KeywordType.ORG: KeywordMARC.ORG,
+    KeywordType.PER: KeywordMARC.PER,
+    KeywordType.EVENT: KeywordMARC.EVENT
+}
+
+URL_SOURCE_MAP = {
+    EntityType.PER: URLSource.VIAF,
+    EntityType.ORG: URLSource.VIAF,
+    EntityType.TITLE: URLSource.VIAF,
+    EntityType.KEYWORD: URLSource.EMS,
+    EntityType.LOC: URLSource.EMS
+}
+
+# Ignore those "keyword types" while linking the
+# rara-subject-indexer results
+KEYWORD_TYPES_TO_IGNORE = [
+    KeywordType.CATEGORY,
+    KeywordType.UDC,
+    KeywordType.UDK
+]
+
+ALLOWED_ENTITY_TYPES = [
+    EntityType.PER,
+    EntityType.ORG,
+    EntityType.KEYWORD,
+    EntityType.LOC,
+    EntityType.TITLE,
+    EntityType.UNK,
+]
+
+
+KEYWORD_TYPE_MAP = {
+    KeywordType.TIME: EntityType.KEYWORD,
+    KeywordType.GENRE: EntityType.KEYWORD,
+    KeywordType.LOC: EntityType.LOC,
+    KeywordType.PER: EntityType.PER,
+    KeywordType.ORG: EntityType.ORG,
+    KeywordType.TOPIC: EntityType.KEYWORD,
+    KeywordType.TITLE: EntityType.TITLE,
+    KeywordType.EVENT: EntityType.ORG
+}
+
+EMS_ENTITY_TYPES = [EntityType.KEYWORD, EntityType.LOC]
+SIERRA_ENTITY_TYPES = [EntityType.PER, EntityType.ORG, EntityType.TITLE]
+VIAF_ENTITY_TYPES = [EntityType.PER, EntityType.ORG, EntityType.TITLE]
+
+# Params for filters
+MIN_AUTHOR_SIMILARITY = 0.95
+YEAR_EXCEPTION_VALUE = True
+
+LOGGER_NAME = "rara-tools-norm-linker"
+LOGGER = logging.getLogger(LOGGER_NAME)
+
+MAIN_TAXONOMY_LANG = "et"

rara_tools/core_formatters/core_formatter.py

@@ -0,0 +1,86 @@
+from typing import List, Tuple, Any
+from rara_tools.core_formatters.formatted_keyword import FormattedKeyword
+from rara_tools.core_formatters.formatted_meta import FormattedAuthor
+from rara_tools.constants.linker import MAIN_TAXONOMY_LANG, KEYWORD_TYPES_TO_IGNORE, EntityType
+
+def get_primary_author(authors: List[dict]) -> str:
+    primary_author = ""
+    for author in authors:
+        if author.get("is_primary", False):
+            primary_author = author.get("name", "")
+    return primary_author
+
+def format_authors(authors: List[dict]) -> List[dict]:
+    formatted_authors = []
+    for author in authors:
+        entity_type = author.get("type", EntityType.UNK)
+
+        formatted_author = FormattedAuthor(
+            object_dict=author,
+            linked_doc=None,
+            entity_type=entity_type
+        ).to_dict()
+        formatted_authors.append(formatted_author)
+    return formatted_authors
+
+def format_sections(sections: List[dict]) -> List[dict]:
+    for section in sections:
+        authors = section.pop("authors", [])
+        titles = section.pop("titles", [])
+        primary_author = get_primary_author(authors)
+        if primary_author:
+            for title in titles:
+                title["author_from_title"] = primary_author
+        section["titles"] = titles
+
+        formatted_authors = format_authors(authors)
+        section["authors"] = formatted_authors
+
+    return sections
+
+def format_meta(meta: dict) -> dict:
+    """ Formats unlinked meta for Kata CORE.
+    """
+
+    meta_to_format = meta.get("meta")
+
+    authors = meta_to_format.pop("authors", [])
+    sections = meta_to_format.pop("sections", [])
+
+    formatted_authors = format_authors(authors)
+    formatted_sections = format_sections(sections)
+
+    if sections and formatted_sections:
+        meta_to_format["sections"] = formatted_sections
+    if authors and formatted_authors:
+        meta_to_format["authors"] = formatted_authors
+
+    meta["meta"] = meta_to_format
+
+    return meta
+
+
+def format_keywords(flat_keywords: List[dict]) -> List[dict]:
+    """ Formats unlinked keywords for Kata CORE.
+    """
+    ignored_keywords = []
+    filtered_keywords = []
+
+    for keyword_dict in flat_keywords:
+        keyword_type = keyword_dict.get("entity_type")
+        if keyword_type in KEYWORD_TYPES_TO_IGNORE:
+            ignored_keywords.append(keyword_dict)
+        else:
+            filtered_keywords.append(keyword_dict)
+
+    formatted_keywords = []
+
+    for keyword_dict in filtered_keywords:
+        formatted_keyword = FormattedKeyword(
+            object_dict=keyword_dict,
+            linked_doc=None,
+            main_taxnomy_lang=MAIN_TAXONOMY_LANG
+        ).to_dict()
+        formatted_keywords.append(formatted_keyword)
+
+    return formatted_keywords

rara_tools/core_formatters/formatted_keyword.py

@@ -0,0 +1,229 @@
+from rara_tools.constants.linker import (
+    LOGGER, URLSource, KeywordSource, EntityType, KeywordType, KeywordMARC,
+    KEYWORD_MARC_MAP,  KEYWORD_TYPES_TO_IGNORE, KEYWORD_TYPE_MAP,
+    EMS_ENTITY_TYPES, SIERRA_ENTITY_TYPES, UNLINKED_KEYWORD_MARC_FIELD,
+    URL_SOURCE_MAP
+)
+from rara_tools.core_formatters.formatted_object import FormattedObject
+from typing import List, Dict, NoReturn, Tuple, Any
+
+class FormattedKeyword(FormattedObject):
+    def __init__(self, object_dict: dict, linked_doc: Any,
+            main_taxnomy_lang: str, url_source_map: str = URL_SOURCE_MAP
+    ) -> NoReturn:
+        super().__init__(
+            object_dict=object_dict,
+            linked_doc=linked_doc,
+            original_entity_key="keyword"
+        )
+
+        self.main_taxnomy_lang: str = main_taxnomy_lang
+
+        self.original_keyword: str = self.original_entity
+        self.score: float = self.object_dict.get("score")
+        self.count: int = self.object_dict.get("count")
+        self.method: str = self.object_dict.get("method")
+        self.model_arch: str = self.object_dict.get("model_arch", self.method)
+        self.keyword_type: str = self.object_dict.get("entity_type")
+
+        self.entity_type: str = KEYWORD_TYPE_MAP.get(self.keyword_type, "")
+        self.url_source_map: dict = url_source_map
+
+        self.__keyword_source: str = ""
+        self.__indicator_1: str = ""
+        self.__indicator_2: str = ""
+        self.__url: str | None = None
+        self.__url_source: str | None = None
+        self.__marc_field: str = ""
+
+        self.__language: str = ""
+        self.__author: str | None = None
+
+    @property
+    def keyword(self) -> str:
+        return self.entity
+
+    @property
+    def keyword_source(self) -> str:
+        if not self.__keyword_source:
+            if not self.is_linked:
+                source = KeywordSource.AI
+            elif self.entity_type in EMS_ENTITY_TYPES:
+                source = KeywordSource.EMS
+            elif self.entity_type in SIERRA_ENTITY_TYPES:
+                if self.linked_doc and self.linked_doc.elastic:
+                    source = KeywordSource.SIERRA
+                elif self.linked_doc and self.linked_doc.viaf:
+                    source = KeywordSource.VIAF
+                else:
+                    source = KeywordSource.AI
+            else:
+                source = KeywordSource.AI
+            self.__keyword_source = source
+        return self.__keyword_source
+
+    @property
+    def indicator1(self) -> str:
+        if not self.__indicator_1:
+            ind1, ind2 = self._get_indicators()
+            self.__indicator_1 = ind1
+            self.__indicator_2 = ind2
+        return self.__indicator_1
+
+    @property
+    def indicator2(self) -> str:
+        if not self.__indicator_2:
+            ind1, ind2 = self._get_indicators()
+            self.__indicator_1 = ind1
+            self.__indicator_2 = ind2
+        return self.__indicator_2
+
+    @property
+    def url(self) -> str:
+        if self.__url == None:
+            url_info = self._get_url_info()
+            self.__url = url_info.get("url")
+            self.__url_source = url_info.get("url_source")
+        return self.__url
+
+    @property
+    def url_source(self) -> str:
+        if self.__url_source == None:
+            url_info = self._get_url_info()
+            self.__url = url_info.get("url")
+            self.__url_source = url_info.get("url_source")
+        return self.__url_source
+
+    @property
+    def marc_field(self) -> int:
+        if not self.__marc_field:
+            # TODO: teoste + isikute loogika!!!!
+            if self.is_linked:
+                marc_field = KEYWORD_MARC_MAP.get(str(self.keyword_type), "")
+            else:
+                marc_field = UNLINKED_KEYWORD_MARC_FIELD
+
+            if self.entity_type == EntityType.TITLE:
+                if self.author:
+                    marc_field = KeywordMARC.TITLE_LINKED
+                else:
+                    marc_field = KeywordMARC.TITLE
+            self.__marc_field = marc_field
+        return self.__marc_field
+
+
+    @property
+    def persons_title(self) -> str:
+        return self.titles
+
+
+    @property
+    def language(self) -> str:
+        if not self.__language:
+            if self.is_linked:
+                self.__language = self.main_taxnomy_lang
+            else:
+                self.__language = self.object_dict.get("language", "")
+        return self.__language
+
+    @property
+    def author(self) -> str:
+        # Only relevant for titles!
+        if self.__author == None:
+            self.__author = ""
+            if self.entity_type == EntityType.TITLE:
+                if self.original_record:
+                    self.__author = self.original_record.author_name
+                elif self.viaf_info:
+                    pass
+                    #self.__author = self.viaf_info.get
+        return self.__author
+
+    def _get_url_info(self) -> dict:
+        """ Finds URL identifier from LinkedDoc based on
+        given entity type.
+
+        Parameters
+        -----------
+        linked_doc: LinkedDoc | None
+            A LinkedDoc class instance.
+        entity_type: str
+            Entity type for detecting correct URL source.
+
+        Returns
+        ----------
+        dict:
+            Dictionary with keys `url` - URL identifier and
+            `url_source` - source of the URL (e.g. "EMS").
+
+        """
+        url_source = self.url_source_map.get(self.entity_type, "")
+        url = ""
+
+        if self.linked_doc:
+            if url_source == URLSource.EMS:
+                url = self.linked_doc.elastic.get("ems_url", "")
+            elif url_source == URLSource.VIAF:
+                url = self.viaf_info.get("viaf_url", "")
+        if not url:
+            url_source = ""
+
+        url_info = {"url": url, "url_source": url_source}
+
+        LOGGER.debug(
+            f"Detected URL info: {url_info}. Used entity_type = {self.entity_type}. " \
+            f"URL source map = {self.url_source_map}."
+        )
+        return url_info
+
+    def _get_indicators(self) -> Tuple[str, str]:
+        """ Find MARC indicators 1 and 2.
+        """
+        ind1 = " "
+        ind2 = " "
+        if self.entity_type in SIERRA_ENTITY_TYPES:
+            if self.entity_type == EntityType.PER:
+                if "," in self.keyword:
+                    ind1 = "1"
+                else:
+                    ind1 = "0"
+            elif self.entity_type == EntityType.ORG:
+                # 1 märksõna esimeseks elemendiks võimupiirkonna nimi, nt:
+                #    (a) Eesti (b) Riigikogu - raske automaatselt määrata
+                # 2 märksõna esimeseks elemendiks nimi pärijärjestuses
+                ind1 = "2"
+            else:
+                ind1 = "0"
+
+            if not self.is_linked:
+                ind2 = "4"
+        elif self.entity_type in EMS_ENTITY_TYPES:
+            ind2 = "4"
+        return (ind1, ind2)
+
+
+    def to_dict(self) -> dict:
+        keyword_dict = {
+            "count": self.count,
+            "dates": self.dates,
+            "entity_type": self.keyword_type,
+            "indicator1": self.indicator1,
+            "indicator2": self.indicator2,
+            "is_linked": self.is_linked,
+            "keyword": self.keyword,
+            "keyword_source": self.keyword_source,
+            "lang": self.language,
+            "location": self.location,
+            "marc_field": self.marc_field,
+            "method": self.method,
+            "model_arch": self.model_arch,
+            "numeration": self.numeration,
+            "organisation_sub_unit": self.organisation_sub_unit,
+            "original_keyword": self.original_keyword,
+            "persons_title": self.persons_title,
+            "score": self.score,
+            "url": self.url,
+            "url_source": self.url_source,
+            "author": self.author
+        }
+        return keyword_dict

rara_tools/core_formatters/formatted_meta.py

@@ -0,0 +1,154 @@
+from rara_tools.constants.linker  import (
+    LOGGER, EntityType
+)
+from rara_tools.core_formatters.formatted_object import FormattedObject
+from typing import List, Dict, NoReturn, Tuple, Any
+
+
+class FormattedTitle(FormattedObject):
+    # TODO: Kas seda on üldse vaja?
+    def __init__(self, object_dict: dict, linked_doc: Any):
+        super().__init__(
+            object_dict=object_dict,
+            linked_doc=linked_doc,
+            original_entity_key="name"
+        )
+
+
+class FormattedAuthor(FormattedObject):
+    def __init__(self, object_dict: dict, linked_doc: Any, entity_type: str):
+        super().__init__(
+            object_dict=object_dict,
+            linked_doc=linked_doc,
+            original_entity_key="name"
+        )
+        self.entity_type: str = entity_type
+
+        self.is_linked: bool = True if self.linked_doc else False # NB! Lisada andmebaasi uus veerg!
+        self.original_name: str = self.original_entity            # NB! Lisada andmebaasi uus veerg
+        self.author_role: str = self.object_dict.get("role")
+        self.is_primary: bool = self.object_dict.get("is_primary")
+
+        self.__primary_author_type: str = None
+
+        self.__name_order_type: str = ""
+        self.__event_sub_unit: str = ""
+        self.__order_number: str = ""
+        self.__sub_title: str = ""
+        self.__additional_info: str = ""
+        self.__publication_type: str = ""
+        self.__publication_language: str = ""
+        #self.__standardized_uri: str = ""
+        self.__viaf_id: str = ""
+
+
+    @property
+    def primary_author_type(self) -> str:
+        if self.__primary_author_type == None:
+            if self.is_primary:
+                if self.entity_type != EntityType.UNK:
+                    self.__primary_author_type = self.entity_type
+                else:
+                    self.__primary_author_type = EntityType.PER
+            else:
+                self.__primary_author_type = ""
+        return self.__primary_author_type
+
+
+    @property
+    def name(self) -> str:
+        return self.entity
+
+    @property
+    def name_order(self) -> str:
+        if not self.__name_order_type:
+            if self.entity_type == EntityType.PER or self.entity_type == EntityType.UNK:
+                if "," in self.name:
+                    ind1 = "1"
+                else:
+                    ind1 = "0"
+            elif self.entity_type == EntityType.ORG:
+                #LOGGER.debug(f"Entity type {self.entity_type} is not {EntityType.PER}.")
+                # 1 märksõna esimeseks elemendiks võimupiirkonna nimi, nt:
+                #    (a) Eesti (b) Riigikogu - raske automaatselt määrata
+                # 2 märksõna esimeseks elemendiks nimi pärijärjestuses
+                ind1 = "2" #????????
+            else:
+                ind1 = "0"
+            self.__name_order_type = ind1
+        return self.__name_order_type
+
+    @property
+    def event_sub_unit(self) -> str:
+        if not self.__event_sub_unit:
+            self.__event_sub_unit = ""
+        return self.__event_sub_unit
+
+
+    @property
+    def order_number(self) -> str:
+        if not self.__order_number:
+            self.__order_number = ""
+        return self.__order_number
+
+    @property
+    def sub_title(self) -> str:
+        if not self.__sub_title:
+            self.__sub_title = ""
+        return self.__sub_title
+
+    @property
+    def additional_info(self) -> str:
+        if not self.__additional_info:
+            self.__additional_info = ""
+        return self.__additional_info
+
+    @property
+    def publication_type(self) -> str:
+        if not self.__publication_type:
+            self.__publication_type = ""
+        return self.__publication_type
+
+    @property
+    def publication_language(self) -> str:
+        if not self.__publication_language:
+            self.__publication_language = ""
+        return self.__publication_language
+
+    @property
+    def standardized_uri(self) -> str:
+        return self.identifier
+
+    @property
+    def viaf_id(self):
+        if not self.__viaf_id:
+            if self.viaf_info:
+                self.__viaf_id = self.viaf_info.get("viaf_url", "")
+            else:
+                self.__viaf_id = ""
+        return self.__viaf_id
+
+    def to_dict(self):
+        author_dict = {
+            "is_linked": self.is_linked,
+            "original_name": self.original_name,
+            "author_role": self.author_role,
+            "is_primary": self.is_primary,
+            "primary_author_type": self.primary_author_type,
+            "name": self.name,
+            "numeration": self.numeration,
+            "organisation_sub_unit": self.organisation_sub_unit,
+            "titles": self.titles,
+            "location": self.location,
+            "dates": self.dates,
+            "name_order_type": self.name_order,
+            "event_sub_unit": self.event_sub_unit,
+            "order_number": self.order_number,
+            "sub_title": self.sub_title,
+            "additional_info": self.additional_info,
+            "publication_type": self.publication_type,
+            "publication_language": self.publication_language,
+            "standardized_uri": self.standardized_uri,
+            "viaf_id": self.viaf_id
+        }
+        return author_dict

rara_tools/core_formatters/formatted_object.py

@@ -0,0 +1,137 @@
+from rara_tools.constants.linker import (
+    LOGGER, URLSource, KeywordSource, EntityType, KeywordType,
+    KEYWORD_MARC_MAP,  KEYWORD_TYPES_TO_IGNORE, KEYWORD_TYPE_MAP,
+    EMS_ENTITY_TYPES, SIERRA_ENTITY_TYPES, UNLINKED_KEYWORD_MARC_FIELD,
+    URL_SOURCE_MAP
+)
+
+from rara_tools.parsers.marc_records.person_record import PersonRecord
+from rara_tools.parsers.marc_records.organization_record import OrganizationRecord
+from rara_tools.parsers.marc_records.title_record import TitleRecord
+from rara_tools.utils import format_date
+from typing import List, Dict, NoReturn, Tuple, Any
+
+
+class FormattedObject:
+    def __init__(self, object_dict: dict, linked_doc: Any, original_entity_key: str):
+        self.object_dict: dict = object_dict
+        self.linked_doc: Any = linked_doc
+        self.viaf_info: dict = self.linked_doc.viaf.get("parsed", {}) if self.linked_doc else {}
+        self.original_entity: str = self.object_dict.get(original_entity_key)
+        self.is_linked: bool = True if self.linked_doc else False
+
+        self.__original_record: PersonRecord | OrganizationRecord | TitleRecord | None = None
+        self.__persons_title: str | None = None
+        self.__dates: str | None = None
+        self.__numeration: str | None = None
+        self.__location: str | None = None
+        self.__organization_sub_unit: str | None = None
+        self.__entity: str | None = None
+        self.__titles: str | None = None
+        self.__identifier: str | None = ""
+
+
+    @property
+    def original_record(self) -> PersonRecord | OrganizationRecord | None:
+        if not self.__original_record and self.linked_doc and self.linked_doc.json:
+            try:
+                if self.entity_type == EntityType.PER:
+                    original_record = PersonRecord(self.linked_doc.json)
+                elif self.entity_type == EntityType.ORG:
+                    original_record = OrganizationRecord(self.linked_doc.json)
+                elif self.entity_type == EntityType.TITLE:
+                    original_record = TitleRecord(self.linked_doc.json)
+                else:
+                    original_record = None
+            except Exception as e:
+                LOGGER.exception(
+                    f"Could not retrieve JSON from LinkedDoc instance. Exception: '{e}'."
+                )
+                original_record = None
+            self.__original_record = original_record
+        return self.__original_record
+
+
+    @property
+    def entity(self) -> str:
+        if self.__entity == None:
+            if self.linked_doc != None:
+                if self.entity_type == EntityType.ORG and self.original_record:
+                    self.__entity = self.original_record.original_name.get("a", "")
+                else:
+                    self.__entity = self.linked_doc.linked_entity
+                if not self.__entity and self.viaf_info:
+                    self.__entity = self.viaf_info.get("name", self.original_entity)
+            else:
+                self.__entity = self.original_entity
+        return self.__entity
+
+
+
+    @property
+    def dates(self) -> str:
+        if self.__dates == None:
+            self.__dates = ""
+            if self.viaf_info:
+                birth_date = format_date(self.viaf_info.get("birth_date", ""))
+                death_date = format_date(self.viaf_info.get("death_date", ""))
+                if not death_date:
+                    death_date = ""
+
+                if birth_date:
+                    self.__dates = f"{birth_date}-{death_date}"
+
+            if self.original_record and not self.__dates:
+                if self.entity_type == EntityType.PER:
+                    self.__dates = self.original_record.life_years
+                elif self.entity_type == EntityType.ORG:
+                    self.__dates = self.original_record.dates
+                elif self.entity_type == EntityType.TITLE:
+                    self.__dates = self.original_record.author_life_years
+
+        return self.__dates
+
+
+    @property
+    def numeration(self) -> str:
+        if self.__numeration == None:
+            self.__numeration = ""
+            if self.original_record:
+                if self.entity_type == EntityType.PER:
+                    self.__numeration = self.original_record.original_name.get("b", "")
+                elif self.entity_type == EntityType.ORG:
+                    self.__numeration = self.original_record.numeration
+        return self.__numeration
+
+    @property
+    def location(self) -> str:
+        if self.__location == None:
+            self.__location = ""
+            if self.entity_type == EntityType.ORG and self.original_record:
+                self.__location = self.original_record.location
+        return self.__location
+
+    @property
+    def organisation_sub_unit(self) -> str:
+        if self.__organization_sub_unit == None:
+            self.__organization_sub_unit = ""
+            if self.entity_type == EntityType.ORG and self.original_record:
+                self.__organization_sub_unit = self.original_record.original_name.get("b", "")
+        return self.__organization_sub_unit
+
+    @property
+    def titles(self) -> str:
+        if self.__titles == None:
+            if self.entity_type == EntityType.PER and self.original_record:
+                self.__titles = self.original_record.name_specification
+            else:
+                self.__titles = ""
+        return self.__titles
+
+    @property
+    def identifier(self) -> str:
+        if self.__identifier == None:
+            self.__identifier = ""
+            if self.original_record:
+                self.__identifier = self.original_record.identifier
+        return self.__identifier

rara_tools/normalizers/viaf.py

@@ -672,6 +672,7 @@ class VIAFClient:
             f"Allowed VIAF sources are: {self.allowed_viaf_sources}."
         )
         viaf_record = None
+        verified_record = None
         viaf_ids = self.get_viaf_ids_by_search_terms(
             search_term=search_term,
             field=field,

rara_tools/utils.py

@@ -2,20 +2,20 @@ from iso639 import Lang
 
 
 def lang_to_iso639_1(lang: str, unk_code: str = "unk") -> str:
-    """ Converts language into ISO-639-1 standard. 
-    Input can be any language code in a valid ISO-639 
-    standard or even a full name of the language, 
+    """ Converts language into ISO-639-1 standard.
+    Input can be any language code in a valid ISO-639
+    standard or even a full name of the language,
     e.g. "Estonian".
-    
+
     Parameters
     -----------
     lang: str
         Language code in any valid ISO-639 standard.
-        
+
     unk_code: str
         Code to return incase of invalid/unsupported
         input language.
-        
+
     Returns
     -------
     Language code in ISO-639-1 standard.
@@ -29,20 +29,20 @@ def lang_to_iso639_1(lang: str, unk_code: str = "unk") -> str:
 
 
 def lang_to_iso639_2(lang: str, unk_code: str = "unk") -> str:
-    """ Converts language into ISO-639-2 standard. 
-    Input can be any language code in a valid ISO-639 
-    standard or even a full name of the language, 
+    """ Converts language into ISO-639-2 standard.
+    Input can be any language code in a valid ISO-639
+    standard or even a full name of the language,
     e.g. "Estonian".
-    
+
     Parameters
     -----------
     lang: str
         Language code in any valid ISO-639 standard.
-        
+
     unk_code: str
         Code to return incase of invalid/unsupported
         input language.
-        
+
     Returns
     -------
     Language code in ISO-639-2 standard.
@@ -59,20 +59,20 @@ def lang_to_iso639_2(lang: str, unk_code: str = "unk") -> str:
 
 
 def lang_to_iso639_3(lang: str, unk_code: str = "unk") -> str:
-    """ Converts language into ISO-639-3 standard. 
-    Input can be any language code in a valid ISO-639 
-    standard or even a full name of the language, 
+    """ Converts language into ISO-639-3 standard.
+    Input can be any language code in a valid ISO-639
+    standard or even a full name of the language,
     e.g. "Estonian".
-    
+
     Parameters
     -----------
     lang: str
         Language code in any valid ISO-639 standard.
         unk_code: str
-        
+
     Code to return incase of invalid/unsupported
         input language.
-        
+
     Returns
     -------
     str
@@ -88,17 +88,39 @@ def lang_to_iso639_3(lang: str, unk_code: str = "unk") -> str:
 
 def ratio_to_percentage(ratio: float) -> str:
     """ Converts ratio to corresponding percentage.
-    
+
     Parameters
     -----------
     ratio: float
         Float in range [0,1]
-    
+
     Returns
     --------
     str
         Percentage corresponding to the float.
-    
+
     """
     percentage = f"{int(ratio*100)}%"
     return percentage
+
+def format_date(original_date: str) -> str:
+    """ Converts date from format %Y-%m-%d into format %d.%m.%Y, e.g:
+    2025-02-12 -> 12.02.2025.
+
+    Parameters
+    -----------
+    original_date: str
+        Original date in format %Y-%m-%d
+
+    Returns
+    ----------
+    str:
+        Date in format %d.%m.%Y
+    """
+    try:
+        date_obj = datetime.strptime(original_date, "%Y-%m-%d")
+        new_date = date_obj.strftime("%d.%m.%Y")
+
+    except:
+        new_date = original_date
+    return new_date

{rara_tools-0.6.16.dist-info → rara_tools-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.6.16
+Version: 0.7.0
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{rara_tools-0.6.16.dist-info → rara_tools-0.7.0.dist-info}/RECORD

@@ -3,24 +3,27 @@ rara_tools/decorators.py,sha256=MjOyvZ5nTkwxwx2JLFEGpKKBysvecFw6EN6UDrSvZLU,2187
 rara_tools/digar_schema_converter.py,sha256=k95U2iRlEA3sh772-v6snhHW6fju6qSTMnvWJ6DpzZk,14254
 rara_tools/elastic.py,sha256=4D9yoyMy6AJIKwhSi2H1usffDHAh2A_IZfv5BtYnBKg,13992
 rara_tools/exceptions.py,sha256=YQyaueUbXeTkJYFDEuN6iWTXMI3eCv5l7PxGp87vg5I,550
-rara_tools/formatters.py,sha256=LTliadjIPZTO4s-44NsumaUdlQlEvqetvWz4bEvwf90,3418
 rara_tools/s3.py,sha256=9ziDXsLjBtFAvsjTPxFddhfvkpA8773rzPJqO7y1N5Q,6415
 rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
-rara_tools/utils.py,sha256=9vSbmuWYU5ydr4lXBKlUKa0xzDccFsaJv4T-XwgUfuY,2578
+rara_tools/utils.py,sha256=1UrxOzo3cxe4juMkDlKWv1VKWMYay5v1pivGci1ajiM,3003
 rara_tools/constants/__init__.py,sha256=r78laM9vyRDAvzDhPvzDlhaX6qPwUUBBtwf1WosrW3o,27
 rara_tools/constants/digitizer.py,sha256=A7FfqqEB4hGJ9t3z8gTFK7hkzCxz44rCOSWx6Pzvwjs,548
 rara_tools/constants/general.py,sha256=jE1aIir_eKbka_q1iCJWRtmyz_xpnTPntbshiWo9eTA,1061
 rara_tools/constants/language_evaluator.py,sha256=3sCSaoS-zXQRY0vJ7UUMuZqbtYQD_quVVbdpgvJjE7I,124
-rara_tools/constants/linker.py,sha256=TQaigi7AUNOqmQPPz3hM8_xXgofrhoQ1taln79LhXQg,550
+rara_tools/constants/linker.py,sha256=yBN9NpUhB3ENz8BapoIfpSHY_xNqwYdqutgQFdc_Cd8,3240
 rara_tools/constants/meta_extractor.py,sha256=adYH8cQqH0ZWYO7clGMiObclXRTGsxWgk3pC1oiHxHE,242
 rara_tools/constants/normalizers.py,sha256=Xs3anDwJHpHeniwx3xoIZyqdEXtO3eb7ouGLLr0CpHw,1344
 rara_tools/constants/parsers.py,sha256=L6nh1Itget9_9DMsliDkh6T25z78eMFPWVkbaU08DwU,5561
 rara_tools/constants/subject_indexer.py,sha256=E2D7pylH6Yey9h2TAvAWQiX5JtKKagsZx2E1Fz_afMI,1967
+rara_tools/core_formatters/core_formatter.py,sha256=HJX7jOi9kaFie_zm0Wzjk0nKF8dRleJpVWbCplFFquo,2760
+rara_tools/core_formatters/formatted_keyword.py,sha256=1-B9IQTycFt69pTy8WZNnfJ2WIMRow3kpEub6igyNQc,7865
+rara_tools/core_formatters/formatted_meta.py,sha256=Zd0oQFLbn6m_wHaWtgxBsu9J7wGyWIpZxb2-8PrR3Wk,5240
+rara_tools/core_formatters/formatted_object.py,sha256=7a499ZmcZXOqtlwxDi6FWHWF5a6HdCsduS22wV3uHIE,5656
 rara_tools/normalizers/__init__.py,sha256=_NqpS5w710DhaURytHq9JpEt8HgYpSPfRDcOtOymJgE,193
 rara_tools/normalizers/authorities.py,sha256=IDtcm0yNZNhv1f-WcdqWFSRzZk_CoKuBFsk6hEPddWM,4513
 rara_tools/normalizers/base.py,sha256=6tLfNdF6FZo8M6j_Q61lXoaF1HdIB1c0SKMatTc-Z64,12014
 rara_tools/normalizers/bibs.py,sha256=4DTS6k37z8qR5B3n7aiCXsT5Z49rLTvQ60lKKr5dyLs,2352
-rara_tools/normalizers/viaf.py,sha256=IA_dIJBA47IBnt0O1E2vKD6UbqhJdrSyg080CXEikKA,24187
+rara_tools/normalizers/viaf.py,sha256=LIeqbJoKtVt_0H1o7XMmhSE0BjF4l-jdAJgX_8Gg9Z4,24218
 rara_tools/parsers/marc_parsers/base_parser.py,sha256=Kdw4aivJf2FkWgIK7pJtHtVXF_G1pjHVQ7IcFItSqy8,1649
 rara_tools/parsers/marc_parsers/ems_parser.py,sha256=LFuhZcVwmHMcJknX9p4ZkO8RdjPdQZ4APGbw8KV6BIs,2024
 rara_tools/parsers/marc_parsers/location_parser.py,sha256=dSU9dQoGV5z0ajhLI1bn3AAghkOr79qKIrX7sO0_4lA,1873
@@ -35,8 +38,8 @@ rara_tools/parsers/marc_records/title_record.py,sha256=XrtJ4gj7wzSaGxNaPtPuawmqq
 rara_tools/parsers/tools/entity_normalizers.py,sha256=VyCy_NowCLpOsL0luQ55IW-Qi-J5oBH0Ofzr7HRFBhM,8949
 rara_tools/parsers/tools/marc_converter.py,sha256=LgSHe-7n7aiDrw2bnsB53r3fXTRFjZXTwBYfTpL0pfs,415
 rara_tools/parsers/tools/russian_transliterator.py,sha256=5ZU66iTqAhr7pmfVqXPAI_cidF43VqqmuN4d7H4_JuA,9770
-rara_tools-0.6.16.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
-rara_tools-0.6.16.dist-info/METADATA,sha256=bdBLSy3aBqH97FHgXbcetqfMI3vX_8NwMaB7fHhOrXo,4080
-rara_tools-0.6.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rara_tools-0.6.16.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
-rara_tools-0.6.16.dist-info/RECORD,,
+rara_tools-0.7.0.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
+rara_tools-0.7.0.dist-info/METADATA,sha256=UmA4_431SdsYqP6IrEv8mr4yL9OoFVGHvpG-mQgs_g8,4079
+rara_tools-0.7.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rara_tools-0.7.0.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
+rara_tools-0.7.0.dist-info/RECORD,,

rara_tools/formatters.py

@@ -1,106 +0,0 @@
-from typing import List, Tuple, Any
-from rara_tools.constants.subject_indexer import (
-    EntityType, KeywordType, KeywordMARC, KeywordSource, URLSource,
-    KEYWORD_TYPE_MAP, KEYWORD_MARC_MAP, KEYWORD_TYPES_TO_IGNORE,
-    EMS_ENTITY_TYPES, SIERRA_ENTITY_TYPES, VIAF_ENTITY_TYPES
-)
-
-def _get_keyword_source(linked_doc: Any, entity_type: str, is_linked: bool
-) -> str:
-    """ Find keyword source.
-    """
-    if not is_linked:
-        source = KeywordSource.AI
-    elif entity_type in EMS_ENTITY_TYPES:
-        source = KeywordSource.EMS
-    elif entity_type in SIERRA_ENTITY_TYPES:
-        if linked_doc and linked_doc.elastic:
-            source = KeywordSource.SIERRA
-        elif linked_doc and linked_doc.viaf:
-            source = KeywordSource.VIAF
-        else:
-            source = KeywordSource.AI
-    else:
-        source = KeywordSource.AI
-    return source
-
-def _find_indicators(entity_type: str, entity: str,
-        is_linked: bool
-) -> Tuple[str, str]:
-    """ Find MARC indicators 1 and 2.
-    """
-    ind1 = " "
-    ind2 = " "
-    if entity_type in SIERRA_ENTITY_TYPES:
-        if entity_type == EntityType.PER:
-            if "," in entity:
-                ind1 = "1"
-            else:
-                ind1 = "0"
-        else:
-            # 1 märksõna esimeseks elemendiks võimupiirkonna nimi, nt:
-            #    (a) Eesti (b) Riigikogu - raske automaatselt määrata
-            # 2 märksõna esimeseks elemendiks nimi pärijärjestuses
-            ind1 = "2"
-        if not is_linked:
-            ind2 = "4"
-    elif entity_type in EMS_ENTITY_TYPES:
-        ind2 = "4"
-    return (ind1, ind2)
-
-
-def format_keywords(flat_keywords: List[dict]) -> dict:
-    """ Formats unlinked keywords for Kata CORE.
-    """
-    ignored_keywords = []
-    filtered_keywords = []
-
-    for keyword_dict in flat_keywords:
-        keyword_type = keyword_dict.get("entity_type")
-        if keyword_type in KEYWORD_TYPES_TO_IGNORE:
-            ignored_keywords.append(keyword_dict)
-        else:
-            filtered_keywords.append(keyword_dict)
-
-    formatted_keywords = {
-        "keywords": [],
-        "other": ignored_keywords
-    }
-
-    for keyword_dict in filtered_keywords:
-        original_keyword = keyword_dict.get("keyword")
-        keyword_type = keyword_dict.get("entity_type")
-        entity_type = KEYWORD_TYPE_MAP.get(keyword_type, "")
-        marc_field = KEYWORD_MARC_MAP.get(str(keyword_type), "")
-        lang = keyword_dict.get("language", "")
-
-        ind1, ind2 = _find_indicators(
-            entity_type=entity_type,
-            entity=original_keyword,
-            is_linked=False
-        )
-        keyword_source = _get_keyword_source(
-            linked_doc=None,
-            is_linked=False,
-            entity_type=entity_type
-        )
-        new_keyword_dict = {
-            "dates": "",
-            "indicator1": ind1,
-            "indicator2": ind2,
-            "is_linked": False,
-            "keyword_source": keyword_source,
-            "lang": lang,
-            "location": "",
-            "marc_field": marc_field,
-            "numeration": "",
-            "organisation_sub_unit": "",
-            "original_keyword": original_keyword,
-            "persons_title": "",
-            "url": "",
-            "url_source": ""
-        }
-        new_keyword_dict.update(keyword_dict)
-        formatted_keywords["keywords"].append(new_keyword_dict)
-
-    return formatted_keywords