rara-tools 0.2.0__tar.gz → 0.3.0__tar.gz

{rara_tools-0.2.0/rara_tools.egg-info → rara_tools-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.2.0
+Version: 0.3.0
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -13,7 +13,6 @@ License-File: LICENSE.md
 Requires-Dist: elasticsearch==8.*
 Requires-Dist: elasticsearch_dsl==8.*
 Requires-Dist: minio==7.*
-Requires-Dist: rara-norm-linker==1.*
 Requires-Dist: requests
 Requires-Dist: iso639-lang
 Requires-Dist: pymarc

rara_tools-0.3.0/VERSION

@@ -0,0 +1 @@
+0.3.0

rara_tools-0.3.0/rara_tools/constants/normalizers.py

@@ -0,0 +1,6 @@
+from pymarc import Indicators
+import os
+
+EMPTY_INDICATORS = Indicators(" ", " ")
+VIAF_ALLOWED_SOURCES = ["LC", "DNB", "LNB", "NLL",
+                        "ERRR", "J9U"]

{rara_tools-0.2.0 → rara_tools-0.3.0/rara_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.2.0
+Version: 0.3.0
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -13,7 +13,6 @@ License-File: LICENSE.md
 Requires-Dist: elasticsearch==8.*
 Requires-Dist: elasticsearch_dsl==8.*
 Requires-Dist: minio==7.*
-Requires-Dist: rara-norm-linker==1.*
 Requires-Dist: requests
 Requires-Dist: iso639-lang
 Requires-Dist: pymarc

{rara_tools-0.2.0 → rara_tools-0.3.0}/rara_tools.egg-info/requires.txt

@@ -1,7 +1,6 @@
 elasticsearch==8.*
 elasticsearch_dsl==8.*
 minio==7.*
-rara-norm-linker==1.*
 requests
 iso639-lang
 pymarc

{rara_tools-0.2.0 → rara_tools-0.3.0}/requirements.txt

@@ -1,7 +1,6 @@
 elasticsearch==8.*
 elasticsearch_dsl==8.*
 minio==7.*
-rara-norm-linker==1.*
 requests
 iso639-lang
 pymarc

{rara_tools-0.2.0 → rara_tools-0.3.0}/tests/test_normalization.py

@@ -1,10 +1,9 @@
 from rara_tools.normalizers import BibRecordNormalizer, AuthoritiesRecordNormalizer
-from tests.test_utils import (get_formatted_sierra_response,
+from tests.test_utils import (get_linker_res_example, get_formatted_sierra_response,
                               check_record_tags_sorted, check_no_dupe_tag_values, check_record_tags_have_values)
 
-from pymarc import Record
 
-import pytest
+from pymarc import Record
 
 import os
 
@@ -19,29 +18,31 @@ EMPTY_SIERRA_RECORDS = [
 ]
 
 REQUIRED_FIELDS = ["667", "925"]  # always included after normalization
-REASON = "Skipped because TEST_LEVEL is set to 'ci'"
+MOCK_LINKER_ONE_FOUND = get_linker_res_example(
+    "oneFound.json")
+MOCK_LINKER_MULTIPLE_FOUND = get_linker_res_example(
+    "multipleFound.json")
+MOCK_LINKER_NOT_FOUND = get_linker_res_example(
+    "notFound.json")
 
 
-@pytest.mark.skipif(TEST_LEVEL == "ci", reason=REASON)
 def test_normalizers_OK():
-    """ Test field editing logic & internals"""
+    """ Test field editing logic & internals """
 
-    entities = [
-        "Paul Keres",  # will find multiple entities
-        "Anton Hansen Tammsaare",
-        "GIBBBERRISH",
-    ]
+    linking_results = [MOCK_LINKER_ONE_FOUND,
+                       MOCK_LINKER_MULTIPLE_FOUND]
 
     test_sierra_data = get_formatted_sierra_response("authorities.json")
 
     normalizer = AuthoritiesRecordNormalizer(
-        entities=entities,
+        linking_results=linking_results,
         sierra_data=test_sierra_data,
     )
+
     assert len(normalizer.records_extra_data) == len(normalizer.data)
 
     normalizer = BibRecordNormalizer(
-        entities=entities,
+        linking_results=linking_results,
         sierra_data=test_sierra_data,
     )
     assert len(normalizer.records_extra_data) == len(normalizer.data)
@@ -146,11 +147,11 @@ def validate_authorities_record_normalized(record: Record, has_viaf_data=False):
         # assert len(field_046.get_subfields("t")) > 0 # activity end
 
 
-@pytest.mark.skipif(TEST_LEVEL == "ci", reason=REASON)
 def test_missing_fields_created_bibrecord_normalization():
+    linking_results = [MOCK_LINKER_ONE_FOUND]
 
     normalizer_entities_only = BibRecordNormalizer(
-        entities=["Eduard Vilde", "Linda Vilde"],  # find one match
+        linking_results=linking_results,
     )
 
     normalizer_sierra_data_only = BibRecordNormalizer(
@@ -172,11 +173,12 @@ def test_missing_fields_created_bibrecord_normalization():
         validate_bibrecord_normalized(record)
 
 
-@pytest.mark.skipif(TEST_LEVEL == "ci", reason=REASON)
 def test_missing_fields_created_authorities_normalization():
 
+    linking_results = [MOCK_LINKER_ONE_FOUND]
+
     normalizer_entities_only = AuthoritiesRecordNormalizer(
-        entities=["Eduard Vilde"],  # find one match
+        linking_results=linking_results,  # find one match
     )
 
     normalizer_sierra_data_only = AuthoritiesRecordNormalizer(
@@ -187,6 +189,7 @@ def test_missing_fields_created_authorities_normalization():
         check_record_tags_have_values(r, ["008", "040",  # SIERRA related
                                           "024", "043", "046"  # VIAF enriched
                                           ] + REQUIRED_FIELDS)
+
         validate_authorities_record_normalized(r, True)
 
     for r in normalizer_sierra_data_only:
@@ -195,7 +198,6 @@ def test_missing_fields_created_authorities_normalization():
         validate_authorities_record_normalized(r)
 
 
-@pytest.mark.skipif(TEST_LEVEL == "ci", reason=REASON)
 def test_normalized_fields_sorted():
 
     unsorted_bibdata = [
@@ -237,7 +239,7 @@ def test_normalized_fields_sorted():
 
     for normalizer in normalizers:
         normalizer = normalizer(
-            entities=[],
+            linking_results=[],
             sierra_data=unsorted_bibdata
         )
 
@@ -246,16 +248,18 @@ def test_normalized_fields_sorted():
             check_record_tags_sorted(r)
 
 
-@pytest.mark.skipif(TEST_LEVEL == "ci", reason=REASON)
 def test_authority_normrecord_found_in_es_and_normalized():
     """ KATA elastic normkirjete seast leitakse 1 vaste & normaliseerija täiendab leitud normkirjet VIAF infoga.
         - valideeri normaliseerimise mapping, mis autori tabelis. Täiendatud väljad ja VIAFist info
         - Valideeri märge lisatud (TODO) """
     # Presume, author name identified and sent to linker
-    name = "Jaan Kross"
+    linker_res = get_linker_res_example(
+        "oneFound.json")  # single result
+    linking_results = [linker_res]
 
+    # 1 result found
     normalizer = AuthoritiesRecordNormalizer(
-        entities=[name]
+        linking_results=linking_results
     )
 
     data = normalizer.data
@@ -267,44 +271,48 @@ def test_authority_normrecord_found_in_es_and_normalized():
         validate_authorities_record_normalized(r, has_viaf_data=True)
 
 
-@pytest.mark.skipif(TEST_LEVEL == "ci", reason=REASON)
 def test_authority_normrecord_not_found_in_es_and_viaf():
     """KATA elastic normkirjete seast vastet ei leitud & linkija sooritab VIAFisse otsingu
         - Üks vaste leiti - luuakse uus normkirje
         - Ei leitud ühtegi vastet, või on leitud vasteid mitu - AI tuvastatud info põhjal uue kirje loomine(TODO)
     """
+    linker_res = get_linker_res_example(
+        "oneFound.json")
+    linking_results = [linker_res]
 
-    # 1 result found
-    normalizer = AuthoritiesRecordNormalizer(entities=["Karl Ristikivi"])
+    normalizer = AuthoritiesRecordNormalizer(
+        linking_results=linking_results)
 
     data = normalizer.data
 
     assert len(data) == 1  # should create new normalized record
 
     # Entities not found, es & VIAF
-    normalizer = AuthoritiesRecordNormalizer(entities=["asdasd#@2"])
+    linking_results = [MOCK_LINKER_NOT_FOUND]
+    normalizer = AuthoritiesRecordNormalizer(linking_results=linking_results)
     data = normalizer.data
-    assert len(data) == 0  # should create new normalized record
+    # should create new normalized record in the future, none for now
+    assert len(data) == 0
 
-    # multiple entities found, skipped
-    normalizer = AuthoritiesRecordNormalizer(entities=["Paul Keres"])
+    linker_res = get_linker_res_example(
+        "multipleFound.json")
+    linking_results = [linker_res]
+    normalizer = AuthoritiesRecordNormalizer(linking_results=linking_results)
     data = normalizer.data
-    assert len(data) == 0  # should not create anything atm
+    # should create new normalized record in the future, none for now
+    assert len(data) == 0
 
 
-@pytest.mark.skipif(TEST_LEVEL == "ci", reason=REASON)
 def test_matching_sierra_record_viaf_id_found():
     """normkirjelt leitakse VIAF ID, vajadusel normi asukoht, kus see ID sisaldub."""
     pass
 
 
-@pytest.mark.skipif(TEST_LEVEL == "ci", reason=REASON)
 def test_matching_sierra_record_viaf_id_not_found():
     """kirjelt VIAF IDd ei leitud, soorita otsing VIAFi pihta, et leida _vastutav isik_?. Loo uus vastavalt otsingu tulemusele."""
     pass
 
 
-@pytest.mark.skipif(TEST_LEVEL == "ci", reason=REASON)
 def test_authorities_normalizer_checks():
     """
     - kontrolli kas tuvastatud nimi on SIERRAst leitud vaste 1XX, 4XX väljadel. Kui pole, siis lisa 4XX väljale.

{rara_tools-0.2.0 → rara_tools-0.3.0}/tests/test_utils.py

@@ -1,11 +1,8 @@
-from tests.const import SIERRA_INPUT_DIR, NORMALIZED_DIR, VIAF_TEST_DATA_DIR
-from rara_tools.constants import VIAF_ALLOWED_SOURCES
+from tests.const import SIERRA_INPUT_DIR, LINKER_DIR
 
 from rara_tools.converters import SierraResponseConverter
 from rara_tools.normalizers.viaf import VIAFRecord, VIAFClient
 
-from rara_linker.linkers.linker import Linker
-
 from pymarc import Record
 from typing import List
 
@@ -25,8 +22,9 @@ def check_record_tags_sorted(record: Record):
 
 
 def check_no_dupe_tag_values(record: Record):
-    repetable_tags = ["024", "035", "400", "670"] 
-    record_tags = [field.tag for field in record.get_fields() if field.tag not in repetable_tags]
+    repetable_tags = ["024", "035", "400", "670"]
+    record_tags = [field.tag for field in record.get_fields()
+                   if field.tag not in repetable_tags]
     assert len(record_tags) == len(set(record_tags))
 
 
@@ -56,7 +54,7 @@ def get_formatted_sierra_response(fname: str):
 def get_viaf_record(id: str, allowed_sources: list):
     """ Fetches VIAF record by ID and returns a VIAFRecord object """
 
-    client = VIAFClient()  # should use Linker instead? not ViafLinker directly
+    client = VIAFClient()
     response = client.get_records_by_viaf_id(id)
 
     viaf_record = VIAFRecord(
@@ -71,7 +69,8 @@ def search_viaf_record(search_term: str, allowed_sources: list):
 
     return VIAFRecord(response, allowed_sources=allowed_sources)
 
-def get_normalized_example(fname: str):
-    with open(os.path.join(NORMALIZED_DIR, fname), "r") as f:
+
+def get_linker_res_example(fname: str):
+    with open(os.path.join(LINKER_DIR, fname), "r") as f:
         data = f.read()
         return json.loads(data)

rara_tools-0.2.0/VERSION

@@ -1 +0,0 @@
-0.2.0

rara_tools-0.2.0/rara_tools/constants/normalizers.py

@@ -1,17 +0,0 @@
-from pymarc import Indicators
-import os
-
-EMPTY_INDICATORS = Indicators(" ", " ")
-VIAF_ALLOWED_SOURCES = ["LC", "DNB", "LNB", "NLL",
-                        "ERRR", "J9U"]
-
-ES_HOST = os.getenv("ELASTIC_TEST_URL", "http://localhost:9200")
-
-LINKER_CONFIG = {
-    "add_viaf_info": True,
-    "vectorizer_data_path": "./vectorizer_data",
-    "per_config": {"es_host": ES_HOST},
-    "org_config": {"es_host": ES_HOST},
-    "loc_config": {"es_host": ES_HOST},
-    "ems_config": {"es_host": ES_HOST},
-}