folio-migration-tools 1.10.0__py3-none-any.whl → 1.10.0b1__py3-none-any.whl

folio_migration_tools/marc_rules_transformation/rules_mapper_base.py

@@ -15,7 +15,6 @@ from folio_uuid.folio_uuid import FOLIONamespaces, FolioUUID
 from folioclient import FolioClient
 from pymarc import Field, Optional, Record, Subfield
 
-from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.custom_exceptions import (
     TransformationFieldMappingError,
     TransformationProcessError,
@@ -300,13 +299,13 @@ class RulesMapperBase(MapperBase):
     def perform_proxy_mapping(self, marc_field):
         proxy_mapping = next(iter(self.mappings.get("880", [])), [])
         if "6" not in marc_field:
-            self.migration_report.add("Field880Mappings", i18n_t("Records without $6"))
+            self.migration_report.add("Field880Mappings", i18n.t("Records without $6"))
             return None
         if not proxy_mapping or not proxy_mapping.get("fieldReplacementBy3Digits", False):
             return None
         if not marc_field["6"][:3] or len(marc_field["6"][:3]) != 3:
             self.migration_report.add(
-                "Field880Mappings", i18n_t("Records with unexpected length in $6")
+                "Field880Mappings", i18n.t("Records with unexpected length in $6")
             )
             return None
         first_three = marc_field["6"][:3]
@@ -321,16 +320,16 @@ class RulesMapperBase(MapperBase):
         )
         self.migration_report.add(
             "Field880Mappings",
-            i18n_t("Source digits")
+            i18n.t("Source digits")
             + f": {marc_field['6']} "
-            + i18n_t("Target field")
+            + i18n.t("Target field")
             + f": {target_field}",
         )
         mappings = self.mappings.get(target_field, {})
         if not mappings:
             self.migration_report.add(
                 "Field880Mappings",
-                i18n_t("Mapping not set up for target field")
+                i18n.t("Mapping not set up for target field")
                 + f": {target_field} ({marc_field['6']})",
             )
         return mappings
@@ -338,7 +337,7 @@ class RulesMapperBase(MapperBase):
     def report_marc_stats(
         self, marc_field: Field, bad_tags, legacy_ids, ignored_subsequent_fields
     ):
-        self.migration_report.add("Trivia", i18n_t("Total number of Tags processed"))
+        self.migration_report.add("Trivia", i18n.t("Total number of Tags processed"))
         self.report_source_and_links(marc_field)
         self.report_bad_tags(marc_field, bad_tags, legacy_ids)
         mapped = marc_field.tag in self.mappings
@@ -352,7 +351,7 @@ class RulesMapperBase(MapperBase):
         for subfield_2 in marc_field.get_subfields("2"):
             self.migration_report.add(
                 "AuthoritySources",
-                i18n_t("Source of heading or term") + f": {subfield_2.split(' ')[0]}",
+                i18n.t("Source of heading or term") + f": {subfield_2.split(' ')[0]}",
             )
         for subfield_0 in marc_field.get_subfields("0"):
             code = ""
@@ -364,7 +363,7 @@ class RulesMapperBase(MapperBase):
                     code = subfield_0[: subfield_0.find(url.path)]
             if code:
                 self.migration_report.add(
-                    "AuthoritySources", i18n_t("$0 base uri or source code") + f": {code}"
+                    "AuthoritySources", i18n.t("$0 base uri or source code") + f": {code}"
                 )
 
     def apply_rules(self, marc_field: pymarc.Field, mapping, legacy_ids):
@@ -403,7 +402,7 @@ class RulesMapperBase(MapperBase):
             )
             trfe.log_it()
             self.migration_report.add_general_statistics(
-                i18n_t("Records failed due to an error. See data issues log for details")
+                i18n.t("Records failed due to an error. See data issues log for details")
             )
         except Exception as exception:
             self.handle_generic_exception(self.parsed_records, exception)
@@ -963,6 +962,7 @@ class RulesMapperBase(MapperBase):
         srs_types = {
             FOLIONamespaces.holdings: FOLIONamespaces.srs_records_holdingsrecord,
             FOLIONamespaces.instances: FOLIONamespaces.srs_records_bib,
+            FOLIONamespaces.authorities: FOLIONamespaces.srs_records_auth,
             FOLIONamespaces.edifact: FOLIONamespaces.srs_records_edifact,
         }
 
@@ -1020,6 +1020,7 @@ class RulesMapperBase(MapperBase):
         record_types = {
             FOLIONamespaces.holdings: "MARC_HOLDING",
             FOLIONamespaces.instances: "MARC_BIB",
+            FOLIONamespaces.authorities: "MARC_AUTHORITY",
             FOLIONamespaces.edifact: "EDIFACT",
         }
 
@@ -1032,6 +1033,10 @@ class RulesMapperBase(MapperBase):
                 "holdingsId": folio_object["id"],
                 "holdingsHrid": folio_object.get("hrid", ""),
             },
+            FOLIONamespaces.authorities: {
+                "authorityId": folio_object["id"],
+                "authorityHrid": marc_record["001"].data,
+            },
             FOLIONamespaces.edifact: {},
         }
 

folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py

@@ -17,7 +17,6 @@ from folioclient import FolioClient
 from pymarc.record import Leader, Record
 from pymarc.field import Field
 
-from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.custom_exceptions import (
     TransformationProcessError,
     TransformationRecordFailedError,
@@ -98,7 +97,7 @@ class BibsRulesMapper(RulesMapperBase):
 
     def handle_leader_05(self, marc_record: Record, legacy_ids: List[str]):
         leader_05 = marc_record.leader[5] or "Empty"
-        self.migration_report.add("RecordStatus", i18n_t("Original value") + f": {leader_05}")
+        self.migration_report.add("RecordStatus", i18n.t("Original value") + f": {leader_05}")
         if leader_05 not in ["a", "c", "d", "n", "p"]:
             marc_record.leader = Leader(f"{marc_record.leader[:5]}c{marc_record.leader[6:]}")
             self.migration_report.add(
@@ -324,7 +323,7 @@ class BibsRulesMapper(RulesMapperBase):
             raise TransformationProcessError("", "No instance_types setup in tenant")
 
         if "336" in marc_record and "b" not in marc_record["336"]:
-            self.migration_report.add("RecourceTypeMapping", i18n_t("Subfield b not in 336"))
+            self.migration_report.add("RecourceTypeMapping", i18n.t("Subfield b not in 336"))
             if "a" in marc_record["336"]:
                 return_id = get_folio_id_by_name(marc_record["336"]["a"])
 

folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -12,7 +12,6 @@ from pymarc import Optional
 from pymarc.field import Field
 from pymarc.record import Record
 
-from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.custom_exceptions import (
     TransformationFieldMappingError,
     TransformationProcessError,
@@ -254,7 +253,7 @@ class RulesMapperHoldings(RulesMapperBase):
             ignored_subsequent_fields (_type_): _description_
             index_or_legacy_ids (_type_): _description_
         """
-        self.migration_report.add("Trivia", i18n_t("Total number of Tags processed"))
+        self.migration_report.add("Trivia", i18n.t("Total number of Tags processed"))
         if marc_field.tag not in self.mappings:
             self.report_legacy_mapping(marc_field.tag, True, False)
         elif marc_field.tag not in ignored_subsequent_fields:
@@ -585,7 +584,7 @@ class RulesMapperHoldings(RulesMapperBase):
                     Helper.log_data_issue(
                         legacy_ids,
                         (
-                            i18n_t("blurbs.HoldingsTypeMapping.title") + " is 'unknown'. "
+                            i18n.t("blurbs.HoldingsTypeMapping.title") + " is 'unknown'. "
                             "(leader 06 is set to 'u') Check if this is correct"
                         ),
                         ldr06,
@@ -599,14 +598,14 @@ class RulesMapperHoldings(RulesMapperBase):
                 folio_holding["holdingsTypeId"] = self.fallback_holdings_type_id
                 self.migration_report.add(
                     "HoldingsTypeMapping",
-                    i18n_t("An Unmapped")
+                    i18n.t("An Unmapped")
                     + f" {ldr06} -> {holdings_type} -> "
-                    + i18n_t("Unmapped"),
+                    + i18n.t("Unmapped"),
                 )
                 Helper.log_data_issue(
                     legacy_ids,
                     (
-                        i18n_t("blurbs.HoldingsTypeMapping.title", locale="en")
+                        i18n.t("blurbs.HoldingsTypeMapping.title", locale="en")
                         + ". leader 06 was unmapped."
                     ),
                     ldr06,

folio_migration_tools/migration_report.py

@@ -1,11 +1,8 @@
 import logging
-import json
 import i18n
 from datetime import datetime
 from datetime import timezone
 
-from folio_migration_tools.i18n_cache import i18n_t
-
 
 class MigrationReport:
     """Class responsible for handling the migration report"""
@@ -50,14 +47,6 @@ class MigrationReport:
         """
         self.add("GeneralStatistics", measure_to_add)
 
-    def _write_json_report(self, report_file):
-        """Writes the raw migration report data to a JSON file.
-
-        Args:
-            report_file: An open file object to write the JSON data to
-        """
-        json.dump(self.report, report_file, indent=2)
-
     def write_migration_report(
         self,
         report_title,
@@ -77,13 +66,13 @@ class MigrationReport:
                 [
                     "# " + report_title,
                     i18n.t("blurbs.Introduction.description"),
-                    "## " + i18n_t("Timings"),
+                    "## " + i18n.t("Timings"),
                     "",
-                    i18n_t("Measure") + " | " + i18n_t("Value"),
+                    i18n.t("Measure") + " | " + i18n.t("Value"),
                     "--- | ---:",
-                    i18n_t("Time Started:") + " | " + datetime.isoformat(time_started),
-                    i18n_t("Time Finished:") + " | " + datetime.isoformat(time_finished),
-                    i18n_t("Elapsed time:") + " | " + str(time_finished - time_started),
+                    i18n.t("Time Started:") + " | " + datetime.isoformat(time_started),
+                    i18n.t("Time Finished:") + " | " + datetime.isoformat(time_finished),
+                    i18n.t("Elapsed time:") + " | " + str(time_finished - time_started),
                 ]
             )
         )
@@ -100,7 +89,7 @@ class MigrationReport:
                         + i18n.t("Click to expand all %{count} things", count=len(self.report[a]))
                         + "</summary>",
                         "",
-                        i18n_t("Measure") + " | " + i18n_t("Count"),
+                        i18n.t("Measure") + " | " + i18n.t("Count"),
                         "--- | ---:",
                     ]
                     + [

folio_migration_tools/migration_tasks/authority_transformer.py

@@ -0,0 +1,118 @@
+import logging
+from typing import Annotated
+from typing import List
+import i18n
+
+from folio_uuid.folio_namespaces import FOLIONamespaces
+from pydantic import Field
+
+from folio_migration_tools.helper import Helper
+from folio_migration_tools.library_configuration import FileDefinition
+from folio_migration_tools.library_configuration import IlsFlavour
+from folio_migration_tools.library_configuration import LibraryConfiguration
+from folio_migration_tools.marc_rules_transformation.marc_file_processor import (
+    MarcFileProcessor,
+)
+from folio_migration_tools.marc_rules_transformation.rules_mapper_authorities import (
+    AuthorityMapper,
+)
+from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
+from folio_migration_tools.task_configuration import AbstractTaskConfiguration
+
+
+class AuthorityTransformer(MigrationTaskBase):
+    class TaskConfiguration(AbstractTaskConfiguration):
+        name: Annotated[
+            str,
+            Field(
+                description=(
+                    "Name of this migration task. The name is being used to call the specific "
+                    "task, and to distinguish tasks of similar types"
+                )
+            ),
+        ]
+        migration_task_type: Annotated[
+            str,
+            Field(
+                title="Migration task type",
+                description=("The type of migration task you want to perform"),
+            ),
+        ]
+        files: Annotated[
+            List[FileDefinition],
+            Field(
+                title="Source files", description=("List of MARC21 files with authority records")
+            ),
+        ]
+        ils_flavour: Annotated[
+            IlsFlavour,
+            Field(
+                title="ILS flavour", description="The type of ILS you are migrating records from."
+            ),
+        ]
+        tags_to_delete: Annotated[
+            List[str],
+            Field(
+                title="Tags to delete from MARC record",
+                description=(
+                    "Tags in the incoming MARC authority that the process should remove "
+                    "before adding them into FOLIO. These tags will be used in the "
+                    "transformation before getting removed."
+                ),
+            ),
+        ] = []
+        create_source_records: Annotated[
+            bool,
+            Field(
+                title="Create source records",
+                description=(
+                    "Controls wheter or not to retain the MARC records in Source Record Storage."
+                ),
+            ),
+        ] = True
+
+    @staticmethod
+    def get_object_type() -> FOLIONamespaces:
+        return FOLIONamespaces.authorities
+
+    def __init__(
+        self,
+        task_config: TaskConfiguration,
+        library_config: LibraryConfiguration,
+        use_logging: bool = True,
+    ):
+        super().__init__(library_config, task_config, use_logging)
+        self.processor: MarcFileProcessor
+        self.check_source_files(
+            self.folder_structure.legacy_records_folder, self.task_configuration.files
+        )
+        self.mapper: AuthorityMapper = AuthorityMapper(
+            self.folio_client, library_config, task_config
+        )
+        self.auth_ids: set = set()
+        logging.info("Init done")
+
+    def do_work(self):
+        self.do_work_marc_transformer()
+
+    def wrap_up(self):
+        logging.info("Done. Transformer Wrapping up...")
+        self.extradata_writer.flush()
+        self.processor.wrap_up()
+        with open(self.folder_structure.migration_reports_file, "w+") as report_file:
+            self.mapper.migration_report.write_migration_report(
+                i18n.t("Authority records transformation report"),
+                report_file,
+                self.start_datetime,
+            )
+            Helper.print_mapping_report(
+                report_file,
+                self.mapper.parsed_records,
+                self.mapper.mapped_folio_fields,
+                self.mapper.mapped_legacy_fields,
+            )
+        logging.info(
+            "Done. Transformation report written to %s",
+            self.folder_structure.migration_reports_file.name,
+        )
+        self.clean_out_empty_logs()