folio-migration-tools 1.9.0a3__tar.gz → 1.9.0a5__tar.gz

{folio_migration_tools-1.9.0a3 → folio_migration_tools-1.9.0a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: folio_migration_tools
-Version: 1.9.0a3
+Version: 1.9.0a5
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 Home-page: https://github.com/FOLIO-FSE/folio_migration_tools
 License: MIT

{folio_migration_tools-1.9.0a3 → folio_migration_tools-1.9.0a5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "folio_migration_tools"
-version = "1.9.0a3"
+version = "1.9.0a5"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = ["Theodor Tolstoy <github.teddes@tolstoy.se>", "Lisa Sjögren", "Brooks Travis", "Jeremy Nelson", "Clinton Bradford"]
 license = "MIT"

{folio_migration_tools-1.9.0a3 → folio_migration_tools-1.9.0a5}/src/folio_migration_tools/mapper_base.py

@@ -24,6 +24,7 @@ from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
     RefDataMapping,
 )
 from folio_migration_tools.migration_report import MigrationReport
+from folio_migration_tools.task_configuration import AbstractTaskConfiguration
 
 
 class MapperBase:
@@ -42,7 +43,7 @@ class MapperBase:
         self.start_datetime = datetime.now(timezone.utc)
         self.folio_client: FolioClient = folio_client
         self.library_configuration: LibraryConfiguration = library_configuration
-
+        self.task_configuration: AbstractTaskConfiguration
         self.mapped_folio_fields: dict = {}
         self.migration_report: MigrationReport = MigrationReport()
         self.num_criticalerrors = 0
@@ -267,7 +268,13 @@ class MapperBase:
             sys.exit(1)
 
     def get_id_map_tuple(self, legacy_id: str, folio_record: dict, object_type: FOLIONamespaces):
-        if object_type == FOLIONamespaces.instances:
+        if all(
+            [
+                object_type == FOLIONamespaces.instances,
+                (not getattr(self.task_configuration, "data_import_marc", False)),
+                getattr(self.task_configuration, "create_source_records", True),
+            ]
+        ):
             return (legacy_id, folio_record["id"], folio_record["hrid"])
         return (legacy_id, folio_record["id"])
 

{folio_migration_tools-1.9.0a3 → folio_migration_tools-1.9.0a5}/src/folio_migration_tools/marc_rules_transformation/marc_file_processor.py

@@ -32,7 +32,7 @@ class MarcFileProcessor:
         self.created_objects_file = created_objects_file
         if mapper.task_configuration.create_source_records:
             self.srs_records_file = open(self.folder_structure.srs_records_path, "w+")
-        if mapper.task_configuration.data_import_marc:
+        if getattr(mapper.task_configuration, "data_import_marc", False):
             self.data_import_marc_file = open(self.folder_structure.data_import_marc_path, "wb+")
         self.unique_001s: set = set()
         self.failed_records_count: int = 0
@@ -87,7 +87,7 @@ class MarcFileProcessor:
                             legacy_ids,
                             self.object_type,
                         )
-                    if self.mapper.task_configuration.data_import_marc:
+                    if getattr(self.mapper.task_configuration, "data_import_marc", False):
                         self.save_marc_record(
                             marc_record,
                             folio_rec,
@@ -271,7 +271,7 @@ class MarcFileProcessor:
             if not self.srs_records_file.seek(0):
                 os.remove(self.srs_records_file.name)
             self.srs_records_file.close()
-        if self.mapper.task_configuration.data_import_marc:
+        if getattr(self.mapper.task_configuration, "data_import_marc", False):
             self.data_import_marc_file.seek(0)
             if not self.data_import_marc_file.read(1):
                 os.remove(self.data_import_marc_file.name)

{folio_migration_tools-1.9.0a3 → folio_migration_tools-1.9.0a5}/src/folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py

@@ -59,7 +59,7 @@ class BibsRulesMapper(RulesMapperBase):
         self.instance_relationship_types: dict = {}
         self.other_mode_of_issuance_id = get_unspecified_mode_of_issuance(self.folio_client)
         self.create_source_records = all(
-            [self.task_configuration.create_source_records, (not self.task_configuration.data_import_marc)]
+            [self.task_configuration.create_source_records, (not getattr(self.task_configuration, "data_import_marc", False))]
         )
         self.data_import_marc = self.task_configuration.data_import_marc
         self.start = time.time()
@@ -118,7 +118,7 @@ class BibsRulesMapper(RulesMapperBase):
         bad_tags = set(self.task_configuration.tags_to_delete)  # "907"
         folio_instance = self.perform_initial_preparation(marc_record, legacy_ids)
         if self.data_import_marc:
-            self.simple_bib_map(marc_record, folio_instance, ignored_subsequent_fields, legacy_ids)
+            self.simple_bib_map(folio_instance, marc_record, ignored_subsequent_fields, legacy_ids)
         else:
             for marc_field in marc_record:
                 self.report_marc_stats(marc_field, bad_tags, legacy_ids, ignored_subsequent_fields)
@@ -152,7 +152,13 @@ class BibsRulesMapper(RulesMapperBase):
             legacy_ids (List[str]): _description_
             file_def (FileDefinition): _description_
         """
-        self.process_marc_field(folio_instnace, marc_record['245'], ignored_subsequent_fields, legacy_ids)
+        try:
+            self.process_marc_field(folio_instnace, marc_record['245'], ignored_subsequent_fields, legacy_ids)
+        except KeyError:
+            raise TransformationRecordFailedError(
+                legacy_ids,
+                "No 245 field in MARC record"
+            )
 
     def perform_additional_parsing(
         self,
@@ -642,20 +648,6 @@ class BibsRulesMapper(RulesMapperBase):
                 ) from e
 
 
-class SimpleBibsRulesMapper(BibsRulesMapper):
-    def __init__(
-        self,
-        folio_client,
-        library_configuration: LibraryConfiguration,
-        task_configuration,
-    ):
-        super().__init__(
-            folio_client,
-            library_configuration,
-            task_configuration,
-        )
-
-
 def get_unspecified_mode_of_issuance(folio_client: FolioClient):
     m_o_is = list(folio_client.modes_of_issuance)
     if not any(m_o_is):

folio_migration_tools-1.9.0a3/src/folio_migration_tools/migration_tasks/bibs_transformer_simple.py

@@ -1,198 +0,0 @@
-import logging
-from typing import Annotated, List
-
-import i18n
-from folio_uuid.folio_namespaces import FOLIONamespaces
-from pydantic import Field
-
-from folio_migration_tools.helper import Helper
-from folio_migration_tools.library_configuration import (
-    FileDefinition,
-    HridHandling,
-    IlsFlavour,
-    LibraryConfiguration,
-)
-from folio_migration_tools.marc_rules_transformation.marc_file_processor import (
-    MarcFileProcessor,
-)
-from folio_migration_tools.marc_rules_transformation.rules_mapper_bibs import (
-    SimpleBibsRulesMapper,
-)
-from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
-from folio_migration_tools.task_configuration import AbstractTaskConfiguration
-
-
-class BibsTransformer(MigrationTaskBase):
-    class TaskConfiguration(AbstractTaskConfiguration):
-        name: Annotated[
-            str,
-            Field(
-                description=(
-                    "Name of this migration task. The name is being used to call the specific "
-                    "task, and to distinguish tasks of similar types"
-                )
-            ),
-        ]
-        migration_task_type: Annotated[
-            str,
-            Field(
-                title="Migration task type",
-                description=("The type of migration task you want to perform."),
-            ),
-        ]
-        files: Annotated[
-            List[FileDefinition],
-            Field(
-                title="Source files",
-                description=("List of MARC21 files with bibliographic records."),
-            ),
-        ]
-        ils_flavour: Annotated[
-            IlsFlavour,
-            Field(
-                title="ILS flavour", description="The type of ILS you are migrating records from."
-            ),
-        ]
-        custom_bib_id_field: Annotated[
-            str,
-            Field(
-                title="Custom BIB ID field",
-                description=(
-                    'A string representing a MARC field with optional subfield indicated by a "$" '
-                    '(eg. "991$a") from which to draw legacy Bib ID. Use this in combination '
-                    'with `ilsFlavour: "custom"`. Defaults to "001", and is ignored for all other '
-                    "ilsFlavours."
-                ),
-            ),
-        ] = "001"
-        add_administrative_notes_with_legacy_ids: Annotated[
-            bool,
-            Field(
-                title="Add administrative notes with legacy IDs",
-                description=(
-                    "If set to true, an Administrative note will be added to the records "
-                    "containing the legacy ID. Use this in order to protect the values from "
-                    "getting overwritten by overlays,"
-                ),
-            ),
-        ] = True
-        tags_to_delete: Annotated[
-            List[str],
-            Field(
-                title="Tags to delete from MARC record",
-                description=(
-                    "Tags in the incoming MARC authority that the process should remove "
-                    "before adding them into FOLIO. These tags will be used in the "
-                    "transformation before getting removed."
-                ),
-            ),
-        ] = []
-        create_source_records: Annotated[
-            bool,
-            Field(
-                title="Create source records",
-                description=(
-                    "Controls wheter or not to retain the MARC records in "
-                    "Source Record Storage."
-                ),
-            ),
-        ] = True
-        parse_cataloged_date: Annotated[
-            bool,
-            Field(
-                title="Parse cataloged date",
-                description=(
-                    "Parse fields mapped to catalogedDate into a FOLIO accepted date string using "
-                    "dateutil.parser. Verify results carefully when using"
-                ),
-            ),
-        ] = False
-        hrid_handling: Annotated[
-            HridHandling,
-            Field(
-                title="HRID Handling",
-                description=(
-                    "Setting to default will make FOLIO generate HRIDs and move the existing "
-                    "001:s into a 035, concatenated with the 003. Choosing preserve001 means "
-                    "the 001:s will remain in place, and that they will also become the HRIDs"
-                ),
-            ),
-        ] = HridHandling.default
-        reset_hrid_settings: Annotated[
-            bool,
-            Field(
-                title="Reset HRID settings",
-                description=(
-                    "Setting to true means the task will "
-                    "reset the HRID counters for this particular record type"
-                ),
-            ),
-        ] = False
-        update_hrid_settings: Annotated[
-            bool,
-            Field(
-                title="Update HRID settings",
-                description="At the end of the run, update FOLIO with the HRID settings",
-            ),
-        ] = True
-        deactivate035_from001: Annotated[
-            bool,
-            Field(
-                title="Create 035 from 001 and 003",
-                description=(
-                    "This deactivates the FOLIO default functionality of moving the previous 001 "
-                    "into a 035, prefixed with the value from 003"
-                ),
-            ),
-        ] = False
-
-    @staticmethod
-    def get_object_type() -> FOLIONamespaces:
-        return FOLIONamespaces.instances
-
-    def __init__(
-        self,
-        task_config: TaskConfiguration,
-        library_config: LibraryConfiguration,
-        folio_client,
-        use_logging: bool = True,
-    ):
-        super().__init__(library_config, task_config, folio_client, use_logging)
-        self.processor: MarcFileProcessor
-        self.check_source_files(
-            self.folder_structure.legacy_records_folder, self.task_configuration.files
-        )
-        self.mapper = SimpleBibsRulesMapper(self.folio_client, library_config, self.task_configuration)
-        self.bib_ids: set = set()
-        if (
-            self.task_configuration.reset_hrid_settings
-            and self.task_configuration.update_hrid_settings
-        ):
-            self.mapper.hrid_handler.reset_instance_hrid_counter()
-        logging.info("Init done")
-
-    def do_work(self):
-        self.do_work_marc_transformer()
-
-    def wrap_up(self):
-        logging.info("Done. Transformer wrapping up...")
-        self.extradata_writer.flush()
-        self.processor.wrap_up()
-        with open(self.folder_structure.migration_reports_file, "w+") as report_file:
-            self.mapper.migration_report.write_migration_report(
-                i18n.t("Bibliographic records transformation report"),
-                report_file,
-                self.start_datetime,
-            )
-            Helper.print_mapping_report(
-                report_file,
-                self.mapper.parsed_records,
-                self.mapper.mapped_folio_fields,
-                self.mapper.mapped_legacy_fields,
-            )
-
-        logging.info(
-            "Done. Transformation report written to %s",
-            self.folder_structure.migration_reports_file.name,
-        )
-        self.clean_out_empty_logs()