folio-migration-tools 1.9.0a6__tar.gz → 1.9.0rc1__tar.gz

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0a6
+Version: 1.9.0rc1
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
@@ -26,7 +26,8 @@ Requires-Dist: pyhumps (>=3.7.3,<4.0.0)
 Requires-Dist: pymarc (>=5.2.3,<6.0.0)
 Requires-Dist: python-dateutil (>=2.8.2,<3.0.0)
 Requires-Dist: python-i18n (>=0.3.9,<0.4.0)
-Project-URL: Homepage, https://github.com/FOLIO-FSE/folio_migration_tools
+Project-URL: Documentation, https://folio-migration-tools.readthedocs.io/en/latest/
+Project-URL: Homepage, https://github.com/folio-fse/folio_migration_tools
 Project-URL: Repository, https://github.com/FOLIO-FSE/folio_migration_tools
 Description-Content-Type: text/markdown
 

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/pyproject.toml

@@ -1,14 +1,26 @@
-[tool.poetry]
+[project]
 name = "folio_migration_tools"
-version = "1.9.0a6"
+version = "1.9.0rc1"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
-authors = ["Theodor Tolstoy <github.teddes@tolstoy.se>", "Lisa Sjögren", "Brooks Travis", "Jeremy Nelson", "Clinton Bradford"]
+authors = [
+	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
+	{name = "Lisa Sjögren"},
+	{name = "Brooks Travis", email = "brooks.travis@gmail.com"},
+	{name = "Jeremy Nelson"},
+	{name = "Clinton Bradford"}
+]
 license = "MIT"
 readme = "README.md"
-homepage = "https://github.com/FOLIO-FSE/folio_migration_tools"
-repository = "https://github.com/FOLIO-FSE/folio_migration_tools"
 keywords = ["FOLIO", "ILS", "LSP", "Library Systems", "MARC21", "Library data"]
 
+[project.urls]
+documentation = "https://folio-migration-tools.readthedocs.io/en/latest/"
+homepage = "https://github.com/folio-fse/folio_migration_tools"
+repository = "https://github.com/FOLIO-FSE/folio_migration_tools"
+
+[project.scripts]
+folio-migration-tools = "folio_migration_tools.__main__:main"
+
 [tool.coverage.report]
 show_missing = false
 fail_under = 10
@@ -21,13 +33,12 @@ branch = true
 source = ["folio_migration_tools"]
 
 [build-system]
-requires = ["setuptools>=42"]
-build-backend = "setuptools.build_meta"
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
 
 [tool.black]
 line-length = 99
 
-
 [tool.isort]
 profile = "black"
 
@@ -65,6 +76,7 @@ myst-parser = "^0.18.1"
 pandas = "^1.5.3"
 types-requests = "^2.28.11.17"
 types-python-dateutil = "^2.8.19.11"
+ipykernel = "^6.29.5"
 
 [tool.poetry.extras]
 docs = ["m2r", "sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "toml"]

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/src/folio_migration_tools/__main__.py

@@ -124,7 +124,8 @@ def main():
         except TransformationProcessError as tpe:
             logging.critical(tpe.message)
             print(f"\n{tpe.message}: {tpe.data_value}")
-            sys.exit("Transformation Failure")
+            print("Task failure. Halting.")
+            sys.exit(1)
         logging.info("Work done. Shutting down")
         sys.exit(0)
     except json.decoder.JSONDecodeError as json_error:
@@ -148,10 +149,17 @@ def main():
         print("Halting")
         sys.exit("JSON Not Matching Spec")
     except httpx.HTTPError as connection_error:
-        print(
-            f"\nConnection Error when connecting to {connection_error.request.url}. "
-            "Are you connectet to the Internet/VPN? Do you need to update DNS settings?"
-        )
+        if hasattr(connection_error, "response"):
+            print(
+                f"\nHTTP Error when connecting to {connection_error.request.url}. "
+                f"Status code: {connection_error.response.status_code}. "
+                f"\nResponse: {connection_error.response.text}"
+            )
+        else:
+            print(
+                f"\nConnection Error when connecting to {connection_error.request.url}. "
+                "Are you connected to the Internet/VPN? Do you need to update DNS settings?"
+            )
         sys.exit("HTTP Not Connecting")
     except FileNotFoundError as fnf_error:
         print(f"\n{fnf_error.strerror}: {fnf_error.filename}")

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/src/folio_migration_tools/library_configuration.py

@@ -62,14 +62,9 @@ class IlsFlavour(str, Enum):
 
 
 class FolioRelease(str, Enum):
-    lotus = "lotus"
-    morning_glory = "morning-glory"
-    nolana = "nolana"
-    orchid = "orchid"
-    poppy = "poppy"
-    quesnelia = "quesnelia"
     ramsons = "ramsons"
     sunflower = "sunflower"
+    trillium = "trillium"
 
 
 class LibraryConfiguration(BaseModel):

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/src/folio_migration_tools/mapper_base.py

@@ -444,7 +444,7 @@ class MapperBase:
                     except IndexError:
                         if call_numbers:
                             bound_with_holding["callNumber"] = call_numbers[0]
-                    except SyntaxError:
+                    except (SyntaxError, ValueError):
                         bound_with_holding["callNumber"] = call_number
                 else:
                     bound_with_holding["callNumber"] = call_number

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/src/folio_migration_tools/mapping_file_transformation/item_mapper.py

@@ -165,7 +165,7 @@ class ItemMapper(MappingFileMapperBase):
                 legacy_item,
                 folio_prop_name,
                 index_or_id,
-                False,
+                self.task_configuration.prevent_permanent_location_map_default,
             )
         elif folio_prop_name == "temporaryLocationId":
             if not self.temp_location_mapping:

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/src/folio_migration_tools/mapping_file_transformation/order_mapper.py

@@ -25,6 +25,8 @@ from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
 
 
 class CompositeOrderMapper(MappingFileMapperBase):
+    VALID_PO_NUMBER_CHARACTERS = r"[A-Za-z0-9]"
+
     def __init__(
         self,
         folio_client: FolioClient,
@@ -364,6 +366,8 @@ class CompositeOrderMapper(MappingFileMapperBase):
             return {}
 
     def perform_additional_mapping(self, index_or_id, composite_order):
+        self.validate_po_number(index_or_id, composite_order.get("poNumber"))
+        
         # Get organization UUID from FOLIO
         composite_order["vendor"] = self.get_folio_organization_uuid(
             index_or_id, composite_order.get("vendor")
@@ -376,6 +380,22 @@ class CompositeOrderMapper(MappingFileMapperBase):
 
         return composite_order
 
+    def validate_po_number(
+            self,
+            index_or_id: str,
+            po_number: str,
+    ):
+        if re.sub(self.VALID_PO_NUMBER_CHARACTERS, "", po_number):
+            self.migration_report.add(
+                "PurchaseOrderVendorLinking",
+                i18n.t("RECORD FAILED: PO number has invalid character(s)"),
+            )
+            raise TransformationRecordFailedError(
+                index_or_id,
+                "Purchase Order number has invalid character(s)",
+                po_number,
+            )
+
     def get_matching_record_from_folio(
         self,
         index_or_id,

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/src/folio_migration_tools/marc_rules_transformation/conditions.py

@@ -846,7 +846,7 @@ class Conditions:
         self.mapper.migration_report.add(
             "StaffOnlyViaIndicator",
             f"{marc_field.tag} indicator1: {ind1} ("
-            + i18n.t("1 is public, all other values are Staff only")
+            + i18n.t("0 is staff-only, all other values are public")
             + ")",
         )
         if ind1 == "0":
@@ -864,4 +864,30 @@ class Conditions:
             raise TransformationProcessError(
                 legacy_id,
                 f"Subject type not found for {parameter['name']} {marc_field}",
-            )
+            )
+
+    def condition_set_subject_source_id(self, legacy_id, value, parameter, marc_field: field.Field):
+        try:
+            t = self.get_ref_data_tuple_by_name(
+                self.folio.folio_get_all("/subject-sources", "subjectSources"), "subject_sources", parameter["name"]
+            )
+            self.mapper.migration_report.add("MappedSubjectSources", t[1])
+            return t[0]
+        except Exception:
+            raise TransformationProcessError(
+                legacy_id,
+                f"Subject source not found for {parameter['name']} {marc_field}",
+            )
+
+    def condition_set_subject_source_id_by_code(self, legacy_id, value, parameter, marc_field: field.Field):
+        try:
+            t = self.get_ref_data_tuple_by_code(
+                self.folio.folio_get_all("/subject-sources", "subjectSources"), "subject_sources", value
+            )
+            self.mapper.migration_report.add("MappedSubjectSources", t[1])
+            return t[0]
+        except Exception:
+            raise TransformationProcessError(
+                legacy_id,
+                f"Subject source not found for {value} {marc_field}",
+            )

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/src/folio_migration_tools/migration_tasks/batch_poster.py

@@ -136,12 +136,12 @@ class BatchPoster(MigrationTaskBase):
     def do_work(self):
         with self.folio_client.get_folio_http_client() as httpx_client:
             self.http_client = httpx_client
-            try:
-                batch = []
-                if self.task_configuration.object_type == "SRS":
-                    self.create_snapshot()
-                with open(self.folder_structure.failed_recs_path, "w") as failed_recs_file:
-                    for file_def in self.task_configuration.files:
+            with open(self.folder_structure.failed_recs_path, "w", encoding='utf-8') as failed_recs_file:
+                try:
+                    batch = []
+                    if self.task_configuration.object_type == "SRS":
+                        self.create_snapshot()
+                    for idx, file_def in enumerate(self.task_configuration.files):
                         path = self.folder_structure.results_folder / file_def.file_name
                         with open(path) as rows:
                             logging.info("Running %s", path)
@@ -172,9 +172,8 @@ class BatchPoster(MigrationTaskBase):
                                             self.processed,
                                             failed_recs_file,
                                         )
-                                        logging.critical("Halting %s", tpe)
-                                        print(f"\n\t{tpe.message}")
-                                        sys.exit(1)
+                                        batch = []
+                                        raise
                                     except TransformationRecordFailedError as exception:
                                         self.handle_generic_exception(
                                             exception,
@@ -184,7 +183,21 @@ class BatchPoster(MigrationTaskBase):
                                             failed_recs_file,
                                         )
                                         batch = []
-
+                                    except (FileNotFoundError, PermissionError) as ose:
+                                        logging.error("Error reading file: %s", ose)
+
+                except Exception as ee:
+                    if "idx" in locals() and self.task_configuration.files[idx:]:
+                        for file in self.task_configuration.files[idx:]:
+                            try:
+                                with open(file, "r") as failed_file:
+                                    failed_file.seek(self.processed)
+                                    failed_recs_file.write(failed_file.read())
+                                    self.processed = 0
+                            except (FileNotFoundError, PermissionError) as ose:
+                                logging.error("Error reading file: %s", ose)
+                    raise ee
+                finally:
                     if self.task_configuration.object_type != "Extradata" and any(batch):
                         try:
                             self.post_batch(batch, failed_recs_file, self.processed)
@@ -193,10 +206,8 @@ class BatchPoster(MigrationTaskBase):
                                 exception, last_row, batch, self.processed, failed_recs_file
                             )
                     logging.info("Done posting %s records. ", (self.processed))
-            except Exception as ee:
-                if self.task_configuration.object_type == "SRS":
-                    self.commit_snapshot()
-                raise ee
+                    if self.task_configuration.object_type == "SRS":
+                        self.commit_snapshot()
 
     def post_record_batch(self, batch, failed_recs_file, row):
         json_rec = json.loads(row.split("\t")[-1])
@@ -339,6 +350,10 @@ class BatchPoster(MigrationTaskBase):
 
     def post_batch(self, batch, failed_recs_file, num_records, recursion_depth=0):
         response = self.do_post(batch)
+        if response.status_code == 401:
+            logging.error("Authorization failed (%s). Fetching new auth token...", response.text)
+            self.folio_client.login()
+            response = self.do_post(batch)
         if response.status_code == 201:
             logging.info(
                 (

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/src/folio_migration_tools/migration_tasks/items_transformer.py

@@ -68,6 +68,15 @@ class ItemsTransformer(MigrationTaskBase):
                 ),
             ),
         ] = ""
+        prevent_permanent_location_map_default: Annotated[
+            bool,
+            Field(
+                title="Prevent permanent location map default",
+                description=(
+                    "Prevent the default mapping of permanent location to the default location."
+                ),
+            ),
+        ] = False
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:

{folio_migration_tools-1.9.0a6 → folio_migration_tools-1.9.0rc1}/src/folio_migration_tools/migration_tasks/loans_migrator.py

@@ -8,11 +8,11 @@ import traceback
 from datetime import datetime, timedelta
 from typing import Optional
 from urllib.error import HTTPError
+from zoneinfo import ZoneInfo
 
 import i18n
 from dateutil import parser as du_parser
 from folio_uuid.folio_namespaces import FOLIONamespaces
-from zoneinfo import ZoneInfo
 
 from folio_migration_tools.circulation_helper import CirculationHelper
 from folio_migration_tools.helper import Helper
@@ -131,34 +131,13 @@ class LoansMigrator(MigrationTaskBase):
         logging.info("Init completed")
 
     def check_smtp_config(self):
-        if self.library_configuration.folio_release.lower() == FolioRelease.morning_glory:
-            logging.warn("DEPRECATED: Morning Glory support will be removed in a future release!")
-            okapi_config_base_path = "/configurations/entries"
-            okapi_config_query = "(module=={}%20and%20configName=={}%20and%20code=={})"
-            okapi_config_limit = 1000
-            okapi_config_module = "SMTP_SERVER"
-            okapi_config_name = "smtp"
-            okapi_config_code = "EMAIL_SMTP_HOST_DISABLED"
-            smtp_config_path = (
-                okapi_config_base_path
-                + "?"
-                + str(okapi_config_limit)
-                + "&query="
-                + okapi_config_query.format(
-                    okapi_config_module, okapi_config_name, okapi_config_code
-                )
-            )
-            smtp_config_disabled = self.folio_client.folio_get_single_object(smtp_config_path)[
-                "configs"
-            ]
-        else:
-            try:
-                smtp_config = self.folio_client.folio_get_single_object("/smtp-configuration")[
-                    "smtpConfigurations"
-                ][0]
-                smtp_config_disabled = "disabled" in smtp_config["host"].lower()
-            except IndexError:
-                smtp_config_disabled = True
+        try:
+            smtp_config = self.folio_client.folio_get_single_object("/smtp-configuration")[
+                "smtpConfigurations"
+            ][0]
+            smtp_config_disabled = "disabled" in smtp_config["host"].lower()
+        except IndexError:
+            smtp_config_disabled = True
         print_smtp_warning()
         if not smtp_config_disabled:
             logging.warn("SMTP connection not disabled...")