folio-migration-tools 1.9.0a5__py3-none-any.whl → 1.9.0a7__py3-none-any.whl

folio_migration_tools/__main__.py

@@ -3,6 +3,7 @@ import logging
 import sys
 from os import environ
 from pathlib import Path
+from sqlite3 import connect
 
 import httpx
 import humps
@@ -124,7 +125,8 @@ def main():
         except TransformationProcessError as tpe:
             logging.critical(tpe.message)
             print(f"\n{tpe.message}: {tpe.data_value}")
-            sys.exit("Transformation Failure")
+            print("Task failure. Halting.")
+            sys.exit(1)
         logging.info("Work done. Shutting down")
         sys.exit(0)
     except json.decoder.JSONDecodeError as json_error:
@@ -148,10 +150,17 @@ def main():
         print("Halting")
         sys.exit("JSON Not Matching Spec")
     except httpx.HTTPError as connection_error:
-        print(
-            f"\nConnection Error when connecting to {connection_error.request.url}. "
-            "Are you connectet to the Internet/VPN? Do you need to update DNS settings?"
-        )
+        if hasattr(connection_error, "response"):
+            print(
+                f"\nHTTP Error when connecting to {connection_error.request.url}. "
+                f"Status code: {connection_error.response.status_code}. "
+                f"\nResponse: {connection_error.response.text}"
+            )
+        else:
+            print(
+                f"\nConnection Error when connecting to {connection_error.request.url}. "
+                "Are you connected to the Internet/VPN? Do you need to update DNS settings?"
+            )
         sys.exit("HTTP Not Connecting")
     except FileNotFoundError as fnf_error:
         print(f"\n{fnf_error.strerror}: {fnf_error.filename}")

folio_migration_tools/library_configuration.py

@@ -62,14 +62,9 @@ class IlsFlavour(str, Enum):
 
 
 class FolioRelease(str, Enum):
-    lotus = "lotus"
-    morning_glory = "morning-glory"
-    nolana = "nolana"
-    orchid = "orchid"
-    poppy = "poppy"
-    quesnelia = "quesnelia"
     ramsons = "ramsons"
     sunflower = "sunflower"
+    trillium = "trillium"
 
 
 class LibraryConfiguration(BaseModel):

folio_migration_tools/mapper_base.py

@@ -444,7 +444,7 @@ class MapperBase:
                     except IndexError:
                         if call_numbers:
                             bound_with_holding["callNumber"] = call_numbers[0]
-                    except SyntaxError:
+                    except (SyntaxError, ValueError):
                         bound_with_holding["callNumber"] = call_number
                 else:
                     bound_with_holding["callNumber"] = call_number

folio_migration_tools/marc_rules_transformation/hrid_handler.py

@@ -6,9 +6,7 @@ import httpx
 import i18n
 from folio_uuid import FOLIONamespaces
 from folioclient import FolioClient
-from pymarc import Field
-from pymarc import Record
-from pymarc import Subfield
+from pymarc import Field, Record, Subfield
 
 from folio_migration_tools.custom_exceptions import TransformationProcessError
 from folio_migration_tools.helper import Helper
@@ -220,7 +218,7 @@ class HRIDHandler:
                 ),
             )
             self.handle_035_generation(
-                marc_record, legacy_ids, self.migration_report, self.deactivate035_from001
+                marc_record, legacy_ids, self.migration_report, self.deactivate035_from001, False
             )
             Helper.log_data_issue(
                 legacy_ids,

folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py

@@ -24,6 +24,7 @@ from folio_migration_tools.custom_exceptions import (
 from folio_migration_tools.helper import Helper
 from folio_migration_tools.library_configuration import (
     FileDefinition,
+    HridHandling,
     IlsFlavour,
     LibraryConfiguration,
 )
@@ -62,6 +63,8 @@ class BibsRulesMapper(RulesMapperBase):
             [self.task_configuration.create_source_records, (not getattr(self.task_configuration, "data_import_marc", False))]
         )
         self.data_import_marc = self.task_configuration.data_import_marc
+        if self.data_import_marc:
+            self.hrid_handler.deactivate035_from001 = True
         self.start = time.time()
 
     def perform_initial_preparation(self, marc_record: pymarc.Record, legacy_ids):
@@ -73,7 +76,7 @@ class BibsRulesMapper(RulesMapperBase):
                 str(legacy_ids[-1]),
             )
         )
-        if self.create_source_records:
+        if self.create_source_records or self.hrid_handler.handling == HridHandling.preserve001:
             self.hrid_handler.handle_hrid(
                 FOLIONamespaces.instances,
                 folio_instance,

folio_migration_tools/migration_tasks/batch_poster.py

@@ -5,6 +5,7 @@ import sys
 import time
 import traceback
 from datetime import datetime
+from math import log
 from typing import Annotated, List
 from uuid import uuid4
 
@@ -136,12 +137,12 @@ class BatchPoster(MigrationTaskBase):
     def do_work(self):
         with self.folio_client.get_folio_http_client() as httpx_client:
             self.http_client = httpx_client
-            try:
-                batch = []
-                if self.task_configuration.object_type == "SRS":
-                    self.create_snapshot()
-                with open(self.folder_structure.failed_recs_path, "w") as failed_recs_file:
-                    for file_def in self.task_configuration.files:
+            with open(self.folder_structure.failed_recs_path, "w", encoding='utf-8') as failed_recs_file:
+                try:
+                    batch = []
+                    if self.task_configuration.object_type == "SRS":
+                        self.create_snapshot()
+                    for idx, file_def in enumerate(self.task_configuration.files):
                         path = self.folder_structure.results_folder / file_def.file_name
                         with open(path) as rows:
                             logging.info("Running %s", path)
@@ -172,9 +173,8 @@ class BatchPoster(MigrationTaskBase):
                                             self.processed,
                                             failed_recs_file,
                                         )
-                                        logging.critical("Halting %s", tpe)
-                                        print(f"\n\t{tpe.message}")
-                                        sys.exit(1)
+                                        batch = []
+                                        raise
                                     except TransformationRecordFailedError as exception:
                                         self.handle_generic_exception(
                                             exception,
@@ -184,7 +184,21 @@ class BatchPoster(MigrationTaskBase):
                                             failed_recs_file,
                                         )
                                         batch = []
-
+                                    except (FileNotFoundError, PermissionError) as ose:
+                                        logging.error("Error reading file: %s", ose)
+
+                except Exception as ee:
+                    if "idx" in locals() and self.task_configuration.files[idx:]:
+                        for file in self.task_configuration.files[idx:]:
+                            try:
+                                with open(file, "r") as failed_file:
+                                    failed_file.seek(self.processed)
+                                    failed_recs_file.write(failed_file.read())
+                                    self.processed = 0
+                            except (FileNotFoundError, PermissionError) as ose:
+                                logging.error("Error reading file: %s", ose)
+                    raise ee
+                finally:
                     if self.task_configuration.object_type != "Extradata" and any(batch):
                         try:
                             self.post_batch(batch, failed_recs_file, self.processed)
@@ -193,10 +207,8 @@ class BatchPoster(MigrationTaskBase):
                                 exception, last_row, batch, self.processed, failed_recs_file
                             )
                     logging.info("Done posting %s records. ", (self.processed))
-            except Exception as ee:
-                if self.task_configuration.object_type == "SRS":
-                    self.commit_snapshot()
-                raise ee
+                    if self.task_configuration.object_type == "SRS":
+                        self.commit_snapshot()
 
     def post_record_batch(self, batch, failed_recs_file, row):
         json_rec = json.loads(row.split("\t")[-1])
@@ -339,6 +351,10 @@ class BatchPoster(MigrationTaskBase):
 
     def post_batch(self, batch, failed_recs_file, num_records, recursion_depth=0):
         response = self.do_post(batch)
+        if response.status_code == 401:
+            logging.error("Authorization failed (%s). Fetching new auth token...", response.text)
+            self.folio_client.login()
+            response = self.do_post(batch)
         if response.status_code == 201:
             logging.info(
                 (

folio_migration_tools/migration_tasks/loans_migrator.py

@@ -8,11 +8,11 @@ import traceback
 from datetime import datetime, timedelta
 from typing import Optional
 from urllib.error import HTTPError
+from zoneinfo import ZoneInfo
 
 import i18n
 from dateutil import parser as du_parser
 from folio_uuid.folio_namespaces import FOLIONamespaces
-from zoneinfo import ZoneInfo
 
 from folio_migration_tools.circulation_helper import CirculationHelper
 from folio_migration_tools.helper import Helper
@@ -131,34 +131,13 @@ class LoansMigrator(MigrationTaskBase):
         logging.info("Init completed")
 
     def check_smtp_config(self):
-        if self.library_configuration.folio_release.lower() == FolioRelease.morning_glory:
-            logging.warn("DEPRECATED: Morning Glory support will be removed in a future release!")
-            okapi_config_base_path = "/configurations/entries"
-            okapi_config_query = "(module=={}%20and%20configName=={}%20and%20code=={})"
-            okapi_config_limit = 1000
-            okapi_config_module = "SMTP_SERVER"
-            okapi_config_name = "smtp"
-            okapi_config_code = "EMAIL_SMTP_HOST_DISABLED"
-            smtp_config_path = (
-                okapi_config_base_path
-                + "?"
-                + str(okapi_config_limit)
-                + "&query="
-                + okapi_config_query.format(
-                    okapi_config_module, okapi_config_name, okapi_config_code
-                )
-            )
-            smtp_config_disabled = self.folio_client.folio_get_single_object(smtp_config_path)[
-                "configs"
-            ]
-        else:
-            try:
-                smtp_config = self.folio_client.folio_get_single_object("/smtp-configuration")[
-                    "smtpConfigurations"
-                ][0]
-                smtp_config_disabled = "disabled" in smtp_config["host"].lower()
-            except IndexError:
-                smtp_config_disabled = True
+        try:
+            smtp_config = self.folio_client.folio_get_single_object("/smtp-configuration")[
+                "smtpConfigurations"
+            ][0]
+            smtp_config_disabled = "disabled" in smtp_config["host"].lower()
+        except IndexError:
+            smtp_config_disabled = True
         print_smtp_warning()
         if not smtp_config_disabled:
             logging.warn("SMTP connection not disabled...")

{folio_migration_tools-1.9.0a5.dist-info → folio_migration_tools-1.9.0a7.dist-info}/METADATA

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: folio_migration_tools
-Version: 1.9.0a5
+Version: 1.9.0a7
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
-Home-page: https://github.com/FOLIO-FSE/folio_migration_tools
 License: MIT
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
 Author: Theodor Tolstoy
@@ -27,6 +26,7 @@ Requires-Dist: pyhumps (>=3.7.3,<4.0.0)
 Requires-Dist: pymarc (>=5.2.3,<6.0.0)
 Requires-Dist: python-dateutil (>=2.8.2,<3.0.0)
 Requires-Dist: python-i18n (>=0.3.9,<0.4.0)
+Project-URL: Homepage, https://github.com/FOLIO-FSE/folio_migration_tools
 Project-URL: Repository, https://github.com/FOLIO-FSE/folio_migration_tools
 Description-Content-Type: text/markdown
 

{folio_migration_tools-1.9.0a5.dist-info → folio_migration_tools-1.9.0a7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 folio_migration_tools/__init__.py,sha256=yTPImroNb0CPuYmG4nm6aAcu5gpFJ7o1uM2dJS-47ec,93
-folio_migration_tools/__main__.py,sha256=ceAx_91SQS-Ab2g7umZpQtpMKWEHy3ivnWK4I3x2tuU,6926
+folio_migration_tools/__main__.py,sha256=etBycP4P9KkbpLcI4sWqwHaVyP8_OQeg-uJOESt5adY,7310
 folio_migration_tools/circulation_helper.py,sha256=2kAkLM6caPiep0ZtBkMICbRDh53KdfdH21oEX1eMRDI,14193
 folio_migration_tools/colors.py,sha256=GP0wdI_GZ2WD5SjrbPN-S3u8vvN_u6rGQIBBcWv_0ZM,227
 folio_migration_tools/config_file_load.py,sha256=zHHa6NDkN6EJiQE4DgjrFQPVKsd70POsfbGkB8308jg,2822
@@ -10,8 +10,8 @@ folio_migration_tools/folder_structure.py,sha256=yyVvbkM9PbczSHNI8vK0Ru7i0x4nbYG
 folio_migration_tools/helper.py,sha256=KkOkNAGO_fuYqxdLrsbLzCJLQHUrFZG1NzD4RmpQ-KM,2804
 folio_migration_tools/holdings_helper.py,sha256=yJpz6aJrKRBiJ1MtT5bs2vXAc88uJuGh2_KDuCySOKc,7559
 folio_migration_tools/i18n_config.py,sha256=3AH_2b9zTsxE4XTe4isM_zYtPJSlK0ix6eBmV7kAYUM,228
-folio_migration_tools/library_configuration.py,sha256=60hgGtyvHhRUKXHvqz41-2V5EtUaySMV20JIKk279N0,3636
-folio_migration_tools/mapper_base.py,sha256=dmwsgeolBWjmzaztms5r1_7OG2PlTcp0D74TQCE6cgk,20303
+folio_migration_tools/library_configuration.py,sha256=JE23VSUKDeCxzj_fOeQwkXF4fwd_y9s_hNy91YtEy7A,3514
+folio_migration_tools/mapper_base.py,sha256=4rtZxmWp1qaAIG2awiJ3RElPM9IVA-n8-Qzzs0R31qc,20317
 folio_migration_tools/mapping_file_transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 folio_migration_tools/mapping_file_transformation/courses_mapper.py,sha256=mJQxxeTn1bCYb2zwFYyXJ6EGZpJ0DsmwOY3nED7D_gQ,8091
 folio_migration_tools/mapping_file_transformation/holdings_mapper.py,sha256=GI9xnN74EsUMAshXKNJ6p9bGPdLtK0PCXqbB3nIxrC8,7207
@@ -26,24 +26,24 @@ folio_migration_tools/mapping_file_transformation/user_mapper.py,sha256=oWuIPRQL
 folio_migration_tools/marc_rules_transformation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 folio_migration_tools/marc_rules_transformation/conditions.py,sha256=MnbMThiOZ6IenXirjMSNkqJ86RtFzbF4cp1T4YcfpyA,36349
 folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py,sha256=lTb5QWEAgwyFHy5vdSK6oDl1Q5v2GnzuV04xWV3p4rc,12401
-folio_migration_tools/marc_rules_transformation/hrid_handler.py,sha256=lEvtJFWe5FoU372nHqTdnVF76qCKZjWgUJqIpiKUHY0,10026
+folio_migration_tools/marc_rules_transformation/hrid_handler.py,sha256=Ihdv0_1q7gL_pZ3HWU3GcfV_jjpIfOLithWk9z_uH3Y,9997
 folio_migration_tools/marc_rules_transformation/loc_language_codes.xml,sha256=ztn2_yKws6qySL4oSsZh7sOjxq5bCC1PhAnXJdtgmJ0,382912
 folio_migration_tools/marc_rules_transformation/marc_file_processor.py,sha256=WkOQRDi7f4PZ5qmVH3Q-1_zdGEKYSvOGC6jixDwDp98,12349
 folio_migration_tools/marc_rules_transformation/marc_reader_wrapper.py,sha256=9ATjYMRAjy0QcXtmNZaHVhHLJ5hE1WUgOcF6KMJjbgo,5309
 folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py,sha256=GFw8j9UtCxnUdLShmPzJa1MpCK8a0NkQIN5C3jyouRs,9604
 folio_migration_tools/marc_rules_transformation/rules_mapper_base.py,sha256=WWSJgYvF9LeY8vh-BtQi7Fm3J-cowUJKWa0Wk2Ge7fc,39358
-folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py,sha256=5BHCEAoTmsJLzG9161Y49-DwA_-Jr44XUGX725-dNKE,28755
+folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py,sha256=ckVeysbpW9s19pmHvogdRFOCouzz17Y6oKJD0_QfQAk,28924
 folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py,sha256=EBxf9Qh5Y0eDOmqYssWfaizxafiXSYDFzWmCuPtdG-8,18226
 folio_migration_tools/migration_report.py,sha256=BkRspM1hwTBnWeqsHamf7yVEofzLj560Q-9G--O00hw,4258
 folio_migration_tools/migration_tasks/__init__.py,sha256=ZkbY_yGyB84Ke8OMlYUzyyBj4cxxNrhMTwQlu_GbdDs,211
 folio_migration_tools/migration_tasks/authority_transformer.py,sha256=AoXg9s-GLO3yEEDCrQV7hc4YVXxwxsdxDdpj1zhHydE,4251
-folio_migration_tools/migration_tasks/batch_poster.py,sha256=iaWhvxdYli0kgsXLHGmFFBMeVXGpAv9eDg02BagyQ2U,29732
+folio_migration_tools/migration_tasks/batch_poster.py,sha256=yQa72SA2sCDCcPgWrdP0jPixG2J6z5_Bhe9NzEDhzBA,30750
 folio_migration_tools/migration_tasks/bibs_transformer.py,sha256=XzlPo-0uuugJA4SM80xOlOj5nDK6OMDXFnAYg80hOBc,7791
 folio_migration_tools/migration_tasks/courses_migrator.py,sha256=dQerp97P3r7wmxK3Ovg6AriO6K_nTr6vA8RKj_XBEt4,5728
 folio_migration_tools/migration_tasks/holdings_csv_transformer.py,sha256=Hwr4YjgNIQpi2N-x8eq-mmRpXAyxYylQjpYubm03-ec,19668
 folio_migration_tools/migration_tasks/holdings_marc_transformer.py,sha256=yN0a8YVNx2P6NswxSylTca0MmNk1shze3PyKXv9JJIw,9547
 folio_migration_tools/migration_tasks/items_transformer.py,sha256=MOpnw1yvNKc6x4ZNjlB1nA2XiXMFdAso-8v2bET5tbE,15011
-folio_migration_tools/migration_tasks/loans_migrator.py,sha256=3OysTLy15lH0SRWoYgkFIlDXXMLn6OElmAk-YlG4z1M,34260
+folio_migration_tools/migration_tasks/loans_migrator.py,sha256=PCjr_bC2ixhPPk_D54m7B871_bQObn5oP_Sow3ocg54,33238
 folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py,sha256=CnmlTge7nChUJ10EiUkriQtJlVxWqglgfhjgneh2_yM,7247
 folio_migration_tools/migration_tasks/migration_task_base.py,sha256=5vVkBqUvCGuGPUyWlnKsLCFtgOQzq4jjBK_TBUHs7BE,13742
 folio_migration_tools/migration_tasks/orders_transformer.py,sha256=dGJlTZi5OHCsK8HS7bda8jcZcnWNc80DHgOOhuM1nhE,11474
@@ -60,7 +60,7 @@ folio_migration_tools/transaction_migration/legacy_request.py,sha256=1ulyFzPQw_I
 folio_migration_tools/transaction_migration/legacy_reserve.py,sha256=rZVtiMBYnt6aI0WxAwPN8fML_MKEUSUYsadCKPTeB4E,1839
 folio_migration_tools/transaction_migration/transaction_result.py,sha256=cTdCN0BnlI9_ZJB2Z3Fdkl9gpymIi-9mGZsRFlQcmDk,656
 folio_migration_tools/translations/en.json,sha256=HOVpkb_T-SN_x0NpDp8gyvV1hMLCui3SsG7ByyIv0OU,38669
-folio_migration_tools-1.9.0a5.dist-info/LICENSE,sha256=PhIEkitVi3ejgq56tt6sWoJIG_zmv82cjjd_aYPPGdI,1072
-folio_migration_tools-1.9.0a5.dist-info/METADATA,sha256=ohkzY98Ct_djuhK-ks9sFehWfuQCsEE42kOOQQeyDR4,7318
-folio_migration_tools-1.9.0a5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-folio_migration_tools-1.9.0a5.dist-info/RECORD,,
+folio_migration_tools-1.9.0a7.dist-info/LICENSE,sha256=PhIEkitVi3ejgq56tt6sWoJIG_zmv82cjjd_aYPPGdI,1072
+folio_migration_tools-1.9.0a7.dist-info/METADATA,sha256=vvVvLOBAoK-NY7na8o2wF7llFGWy74ExfyZBgbKpwYA,7330
+folio_migration_tools-1.9.0a7.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+folio_migration_tools-1.9.0a7.dist-info/RECORD,,

{folio_migration_tools-1.9.0a5.dist-info → folio_migration_tools-1.9.0a7.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 2.0.1
 Root-Is-Purelib: true
 Tag: py3-none-any