folio-migration-tools 1.10.0b7__tar.gz → 1.10.2__tar.gz

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: folio-migration-tools
-Version: 1.10.0b7
+Version: 1.10.2
 Summary: A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP
 Keywords: FOLIO,ILS,LSP,Library Systems,MARC21,Library data
 Author: Theodor Tolstoy, Lisa Sjögren, Brooks Travis, Jeremy Nelson, Clinton Bradford
@@ -18,6 +18,7 @@ Requires-Dist: deepdiff>=6.2.3,<7.0.0
 Requires-Dist: pyaml>=21.10.1,<22.0.0
 Requires-Dist: python-i18n>=0.3.9,<1.0.0
 Requires-Dist: art>=6.5,<7.0
+Requires-Dist: folio-data-import>=0.5.0,<0.6.0
 Requires-Python: >=3.10, <4.0
 Project-URL: Documentation, https://folio-migration-tools.readthedocs.io/en/latest/
 Project-URL: Homepage, https://github.com/folio-fse/folio_migration_tools

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.10.0b7"
+version = "1.10.2"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -26,6 +26,7 @@ dependencies = [
     "pyaml>=21.10.1,<22.0.0",
     "python-i18n>=0.3.9,<1.0.0",
     "art>=6.5,<7.0",
+    "folio-data-import>=0.5.0,<0.6.0",
 ]
 
 [dependency-groups]

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_report.py

@@ -50,7 +50,7 @@ class MigrationReport:
         """
         self.add("GeneralStatistics", measure_to_add)
 
-    def _write_json_report(self, report_file):
+    def write_json_report(self, report_file):
         """Writes the raw migration report data to a JSON file.
 
         Args:

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/batch_poster.py

@@ -2,6 +2,7 @@ import asyncio
 import copy
 import json
 import logging
+import os
 import re
 import sys
 import traceback
@@ -249,6 +250,10 @@ class BatchPoster(MigrationTaskBase):
         self.num_posted = 0
         self.starting_record_count_in_folio: Optional[int] = None
         self.finished_record_count_in_folio: Optional[int] = None
+        # Create a semaphore to limit concurrent async requests
+        self.semaphore = asyncio.Semaphore(
+            int(os.environ.get("FOLIO_MAX_CONCURRENT_REQUESTS", 10))
+        )
 
     def do_work(self):  # noqa: C901
         with open(
@@ -520,7 +525,8 @@ class BatchPoster(MigrationTaskBase):
 
         for attempt in range(retries):
             try:
-                return await self.folio_client.folio_get_async(url, query_params=params)
+                async with self.semaphore:
+                    return await self.folio_client.folio_get_async(url, query_params=params)
 
             except folioclient.FolioConnectionError as e:
                 # Network/connection errors - always retry
@@ -901,6 +907,8 @@ class BatchPoster(MigrationTaskBase):
                 report_file,
                 self.start_datetime,
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.migration_report.write_json_report(raw_report_file)
         self.clean_out_empty_logs()
 
     def rerun_run(self):

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/bibs_transformer.py

@@ -163,6 +163,8 @@ class BibsTransformer(MigrationTaskBase):
                 self.mapper.mapped_folio_fields,
                 self.mapper.mapped_legacy_fields,
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.mapper.migration_report.write_json_report(raw_report_file)
 
         logging.info(
             "Done. Transformation report written to %s",

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/courses_migrator.py

@@ -178,6 +178,8 @@ class CoursesMigrator(MigrationTaskBase):
             self.mapper.migration_report.write_migration_report(
                 i18n.t("Courses migration report"), report_file, self.mapper.start_datetime
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.mapper.migration_report.write_json_report(raw_report_file)
         self.clean_out_empty_logs()
 
 

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/holdings_csv_transformer.py

@@ -368,6 +368,8 @@ class HoldingsCsvTransformer(MigrationTaskBase):
                 self.mapper.mapped_folio_fields,
                 self.mapper.mapped_legacy_fields,
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.mapper.migration_report.write_json_report(raw_report_file)
         logging.info("All done!")
         self.clean_out_empty_logs()
 

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/holdings_marc_transformer.py

@@ -360,6 +360,8 @@ class HoldingsMarcTransformer(MigrationTaskBase):
                 self.mapper.mapped_folio_fields,
                 self.mapper.mapped_legacy_fields,
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.mapper.migration_report.write_json_report(raw_report_file)
 
         logging.info(
             "Done. Transformation report written to %s",

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/items_transformer.py

@@ -481,5 +481,7 @@ class ItemsTransformer(MigrationTaskBase):
                 self.mapper.mapped_folio_fields,
                 self.mapper.mapped_legacy_fields,
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.mapper.migration_report.write_json_report(raw_report_file)
         self.clean_out_empty_logs()
         logging.info("All done!")

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/loans_migrator.py

@@ -314,6 +314,8 @@ class LoansMigrator(MigrationTaskBase):
             self.migration_report.write_migration_report(
                 i18n_t("Loans migration report"), report_file, self.start_datetime
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.migration_report.write_json_report(raw_report_file)
         self.clean_out_empty_logs()
 
     def write_failed_loans_to_file(self):

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py

@@ -183,5 +183,7 @@ class ManualFeeFinesTransformer(MigrationTaskBase):
                 self.mapper.mapped_folio_fields,
                 self.mapper.mapped_legacy_fields,
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.mapper.migration_report.write_json_report(raw_report_file)
 
         self.clean_out_empty_logs()

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/orders_transformer.py

@@ -328,6 +328,8 @@ class OrdersTransformer(MigrationTaskBase):
                 self.mapper.mapped_folio_fields,
                 self.mapper.mapped_legacy_fields,
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.mapper.migration_report.write_json_report(raw_report_file)
         logging.info("All done!")
 
     def merge_into_orders_with_embedded_pols(self, folio_rec, results_file):

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/organization_transformer.py

@@ -299,6 +299,8 @@ class OrganizationTransformer(MigrationTaskBase):
             self.mapper.save_id_map_file(
                 self.folder_structure.organizations_id_map_path, self.organizations_id_map
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.mapper.migration_report.write_json_report(raw_report_file)
         self.clean_out_empty_logs()
 
         logging.info("All done!")

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/requests_migrator.py

@@ -236,6 +236,8 @@ class RequestsMigrator(MigrationTaskBase):
             self.migration_report.write_migration_report(
                 i18n_t("Requests migration report"), report_file, self.start_datetime
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.migration_report.write_json_report(raw_report_file)
         self.clean_out_empty_logs()
 
     def write_failed_request_to_file(self):

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/reserves_migrator.py

@@ -126,6 +126,8 @@ class ReservesMigrator(MigrationTaskBase):
             self.migration_report.write_migration_report(
                 i18n_t("Reserves migration report"), report_file, self.start_datetime
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.migration_report.write_json_report(raw_report_file)
         self.clean_out_empty_logs()
 
     def write_failed_reserves_to_file(self):

{folio_migration_tools-1.10.0b7 → folio_migration_tools-1.10.2}/src/folio_migration_tools/migration_tasks/user_transformer.py

@@ -265,6 +265,8 @@ class UserTransformer(MigrationTaskBase):
                 self.mapper.mapped_folio_fields,
                 self.mapper.mapped_legacy_fields,
             )
+        with open(self.folder_structure.migration_reports_raw_file, "w") as raw_report_file:
+            self.mapper.migration_report.write_json_report(raw_report_file)
         logging.info("All done!")
         self.clean_out_empty_logs()