folio-migration-tools 1.9.0rc12__py3-none-any.whl → 1.9.1__py3-none-any.whl

folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py

@@ -1,12 +1,14 @@
 import copy
 import json
 import logging
+import re
 from typing import Dict, List, Set
 
 import i18n
 from folio_uuid.folio_namespaces import FOLIONamespaces
 from folio_uuid.folio_uuid import FolioUUID
 from folioclient import FolioClient
+from pymarc import Optional
 from pymarc.field import Field
 from pymarc.record import Record
 
@@ -40,20 +42,21 @@ class RulesMapperHoldings(RulesMapperBase):
         library_configuration: LibraryConfiguration,
         parent_id_map: dict,
         boundwith_relationship_map_rows: List[Dict],
+        statistical_codes_map: Optional[Dict] = None,
     ):
-        self.task_configuration = task_configuration
         self.conditions = Conditions(
             folio_client,
             self,
             "holdings",
             library_configuration.folio_release,
-            self.task_configuration.default_call_number_type_name,
+            task_configuration.default_call_number_type_name,
         )
         self.folio = folio_client
         super().__init__(
             folio_client,
             library_configuration,
             task_configuration,
+            statistical_codes_map,
             self.fetch_holdings_schema(folio_client),
             self.conditions,
             parent_id_map,
@@ -297,6 +300,10 @@ class RulesMapperHoldings(RulesMapperBase):
                 "",
             )
         self.handle_suppression(folio_holding, file_def, True)
+        # First, map statistical codes from MARC fields and FileDefinitions to FOLIO statistical codes.
+        # Then, convert the mapped statistical codes to their corresponding code IDs.
+        self.map_statistical_codes(folio_holding, file_def, marc_record)
+        self.map_statistical_code_ids(legacy_ids, folio_holding)
         self.set_source_id(self.create_source_records, folio_holding, self.holdingssources, file_def)
 
     def pick_first_location_if_many(self, folio_holding: Dict, legacy_ids: List[str]):
@@ -387,12 +394,29 @@ class RulesMapperHoldings(RulesMapperBase):
             ) from ee
         return [
             {
-                "note": "\n".join(mrk_statement_notes),
+                "note": chunk,
                 "holdingsNoteTypeId": holdings_note_type_id,
                 "staffOnly": True,
-            }
+            } for chunk in self.split_mrk_by_max_note_size("\n".join(mrk_statement_notes))
         ]
 
+    @staticmethod
+    def split_mrk_by_max_note_size(s: str, max_chunk_size: int = 32000) -> List[str]:
+        lines = s.splitlines(keepends=True)
+        chunks = []
+        current_chunk = ""
+        for line in lines:
+            # If adding this line would exceed the limit, start a new chunk
+            if len(current_chunk) + len(line) > max_chunk_size:
+                if current_chunk:
+                    chunks.append(current_chunk)
+                current_chunk = line
+            else:
+                current_chunk += line
+        if current_chunk:
+            chunks.append(current_chunk)
+        return chunks
+
     def add_mfhd_as_mrk_note(self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]):
         """Adds the MFHD as a note to the holdings record
 
@@ -418,12 +442,29 @@ class RulesMapperHoldings(RulesMapperBase):
                 ) from ee
             folio_holding["notes"] = folio_holding.get("notes", []) + [
                 {
-                    "note": str(marc_record),
+                    "note": chunk,
                     "holdingsNoteTypeId": holdings_note_type_id,
                     "staffOnly": True,
-                }
+                } for chunk in self.split_mrk_by_max_note_size(str(marc_record))
             ]
 
+    @staticmethod
+    def split_mrc_by_max_note_size(data: bytes, sep: bytes = b"\x1e", max_chunk_size: int = 32000) -> List[bytes]:
+        # Split data into segments, each ending with the separator (except possibly the last)
+        pattern = re.compile(b'(.*?' + re.escape(sep) + b'|.+?$)', re.DOTALL)
+        parts = [m.group(0) for m in pattern.finditer(data) if m.group(0)]
+        chunks = []
+        current_chunk = b""
+        for part in parts:
+            if len(current_chunk) + len(part) > max_chunk_size and current_chunk:
+                chunks.append(current_chunk)
+                current_chunk = part
+            else:
+                current_chunk += part
+        if current_chunk:
+            chunks.append(current_chunk)
+        return chunks
+
     def add_mfhd_as_mrc_note(self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]):
         """Adds the MFHD as a note to the holdings record
 
@@ -449,10 +490,10 @@ class RulesMapperHoldings(RulesMapperBase):
                 ) from ee
             folio_holding["notes"] = folio_holding.get("notes", []) + [
                 {
-                    "note": marc_record.as_marc().decode("utf-8"),
+                    "note": chunk.decode("utf-8"),
                     "holdingsNoteTypeId": holdings_note_type_id,
                     "staffOnly": True,
-                }
+                } for chunk in self.split_mrc_by_max_note_size(marc_record.as_marc())
             ]
 
     def wrap_up(self):
@@ -582,7 +623,7 @@ class RulesMapperHoldings(RulesMapperBase):
                 "", "Column BIB_ID missing from Boundwith relationship map", ""
             )
 
-    def setup_boundwith_relationship_map(self, boundwith_relationship_map: List[Dict]):
+    def setup_boundwith_relationship_map(self, boundwith_relationship_map_list: List[Dict]):
         """
         Creates a map of MFHD_ID to BIB_ID for boundwith relationships.
 
@@ -597,7 +638,7 @@ class RulesMapperHoldings(RulesMapperBase):
             TransformationRecordFailedError: If BIB_ID is not in the instance id map.
         """
         new_map = {}
-        for idx, entry in enumerate(boundwith_relationship_map):
+        for idx, entry in enumerate(boundwith_relationship_map_list):
             self.verity_boundwith_map_entry(entry)
             mfhd_uuid = str(
                 FolioUUID(

folio_migration_tools/migration_tasks/batch_poster.py

@@ -6,7 +6,7 @@ import sys
 import time
 import traceback
 from datetime import datetime
-from typing import Annotated, List
+from typing import Annotated, List, Optional
 from uuid import uuid4
 
 import httpx
@@ -173,11 +173,13 @@ class BatchPoster(MigrationTaskBase):
         self.num_posted = 0
         self.okapi_headers = self.folio_client.okapi_headers
         self.http_client = None
+        self.starting_record_count_in_folio: Optional[int] = None
 
     def do_work(self):
         with self.folio_client.get_folio_http_client() as httpx_client:
             self.http_client = httpx_client
             with open(self.folder_structure.failed_recs_path, "w", encoding='utf-8') as failed_recs_file:
+                self.get_starting_record_count()
                 try:
                     batch = []
                     if self.task_configuration.object_type == "SRS":
@@ -317,6 +319,8 @@ class BatchPoster(MigrationTaskBase):
                 updates[record["id"]] = {
                             "_version": record["_version"],
                         }
+                if "hrid" in record:
+                    updates[record["id"]]["hrid"] = record["hrid"]
                 if "status" in record:
                     updates[record["id"]]["status"] = record["status"]
                 if "lastCheckIn" in record:
@@ -604,6 +608,42 @@ class BatchPoster(MigrationTaskBase):
         else:
             return httpx.post(url, headers=self.okapi_headers, json=payload, params=self.query_params, timeout=None)
 
+    def get_current_record_count_in_folio(self):
+        if "query_endpoint" in self.api_info:
+            url = f"{self.folio_client.gateway_url}{self.api_info['query_endpoint']}"
+            query_params = {"query": "cql.allRecords=1", "limit": 0}
+            if self.http_client and not self.http_client.is_closed:
+                res = self.http_client.get(url, headers=self.folio_client.okapi_headers, params=query_params)
+            else:
+                res = httpx.get(url, headers=self.okapi_headers, params=query_params, timeout=None)
+            try:
+                res.raise_for_status()
+                return res.json()["totalRecords"]
+            except httpx.HTTPStatusError:
+                logging.error("Failed to get current record count. HTTP %s", res.status_code)
+                return 0
+            except KeyError:
+                logging.error(f"Failed to get current record count. No 'totalRecords' in response: {res.json()}")
+                return 0
+        else:
+            raise ValueError(
+                "No 'query_endpoint' available for %s. Cannot get current record count.", self.task_configuration.object_type
+            )
+
+    def get_starting_record_count(self):
+        if "query_endpoint" in self.api_info and not self.starting_record_count_in_folio:
+            logging.info("Getting starting record count in FOLIO")
+            self.starting_record_count_in_folio = self.get_current_record_count_in_folio()
+        else:
+            logging.info("No query_endpoint available for %s. Cannot get starting record count.", self.task_configuration.object_type)
+
+    def get_finished_record_count(self):
+        if "query_endpoint" in self.api_info:
+            logging.info("Getting finished record count in FOLIO")
+            self.finished_record_count_in_folio = self.get_current_record_count_in_folio()
+        else:
+            logging.info("No query_endpoint available for %s. Cannot get ending record count.", self.task_configuration.object_type)
+
     def wrap_up(self):
         logging.info("Done. Wrapping up")
         self.extradata_writer.flush()
@@ -621,11 +661,34 @@ class BatchPoster(MigrationTaskBase):
             )
         else:
             logging.info("Done posting %s records. %s failed", self.num_posted, self.num_failures)
-
+        if self.starting_record_count_in_folio:
+            self.get_finished_record_count()
+            total_on_server = self.finished_record_count_in_folio - self.starting_record_count_in_folio
+            discrepancy = self.processed - self.num_failures - total_on_server
+            if discrepancy != 0:
+                logging.error(
+                    (
+                        "Discrepancy in record count. "
+                        "Starting record count: %s. Finished record count: %s. "
+                        "Records posted: %s. Discrepancy: %s"
+                    ),
+                    self.starting_record_count_in_folio,
+                    self.finished_record_count_in_folio,
+                    self.num_posted - self.num_failures,
+                    discrepancy,
+                )
+        else:
+            discrepancy = 0
         run = "second time" if self.performing_rerun else "first time"
         self.migration_report.set("GeneralStatistics", f"Records processed {run}", self.processed)
         self.migration_report.set("GeneralStatistics", f"Records posted {run}", self.num_posted)
         self.migration_report.set("GeneralStatistics", f"Failed to post {run}", self.num_failures)
+        if discrepancy:
+            self.migration_report.set(
+                "GeneralStatistics",
+                f"Discrepancy in record count {run}",
+                discrepancy,
+            )
         self.rerun_run()
         with open(self.folder_structure.migration_reports_file, "w+") as report_file:
             self.migration_report.write_migration_report(

folio_migration_tools/migration_tasks/bibs_transformer.py

@@ -7,8 +7,6 @@ from pydantic import Field
 
 from folio_migration_tools.helper import Helper
 from folio_migration_tools.library_configuration import (
-    FileDefinition,
-    HridHandling,
     IlsFlavour,
     LibraryConfiguration,
 )
@@ -116,11 +114,23 @@ class BibsTransformer(MigrationTaskBase):
         use_logging: bool = True,
     ):
         super().__init__(library_config, task_config, folio_client, use_logging)
+        self.task_config = task_config
+        self.task_configuration = self.task_config
+        if self.task_config.statistical_codes_map_file_name:
+            statcode_mapping = self.load_ref_data_mapping_file(
+                "statisticalCodeIds",
+                self.folder_structure.mapping_files_folder
+                / self.task_config.statistical_codes_map_file_name,
+                [],
+                False,
+            )
+        else:
+            statcode_mapping = None
         self.processor: MarcFileProcessor
         self.check_source_files(
             self.folder_structure.legacy_records_folder, self.task_configuration.files
         )
-        self.mapper = BibsRulesMapper(self.folio_client, library_config, self.task_configuration)
+        self.mapper = BibsRulesMapper(self.folio_client, library_config, self.task_configuration, statcode_mapping)
         self.bib_ids: set = set()
         if (
             self.task_configuration.reset_hrid_settings

folio_migration_tools/migration_tasks/holdings_csv_transformer.py

@@ -160,6 +160,16 @@ class HoldingsCsvTransformer(MigrationTaskBase):
                 ),
             ),
         ] = True
+        statistical_codes_map_file_name: Annotated[
+            Optional[str],
+            Field(
+                title="Statistical code map file name",
+                description=(
+                    "Path to the file containing the mapping of statistical codes. "
+                    "The file should be in TSV format with legacy_stat_code and folio_code columns."
+                ),
+            ),
+        ] = ""
 
     @staticmethod
     def get_object_type() -> FOLIONamespaces:
@@ -174,16 +184,27 @@ class HoldingsCsvTransformer(MigrationTaskBase):
     ):
         super().__init__(library_config, task_config, folio_client, use_logging)
         self.fallback_holdings_type = None
+        self.folio_keys, self.holdings_field_map = self.load_mapped_fields()
+        if any(k for k in self.folio_keys if k.startswith("statisticalCodeIds")):
+            statcode_mapping = self.load_ref_data_mapping_file(
+                "statisticalCodeIds",
+                self.folder_structure.mapping_files_folder
+                / self.task_configuration.statistical_codes_map_file_name,
+                self.folio_keys,
+                False,
+            )
+        else:
+            statcode_mapping = None
         try:
-            self.task_config = task_config
             self.bound_with_keys = set()
             self.mapper = HoldingsMapper(
                 self.folio_client,
-                self.load_mapped_fields(),
+                self.holdings_field_map,
                 self.load_location_map(),
                 self.load_call_number_type_map(),
                 self.load_instance_id_map(True),
                 library_config,
+                statcode_mapping,
             )
             self.holdings = {}
             self.total_records = 0
@@ -196,19 +217,19 @@ class HoldingsCsvTransformer(MigrationTaskBase):
             logging.info("%s\tholdings types in tenant", len(self.holdings_types))
             self.validate_merge_criterias()
             self.check_source_files(
-                self.folder_structure.data_folder / "items", self.task_config.files
+                self.folder_structure.data_folder / "items", self.task_configuration.files
             )
             self.fallback_holdings_type = next(
                 h
                 for h in self.holdings_types
-                if h["id"] == self.task_config.fallback_holdings_type_id
+                if h["id"] == self.task_configuration.fallback_holdings_type_id
             )
             if not self.fallback_holdings_type:
                 raise TransformationProcessError(
                     "",
                     (
                         "Holdings type with ID "
-                        f"{self.task_config.fallback_holdings_type_id} "
+                        f"{self.task_configuration.fallback_holdings_type_id} "
                         "not found in FOLIO."
                     ),
                 )
@@ -216,15 +237,15 @@ class HoldingsCsvTransformer(MigrationTaskBase):
                 "%s will be used as default holdings type",
                 self.fallback_holdings_type["name"],
             )
-            if any(self.task_config.previously_generated_holdings_files):
-                for file_name in self.task_config.previously_generated_holdings_files:
+            if any(self.task_configuration.previously_generated_holdings_files):
+                for file_name in self.task_configuration.previously_generated_holdings_files:
                     logging.info("Processing %s", file_name)
                     self.holdings.update(
                         HoldingsHelper.load_previously_generated_holdings(
                             self.folder_structure.results_folder / file_name,
-                            self.task_config.holdings_merge_criteria,
+                            self.task_configuration.holdings_merge_criteria,
                             self.mapper.migration_report,
-                            self.task_config.holdings_type_uuid_for_boundwiths,
+                            self.task_configuration.holdings_type_uuid_for_boundwiths,
                         )
                     )
 
@@ -260,7 +281,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
     def load_call_number_type_map(self):
         with open(
             self.folder_structure.mapping_files_folder
-            / self.task_config.call_number_type_map_file_name,
+            / self.task_configuration.call_number_type_map_file_name,
             "r",
         ) as callnumber_type_map_f:
             return self.load_ref_data_map_from_file(
@@ -269,7 +290,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
 
     def load_location_map(self):
         with open(
-            self.folder_structure.mapping_files_folder / self.task_config.location_map_file_name
+            self.folder_structure.mapping_files_folder / self.task_configuration.location_map_file_name
         ) as location_map_f:
             return self.load_ref_data_map_from_file(
                 location_map_f, "Found %s rows in location map"
@@ -283,7 +304,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
 
     def load_mapped_fields(self):
         with open(
-            self.folder_structure.mapping_files_folder / self.task_config.holdings_map_file_name
+            self.folder_structure.mapping_files_folder / self.task_configuration.holdings_map_file_name
         ) as holdings_mapper_f:
             holdings_map = json.load(holdings_mapper_f)
             logging.info("%s fields in holdings mapping file map", len(holdings_map["data"]))
@@ -294,11 +315,11 @@ class HoldingsCsvTransformer(MigrationTaskBase):
                 "%s mapped fields in holdings mapping file map",
                 len(list(mapped_fields)),
             )
-            return holdings_map
+            return mapped_fields, holdings_map
 
     def do_work(self):
         logging.info("Starting....")
-        for file_def in self.task_config.files:
+        for file_def in self.task_configuration.files:
             logging.info("Processing %s", file_def.file_name)
             try:
                 self.process_single_file(file_def)
@@ -311,7 +332,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
                 print(f"\n{error_str}\nHalting")
                 sys.exit(1)
         logging.info(
-            f"processed {self.total_records:,} records in {len(self.task_config.files)} files"
+            f"processed {self.total_records:,} records in {len(self.task_configuration.files)} files"
         )
 
     def wrap_up(self):
@@ -357,8 +378,8 @@ class HoldingsCsvTransformer(MigrationTaskBase):
         holdings_schema = self.folio_client.get_holdings_schema()
         properties = holdings_schema["properties"].keys()
         logging.info(properties)
-        logging.info(self.task_config.holdings_merge_criteria)
-        res = [mc for mc in self.task_config.holdings_merge_criteria if mc not in properties]
+        logging.info(self.task_configuration.holdings_merge_criteria)
+        res = [mc for mc in self.task_configuration.holdings_merge_criteria if mc not in properties]
         if any(res):
             logging.critical(
                 (
@@ -426,7 +447,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
             raise TransformationRecordFailedError(legacy_id, "No instance id in parsed record", "")
 
         for folio_holding in holdings_from_row:
-            self.mapper.perform_additional_mappings(folio_holding, file_def)
+            self.mapper.perform_additional_mappings(legacy_id, folio_holding, file_def)
             self.merge_holding_in(folio_holding, all_instance_ids, legacy_id)
         self.mapper.report_folio_mapping(folio_holding, self.mapper.schema)
 
@@ -436,7 +457,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
             self.mapper.create_bound_with_holdings(
                 folio_holding,
                 folio_holding["instanceId"],
-                self.task_config.holdings_type_uuid_for_boundwiths,
+                self.task_configuration.holdings_type_uuid_for_boundwiths,
             )
         )
 
@@ -480,9 +501,9 @@ class HoldingsCsvTransformer(MigrationTaskBase):
             # Regular holding. Merge according to criteria
             new_holding_key = HoldingsHelper.to_key(
                 incoming_holding,
-                self.task_config.holdings_merge_criteria,
+                self.task_configuration.holdings_merge_criteria,
                 self.mapper.migration_report,
-                self.task_config.holdings_type_uuid_for_boundwiths,
+                self.task_configuration.holdings_type_uuid_for_boundwiths,
             )
             if self.holdings.get(new_holding_key, None):
                 self.mapper.migration_report.add_general_statistics(

folio_migration_tools/migration_tasks/holdings_marc_transformer.py

@@ -216,7 +216,16 @@ class HoldingsMarcTransformer(MigrationTaskBase):
     ):
         csv.register_dialect("tsv", delimiter="\t")
         super().__init__(library_config, task_config, folio_client, use_logging)
-        self.task_config = task_config
+        if self.task_configuration.statistical_codes_map_file_name:
+            statcode_mapping = self.load_ref_data_mapping_file(
+                "statisticalCodeIds",
+                self.folder_structure.mapping_files_folder
+                / self.task_configuration.statistical_codes_map_file_name,
+                [],
+                False,
+            )
+        else:
+            statcode_mapping = None
         self.holdings_types = list(
             self.folio_client.folio_get_all("/holdings-types", "holdingsTypes")
         )
@@ -224,7 +233,7 @@ class HoldingsMarcTransformer(MigrationTaskBase):
             (
                 h
                 for h in self.holdings_types
-                if h["id"] == self.task_config.fallback_holdings_type_id
+                if h["id"] == self.task_configuration.fallback_holdings_type_id
             ),
             {"name": ""},
         )
@@ -232,7 +241,7 @@ class HoldingsMarcTransformer(MigrationTaskBase):
             raise TransformationProcessError(
                 "",
                 (
-                    f"Holdings type with ID {self.task_config.fallback_holdings_type_id}"
+                    f"Holdings type with ID {self.task_configuration.fallback_holdings_type_id}"
                     " not found in FOLIO."
                 ),
             )
@@ -243,11 +252,11 @@ class HoldingsMarcTransformer(MigrationTaskBase):
 
         # Load Boundwith relationship map
         self.boundwith_relationship_map_rows = []
-        if self.task_config.boundwith_relationship_file_path:
+        if self.task_configuration.boundwith_relationship_file_path:
             try:
                 with open(
                     self.folder_structure.legacy_records_folder
-                    / self.task_config.boundwith_relationship_file_path
+                    / self.task_configuration.boundwith_relationship_file_path
                 ) as boundwith_relationship_file:
                     self.boundwith_relationship_map_rows = list(
                         csv.DictReader(boundwith_relationship_file, dialect="tsv")
@@ -260,28 +269,29 @@ class HoldingsMarcTransformer(MigrationTaskBase):
                 raise TransformationProcessError(
                     "",
                     i18n.t("Provided boundwith relationship file not found"),
-                    self.task_config.boundwith_relationship_file_path,
+                    self.task_configuration.boundwith_relationship_file_path,
                 )
 
         location_map_path = (
             self.folder_structure.mapping_files_folder
-            / self.task_config.location_map_file_name
+            / self.task_configuration.location_map_file_name
         )
         with open(location_map_path) as location_map_file:
             self.location_map = list(csv.DictReader(location_map_file, dialect="tsv"))
             logging.info("Locations in map: %s", len(self.location_map))
 
         self.check_source_files(
-            self.folder_structure.legacy_records_folder, self.task_config.files
+            self.folder_structure.legacy_records_folder, self.task_configuration.files
         )
         self.instance_id_map = self.load_instance_id_map(True)
         self.mapper = RulesMapperHoldings(
             self.folio_client,
             self.location_map,
-            self.task_config,
+            self.task_configuration,
             self.library_configuration,
             self.instance_id_map,
             self.boundwith_relationship_map_rows,
+            statcode_mapping
         )
         self.add_supplemental_mfhd_mappings()
         if (
@@ -293,12 +303,12 @@ class HoldingsMarcTransformer(MigrationTaskBase):
         logging.info("Init done")
 
     def add_supplemental_mfhd_mappings(self):
-        if self.task_config.supplemental_mfhd_mapping_rules_file:
+        if self.task_configuration.supplemental_mfhd_mapping_rules_file:
             try:
                 with open(
                     (
                         self.folder_structure.mapping_files_folder
-                        / self.task_config.supplemental_mfhd_mapping_rules_file
+                        / self.task_configuration.supplemental_mfhd_mapping_rules_file
                     ),
                     "r",
                 ) as new_rules_file:
@@ -313,7 +323,7 @@ class HoldingsMarcTransformer(MigrationTaskBase):
                 raise TransformationProcessError(
                     "",
                     "Provided supplemental MFHD mapping rules file not found",
-                    self.task_config.supplemental_mfhd_mapping_rules_file,
+                    self.task_configuration.supplemental_mfhd_mapping_rules_file,
                 )
         else:
             new_rules = {}

folio_migration_tools/migration_tasks/items_transformer.py

@@ -124,10 +124,10 @@ class ItemsTransformer(MigrationTaskBase):
         statistical_codes_map_file_name: Annotated[
             Optional[str],
             Field(
-                title="Statistical codes map file name",
+                title="Statistical code map file name",
                 description=(
-                    "File name for statistical codes map. "
-                    "Empty string by default."
+                    "Path to the file containing the mapping of statistical codes. "
+                    "The file should be in TSV format with legacy_stat_code and folio_code columns."
                 ),
             ),
         ] = ""
@@ -205,6 +205,7 @@ class ItemsTransformer(MigrationTaskBase):
         csv.register_dialect("tsv", delimiter="\t")
         super().__init__(library_config, task_config, folio_client, use_logging)
         self.task_config = task_config
+        self.task_configuration = self.task_config
         self.check_source_files(
             self.folder_structure.legacy_records_folder, self.task_config.files
         )
@@ -353,7 +354,7 @@ class ItemsTransformer(MigrationTaskBase):
                         record, f"row {idx}", FOLIONamespaces.items
                     )
 
-                    self.mapper.perform_additional_mappings(folio_rec, file_def)
+                    self.mapper.perform_additional_mappings(legacy_id, folio_rec, file_def)
                     self.handle_circiulation_notes(folio_rec, self.folio_client.current_user)
                     self.handle_notes(folio_rec)
                     if folio_rec["holdingsRecordId"] in self.boundwith_relationship_map:

folio_migration_tools/migration_tasks/migration_task_base.py

@@ -455,7 +455,7 @@ class MigrationTaskBase:
             logging.info("No mapping setup for %s", folio_property_name)
             logging.info("%s will have default mapping if any ", folio_property_name)
             logging.info(
-                "Add a file named %s and add the field to the item.mapping.json file.",
+                "Add a file named %s and add the field to the field mapping json file.",
                 map_file_path,
             )
             return None
@@ -522,6 +522,27 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
             ),
         ),
     ] = False
+    statistical_codes_map_file_name: Annotated[
+        Optional[str],
+        Field(
+            title="Statistical code map file name",
+            description=(
+                "Path to the file containing the mapping of statistical codes. "
+                "The file should be in TSV format with legacy_stat_code and folio_code columns."
+            ),
+        ),
+    ] = ""
+    statistical_code_mapping_fields: Annotated[
+        List[str],
+        Field(
+            title="Statistical code mapping fields",
+            description=(
+                "List of fields + subfields to be used for mapping statistical codes. "
+                "Subfields should be delimited by a \"$\" (eg. 907$a). Single repeating subfields "
+                "will be treated as unique values. Multiple subfields will be concatenated together with a space."
+            ),
+        ),
+    ] = []
 
 class ExcludeLevelFilter(logging.Filter):
     def __init__(self, level):

folio_migration_tools/migration_tasks/orders_transformer.py

@@ -157,6 +157,7 @@ class OrdersTransformer(MigrationTaskBase):
         super().__init__(library_config, task_config, folio_client, use_logging)
         self.object_type_name = self.get_object_type().name
         self.task_config = task_config
+        self.task_configuration = self.task_config
         self.files = self.list_source_files()
         self.total_records = 0
         self.current_folio_record: dict = {}
@@ -175,6 +176,7 @@ class OrdersTransformer(MigrationTaskBase):
         self.mapper = CompositeOrderMapper(
             self.folio_client,
             self.library_configuration,
+            self.task_configuration,
             self.orders_map,
             self.load_id_map(self.folder_structure.organizations_id_map_path, True),
             self.load_instance_id_map(True),

folio_migration_tools/migration_tasks/user_transformer.py

@@ -119,6 +119,7 @@ class UserTransformer(MigrationTaskBase):
     ):
         super().__init__(library_config, task_config, folio_client, use_logging)
         self.task_config = task_config
+        self.task_configuration = self.task_config
         self.total_records = 0
 
         self.user_map = self.setup_records_map(