folio-migration-tools 1.9.2__py3-none-any.whl → 1.9.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. folio_migration_tools/mapping_file_transformation/user_mapper.py +39 -31
  2. folio_migration_tools/marc_rules_transformation/conditions.py +239 -30
  3. folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +99 -65
  4. folio_migration_tools/marc_rules_transformation/marc_file_processor.py +6 -1
  5. folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +2 -2
  6. folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +7 -1
  7. folio_migration_tools/migration_tasks/batch_poster.py +201 -51
  8. folio_migration_tools/migration_tasks/holdings_marc_transformer.py +22 -11
  9. folio_migration_tools/migration_tasks/items_transformer.py +25 -10
  10. folio_migration_tools/migration_tasks/loans_migrator.py +238 -77
  11. folio_migration_tools/migration_tasks/organization_transformer.py +1 -0
  12. folio_migration_tools/task_configuration.py +3 -1
  13. folio_migration_tools/test_infrastructure/mocked_classes.py +5 -0
  14. folio_migration_tools/transaction_migration/legacy_loan.py +30 -10
  15. folio_migration_tools/translations/en.json +19 -1
  16. {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/METADATA +2 -1
  17. {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/RECORD +20 -20
  18. {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/LICENSE +0 -0
  19. {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/WHEEL +0 -0
  20. {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/entry_points.txt +0 -0
@@ -33,9 +33,9 @@ def write_failed_batch_to_file(batch, file):
33
33
 
34
34
 
35
35
  class BatchPoster(MigrationTaskBase):
36
- """Batchposter
36
+ """BatchPoster
37
37
 
38
- Args:
38
+ Parents:
39
39
  MigrationTaskBase (_type_): _description_
40
40
 
41
41
  Raises:
@@ -131,6 +131,56 @@ class BatchPoster(MigrationTaskBase):
131
131
  ),
132
132
  ),
133
133
  ] = False
134
+ preserve_statistical_codes: Annotated[
135
+ bool,
136
+ Field(
137
+ title="Preserve statistical codes",
138
+ description=(
139
+ "Toggles whether or not to preserve statistical codes "
140
+ "during the upsert process. Defaults to False"
141
+ ),
142
+ ),
143
+ ] = False
144
+ preserve_administrative_notes: Annotated[
145
+ bool,
146
+ Field(
147
+ title="Preserve administrative notes",
148
+ description=(
149
+ "Toggles whether or not to preserve administrative notes "
150
+ "during the upsert process. Defaults to False"
151
+ ),
152
+ ),
153
+ ] = False
154
+ preserve_temporary_locations: Annotated[
155
+ bool,
156
+ Field(
157
+ title="Preserve temporary locations",
158
+ description=(
159
+ "Toggles whether or not to preserve temporary locations "
160
+ "on items during the upsert process. Defaults to False"
161
+ ),
162
+ ),
163
+ ] = False
164
+ preserve_temporary_loan_types: Annotated[
165
+ bool,
166
+ Field(
167
+ title="Preserve temporary loan types",
168
+ description=(
169
+ "Toggles whether or not to preserve temporary loan types "
170
+ "on items during the upsert process. Defaults to False"
171
+ ),
172
+ ),
173
+ ] = False
174
+ preserve_item_status: Annotated[
175
+ bool,
176
+ Field(
177
+ title="Preserve item status",
178
+ description=(
179
+ "Toggles whether or not to preserve item status "
180
+ "on items during the upsert process. Defaults to False"
181
+ ),
182
+ ),
183
+ ] = True
134
184
 
135
185
  task_configuration: TaskConfiguration
136
186
 
@@ -158,7 +208,8 @@ class BatchPoster(MigrationTaskBase):
158
208
  if self.api_info["supports_upsert"]:
159
209
  self.query_params["upsert"] = self.task_configuration.upsert
160
210
  elif self.task_configuration.upsert and not self.api_info["supports_upsert"]:
161
- logging.info("Upsert is not supported for this object type. Query parameter will not be set.")
211
+ logging.info(
212
+ "Upsert is not supported for this object type. Query parameter will not be set.")
162
213
  self.snapshot_id = str(uuid4())
163
214
  self.failed_objects: list = []
164
215
  self.batch_size = self.task_configuration.batch_size
@@ -174,11 +225,14 @@ class BatchPoster(MigrationTaskBase):
174
225
  self.okapi_headers = self.folio_client.okapi_headers
175
226
  self.http_client = None
176
227
  self.starting_record_count_in_folio: Optional[int] = None
228
+ self.finished_record_count_in_folio: Optional[int] = None
177
229
 
178
230
  def do_work(self):
179
231
  with self.folio_client.get_folio_http_client() as httpx_client:
180
232
  self.http_client = httpx_client
181
- with open(self.folder_structure.failed_recs_path, "w", encoding='utf-8') as failed_recs_file:
233
+ with open(
234
+ self.folder_structure.failed_recs_path, "w", encoding='utf-8'
235
+ ) as failed_recs_file:
182
236
  self.get_starting_record_count()
183
237
  try:
184
238
  batch = []
@@ -249,7 +303,7 @@ class BatchPoster(MigrationTaskBase):
249
303
  self.handle_generic_exception(
250
304
  exception, last_row, batch, self.processed, failed_recs_file
251
305
  )
252
- logging.info("Done posting %s records. ", (self.processed))
306
+ logging.info("Done posting %s records. ", self.processed)
253
307
  if self.task_configuration.object_type == "SRS":
254
308
  self.commit_snapshot()
255
309
 
@@ -276,7 +330,7 @@ class BatchPoster(MigrationTaskBase):
276
330
 
277
331
  async def set_version_async(self, batch, query_api, object_type) -> None:
278
332
  """
279
- Fetches the current version of the records in the batch, if the record exists in FOLIO
333
+ Fetches the current version of the records in the batch if the record exists in FOLIO
280
334
 
281
335
  Args:
282
336
  batch (list): List of records to fetch versions for
@@ -288,7 +342,7 @@ class BatchPoster(MigrationTaskBase):
288
342
  """
289
343
  fetch_batch_size = 90
290
344
  fetch_tasks = []
291
- updates = {}
345
+ existing_records = {}
292
346
  async with httpx.AsyncClient(base_url=self.folio_client.gateway_url) as client:
293
347
  for i in range(0, len(batch), fetch_batch_size):
294
348
  batch_slice = batch[i:i + fetch_batch_size]
@@ -297,7 +351,10 @@ class BatchPoster(MigrationTaskBase):
297
351
  client,
298
352
  query_api,
299
353
  params={
300
- "query": f"id==({' OR '.join([record['id'] for record in batch_slice if 'id' in record])})",
354
+ "query": (
355
+ "id==("
356
+ f"{' OR '.join([r['id'] for r in batch_slice if 'id' in r])})"
357
+ ),
301
358
  "limit": fetch_batch_size
302
359
  },
303
360
  )
@@ -306,37 +363,91 @@ class BatchPoster(MigrationTaskBase):
306
363
  responses = await asyncio.gather(*fetch_tasks)
307
364
 
308
365
  for response in responses:
309
- self.update_record_versions(object_type, updates, response)
366
+ self.collect_existing_records_for_upsert(object_type, response, existing_records)
310
367
  for record in batch:
311
- if record["id"] in updates:
312
- record.update(updates[record["id"]])
368
+ if record["id"] in existing_records:
369
+ self.prepare_record_for_upsert(record, existing_records[record["id"]])
370
+
371
+ def handle_source_marc(self, new_record: dict, existing_record: dict):
372
+ updates = {}
373
+ updates.update(existing_record)
374
+ self.handle_upsert_for_administrative_notes(updates)
375
+ self.handle_upsert_for_statistical_codes(updates)
376
+ keep_new = {k: v for k, v in new_record.items() if k in ["statisticalCodeIds", "administrativeNotes"]}
377
+ if "instanceStatusId" in new_record:
378
+ updates["instanceStatusId"] = new_record["instanceStatusId"]
379
+ for k, v in keep_new.items():
380
+ updates[k] = list(dict.fromkeys(updates.get(k, []) + v))
381
+ new_record.update(updates)
313
382
 
314
383
  @staticmethod
315
- def update_record_versions(object_type, updates, response):
384
+ def collect_existing_records_for_upsert(object_type: str, response: httpx.Response, existing_records: dict):
316
385
  if response.status_code == 200:
317
386
  response_json = response.json()
318
387
  for record in response_json[object_type]:
319
- updates[record["id"]] = {
320
- "_version": record["_version"],
321
- }
322
- if "hrid" in record:
323
- updates[record["id"]]["hrid"] = record["hrid"]
324
- if "status" in record:
325
- updates[record["id"]]["status"] = record["status"]
326
- if "lastCheckIn" in record:
327
- updates[record["id"]]["lastCheckIn"] = record["lastCheckIn"]
388
+ existing_records[record["id"]] = record
328
389
  else:
329
390
  logging.error(
330
- "Failed to fetch current records. HTTP %s\t%s",
331
- response.status_code,
332
- response.text,
333
- )
391
+ "Failed to fetch current records. HTTP %s\t%s",
392
+ response.status_code,
393
+ response.text,
394
+ )
395
+
396
+ def handle_upsert_for_statistical_codes(self, updates: dict):
397
+ if not self.task_configuration.preserve_statistical_codes:
398
+ updates.pop("statisticalCodeIds", None)
399
+
400
+ def handle_upsert_for_administrative_notes(self, updates: dict):
401
+ if not self.task_configuration.preserve_administrative_notes:
402
+ updates.pop("administrativeNotes", None)
403
+
404
+ def handle_upsert_for_temporary_locations(self, updates: dict):
405
+ if not self.task_configuration.preserve_temporary_locations:
406
+ updates.pop("temporaryLocationId", None)
407
+
408
+ def handle_upsert_for_temporary_loan_types(self, updates: dict):
409
+ if not self.task_configuration.preserve_temporary_loan_types:
410
+ updates.pop("temporaryLoanTypeId", None)
411
+
412
+ def keep_existing_fields(self, updates: dict, existing_record: dict):
413
+ keep_existing_fields = ["hrid", "lastCheckIn"]
414
+ if self.task_configuration.preserve_item_status:
415
+ keep_existing_fields.append("status")
416
+ for key in keep_existing_fields:
417
+ if key in existing_record:
418
+ updates[key] = existing_record[key]
419
+
420
+ def prepare_record_for_upsert(self, new_record: dict, existing_record: dict):
421
+ if "source" in existing_record and "MARC" in existing_record["source"]:
422
+ self.handle_source_marc(new_record, existing_record)
423
+ else:
424
+ updates = {
425
+ "_version": existing_record["_version"],
426
+ }
427
+ self.keep_existing_fields(updates, existing_record)
428
+ keep_new = {k: v for k, v in new_record.items() if k in ["statisticalCodeIds", "administrativeNotes"]}
429
+ self.handle_upsert_for_statistical_codes(existing_record)
430
+ self.handle_upsert_for_administrative_notes(existing_record)
431
+ self.handle_upsert_for_temporary_locations(existing_record)
432
+ self.handle_upsert_for_temporary_loan_types(existing_record)
433
+ for k, v in keep_new.items():
434
+ updates[k] = list(dict.fromkeys(existing_record.get(k, []) + v))
435
+ for key in [
436
+ "temporaryLocationId",
437
+ "temporaryLoanTypeId",
438
+ ]:
439
+ if key in existing_record:
440
+ updates[key] = existing_record[key]
441
+ new_record.update(updates)
334
442
 
335
- async def get_with_retry(self, client: httpx.AsyncClient, url: str, params: dict = {}):
443
+ async def get_with_retry(self, client: httpx.AsyncClient, url: str, params=None):
444
+ if params is None:
445
+ params = {}
336
446
  retries = 3
337
447
  for attempt in range(retries):
338
448
  try:
339
- response = await client.get(url, params=params, headers=self.folio_client.okapi_headers)
449
+ response = await client.get(
450
+ url, params=params, headers=self.folio_client.okapi_headers)
340
451
  response.raise_for_status()
341
452
  return response
342
453
  except httpx.HTTPError as e:
@@ -477,8 +588,8 @@ class BatchPoster(MigrationTaskBase):
477
588
  )
478
589
  logging.info(last_row)
479
590
  logging.info("=========Stack trace==============")
480
- traceback.logging.info_exc() # type: ignore
481
- logging.info("=======================", flush=True)
591
+ traceback.logging.info_exc() # type: ignore
592
+ logging.info("=======================")
482
593
 
483
594
  def post_batch(self, batch, failed_recs_file, num_records, recursion_depth=0):
484
595
  if self.query_params.get("upsert", False) and self.api_info.get("query_endpoint", ""):
@@ -514,7 +625,7 @@ class BatchPoster(MigrationTaskBase):
514
625
  )
515
626
  write_failed_batch_to_file(batch, failed_recs_file)
516
627
  if json_report.get("failedUsers", []):
517
- logging.error("Errormessage: %s", json_report.get("error", []))
628
+ logging.error("Error message: %s", json_report.get("error", []))
518
629
  for failed_user in json_report.get("failedUsers"):
519
630
  logging.error(
520
631
  "User failed. %s\t%s\t%s",
@@ -581,8 +692,8 @@ class BatchPoster(MigrationTaskBase):
581
692
  resp = json.dumps(response, indent=4)
582
693
  except TypeError:
583
694
  resp = response
584
- except Exception:
585
- logging.exception("something unexpected happened")
695
+ except Exception as e:
696
+ logging.exception(f"something unexpected happened, {e}")
586
697
  resp = response
587
698
  raise TransformationRecordFailedError(
588
699
  "",
@@ -603,17 +714,29 @@ class BatchPoster(MigrationTaskBase):
603
714
  payload = {self.api_info["object_name"]: batch}
604
715
  if self.http_client and not self.http_client.is_closed:
605
716
  return self.http_client.post(
606
- url, json=payload, headers=self.folio_client.okapi_headers, params=self.query_params
717
+ url,
718
+ json=payload,
719
+ headers=self.folio_client.okapi_headers,
720
+ params=self.query_params
607
721
  )
608
722
  else:
609
- return httpx.post(url, headers=self.okapi_headers, json=payload, params=self.query_params, timeout=None)
723
+ return httpx.post(
724
+ url,
725
+ headers=self.okapi_headers,
726
+ json=payload,
727
+ params=self.query_params,
728
+ timeout=None)
610
729
 
611
730
  def get_current_record_count_in_folio(self):
612
731
  if "query_endpoint" in self.api_info:
613
732
  url = f"{self.folio_client.gateway_url}{self.api_info['query_endpoint']}"
614
733
  query_params = {"query": "cql.allRecords=1", "limit": 0}
615
734
  if self.http_client and not self.http_client.is_closed:
616
- res = self.http_client.get(url, headers=self.folio_client.okapi_headers, params=query_params)
735
+ res = self.http_client.get(
736
+ url,
737
+ headers=self.folio_client.okapi_headers,
738
+ params=query_params
739
+ )
617
740
  else:
618
741
  res = httpx.get(url, headers=self.okapi_headers, params=query_params, timeout=None)
619
742
  try:
@@ -623,11 +746,15 @@ class BatchPoster(MigrationTaskBase):
623
746
  logging.error("Failed to get current record count. HTTP %s", res.status_code)
624
747
  return 0
625
748
  except KeyError:
626
- logging.error(f"Failed to get current record count. No 'totalRecords' in response: {res.json()}")
749
+ logging.error(
750
+ "Failed to get current record count. "
751
+ f"No 'totalRecords' in response: {res.json()}"
752
+ )
627
753
  return 0
628
754
  else:
629
755
  raise ValueError(
630
- "No 'query_endpoint' available for %s. Cannot get current record count.", self.task_configuration.object_type
756
+ "No 'query_endpoint' available for %s. Cannot get current record count.",
757
+ self.task_configuration.object_type
631
758
  )
632
759
 
633
760
  def get_starting_record_count(self):
@@ -635,14 +762,20 @@ class BatchPoster(MigrationTaskBase):
635
762
  logging.info("Getting starting record count in FOLIO")
636
763
  self.starting_record_count_in_folio = self.get_current_record_count_in_folio()
637
764
  else:
638
- logging.info("No query_endpoint available for %s. Cannot get starting record count.", self.task_configuration.object_type)
765
+ logging.info(
766
+ "No query_endpoint available for %s. Cannot get starting record count.",
767
+ self.task_configuration.object_type
768
+ )
639
769
 
640
770
  def get_finished_record_count(self):
641
771
  if "query_endpoint" in self.api_info:
642
772
  logging.info("Getting finished record count in FOLIO")
643
773
  self.finished_record_count_in_folio = self.get_current_record_count_in_folio()
644
774
  else:
645
- logging.info("No query_endpoint available for %s. Cannot get ending record count.", self.task_configuration.object_type)
775
+ logging.info(
776
+ "No query_endpoint available for %s. Cannot get ending record count.",
777
+ self.task_configuration.object_type
778
+ )
646
779
 
647
780
  def wrap_up(self):
648
781
  logging.info("Done. Wrapping up")
@@ -663,7 +796,9 @@ class BatchPoster(MigrationTaskBase):
663
796
  logging.info("Done posting %s records. %s failed", self.num_posted, self.num_failures)
664
797
  if self.starting_record_count_in_folio:
665
798
  self.get_finished_record_count()
666
- total_on_server = self.finished_record_count_in_folio - self.starting_record_count_in_folio
799
+ total_on_server = (
800
+ self.finished_record_count_in_folio - self.starting_record_count_in_folio
801
+ )
667
802
  discrepancy = self.processed - self.num_failures - total_on_server
668
803
  if discrepancy != 0:
669
804
  logging.error(
@@ -712,7 +847,10 @@ class BatchPoster(MigrationTaskBase):
712
847
  temp_report = copy.deepcopy(self.migration_report)
713
848
  temp_start = self.start_datetime
714
849
  self.task_configuration.rerun_failed_records = False
715
- self.__init__(self.task_configuration, self.library_configuration, self.folio_client)
850
+ self.__init__(
851
+ self.task_configuration,
852
+ self.library_configuration,
853
+ self.folio_client)
716
854
  self.performing_rerun = True
717
855
  self.migration_report = temp_report
718
856
  self.start_datetime = temp_start
@@ -747,9 +885,11 @@ class BatchPoster(MigrationTaskBase):
747
885
  res = httpx.post(url, headers=self.okapi_headers, json=snapshot, timeout=None)
748
886
  res.raise_for_status()
749
887
  logging.info("Posted Snapshot to FOLIO: %s", json.dumps(snapshot, indent=4))
750
- get_url = f"{self.folio_client.gateway_url}/source-storage/snapshots/{self.snapshot_id}"
751
- getted = False
752
- while not getted:
888
+ get_url = (
889
+ f"{self.folio_client.gateway_url}/source-storage/snapshots/{self.snapshot_id}"
890
+ )
891
+ got = False
892
+ while not got:
753
893
  logging.info("Sleeping while waiting for the snapshot to get created")
754
894
  time.sleep(5)
755
895
  if self.http_client and not self.http_client.is_closed:
@@ -757,11 +897,14 @@ class BatchPoster(MigrationTaskBase):
757
897
  else:
758
898
  res = httpx.get(get_url, headers=self.okapi_headers, timeout=None)
759
899
  if res.status_code == 200:
760
- getted = True
900
+ got = True
761
901
  else:
762
902
  logging.info(res.status_code)
763
- except Exception:
764
- logging.exception("Could not post the snapshot")
903
+ except httpx.HTTPStatusError as exc:
904
+ logging.exception("HTTP error occurred while posting the snapshot: %s", exc)
905
+ sys.exit(1)
906
+ except Exception as exc:
907
+ logging.exception("Could not post the snapshot: %s", exc)
765
908
  sys.exit(1)
766
909
 
767
910
  def commit_snapshot(self):
@@ -776,11 +919,15 @@ class BatchPoster(MigrationTaskBase):
776
919
  res = httpx.put(url, headers=self.okapi_headers, json=snapshot, timeout=None)
777
920
  res.raise_for_status()
778
921
  logging.info("Posted Committed snapshot to FOLIO: %s", json.dumps(snapshot, indent=4))
779
- except Exception:
922
+ except httpx.HTTPStatusError as exc:
923
+ logging.exception("HTTP error occurred while posting the snapshot: %s", exc)
924
+ sys.exit(1)
925
+ except Exception as exc:
780
926
  logging.exception(
781
927
  "Could not commit snapshot with id %s. Post this to /source-storage/snapshots/%s:",
782
928
  self.snapshot_id,
783
929
  self.snapshot_id,
930
+ exc,
784
931
  )
785
932
  logging.info("%s", json.dumps(snapshot, indent=4))
786
933
  sys.exit(1)
@@ -891,8 +1038,11 @@ def get_api_info(object_type: str, use_safe: bool = True):
891
1038
  try:
892
1039
  return choices[object_type]
893
1040
  except KeyError:
894
- key_string = ",".join(choices.keys())
895
- logging.error(f"Wrong type. Only one of {key_string} are allowed")
1041
+ key_string = ", ".join(choices.keys())
1042
+ logging.error(
1043
+ f"Wrong type. Only one of {key_string} are allowed, "
1044
+ f"received {object_type=} instead"
1045
+ )
896
1046
  logging.error("Halting")
897
1047
  sys.exit(1)
898
1048
 
@@ -908,7 +1058,7 @@ def chunks(records, number_of_chunks):
908
1058
  _type_: _description_
909
1059
  """
910
1060
  for i in range(0, len(records), number_of_chunks):
911
- yield records[i : i + number_of_chunks]
1061
+ yield records[i: i + number_of_chunks]
912
1062
 
913
1063
 
914
1064
  def get_human_readable(size, precision=2):
@@ -1,5 +1,3 @@
1
- '''Main "script."'''
2
-
3
1
  import csv
4
2
  import json
5
3
  import logging
@@ -19,7 +17,10 @@ from folio_migration_tools.library_configuration import (
19
17
  from folio_migration_tools.marc_rules_transformation.rules_mapper_holdings import (
20
18
  RulesMapperHoldings,
21
19
  )
22
- from folio_migration_tools.migration_tasks.migration_task_base import MarcTaskConfigurationBase, MigrationTaskBase
20
+ from folio_migration_tools.migration_tasks.migration_task_base import (
21
+ MarcTaskConfigurationBase,
22
+ MigrationTaskBase
23
+ )
23
24
 
24
25
 
25
26
  class HoldingsMarcTransformer(MigrationTaskBase):
@@ -37,14 +38,18 @@ class HoldingsMarcTransformer(MigrationTaskBase):
37
38
  str,
38
39
  Field(
39
40
  title="Migration task type",
40
- description=("The type of migration task you want to perform"),
41
+ description=(
42
+ "The type of migration task you want to perform"
43
+ ),
41
44
  ),
42
45
  ]
43
46
  files: Annotated[
44
47
  List[FileDefinition],
45
48
  Field(
46
49
  title="Source files",
47
- description=("List of MARC21 files with holdings records"),
50
+ description=(
51
+ "List of MARC21 files with holdings records"
52
+ ),
48
53
  ),
49
54
  ]
50
55
  hrid_handling: Annotated[
@@ -125,8 +130,8 @@ class HoldingsMarcTransformer(MigrationTaskBase):
125
130
  default_call_number_type_name: Annotated[
126
131
  str,
127
132
  Field(
128
- title="Default callnumber type name",
129
- description="The name of the callnumber type that will be used as fallback",
133
+ title="Default call_number type name",
134
+ description="The name of the call_number type that will be used as fallback",
130
135
  ),
131
136
  ]
132
137
  fallback_holdings_type_id: Annotated[
@@ -140,7 +145,10 @@ class HoldingsMarcTransformer(MigrationTaskBase):
140
145
  str,
141
146
  Field(
142
147
  title="Supplemental MFHD mapping rules file",
143
- description="The name of the file in the mapping_files directory containing supplemental MFHD mapping rules",
148
+ description=(
149
+ "The name of the file in the mapping_files directory "
150
+ "containing supplemental MFHD mapping rules"
151
+ ),
144
152
  ),
145
153
  ] = ""
146
154
  include_mrk_statements: Annotated[
@@ -148,8 +156,10 @@ class HoldingsMarcTransformer(MigrationTaskBase):
148
156
  Field(
149
157
  title="Include MARC statements (MRK-format) as staff-only Holdings notes",
150
158
  description=(
151
- "If set to true, the MARC statements will be included in the output as MARC Maker format fields. "
152
- "If set to false (default), the MARC statements will not be included in the output."
159
+ "If set to true, the MARC statements "
160
+ "will be included in the output as MARC Maker format fields. "
161
+ "If set to false (default), the MARC statements "
162
+ "will not be included in the output."
153
163
  ),
154
164
  ),
155
165
  ] = False
@@ -188,7 +198,8 @@ class HoldingsMarcTransformer(MigrationTaskBase):
188
198
  title="Include MARC Record (as MARC21 decoded string) as note",
189
199
  description=(
190
200
  "If set to true, the MARC record will be included in the output as a "
191
- "decoded binary MARC21 record. If set to false (default), the MARC record will not be "
201
+ "decoded binary MARC21 record. If set to false (default), "
202
+ "the MARC record will not be "
192
203
  "included in the output."
193
204
  ),
194
205
  ),
@@ -127,7 +127,8 @@ class ItemsTransformer(MigrationTaskBase):
127
127
  title="Statistical code map file name",
128
128
  description=(
129
129
  "Path to the file containing the mapping of statistical codes. "
130
- "The file should be in TSV format with legacy_stat_code and folio_code columns."
130
+ "The file should be in TSV format with legacy_stat_code "
131
+ "and folio_code columns."
131
132
  ),
132
133
  ),
133
134
  ] = ""
@@ -217,7 +218,10 @@ class ItemsTransformer(MigrationTaskBase):
217
218
  self.folio_keys = MappingFileMapperBase.get_mapped_folio_properties_from_map(
218
219
  self.items_map
219
220
  )
220
- if any(k for k in self.folio_keys if k.startswith("statisticalCodeIds")):
221
+ if (
222
+ any(k for k in self.folio_keys if k.startswith("statisticalCodeIds"))
223
+ or any(getattr(k, "statistical_code", "") for k in self.task_configuration.files)
224
+ ):
221
225
  statcode_mapping = self.load_ref_data_mapping_file(
222
226
  "statisticalCodeIds",
223
227
  self.folder_structure.mapping_files_folder
@@ -355,7 +359,7 @@ class ItemsTransformer(MigrationTaskBase):
355
359
  )
356
360
 
357
361
  self.mapper.perform_additional_mappings(legacy_id, folio_rec, file_def)
358
- self.handle_circiulation_notes(folio_rec, self.folio_client.current_user)
362
+ self.handle_circulation_notes(folio_rec, self.folio_client.current_user)
359
363
  self.handle_notes(folio_rec)
360
364
  if folio_rec["holdingsRecordId"] in self.boundwith_relationship_map:
361
365
  for idx_, instance_id in enumerate(
@@ -373,7 +377,7 @@ class ItemsTransformer(MigrationTaskBase):
373
377
  if idx == 0:
374
378
  logging.info("First FOLIO record:")
375
379
  logging.info(json.dumps(folio_rec, indent=4))
376
- # TODO: turn this into a asynchrounous task
380
+ # TODO: turn this into a asynchronous task
377
381
  Helper.write_to_file(results_file, folio_rec)
378
382
  self.mapper.migration_report.add_general_statistics(
379
383
  i18n.t("Number of records written to disk")
@@ -388,8 +392,8 @@ class ItemsTransformer(MigrationTaskBase):
388
392
  logging.fatal(attribute_error)
389
393
  logging.info("Quitting...")
390
394
  sys.exit(1)
391
- except Exception as excepion:
392
- self.mapper.handle_generic_exception(idx, excepion)
395
+ except Exception as exception:
396
+ self.mapper.handle_generic_exception(idx, exception)
393
397
  self.mapper.migration_report.add(
394
398
  "GeneralStatistics",
395
399
  i18n.t("Number of Legacy items in %{container}", container=file_def),
@@ -425,14 +429,14 @@ class ItemsTransformer(MigrationTaskBase):
425
429
  del folio_object["notes"]
426
430
 
427
431
  @staticmethod
428
- def handle_circiulation_notes(folio_rec, current_user_uuid):
432
+ def handle_circulation_notes(folio_rec, current_user_uuid):
429
433
  if not folio_rec.get("circulationNotes", []):
430
434
  return
431
435
  filtered_notes = []
432
436
  for circ_note in folio_rec.get("circulationNotes", []):
433
437
  if circ_note.get("noteType", "") not in ["Check in", "Check out"]:
434
438
  raise TransformationProcessError(
435
- "", "Circulation Note types are not mapped correclty"
439
+ "", "Circulation Note types are not mapped correctly"
436
440
  )
437
441
  if circ_note.get("note", ""):
438
442
  circ_note["id"] = str(uuid.uuid4())
@@ -455,11 +459,22 @@ class ItemsTransformer(MigrationTaskBase):
455
459
  json.loads(x) for x in boundwith_relationship_file
456
460
  )
457
461
  logging.info(
458
- "Rows in Bound with relationship map: %s", len(self.boundwith_relationship_map)
462
+ "Rows in Bound with relationship map: %s",
463
+ len(self.boundwith_relationship_map)
459
464
  )
460
465
  except FileNotFoundError:
461
466
  raise TransformationProcessError(
462
- "", "Boundwith relationship file specified, but relationships file from holdings transformation not found. ", self.folder_structure.boundwith_relationships_map_path
467
+ "",
468
+ "Boundwith relationship file specified, but relationships file "
469
+ "from holdings transformation not found.",
470
+ self.folder_structure.boundwith_relationships_map_path
471
+ )
472
+ except ValueError:
473
+ raise TransformationProcessError(
474
+ "",
475
+ "Boundwith relationship file specified, but relationships file "
476
+ "from holdings transformation is not a valid line JSON.",
477
+ self.folder_structure.boundwith_relationships_map_path,
463
478
  )
464
479
 
465
480
  def wrap_up(self):