folio-migration-tools 1.9.2__py3-none-any.whl → 1.9.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/mapping_file_transformation/user_mapper.py +39 -31
- folio_migration_tools/marc_rules_transformation/conditions.py +239 -30
- folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +99 -65
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +6 -1
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +2 -2
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +7 -1
- folio_migration_tools/migration_tasks/batch_poster.py +201 -51
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +22 -11
- folio_migration_tools/migration_tasks/items_transformer.py +25 -10
- folio_migration_tools/migration_tasks/loans_migrator.py +238 -77
- folio_migration_tools/migration_tasks/organization_transformer.py +1 -0
- folio_migration_tools/task_configuration.py +3 -1
- folio_migration_tools/test_infrastructure/mocked_classes.py +5 -0
- folio_migration_tools/transaction_migration/legacy_loan.py +30 -10
- folio_migration_tools/translations/en.json +19 -1
- {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/METADATA +2 -1
- {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/RECORD +20 -20
- {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/LICENSE +0 -0
- {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/WHEEL +0 -0
- {folio_migration_tools-1.9.2.dist-info → folio_migration_tools-1.9.4.dist-info}/entry_points.txt +0 -0
|
@@ -33,9 +33,9 @@ def write_failed_batch_to_file(batch, file):
|
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
class BatchPoster(MigrationTaskBase):
|
|
36
|
-
"""
|
|
36
|
+
"""BatchPoster
|
|
37
37
|
|
|
38
|
-
|
|
38
|
+
Parents:
|
|
39
39
|
MigrationTaskBase (_type_): _description_
|
|
40
40
|
|
|
41
41
|
Raises:
|
|
@@ -131,6 +131,56 @@ class BatchPoster(MigrationTaskBase):
|
|
|
131
131
|
),
|
|
132
132
|
),
|
|
133
133
|
] = False
|
|
134
|
+
preserve_statistical_codes: Annotated[
|
|
135
|
+
bool,
|
|
136
|
+
Field(
|
|
137
|
+
title="Preserve statistical codes",
|
|
138
|
+
description=(
|
|
139
|
+
"Toggles whether or not to preserve statistical codes "
|
|
140
|
+
"during the upsert process. Defaults to False"
|
|
141
|
+
),
|
|
142
|
+
),
|
|
143
|
+
] = False
|
|
144
|
+
preserve_administrative_notes: Annotated[
|
|
145
|
+
bool,
|
|
146
|
+
Field(
|
|
147
|
+
title="Preserve administrative notes",
|
|
148
|
+
description=(
|
|
149
|
+
"Toggles whether or not to preserve administrative notes "
|
|
150
|
+
"during the upsert process. Defaults to False"
|
|
151
|
+
),
|
|
152
|
+
),
|
|
153
|
+
] = False
|
|
154
|
+
preserve_temporary_locations: Annotated[
|
|
155
|
+
bool,
|
|
156
|
+
Field(
|
|
157
|
+
title="Preserve temporary locations",
|
|
158
|
+
description=(
|
|
159
|
+
"Toggles whether or not to preserve temporary locations "
|
|
160
|
+
"on items during the upsert process. Defaults to False"
|
|
161
|
+
),
|
|
162
|
+
),
|
|
163
|
+
] = False
|
|
164
|
+
preserve_temporary_loan_types: Annotated[
|
|
165
|
+
bool,
|
|
166
|
+
Field(
|
|
167
|
+
title="Preserve temporary loan types",
|
|
168
|
+
description=(
|
|
169
|
+
"Toggles whether or not to preserve temporary loan types "
|
|
170
|
+
"on items during the upsert process. Defaults to False"
|
|
171
|
+
),
|
|
172
|
+
),
|
|
173
|
+
] = False
|
|
174
|
+
preserve_item_status: Annotated[
|
|
175
|
+
bool,
|
|
176
|
+
Field(
|
|
177
|
+
title="Preserve item status",
|
|
178
|
+
description=(
|
|
179
|
+
"Toggles whether or not to preserve item status "
|
|
180
|
+
"on items during the upsert process. Defaults to False"
|
|
181
|
+
),
|
|
182
|
+
),
|
|
183
|
+
] = True
|
|
134
184
|
|
|
135
185
|
task_configuration: TaskConfiguration
|
|
136
186
|
|
|
@@ -158,7 +208,8 @@ class BatchPoster(MigrationTaskBase):
|
|
|
158
208
|
if self.api_info["supports_upsert"]:
|
|
159
209
|
self.query_params["upsert"] = self.task_configuration.upsert
|
|
160
210
|
elif self.task_configuration.upsert and not self.api_info["supports_upsert"]:
|
|
161
|
-
logging.info(
|
|
211
|
+
logging.info(
|
|
212
|
+
"Upsert is not supported for this object type. Query parameter will not be set.")
|
|
162
213
|
self.snapshot_id = str(uuid4())
|
|
163
214
|
self.failed_objects: list = []
|
|
164
215
|
self.batch_size = self.task_configuration.batch_size
|
|
@@ -174,11 +225,14 @@ class BatchPoster(MigrationTaskBase):
|
|
|
174
225
|
self.okapi_headers = self.folio_client.okapi_headers
|
|
175
226
|
self.http_client = None
|
|
176
227
|
self.starting_record_count_in_folio: Optional[int] = None
|
|
228
|
+
self.finished_record_count_in_folio: Optional[int] = None
|
|
177
229
|
|
|
178
230
|
def do_work(self):
|
|
179
231
|
with self.folio_client.get_folio_http_client() as httpx_client:
|
|
180
232
|
self.http_client = httpx_client
|
|
181
|
-
with open(
|
|
233
|
+
with open(
|
|
234
|
+
self.folder_structure.failed_recs_path, "w", encoding='utf-8'
|
|
235
|
+
) as failed_recs_file:
|
|
182
236
|
self.get_starting_record_count()
|
|
183
237
|
try:
|
|
184
238
|
batch = []
|
|
@@ -249,7 +303,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
249
303
|
self.handle_generic_exception(
|
|
250
304
|
exception, last_row, batch, self.processed, failed_recs_file
|
|
251
305
|
)
|
|
252
|
-
logging.info("Done posting %s records. ",
|
|
306
|
+
logging.info("Done posting %s records. ", self.processed)
|
|
253
307
|
if self.task_configuration.object_type == "SRS":
|
|
254
308
|
self.commit_snapshot()
|
|
255
309
|
|
|
@@ -276,7 +330,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
276
330
|
|
|
277
331
|
async def set_version_async(self, batch, query_api, object_type) -> None:
|
|
278
332
|
"""
|
|
279
|
-
Fetches the current version of the records in the batch
|
|
333
|
+
Fetches the current version of the records in the batch if the record exists in FOLIO
|
|
280
334
|
|
|
281
335
|
Args:
|
|
282
336
|
batch (list): List of records to fetch versions for
|
|
@@ -288,7 +342,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
288
342
|
"""
|
|
289
343
|
fetch_batch_size = 90
|
|
290
344
|
fetch_tasks = []
|
|
291
|
-
|
|
345
|
+
existing_records = {}
|
|
292
346
|
async with httpx.AsyncClient(base_url=self.folio_client.gateway_url) as client:
|
|
293
347
|
for i in range(0, len(batch), fetch_batch_size):
|
|
294
348
|
batch_slice = batch[i:i + fetch_batch_size]
|
|
@@ -297,7 +351,10 @@ class BatchPoster(MigrationTaskBase):
|
|
|
297
351
|
client,
|
|
298
352
|
query_api,
|
|
299
353
|
params={
|
|
300
|
-
"query":
|
|
354
|
+
"query": (
|
|
355
|
+
"id==("
|
|
356
|
+
f"{' OR '.join([r['id'] for r in batch_slice if 'id' in r])})"
|
|
357
|
+
),
|
|
301
358
|
"limit": fetch_batch_size
|
|
302
359
|
},
|
|
303
360
|
)
|
|
@@ -306,37 +363,91 @@ class BatchPoster(MigrationTaskBase):
|
|
|
306
363
|
responses = await asyncio.gather(*fetch_tasks)
|
|
307
364
|
|
|
308
365
|
for response in responses:
|
|
309
|
-
self.
|
|
366
|
+
self.collect_existing_records_for_upsert(object_type, response, existing_records)
|
|
310
367
|
for record in batch:
|
|
311
|
-
if record["id"] in
|
|
312
|
-
|
|
368
|
+
if record["id"] in existing_records:
|
|
369
|
+
self.prepare_record_for_upsert(record, existing_records[record["id"]])
|
|
370
|
+
|
|
371
|
+
def handle_source_marc(self, new_record: dict, existing_record: dict):
|
|
372
|
+
updates = {}
|
|
373
|
+
updates.update(existing_record)
|
|
374
|
+
self.handle_upsert_for_administrative_notes(updates)
|
|
375
|
+
self.handle_upsert_for_statistical_codes(updates)
|
|
376
|
+
keep_new = {k: v for k, v in new_record.items() if k in ["statisticalCodeIds", "administrativeNotes"]}
|
|
377
|
+
if "instanceStatusId" in new_record:
|
|
378
|
+
updates["instanceStatusId"] = new_record["instanceStatusId"]
|
|
379
|
+
for k, v in keep_new.items():
|
|
380
|
+
updates[k] = list(dict.fromkeys(updates.get(k, []) + v))
|
|
381
|
+
new_record.update(updates)
|
|
313
382
|
|
|
314
383
|
@staticmethod
|
|
315
|
-
def
|
|
384
|
+
def collect_existing_records_for_upsert(object_type: str, response: httpx.Response, existing_records: dict):
|
|
316
385
|
if response.status_code == 200:
|
|
317
386
|
response_json = response.json()
|
|
318
387
|
for record in response_json[object_type]:
|
|
319
|
-
|
|
320
|
-
"_version": record["_version"],
|
|
321
|
-
}
|
|
322
|
-
if "hrid" in record:
|
|
323
|
-
updates[record["id"]]["hrid"] = record["hrid"]
|
|
324
|
-
if "status" in record:
|
|
325
|
-
updates[record["id"]]["status"] = record["status"]
|
|
326
|
-
if "lastCheckIn" in record:
|
|
327
|
-
updates[record["id"]]["lastCheckIn"] = record["lastCheckIn"]
|
|
388
|
+
existing_records[record["id"]] = record
|
|
328
389
|
else:
|
|
329
390
|
logging.error(
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
391
|
+
"Failed to fetch current records. HTTP %s\t%s",
|
|
392
|
+
response.status_code,
|
|
393
|
+
response.text,
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
def handle_upsert_for_statistical_codes(self, updates: dict):
|
|
397
|
+
if not self.task_configuration.preserve_statistical_codes:
|
|
398
|
+
updates.pop("statisticalCodeIds", None)
|
|
399
|
+
|
|
400
|
+
def handle_upsert_for_administrative_notes(self, updates: dict):
|
|
401
|
+
if not self.task_configuration.preserve_administrative_notes:
|
|
402
|
+
updates.pop("administrativeNotes", None)
|
|
403
|
+
|
|
404
|
+
def handle_upsert_for_temporary_locations(self, updates: dict):
|
|
405
|
+
if not self.task_configuration.preserve_temporary_locations:
|
|
406
|
+
updates.pop("temporaryLocationId", None)
|
|
407
|
+
|
|
408
|
+
def handle_upsert_for_temporary_loan_types(self, updates: dict):
|
|
409
|
+
if not self.task_configuration.preserve_temporary_loan_types:
|
|
410
|
+
updates.pop("temporaryLoanTypeId", None)
|
|
411
|
+
|
|
412
|
+
def keep_existing_fields(self, updates: dict, existing_record: dict):
|
|
413
|
+
keep_existing_fields = ["hrid", "lastCheckIn"]
|
|
414
|
+
if self.task_configuration.preserve_item_status:
|
|
415
|
+
keep_existing_fields.append("status")
|
|
416
|
+
for key in keep_existing_fields:
|
|
417
|
+
if key in existing_record:
|
|
418
|
+
updates[key] = existing_record[key]
|
|
419
|
+
|
|
420
|
+
def prepare_record_for_upsert(self, new_record: dict, existing_record: dict):
|
|
421
|
+
if "source" in existing_record and "MARC" in existing_record["source"]:
|
|
422
|
+
self.handle_source_marc(new_record, existing_record)
|
|
423
|
+
else:
|
|
424
|
+
updates = {
|
|
425
|
+
"_version": existing_record["_version"],
|
|
426
|
+
}
|
|
427
|
+
self.keep_existing_fields(updates, existing_record)
|
|
428
|
+
keep_new = {k: v for k, v in new_record.items() if k in ["statisticalCodeIds", "administrativeNotes"]}
|
|
429
|
+
self.handle_upsert_for_statistical_codes(existing_record)
|
|
430
|
+
self.handle_upsert_for_administrative_notes(existing_record)
|
|
431
|
+
self.handle_upsert_for_temporary_locations(existing_record)
|
|
432
|
+
self.handle_upsert_for_temporary_loan_types(existing_record)
|
|
433
|
+
for k, v in keep_new.items():
|
|
434
|
+
updates[k] = list(dict.fromkeys(existing_record.get(k, []) + v))
|
|
435
|
+
for key in [
|
|
436
|
+
"temporaryLocationId",
|
|
437
|
+
"temporaryLoanTypeId",
|
|
438
|
+
]:
|
|
439
|
+
if key in existing_record:
|
|
440
|
+
updates[key] = existing_record[key]
|
|
441
|
+
new_record.update(updates)
|
|
334
442
|
|
|
335
|
-
async def get_with_retry(self, client: httpx.AsyncClient, url: str, params
|
|
443
|
+
async def get_with_retry(self, client: httpx.AsyncClient, url: str, params=None):
|
|
444
|
+
if params is None:
|
|
445
|
+
params = {}
|
|
336
446
|
retries = 3
|
|
337
447
|
for attempt in range(retries):
|
|
338
448
|
try:
|
|
339
|
-
response = await client.get(
|
|
449
|
+
response = await client.get(
|
|
450
|
+
url, params=params, headers=self.folio_client.okapi_headers)
|
|
340
451
|
response.raise_for_status()
|
|
341
452
|
return response
|
|
342
453
|
except httpx.HTTPError as e:
|
|
@@ -477,8 +588,8 @@ class BatchPoster(MigrationTaskBase):
|
|
|
477
588
|
)
|
|
478
589
|
logging.info(last_row)
|
|
479
590
|
logging.info("=========Stack trace==============")
|
|
480
|
-
traceback.logging.info_exc()
|
|
481
|
-
logging.info("======================="
|
|
591
|
+
traceback.logging.info_exc() # type: ignore
|
|
592
|
+
logging.info("=======================")
|
|
482
593
|
|
|
483
594
|
def post_batch(self, batch, failed_recs_file, num_records, recursion_depth=0):
|
|
484
595
|
if self.query_params.get("upsert", False) and self.api_info.get("query_endpoint", ""):
|
|
@@ -514,7 +625,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
514
625
|
)
|
|
515
626
|
write_failed_batch_to_file(batch, failed_recs_file)
|
|
516
627
|
if json_report.get("failedUsers", []):
|
|
517
|
-
logging.error("
|
|
628
|
+
logging.error("Error message: %s", json_report.get("error", []))
|
|
518
629
|
for failed_user in json_report.get("failedUsers"):
|
|
519
630
|
logging.error(
|
|
520
631
|
"User failed. %s\t%s\t%s",
|
|
@@ -581,8 +692,8 @@ class BatchPoster(MigrationTaskBase):
|
|
|
581
692
|
resp = json.dumps(response, indent=4)
|
|
582
693
|
except TypeError:
|
|
583
694
|
resp = response
|
|
584
|
-
except Exception:
|
|
585
|
-
logging.exception("something unexpected happened")
|
|
695
|
+
except Exception as e:
|
|
696
|
+
logging.exception(f"something unexpected happened, {e}")
|
|
586
697
|
resp = response
|
|
587
698
|
raise TransformationRecordFailedError(
|
|
588
699
|
"",
|
|
@@ -603,17 +714,29 @@ class BatchPoster(MigrationTaskBase):
|
|
|
603
714
|
payload = {self.api_info["object_name"]: batch}
|
|
604
715
|
if self.http_client and not self.http_client.is_closed:
|
|
605
716
|
return self.http_client.post(
|
|
606
|
-
url,
|
|
717
|
+
url,
|
|
718
|
+
json=payload,
|
|
719
|
+
headers=self.folio_client.okapi_headers,
|
|
720
|
+
params=self.query_params
|
|
607
721
|
)
|
|
608
722
|
else:
|
|
609
|
-
return httpx.post(
|
|
723
|
+
return httpx.post(
|
|
724
|
+
url,
|
|
725
|
+
headers=self.okapi_headers,
|
|
726
|
+
json=payload,
|
|
727
|
+
params=self.query_params,
|
|
728
|
+
timeout=None)
|
|
610
729
|
|
|
611
730
|
def get_current_record_count_in_folio(self):
|
|
612
731
|
if "query_endpoint" in self.api_info:
|
|
613
732
|
url = f"{self.folio_client.gateway_url}{self.api_info['query_endpoint']}"
|
|
614
733
|
query_params = {"query": "cql.allRecords=1", "limit": 0}
|
|
615
734
|
if self.http_client and not self.http_client.is_closed:
|
|
616
|
-
res = self.http_client.get(
|
|
735
|
+
res = self.http_client.get(
|
|
736
|
+
url,
|
|
737
|
+
headers=self.folio_client.okapi_headers,
|
|
738
|
+
params=query_params
|
|
739
|
+
)
|
|
617
740
|
else:
|
|
618
741
|
res = httpx.get(url, headers=self.okapi_headers, params=query_params, timeout=None)
|
|
619
742
|
try:
|
|
@@ -623,11 +746,15 @@ class BatchPoster(MigrationTaskBase):
|
|
|
623
746
|
logging.error("Failed to get current record count. HTTP %s", res.status_code)
|
|
624
747
|
return 0
|
|
625
748
|
except KeyError:
|
|
626
|
-
logging.error(
|
|
749
|
+
logging.error(
|
|
750
|
+
"Failed to get current record count. "
|
|
751
|
+
f"No 'totalRecords' in response: {res.json()}"
|
|
752
|
+
)
|
|
627
753
|
return 0
|
|
628
754
|
else:
|
|
629
755
|
raise ValueError(
|
|
630
|
-
"No 'query_endpoint' available for %s. Cannot get current record count.",
|
|
756
|
+
"No 'query_endpoint' available for %s. Cannot get current record count.",
|
|
757
|
+
self.task_configuration.object_type
|
|
631
758
|
)
|
|
632
759
|
|
|
633
760
|
def get_starting_record_count(self):
|
|
@@ -635,14 +762,20 @@ class BatchPoster(MigrationTaskBase):
|
|
|
635
762
|
logging.info("Getting starting record count in FOLIO")
|
|
636
763
|
self.starting_record_count_in_folio = self.get_current_record_count_in_folio()
|
|
637
764
|
else:
|
|
638
|
-
logging.info(
|
|
765
|
+
logging.info(
|
|
766
|
+
"No query_endpoint available for %s. Cannot get starting record count.",
|
|
767
|
+
self.task_configuration.object_type
|
|
768
|
+
)
|
|
639
769
|
|
|
640
770
|
def get_finished_record_count(self):
|
|
641
771
|
if "query_endpoint" in self.api_info:
|
|
642
772
|
logging.info("Getting finished record count in FOLIO")
|
|
643
773
|
self.finished_record_count_in_folio = self.get_current_record_count_in_folio()
|
|
644
774
|
else:
|
|
645
|
-
logging.info(
|
|
775
|
+
logging.info(
|
|
776
|
+
"No query_endpoint available for %s. Cannot get ending record count.",
|
|
777
|
+
self.task_configuration.object_type
|
|
778
|
+
)
|
|
646
779
|
|
|
647
780
|
def wrap_up(self):
|
|
648
781
|
logging.info("Done. Wrapping up")
|
|
@@ -663,7 +796,9 @@ class BatchPoster(MigrationTaskBase):
|
|
|
663
796
|
logging.info("Done posting %s records. %s failed", self.num_posted, self.num_failures)
|
|
664
797
|
if self.starting_record_count_in_folio:
|
|
665
798
|
self.get_finished_record_count()
|
|
666
|
-
total_on_server =
|
|
799
|
+
total_on_server = (
|
|
800
|
+
self.finished_record_count_in_folio - self.starting_record_count_in_folio
|
|
801
|
+
)
|
|
667
802
|
discrepancy = self.processed - self.num_failures - total_on_server
|
|
668
803
|
if discrepancy != 0:
|
|
669
804
|
logging.error(
|
|
@@ -712,7 +847,10 @@ class BatchPoster(MigrationTaskBase):
|
|
|
712
847
|
temp_report = copy.deepcopy(self.migration_report)
|
|
713
848
|
temp_start = self.start_datetime
|
|
714
849
|
self.task_configuration.rerun_failed_records = False
|
|
715
|
-
self.__init__(
|
|
850
|
+
self.__init__(
|
|
851
|
+
self.task_configuration,
|
|
852
|
+
self.library_configuration,
|
|
853
|
+
self.folio_client)
|
|
716
854
|
self.performing_rerun = True
|
|
717
855
|
self.migration_report = temp_report
|
|
718
856
|
self.start_datetime = temp_start
|
|
@@ -747,9 +885,11 @@ class BatchPoster(MigrationTaskBase):
|
|
|
747
885
|
res = httpx.post(url, headers=self.okapi_headers, json=snapshot, timeout=None)
|
|
748
886
|
res.raise_for_status()
|
|
749
887
|
logging.info("Posted Snapshot to FOLIO: %s", json.dumps(snapshot, indent=4))
|
|
750
|
-
get_url =
|
|
751
|
-
|
|
752
|
-
|
|
888
|
+
get_url = (
|
|
889
|
+
f"{self.folio_client.gateway_url}/source-storage/snapshots/{self.snapshot_id}"
|
|
890
|
+
)
|
|
891
|
+
got = False
|
|
892
|
+
while not got:
|
|
753
893
|
logging.info("Sleeping while waiting for the snapshot to get created")
|
|
754
894
|
time.sleep(5)
|
|
755
895
|
if self.http_client and not self.http_client.is_closed:
|
|
@@ -757,11 +897,14 @@ class BatchPoster(MigrationTaskBase):
|
|
|
757
897
|
else:
|
|
758
898
|
res = httpx.get(get_url, headers=self.okapi_headers, timeout=None)
|
|
759
899
|
if res.status_code == 200:
|
|
760
|
-
|
|
900
|
+
got = True
|
|
761
901
|
else:
|
|
762
902
|
logging.info(res.status_code)
|
|
763
|
-
except
|
|
764
|
-
logging.exception("
|
|
903
|
+
except httpx.HTTPStatusError as exc:
|
|
904
|
+
logging.exception("HTTP error occurred while posting the snapshot: %s", exc)
|
|
905
|
+
sys.exit(1)
|
|
906
|
+
except Exception as exc:
|
|
907
|
+
logging.exception("Could not post the snapshot: %s", exc)
|
|
765
908
|
sys.exit(1)
|
|
766
909
|
|
|
767
910
|
def commit_snapshot(self):
|
|
@@ -776,11 +919,15 @@ class BatchPoster(MigrationTaskBase):
|
|
|
776
919
|
res = httpx.put(url, headers=self.okapi_headers, json=snapshot, timeout=None)
|
|
777
920
|
res.raise_for_status()
|
|
778
921
|
logging.info("Posted Committed snapshot to FOLIO: %s", json.dumps(snapshot, indent=4))
|
|
779
|
-
except
|
|
922
|
+
except httpx.HTTPStatusError as exc:
|
|
923
|
+
logging.exception("HTTP error occurred while posting the snapshot: %s", exc)
|
|
924
|
+
sys.exit(1)
|
|
925
|
+
except Exception as exc:
|
|
780
926
|
logging.exception(
|
|
781
927
|
"Could not commit snapshot with id %s. Post this to /source-storage/snapshots/%s:",
|
|
782
928
|
self.snapshot_id,
|
|
783
929
|
self.snapshot_id,
|
|
930
|
+
exc,
|
|
784
931
|
)
|
|
785
932
|
logging.info("%s", json.dumps(snapshot, indent=4))
|
|
786
933
|
sys.exit(1)
|
|
@@ -891,8 +1038,11 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
891
1038
|
try:
|
|
892
1039
|
return choices[object_type]
|
|
893
1040
|
except KeyError:
|
|
894
|
-
key_string = ",".join(choices.keys())
|
|
895
|
-
logging.error(
|
|
1041
|
+
key_string = ", ".join(choices.keys())
|
|
1042
|
+
logging.error(
|
|
1043
|
+
f"Wrong type. Only one of {key_string} are allowed, "
|
|
1044
|
+
f"received {object_type=} instead"
|
|
1045
|
+
)
|
|
896
1046
|
logging.error("Halting")
|
|
897
1047
|
sys.exit(1)
|
|
898
1048
|
|
|
@@ -908,7 +1058,7 @@ def chunks(records, number_of_chunks):
|
|
|
908
1058
|
_type_: _description_
|
|
909
1059
|
"""
|
|
910
1060
|
for i in range(0, len(records), number_of_chunks):
|
|
911
|
-
yield records[i
|
|
1061
|
+
yield records[i: i + number_of_chunks]
|
|
912
1062
|
|
|
913
1063
|
|
|
914
1064
|
def get_human_readable(size, precision=2):
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
'''Main "script."'''
|
|
2
|
-
|
|
3
1
|
import csv
|
|
4
2
|
import json
|
|
5
3
|
import logging
|
|
@@ -19,7 +17,10 @@ from folio_migration_tools.library_configuration import (
|
|
|
19
17
|
from folio_migration_tools.marc_rules_transformation.rules_mapper_holdings import (
|
|
20
18
|
RulesMapperHoldings,
|
|
21
19
|
)
|
|
22
|
-
from folio_migration_tools.migration_tasks.migration_task_base import
|
|
20
|
+
from folio_migration_tools.migration_tasks.migration_task_base import (
|
|
21
|
+
MarcTaskConfigurationBase,
|
|
22
|
+
MigrationTaskBase
|
|
23
|
+
)
|
|
23
24
|
|
|
24
25
|
|
|
25
26
|
class HoldingsMarcTransformer(MigrationTaskBase):
|
|
@@ -37,14 +38,18 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
37
38
|
str,
|
|
38
39
|
Field(
|
|
39
40
|
title="Migration task type",
|
|
40
|
-
description=(
|
|
41
|
+
description=(
|
|
42
|
+
"The type of migration task you want to perform"
|
|
43
|
+
),
|
|
41
44
|
),
|
|
42
45
|
]
|
|
43
46
|
files: Annotated[
|
|
44
47
|
List[FileDefinition],
|
|
45
48
|
Field(
|
|
46
49
|
title="Source files",
|
|
47
|
-
description=(
|
|
50
|
+
description=(
|
|
51
|
+
"List of MARC21 files with holdings records"
|
|
52
|
+
),
|
|
48
53
|
),
|
|
49
54
|
]
|
|
50
55
|
hrid_handling: Annotated[
|
|
@@ -125,8 +130,8 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
125
130
|
default_call_number_type_name: Annotated[
|
|
126
131
|
str,
|
|
127
132
|
Field(
|
|
128
|
-
title="Default
|
|
129
|
-
description="The name of the
|
|
133
|
+
title="Default call_number type name",
|
|
134
|
+
description="The name of the call_number type that will be used as fallback",
|
|
130
135
|
),
|
|
131
136
|
]
|
|
132
137
|
fallback_holdings_type_id: Annotated[
|
|
@@ -140,7 +145,10 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
140
145
|
str,
|
|
141
146
|
Field(
|
|
142
147
|
title="Supplemental MFHD mapping rules file",
|
|
143
|
-
description=
|
|
148
|
+
description=(
|
|
149
|
+
"The name of the file in the mapping_files directory "
|
|
150
|
+
"containing supplemental MFHD mapping rules"
|
|
151
|
+
),
|
|
144
152
|
),
|
|
145
153
|
] = ""
|
|
146
154
|
include_mrk_statements: Annotated[
|
|
@@ -148,8 +156,10 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
148
156
|
Field(
|
|
149
157
|
title="Include MARC statements (MRK-format) as staff-only Holdings notes",
|
|
150
158
|
description=(
|
|
151
|
-
"If set to true, the MARC statements
|
|
152
|
-
"
|
|
159
|
+
"If set to true, the MARC statements "
|
|
160
|
+
"will be included in the output as MARC Maker format fields. "
|
|
161
|
+
"If set to false (default), the MARC statements "
|
|
162
|
+
"will not be included in the output."
|
|
153
163
|
),
|
|
154
164
|
),
|
|
155
165
|
] = False
|
|
@@ -188,7 +198,8 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
188
198
|
title="Include MARC Record (as MARC21 decoded string) as note",
|
|
189
199
|
description=(
|
|
190
200
|
"If set to true, the MARC record will be included in the output as a "
|
|
191
|
-
"decoded binary MARC21 record. If set to false (default),
|
|
201
|
+
"decoded binary MARC21 record. If set to false (default), "
|
|
202
|
+
"the MARC record will not be "
|
|
192
203
|
"included in the output."
|
|
193
204
|
),
|
|
194
205
|
),
|
|
@@ -127,7 +127,8 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
127
127
|
title="Statistical code map file name",
|
|
128
128
|
description=(
|
|
129
129
|
"Path to the file containing the mapping of statistical codes. "
|
|
130
|
-
"The file should be in TSV format with legacy_stat_code
|
|
130
|
+
"The file should be in TSV format with legacy_stat_code "
|
|
131
|
+
"and folio_code columns."
|
|
131
132
|
),
|
|
132
133
|
),
|
|
133
134
|
] = ""
|
|
@@ -217,7 +218,10 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
217
218
|
self.folio_keys = MappingFileMapperBase.get_mapped_folio_properties_from_map(
|
|
218
219
|
self.items_map
|
|
219
220
|
)
|
|
220
|
-
if
|
|
221
|
+
if (
|
|
222
|
+
any(k for k in self.folio_keys if k.startswith("statisticalCodeIds"))
|
|
223
|
+
or any(getattr(k, "statistical_code", "") for k in self.task_configuration.files)
|
|
224
|
+
):
|
|
221
225
|
statcode_mapping = self.load_ref_data_mapping_file(
|
|
222
226
|
"statisticalCodeIds",
|
|
223
227
|
self.folder_structure.mapping_files_folder
|
|
@@ -355,7 +359,7 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
355
359
|
)
|
|
356
360
|
|
|
357
361
|
self.mapper.perform_additional_mappings(legacy_id, folio_rec, file_def)
|
|
358
|
-
self.
|
|
362
|
+
self.handle_circulation_notes(folio_rec, self.folio_client.current_user)
|
|
359
363
|
self.handle_notes(folio_rec)
|
|
360
364
|
if folio_rec["holdingsRecordId"] in self.boundwith_relationship_map:
|
|
361
365
|
for idx_, instance_id in enumerate(
|
|
@@ -373,7 +377,7 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
373
377
|
if idx == 0:
|
|
374
378
|
logging.info("First FOLIO record:")
|
|
375
379
|
logging.info(json.dumps(folio_rec, indent=4))
|
|
376
|
-
# TODO: turn this into a
|
|
380
|
+
# TODO: turn this into a asynchronous task
|
|
377
381
|
Helper.write_to_file(results_file, folio_rec)
|
|
378
382
|
self.mapper.migration_report.add_general_statistics(
|
|
379
383
|
i18n.t("Number of records written to disk")
|
|
@@ -388,8 +392,8 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
388
392
|
logging.fatal(attribute_error)
|
|
389
393
|
logging.info("Quitting...")
|
|
390
394
|
sys.exit(1)
|
|
391
|
-
except Exception as
|
|
392
|
-
self.mapper.handle_generic_exception(idx,
|
|
395
|
+
except Exception as exception:
|
|
396
|
+
self.mapper.handle_generic_exception(idx, exception)
|
|
393
397
|
self.mapper.migration_report.add(
|
|
394
398
|
"GeneralStatistics",
|
|
395
399
|
i18n.t("Number of Legacy items in %{container}", container=file_def),
|
|
@@ -425,14 +429,14 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
425
429
|
del folio_object["notes"]
|
|
426
430
|
|
|
427
431
|
@staticmethod
|
|
428
|
-
def
|
|
432
|
+
def handle_circulation_notes(folio_rec, current_user_uuid):
|
|
429
433
|
if not folio_rec.get("circulationNotes", []):
|
|
430
434
|
return
|
|
431
435
|
filtered_notes = []
|
|
432
436
|
for circ_note in folio_rec.get("circulationNotes", []):
|
|
433
437
|
if circ_note.get("noteType", "") not in ["Check in", "Check out"]:
|
|
434
438
|
raise TransformationProcessError(
|
|
435
|
-
"", "Circulation Note types are not mapped
|
|
439
|
+
"", "Circulation Note types are not mapped correctly"
|
|
436
440
|
)
|
|
437
441
|
if circ_note.get("note", ""):
|
|
438
442
|
circ_note["id"] = str(uuid.uuid4())
|
|
@@ -455,11 +459,22 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
455
459
|
json.loads(x) for x in boundwith_relationship_file
|
|
456
460
|
)
|
|
457
461
|
logging.info(
|
|
458
|
-
|
|
462
|
+
"Rows in Bound with relationship map: %s",
|
|
463
|
+
len(self.boundwith_relationship_map)
|
|
459
464
|
)
|
|
460
465
|
except FileNotFoundError:
|
|
461
466
|
raise TransformationProcessError(
|
|
462
|
-
|
|
467
|
+
"",
|
|
468
|
+
"Boundwith relationship file specified, but relationships file "
|
|
469
|
+
"from holdings transformation not found.",
|
|
470
|
+
self.folder_structure.boundwith_relationships_map_path
|
|
471
|
+
)
|
|
472
|
+
except ValueError:
|
|
473
|
+
raise TransformationProcessError(
|
|
474
|
+
"",
|
|
475
|
+
"Boundwith relationship file specified, but relationships file "
|
|
476
|
+
"from holdings transformation is not a valid line JSON.",
|
|
477
|
+
self.folder_structure.boundwith_relationships_map_path,
|
|
463
478
|
)
|
|
464
479
|
|
|
465
480
|
def wrap_up(self):
|