folio-migration-tools 1.9.0rc9__py3-none-any.whl → 1.9.0rc11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__main__.py +23 -5
- folio_migration_tools/circulation_helper.py +3 -3
- folio_migration_tools/library_configuration.py +54 -6
- folio_migration_tools/mapper_base.py +2 -2
- folio_migration_tools/marc_rules_transformation/hrid_handler.py +1 -1
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +21 -23
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +3 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +52 -37
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +131 -21
- folio_migration_tools/migration_tasks/batch_poster.py +7 -7
- folio_migration_tools/migration_tasks/bibs_transformer.py +6 -59
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +61 -23
- folio_migration_tools/migration_tasks/loans_migrator.py +5 -5
- folio_migration_tools/migration_tasks/migration_task_base.py +64 -1
- folio_migration_tools/migration_tasks/reserves_migrator.py +1 -1
- folio_migration_tools/task_configuration.py +18 -1
- folio_migration_tools/test_infrastructure/mocked_classes.py +94 -0
- folio_migration_tools/transaction_migration/legacy_loan.py +14 -12
- folio_migration_tools/transaction_migration/legacy_reserve.py +1 -1
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/METADATA +2 -2
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/RECORD +24 -24
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/LICENSE +0 -0
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/WHEEL +0 -0
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/entry_points.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import copy
|
|
2
2
|
import json
|
|
3
3
|
import logging
|
|
4
|
-
from typing import List
|
|
4
|
+
from typing import Dict, List, Set
|
|
5
5
|
|
|
6
6
|
import i18n
|
|
7
7
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
@@ -34,7 +34,7 @@ from folio_migration_tools.marc_rules_transformation.rules_mapper_base import (
|
|
|
34
34
|
class RulesMapperHoldings(RulesMapperBase):
|
|
35
35
|
def __init__(
|
|
36
36
|
self,
|
|
37
|
-
folio_client,
|
|
37
|
+
folio_client: FolioClient,
|
|
38
38
|
location_map,
|
|
39
39
|
task_configuration,
|
|
40
40
|
library_configuration: LibraryConfiguration,
|
|
@@ -235,10 +235,10 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
235
235
|
|
|
236
236
|
def process_marc_field(
|
|
237
237
|
self,
|
|
238
|
-
folio_holding:
|
|
238
|
+
folio_holding: Dict,
|
|
239
239
|
marc_field: Field,
|
|
240
|
-
ignored_subsequent_fields,
|
|
241
|
-
index_or_legacy_ids,
|
|
240
|
+
ignored_subsequent_fields: Set,
|
|
241
|
+
index_or_legacy_ids: List[str],
|
|
242
242
|
):
|
|
243
243
|
"""This overwrites the implementation for Auth and instances
|
|
244
244
|
|
|
@@ -261,7 +261,7 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
261
261
|
ignored_subsequent_fields.add(marc_field.tag)
|
|
262
262
|
|
|
263
263
|
def perform_additional_mapping(
|
|
264
|
-
self, marc_record: Record, folio_holding, legacy_ids: List[str], file_def: FileDefinition
|
|
264
|
+
self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str], file_def: FileDefinition
|
|
265
265
|
):
|
|
266
266
|
"""_summary_
|
|
267
267
|
|
|
@@ -278,12 +278,11 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
278
278
|
self.set_default_call_number_type_if_empty(folio_holding)
|
|
279
279
|
self.pick_first_location_if_many(folio_holding, legacy_ids)
|
|
280
280
|
self.parse_coded_holdings_statements(marc_record, folio_holding, legacy_ids)
|
|
281
|
+
self.add_mfhd_as_mrk_note(marc_record, folio_holding, legacy_ids)
|
|
282
|
+
self.add_mfhd_as_mrc_note(marc_record, folio_holding, legacy_ids)
|
|
281
283
|
HoldingsHelper.handle_notes(folio_holding)
|
|
282
|
-
create_source_records = all(
|
|
283
|
-
[file_def.create_source_records, self.task_configuration.create_source_records]
|
|
284
|
-
)
|
|
285
284
|
if (
|
|
286
|
-
create_source_records
|
|
285
|
+
all([file_def.create_source_records, self.create_source_records])
|
|
287
286
|
or self.task_configuration.hrid_handling == HridHandling.preserve001
|
|
288
287
|
):
|
|
289
288
|
self.hrid_handler.handle_hrid(
|
|
@@ -298,9 +297,9 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
298
297
|
"",
|
|
299
298
|
)
|
|
300
299
|
self.handle_suppression(folio_holding, file_def, True)
|
|
301
|
-
self.set_source_id(self.
|
|
300
|
+
self.set_source_id(self.create_source_records, folio_holding, self.holdingssources, file_def)
|
|
302
301
|
|
|
303
|
-
def pick_first_location_if_many(self, folio_holding, legacy_ids: List[str]):
|
|
302
|
+
def pick_first_location_if_many(self, folio_holding: Dict, legacy_ids: List[str]):
|
|
304
303
|
if " " in folio_holding.get("permanentLocationId", ""):
|
|
305
304
|
Helper.log_data_issue(
|
|
306
305
|
legacy_ids,
|
|
@@ -312,14 +311,14 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
312
311
|
]
|
|
313
312
|
|
|
314
313
|
@staticmethod
|
|
315
|
-
def set_source_id(
|
|
316
|
-
if file_def.create_source_records and
|
|
314
|
+
def set_source_id(create_source_records: bool, folio_rec: Dict, holdingssources: Dict, file_def: FileDefinition):
|
|
315
|
+
if file_def.create_source_records and create_source_records:
|
|
317
316
|
folio_rec["sourceId"] = holdingssources.get("MARC")
|
|
318
317
|
else:
|
|
319
318
|
folio_rec["sourceId"] = holdingssources.get("FOLIO")
|
|
320
319
|
|
|
321
320
|
def parse_coded_holdings_statements(
|
|
322
|
-
self, marc_record: Record, folio_holding, legacy_ids: List[str]
|
|
321
|
+
self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]
|
|
323
322
|
):
|
|
324
323
|
# TODO: Should one be able to switch these things off?
|
|
325
324
|
a = {
|
|
@@ -344,6 +343,117 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
344
343
|
except TransformationFieldMappingError as tfme:
|
|
345
344
|
Helper.log_data_issue(tfme.index_or_id, tfme.message, tfme.data_value)
|
|
346
345
|
self.migration_report.add("FieldMappingErrors", tfme.message)
|
|
346
|
+
self.collect_mrk_statement_notes(marc_record, folio_holding, legacy_ids)
|
|
347
|
+
|
|
348
|
+
def collect_mrk_statement_notes(self, marc_record, folio_holding, legacy_ids):
|
|
349
|
+
"""Collects MFHD holdings statements as MARC Maker field strings in a FOLIO holdings note
|
|
350
|
+
and adds them to the FOLIO holdings record.
|
|
351
|
+
|
|
352
|
+
This is done to preserve the information in the MARC record for future reference.
|
|
353
|
+
|
|
354
|
+
Args:
|
|
355
|
+
marc_record (Record): PyMARC record
|
|
356
|
+
folio_holding (Dict): FOLIO holdings record
|
|
357
|
+
|
|
358
|
+
"""
|
|
359
|
+
if self.task_configuration.include_mrk_statements:
|
|
360
|
+
mrk_statement_notes = []
|
|
361
|
+
for field in marc_record.get_fields("853", "854", "855", "863", "864", "865", "866", "867", "868"):
|
|
362
|
+
mrk_statement_notes.append(str(field))
|
|
363
|
+
if mrk_statement_notes:
|
|
364
|
+
folio_holding["notes"] = folio_holding.get("notes", []) + self.add_mrk_statements_note(mrk_statement_notes, legacy_ids)
|
|
365
|
+
|
|
366
|
+
def add_mrk_statements_note(self, mrk_statement_notes: List[str], legacy_ids) -> List[Dict]:
|
|
367
|
+
"""Creates a note from the MRK statements
|
|
368
|
+
|
|
369
|
+
Args:
|
|
370
|
+
mrk_statement_notes (List[str]): A list of MFHD holdings statements as MRK strings
|
|
371
|
+
|
|
372
|
+
Returns:
|
|
373
|
+
List: A list containing the FOLIO holdings note object (Dict)
|
|
374
|
+
"""
|
|
375
|
+
holdings_note_type_tuple = self.conditions.get_ref_data_tuple_by_name(
|
|
376
|
+
self.folio.holding_note_types, "holding_note_types", self.task_configuration.mrk_holdings_note_type
|
|
377
|
+
)
|
|
378
|
+
try:
|
|
379
|
+
holdings_note_type_id = holdings_note_type_tuple[0]
|
|
380
|
+
except Exception as ee:
|
|
381
|
+
logging.error(ee)
|
|
382
|
+
raise TransformationRecordFailedError(
|
|
383
|
+
legacy_ids,
|
|
384
|
+
f'Holdings note type mapping error.\tNote type name: {self.task_configuration.mrk_holdings_note_type}\t'
|
|
385
|
+
f"MFHD holdings statement note type not found in FOLIO.",
|
|
386
|
+
self.task_configuration.mrk_holdings_note_type,
|
|
387
|
+
) from ee
|
|
388
|
+
return [
|
|
389
|
+
{
|
|
390
|
+
"note": "\n".join(mrk_statement_notes),
|
|
391
|
+
"holdingsNoteTypeId": holdings_note_type_id,
|
|
392
|
+
"staffOnly": True,
|
|
393
|
+
}
|
|
394
|
+
]
|
|
395
|
+
|
|
396
|
+
def add_mfhd_as_mrk_note(self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]):
|
|
397
|
+
"""Adds the MFHD as a note to the holdings record
|
|
398
|
+
|
|
399
|
+
This is done to preserve the information in the MARC record for future reference.
|
|
400
|
+
|
|
401
|
+
Args:
|
|
402
|
+
marc_record (Record): PyMARC record
|
|
403
|
+
folio_holding (Dict): FOLIO holdings record
|
|
404
|
+
"""
|
|
405
|
+
holdings_note_type_tuple = self.conditions.get_ref_data_tuple_by_name(
|
|
406
|
+
self.folio.holding_note_types, "holding_note_types", self.task_configuration.mfhd_mrk_note_type
|
|
407
|
+
)
|
|
408
|
+
try:
|
|
409
|
+
holdings_note_type_id = holdings_note_type_tuple[0]
|
|
410
|
+
except Exception as ee:
|
|
411
|
+
logging.error(ee)
|
|
412
|
+
raise TransformationRecordFailedError(
|
|
413
|
+
legacy_ids,
|
|
414
|
+
f'Holdings note type mapping error.\tNote type name: {self.task_configuration.mfhd_mrk_note_type}\t'
|
|
415
|
+
f"Note type not found in FOLIO.",
|
|
416
|
+
self.task_configuration.mfhd_mrk_note_type,
|
|
417
|
+
) from ee
|
|
418
|
+
if self.task_configuration.include_mfhd_mrk_as_note:
|
|
419
|
+
folio_holding["notes"] = folio_holding.get("notes", []) + [
|
|
420
|
+
{
|
|
421
|
+
"note": str(marc_record),
|
|
422
|
+
"holdingsNoteTypeId": holdings_note_type_id,
|
|
423
|
+
"staffOnly": True,
|
|
424
|
+
}
|
|
425
|
+
]
|
|
426
|
+
|
|
427
|
+
def add_mfhd_as_mrc_note(self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]):
|
|
428
|
+
"""Adds the MFHD as a note to the holdings record
|
|
429
|
+
|
|
430
|
+
This is done to preserve the information in the MARC record for future reference.
|
|
431
|
+
|
|
432
|
+
Args:
|
|
433
|
+
marc_record (Record): PyMARC record
|
|
434
|
+
folio_holding (Dict): FOLIO holdings record
|
|
435
|
+
"""
|
|
436
|
+
holdings_note_type_tuple = self.conditions.get_ref_data_tuple_by_name(
|
|
437
|
+
self.folio.holding_note_types, "holding_note_types", self.task_configuration.mfhd_mrc_note_type
|
|
438
|
+
)
|
|
439
|
+
try:
|
|
440
|
+
holdings_note_type_id = holdings_note_type_tuple[0]
|
|
441
|
+
except Exception as ee:
|
|
442
|
+
logging.error(ee)
|
|
443
|
+
raise TransformationRecordFailedError(
|
|
444
|
+
legacy_ids,
|
|
445
|
+
f'Holdings note type mapping error.\tNote type name: {self.task_configuration.mfhd_mrc_note_type}\t'
|
|
446
|
+
f"Note type not found in FOLIO.",
|
|
447
|
+
self.task_configuration.mfhd_mrc_note_type,
|
|
448
|
+
) from ee
|
|
449
|
+
if self.task_configuration.include_mfhd_mrc_as_note:
|
|
450
|
+
folio_holding["notes"] = folio_holding.get("notes", []) + [
|
|
451
|
+
{
|
|
452
|
+
"note": marc_record.as_marc().decode("utf-8"),
|
|
453
|
+
"holdingsNoteTypeId": holdings_note_type_id,
|
|
454
|
+
"staffOnly": True,
|
|
455
|
+
}
|
|
456
|
+
]
|
|
347
457
|
|
|
348
458
|
def wrap_up(self):
|
|
349
459
|
logging.info("Mapper wrapping up")
|
|
@@ -351,7 +461,7 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
351
461
|
x.create_source_records for x in self.task_configuration.files
|
|
352
462
|
]
|
|
353
463
|
if all(source_file_create_source_records):
|
|
354
|
-
create_source_records = self.
|
|
464
|
+
create_source_records = self.create_source_records
|
|
355
465
|
else:
|
|
356
466
|
logging.info(
|
|
357
467
|
"If all source files have create_source_records set to false, "
|
|
@@ -369,7 +479,7 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
369
479
|
logging.info("Fetching HoldingsRecord schema...")
|
|
370
480
|
return folio_client.get_holdings_schema()
|
|
371
481
|
|
|
372
|
-
def set_holdings_type(self, marc_record: Record, folio_holding, legacy_ids: List[str]):
|
|
482
|
+
def set_holdings_type(self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]):
|
|
373
483
|
# Holdings type mapping
|
|
374
484
|
ldr06 = marc_record.leader[6]
|
|
375
485
|
# TODO: map this better
|
|
@@ -425,7 +535,7 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
425
535
|
ldr06,
|
|
426
536
|
)
|
|
427
537
|
|
|
428
|
-
def set_default_call_number_type_if_empty(self, folio_holding):
|
|
538
|
+
def set_default_call_number_type_if_empty(self, folio_holding: Dict):
|
|
429
539
|
if not folio_holding.get("callNumberTypeId", ""):
|
|
430
540
|
folio_holding["callNumberTypeId"] = self.conditions.default_call_number_type["id"]
|
|
431
541
|
|
|
@@ -462,7 +572,7 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
462
572
|
)
|
|
463
573
|
return results
|
|
464
574
|
|
|
465
|
-
def verity_boundwith_map_entry(self, entry):
|
|
575
|
+
def verity_boundwith_map_entry(self, entry: Dict):
|
|
466
576
|
if "MFHD_ID" not in entry or not entry.get("MFHD_ID", ""):
|
|
467
577
|
raise TransformationProcessError(
|
|
468
578
|
"", "Column MFHD_ID missing from Boundwith relationship map", ""
|
|
@@ -472,7 +582,7 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
472
582
|
"", "Column BIB_ID missing from Boundwith relationship map", ""
|
|
473
583
|
)
|
|
474
584
|
|
|
475
|
-
def setup_boundwith_relationship_map(self, boundwith_relationship_map):
|
|
585
|
+
def setup_boundwith_relationship_map(self, boundwith_relationship_map: List[Dict]):
|
|
476
586
|
"""
|
|
477
587
|
Creates a map of MFHD_ID to BIB_ID for boundwith relationships.
|
|
478
588
|
|
|
@@ -503,7 +613,7 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
503
613
|
self.handle_transformation_record_failed_error(idx, trfe)
|
|
504
614
|
return new_map
|
|
505
615
|
|
|
506
|
-
def get_bw_instance_id_map_tuple(self, entry):
|
|
616
|
+
def get_bw_instance_id_map_tuple(self, entry: Dict):
|
|
507
617
|
try:
|
|
508
618
|
return self.parent_id_map[entry["BIB_ID"]]
|
|
509
619
|
except KeyError:
|
|
@@ -287,7 +287,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
287
287
|
fetch_batch_size = 90
|
|
288
288
|
fetch_tasks = []
|
|
289
289
|
updates = {}
|
|
290
|
-
async with httpx.AsyncClient(base_url=self.folio_client.
|
|
290
|
+
async with httpx.AsyncClient(base_url=self.folio_client.gateway_url) as client:
|
|
291
291
|
for i in range(0, len(batch), fetch_batch_size):
|
|
292
292
|
batch_slice = batch[i:i + fetch_batch_size]
|
|
293
293
|
fetch_tasks.append(
|
|
@@ -360,7 +360,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
360
360
|
def post_extra_data(self, row: str, num_records: int, failed_recs_file):
|
|
361
361
|
(object_name, data) = row.split("\t")
|
|
362
362
|
endpoint = self.get_extradata_endpoint(self.task_configuration, object_name, data)
|
|
363
|
-
url = f"{self.folio_client.
|
|
363
|
+
url = f"{self.folio_client.gateway_url}/{endpoint}"
|
|
364
364
|
body = data
|
|
365
365
|
response = self.post_objects(url, body)
|
|
366
366
|
if response.status_code == 201:
|
|
@@ -415,7 +415,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
415
415
|
if self.api_info["is_batch"]:
|
|
416
416
|
raise TypeError("This record type supports batch processing, use post_batch method")
|
|
417
417
|
api_endpoint = self.api_info.get("api_endpoint")
|
|
418
|
-
url = f"{self.folio_client.
|
|
418
|
+
url = f"{self.folio_client.gateway_url}{api_endpoint}"
|
|
419
419
|
response = self.post_objects(url, row)
|
|
420
420
|
if response.status_code == 201:
|
|
421
421
|
self.num_posted += 1
|
|
@@ -590,7 +590,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
590
590
|
|
|
591
591
|
def do_post(self, batch):
|
|
592
592
|
path = self.api_info["api_endpoint"]
|
|
593
|
-
url = self.folio_client.
|
|
593
|
+
url = self.folio_client.gateway_url + path
|
|
594
594
|
if self.api_info["object_name"] == "users":
|
|
595
595
|
payload = {self.api_info["object_name"]: list(batch), "totalRecords": len(batch)}
|
|
596
596
|
elif self.api_info["total_records"]:
|
|
@@ -675,7 +675,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
675
675
|
"processingStartedDate": datetime.utcnow().isoformat(timespec="milliseconds"),
|
|
676
676
|
}
|
|
677
677
|
try:
|
|
678
|
-
url = f"{self.folio_client.
|
|
678
|
+
url = f"{self.folio_client.gateway_url}/source-storage/snapshots"
|
|
679
679
|
if self.http_client and not self.http_client.is_closed:
|
|
680
680
|
res = self.http_client.post(
|
|
681
681
|
url, json=snapshot, headers=self.folio_client.okapi_headers
|
|
@@ -684,7 +684,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
684
684
|
res = httpx.post(url, headers=self.okapi_headers, json=snapshot, timeout=None)
|
|
685
685
|
res.raise_for_status()
|
|
686
686
|
logging.info("Posted Snapshot to FOLIO: %s", json.dumps(snapshot, indent=4))
|
|
687
|
-
get_url = f"{self.folio_client.
|
|
687
|
+
get_url = f"{self.folio_client.gateway_url}/source-storage/snapshots/{self.snapshot_id}"
|
|
688
688
|
getted = False
|
|
689
689
|
while not getted:
|
|
690
690
|
logging.info("Sleeping while waiting for the snapshot to get created")
|
|
@@ -704,7 +704,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
704
704
|
def commit_snapshot(self):
|
|
705
705
|
snapshot = {"jobExecutionId": self.snapshot_id, "status": "COMMITTED"}
|
|
706
706
|
try:
|
|
707
|
-
url = f"{self.folio_client.
|
|
707
|
+
url = f"{self.folio_client.gateway_url}/source-storage/snapshots/{self.snapshot_id}"
|
|
708
708
|
if self.http_client and not self.http_client.is_closed:
|
|
709
709
|
res = self.http_client.put(
|
|
710
710
|
url, json=snapshot, headers=self.folio_client.okapi_headers
|
|
@@ -18,39 +18,17 @@ from folio_migration_tools.marc_rules_transformation.marc_file_processor import
|
|
|
18
18
|
from folio_migration_tools.marc_rules_transformation.rules_mapper_bibs import (
|
|
19
19
|
BibsRulesMapper,
|
|
20
20
|
)
|
|
21
|
-
from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
|
|
22
|
-
from folio_migration_tools.task_configuration import AbstractTaskConfiguration
|
|
21
|
+
from folio_migration_tools.migration_tasks.migration_task_base import MarcTaskConfigurationBase, MigrationTaskBase
|
|
23
22
|
|
|
24
23
|
|
|
25
24
|
class BibsTransformer(MigrationTaskBase):
|
|
26
|
-
class TaskConfiguration(
|
|
27
|
-
name: Annotated[
|
|
28
|
-
str,
|
|
29
|
-
Field(
|
|
30
|
-
description=(
|
|
31
|
-
"Name of this migration task. The name is being used to call the specific "
|
|
32
|
-
"task, and to distinguish tasks of similar types"
|
|
33
|
-
)
|
|
34
|
-
),
|
|
35
|
-
]
|
|
36
|
-
migration_task_type: Annotated[
|
|
37
|
-
str,
|
|
38
|
-
Field(
|
|
39
|
-
title="Migration task type",
|
|
40
|
-
description=("The type of migration task you want to perform."),
|
|
41
|
-
),
|
|
42
|
-
]
|
|
43
|
-
files: Annotated[
|
|
44
|
-
List[FileDefinition],
|
|
45
|
-
Field(
|
|
46
|
-
title="Source files",
|
|
47
|
-
description=("List of MARC21 files with bibliographic records."),
|
|
48
|
-
),
|
|
49
|
-
]
|
|
25
|
+
class TaskConfiguration(MarcTaskConfigurationBase):
|
|
50
26
|
ils_flavour: Annotated[
|
|
51
27
|
IlsFlavour,
|
|
52
28
|
Field(
|
|
53
|
-
title="ILS flavour",
|
|
29
|
+
title="ILS flavour",
|
|
30
|
+
description="The type of ILS you are migrating records from.",
|
|
31
|
+
alias="ils_flavor"
|
|
54
32
|
),
|
|
55
33
|
]
|
|
56
34
|
custom_bib_id_field: Annotated[
|
|
@@ -87,16 +65,6 @@ class BibsTransformer(MigrationTaskBase):
|
|
|
87
65
|
),
|
|
88
66
|
),
|
|
89
67
|
] = []
|
|
90
|
-
create_source_records: Annotated[
|
|
91
|
-
bool,
|
|
92
|
-
Field(
|
|
93
|
-
title="Create source records",
|
|
94
|
-
description=(
|
|
95
|
-
"Controls wheter or not to retain the MARC records in "
|
|
96
|
-
"Source Record Storage."
|
|
97
|
-
),
|
|
98
|
-
),
|
|
99
|
-
] = True
|
|
100
68
|
data_import_marc: Annotated[
|
|
101
69
|
bool,
|
|
102
70
|
Field(
|
|
@@ -107,7 +75,7 @@ class BibsTransformer(MigrationTaskBase):
|
|
|
107
75
|
"of FOLIO instance records (and optional SRS records) will be generated."
|
|
108
76
|
),
|
|
109
77
|
)
|
|
110
|
-
] =
|
|
78
|
+
] = True
|
|
111
79
|
parse_cataloged_date: Annotated[
|
|
112
80
|
bool,
|
|
113
81
|
Field(
|
|
@@ -118,17 +86,6 @@ class BibsTransformer(MigrationTaskBase):
|
|
|
118
86
|
),
|
|
119
87
|
),
|
|
120
88
|
] = False
|
|
121
|
-
hrid_handling: Annotated[
|
|
122
|
-
HridHandling,
|
|
123
|
-
Field(
|
|
124
|
-
title="HRID Handling",
|
|
125
|
-
description=(
|
|
126
|
-
"Setting to default will make FOLIO generate HRIDs and move the existing "
|
|
127
|
-
"001:s into a 035, concatenated with the 003. Choosing preserve001 means "
|
|
128
|
-
"the 001:s will remain in place, and that they will also become the HRIDs"
|
|
129
|
-
),
|
|
130
|
-
),
|
|
131
|
-
] = HridHandling.default
|
|
132
89
|
reset_hrid_settings: Annotated[
|
|
133
90
|
bool,
|
|
134
91
|
Field(
|
|
@@ -146,16 +103,6 @@ class BibsTransformer(MigrationTaskBase):
|
|
|
146
103
|
description="At the end of the run, update FOLIO with the HRID settings",
|
|
147
104
|
),
|
|
148
105
|
] = True
|
|
149
|
-
deactivate035_from001: Annotated[
|
|
150
|
-
bool,
|
|
151
|
-
Field(
|
|
152
|
-
title="Create 035 from 001 and 003",
|
|
153
|
-
description=(
|
|
154
|
-
"This deactivates the FOLIO default functionality of moving the previous 001 "
|
|
155
|
-
"into a 035, prefixed with the value from 003"
|
|
156
|
-
),
|
|
157
|
-
),
|
|
158
|
-
] = False
|
|
159
106
|
|
|
160
107
|
@staticmethod
|
|
161
108
|
def get_object_type() -> FOLIONamespaces:
|
|
@@ -19,12 +19,11 @@ from folio_migration_tools.library_configuration import (
|
|
|
19
19
|
from folio_migration_tools.marc_rules_transformation.rules_mapper_holdings import (
|
|
20
20
|
RulesMapperHoldings,
|
|
21
21
|
)
|
|
22
|
-
from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
|
|
23
|
-
from folio_migration_tools.task_configuration import AbstractTaskConfiguration
|
|
22
|
+
from folio_migration_tools.migration_tasks.migration_task_base import MarcTaskConfigurationBase, MigrationTaskBase
|
|
24
23
|
|
|
25
24
|
|
|
26
25
|
class HoldingsMarcTransformer(MigrationTaskBase):
|
|
27
|
-
class TaskConfiguration(
|
|
26
|
+
class TaskConfiguration(MarcTaskConfigurationBase):
|
|
28
27
|
name: Annotated[
|
|
29
28
|
str,
|
|
30
29
|
Field(
|
|
@@ -59,16 +58,6 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
59
58
|
),
|
|
60
59
|
),
|
|
61
60
|
] = HridHandling.default
|
|
62
|
-
deactivate035_from001: Annotated[
|
|
63
|
-
bool,
|
|
64
|
-
Field(
|
|
65
|
-
title="Create 035 from 001 and 003",
|
|
66
|
-
description=(
|
|
67
|
-
"This deactivates the FOLIO default functionality of moving the previous 001 "
|
|
68
|
-
"into a 035, prefixed with the value from 003"
|
|
69
|
-
),
|
|
70
|
-
),
|
|
71
|
-
] = False
|
|
72
61
|
holdings_type_uuid_for_boundwiths: Annotated[
|
|
73
62
|
str,
|
|
74
63
|
Field(
|
|
@@ -89,16 +78,6 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
89
78
|
),
|
|
90
79
|
),
|
|
91
80
|
] = ""
|
|
92
|
-
create_source_records: Annotated[
|
|
93
|
-
bool,
|
|
94
|
-
Field(
|
|
95
|
-
title="Create source records",
|
|
96
|
-
description=(
|
|
97
|
-
"Controls wheter or not to retain the MARC records in "
|
|
98
|
-
"Source Record Storage."
|
|
99
|
-
),
|
|
100
|
-
),
|
|
101
|
-
] = True
|
|
102
81
|
update_hrid_settings: Annotated[
|
|
103
82
|
bool,
|
|
104
83
|
Field(
|
|
@@ -164,6 +143,65 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
164
143
|
description="The name of the file in the mapping_files directory containing supplemental MFHD mapping rules",
|
|
165
144
|
),
|
|
166
145
|
] = ""
|
|
146
|
+
include_mrk_statements: Annotated[
|
|
147
|
+
bool,
|
|
148
|
+
Field(
|
|
149
|
+
title="Include MARC statements (MRK-format) as staff-only Holdings notes",
|
|
150
|
+
description=(
|
|
151
|
+
"If set to true, the MARC statements will be included in the output as MARC Maker format fields. "
|
|
152
|
+
"If set to false (default), the MARC statements will not be included in the output."
|
|
153
|
+
),
|
|
154
|
+
),
|
|
155
|
+
] = False
|
|
156
|
+
mrk_holdings_note_type: Annotated[
|
|
157
|
+
str,
|
|
158
|
+
Field(
|
|
159
|
+
title="MARC Holdings Note type",
|
|
160
|
+
description=(
|
|
161
|
+
"The name of the note type to use for MARC (MRK) statements. "
|
|
162
|
+
),
|
|
163
|
+
),
|
|
164
|
+
] = "Original MARC holdings statements"
|
|
165
|
+
include_mfhd_mrk_as_note: Annotated[
|
|
166
|
+
bool,
|
|
167
|
+
Field(
|
|
168
|
+
title="Include MARC Record (as MARC Maker Representation) as note",
|
|
169
|
+
description=(
|
|
170
|
+
"If set to true, the MARC statements will be included in the output as a "
|
|
171
|
+
"(MRK) note. If set to false (default), the MARC statements will not be "
|
|
172
|
+
"included in the output."
|
|
173
|
+
),
|
|
174
|
+
),
|
|
175
|
+
] = False
|
|
176
|
+
mfhd_mrk_note_type: Annotated[
|
|
177
|
+
str,
|
|
178
|
+
Field(
|
|
179
|
+
title="MARC Record (as MARC Maker Representation) note type",
|
|
180
|
+
description=(
|
|
181
|
+
"The name of the note type to use for MFHD (MRK) note. "
|
|
182
|
+
),
|
|
183
|
+
),
|
|
184
|
+
] = "Original MFHD Record"
|
|
185
|
+
include_mfhd_mrc_as_note: Annotated[
|
|
186
|
+
bool,
|
|
187
|
+
Field(
|
|
188
|
+
title="Include MARC Record (as MARC21 decoded string) as note",
|
|
189
|
+
description=(
|
|
190
|
+
"If set to true, the MARC record will be included in the output as a "
|
|
191
|
+
"decoded binary MARC21 record. If set to false (default), the MARC record will not be "
|
|
192
|
+
"included in the output."
|
|
193
|
+
),
|
|
194
|
+
),
|
|
195
|
+
] = False
|
|
196
|
+
mfhd_mrc_note_type: Annotated[
|
|
197
|
+
str,
|
|
198
|
+
Field(
|
|
199
|
+
title="MARC Record (as MARC21 decoded string) note type",
|
|
200
|
+
description=(
|
|
201
|
+
"The name of the note type to use for MFHD (MRC) note. "
|
|
202
|
+
),
|
|
203
|
+
),
|
|
204
|
+
] = "Original MFHD (MARC21)"
|
|
167
205
|
|
|
168
206
|
@staticmethod
|
|
169
207
|
def get_object_type() -> FOLIONamespaces:
|
|
@@ -525,7 +525,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
525
525
|
loan_to_put["dueDate"] = due_date.isoformat()
|
|
526
526
|
loan_to_put["loanDate"] = out_date.isoformat()
|
|
527
527
|
loan_to_put["renewalCount"] = renewal_count
|
|
528
|
-
url = f"{self.folio_client.
|
|
528
|
+
url = f"{self.folio_client.gateway_url}/circulation/loans/{loan_to_put['id']}"
|
|
529
529
|
req = self.http_client.put(
|
|
530
530
|
url,
|
|
531
531
|
headers=self.folio_client.okapi_headers,
|
|
@@ -608,7 +608,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
608
608
|
try:
|
|
609
609
|
# Get Item by barcode, update status.
|
|
610
610
|
item_path = f'item-storage/items?query=(barcode=="{legacy_loan.item_barcode}")'
|
|
611
|
-
item_url = f"{self.folio_client.
|
|
611
|
+
item_url = f"{self.folio_client.gateway_url}/{item_path}"
|
|
612
612
|
resp = self.http_client.get(item_url, headers=self.folio_client.okapi_headers)
|
|
613
613
|
resp.raise_for_status()
|
|
614
614
|
data = resp.json()
|
|
@@ -667,14 +667,14 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
667
667
|
self.folio_put_post(url, user, "PUT", i18n.t("Update user"))
|
|
668
668
|
|
|
669
669
|
def get_user_by_barcode(self, barcode):
|
|
670
|
-
url = f'{self.folio_client.
|
|
670
|
+
url = f'{self.folio_client.gateway_url}/users?query=(barcode=="{barcode}")'
|
|
671
671
|
resp = self.http_client.get(url, headers=self.folio_client.okapi_headers)
|
|
672
672
|
resp.raise_for_status()
|
|
673
673
|
data = resp.json()
|
|
674
674
|
return data["users"][0]
|
|
675
675
|
|
|
676
676
|
def folio_put_post(self, url, data_dict, verb, action_description=""):
|
|
677
|
-
full_url = f"{self.folio_client.
|
|
677
|
+
full_url = f"{self.folio_client.gateway_url}{url}"
|
|
678
678
|
try:
|
|
679
679
|
if verb == "PUT":
|
|
680
680
|
resp = self.http_client.put(
|
|
@@ -729,7 +729,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
729
729
|
def change_due_date(self, folio_loan, legacy_loan):
|
|
730
730
|
try:
|
|
731
731
|
api_path = f"{folio_loan['id']}/change-due-date"
|
|
732
|
-
api_url = f"{self.folio_client.
|
|
732
|
+
api_url = f"{self.folio_client.gateway_url}/circulation/loans/{api_path}"
|
|
733
733
|
body = {"dueDate": du_parser.isoparse(str(legacy_loan.due_date)).isoformat()}
|
|
734
734
|
req = self.http_client.post(
|
|
735
735
|
api_url, headers=self.folio_client.okapi_headers, json=body
|
|
@@ -9,11 +9,12 @@ from abc import abstractmethod
|
|
|
9
9
|
from datetime import datetime, timezone
|
|
10
10
|
from genericpath import isfile
|
|
11
11
|
from pathlib import Path
|
|
12
|
-
from typing import Optional
|
|
12
|
+
from typing import Annotated, List, Optional
|
|
13
13
|
|
|
14
14
|
import folioclient
|
|
15
15
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
16
16
|
from folioclient import FolioClient
|
|
17
|
+
from pydantic import Field
|
|
17
18
|
|
|
18
19
|
from folio_migration_tools import library_configuration, task_configuration
|
|
19
20
|
from folio_migration_tools.custom_exceptions import (
|
|
@@ -460,6 +461,68 @@ class MigrationTaskBase:
|
|
|
460
461
|
return None
|
|
461
462
|
|
|
462
463
|
|
|
464
|
+
class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
|
|
465
|
+
"""
|
|
466
|
+
Base class for MARC task configurations.
|
|
467
|
+
|
|
468
|
+
Attributes:
|
|
469
|
+
files (List[library_configuration.FileDefinition]):
|
|
470
|
+
List of MARC21 files to be processed.
|
|
471
|
+
|
|
472
|
+
create_source_records (bool):
|
|
473
|
+
Controls whether or not to retain the MARC records in Source Record Storage.
|
|
474
|
+
Default is False, meaning MARC records will not be retained.
|
|
475
|
+
|
|
476
|
+
hrid_handling (library_configuration.HridHandling):
|
|
477
|
+
Determines how HRIDs are handled.
|
|
478
|
+
- 'default': FOLIO generates HRIDs and moves existing 001 fields into a 035 field, concatenated with the 003 field.
|
|
479
|
+
- 'preserve001': Keeps the 001 fields in place and uses them as HRIDs.
|
|
480
|
+
Default is 'default'.
|
|
481
|
+
|
|
482
|
+
deactivate035_from001 (bool):
|
|
483
|
+
Disables the default FOLIO functionality of moving the previous 001 field into a 035 field, prefixed with the value from 003.
|
|
484
|
+
Default is False, meaning the functionality remains active.
|
|
485
|
+
"""
|
|
486
|
+
|
|
487
|
+
files: Annotated[
|
|
488
|
+
List[library_configuration.FileDefinition],
|
|
489
|
+
Field(
|
|
490
|
+
title="Source files",
|
|
491
|
+
description=("List of MARC21 files with bibliographic records."),
|
|
492
|
+
),
|
|
493
|
+
]
|
|
494
|
+
create_source_records: Annotated[
|
|
495
|
+
bool,
|
|
496
|
+
Field(
|
|
497
|
+
title="Create source records",
|
|
498
|
+
description=(
|
|
499
|
+
"Controls whether or not to retain the MARC records in "
|
|
500
|
+
"Source Record Storage."
|
|
501
|
+
),
|
|
502
|
+
),
|
|
503
|
+
] = False
|
|
504
|
+
hrid_handling: Annotated[
|
|
505
|
+
library_configuration.HridHandling,
|
|
506
|
+
Field(
|
|
507
|
+
title="HRID Handling",
|
|
508
|
+
description=(
|
|
509
|
+
"Setting to default will make FOLIO generate HRIDs and move the existing "
|
|
510
|
+
"001:s into a 035, concatenated with the 003. Choosing preserve001 means "
|
|
511
|
+
"the 001:s will remain in place, and that they will also become the HRIDs"
|
|
512
|
+
),
|
|
513
|
+
),
|
|
514
|
+
] = library_configuration.HridHandling.default
|
|
515
|
+
deactivate035_from001: Annotated[
|
|
516
|
+
bool,
|
|
517
|
+
Field(
|
|
518
|
+
title="Create 035 from 001 and 003",
|
|
519
|
+
description=(
|
|
520
|
+
"This deactivates the FOLIO default functionality of moving the previous 001 "
|
|
521
|
+
"into a 035, prefixed with the value from 003"
|
|
522
|
+
),
|
|
523
|
+
),
|
|
524
|
+
] = False
|
|
525
|
+
|
|
463
526
|
class ExcludeLevelFilter(logging.Filter):
|
|
464
527
|
def __init__(self, level):
|
|
465
528
|
super().__init__()
|
|
@@ -190,7 +190,7 @@ class ReservesMigrator(MigrationTaskBase):
|
|
|
190
190
|
sys.exit(1)
|
|
191
191
|
|
|
192
192
|
def folio_put_post(self, url, data_dict, verb, action_description=""):
|
|
193
|
-
full_url = f"{self.folio_client.
|
|
193
|
+
full_url = f"{self.folio_client.gateway_url}{url}"
|
|
194
194
|
try:
|
|
195
195
|
if verb == "PUT":
|
|
196
196
|
resp = httpx.put(
|