folio-migration-tools 1.9.10__py3-none-any.whl → 1.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__init__.py +3 -4
- folio_migration_tools/__main__.py +53 -31
- folio_migration_tools/circulation_helper.py +118 -108
- folio_migration_tools/custom_dict.py +2 -2
- folio_migration_tools/custom_exceptions.py +4 -5
- folio_migration_tools/folder_structure.py +17 -7
- folio_migration_tools/helper.py +8 -7
- folio_migration_tools/holdings_helper.py +4 -3
- folio_migration_tools/i18n_cache.py +79 -0
- folio_migration_tools/library_configuration.py +77 -37
- folio_migration_tools/mapper_base.py +45 -31
- folio_migration_tools/mapping_file_transformation/courses_mapper.py +1 -1
- folio_migration_tools/mapping_file_transformation/holdings_mapper.py +7 -3
- folio_migration_tools/mapping_file_transformation/item_mapper.py +13 -26
- folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py +1 -2
- folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +13 -11
- folio_migration_tools/mapping_file_transformation/order_mapper.py +6 -5
- folio_migration_tools/mapping_file_transformation/organization_mapper.py +3 -3
- folio_migration_tools/mapping_file_transformation/user_mapper.py +47 -28
- folio_migration_tools/marc_rules_transformation/conditions.py +82 -97
- folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +13 -5
- folio_migration_tools/marc_rules_transformation/hrid_handler.py +3 -2
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +26 -24
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +56 -51
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +28 -17
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +68 -37
- folio_migration_tools/migration_report.py +18 -7
- folio_migration_tools/migration_tasks/batch_poster.py +285 -354
- folio_migration_tools/migration_tasks/bibs_transformer.py +14 -9
- folio_migration_tools/migration_tasks/courses_migrator.py +2 -3
- folio_migration_tools/migration_tasks/holdings_csv_transformer.py +23 -24
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +14 -24
- folio_migration_tools/migration_tasks/items_transformer.py +23 -34
- folio_migration_tools/migration_tasks/loans_migrator.py +67 -144
- folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py +3 -3
- folio_migration_tools/migration_tasks/migration_task_base.py +47 -60
- folio_migration_tools/migration_tasks/orders_transformer.py +25 -42
- folio_migration_tools/migration_tasks/organization_transformer.py +9 -18
- folio_migration_tools/migration_tasks/requests_migrator.py +21 -24
- folio_migration_tools/migration_tasks/reserves_migrator.py +6 -5
- folio_migration_tools/migration_tasks/user_transformer.py +25 -20
- folio_migration_tools/task_configuration.py +6 -7
- folio_migration_tools/transaction_migration/legacy_loan.py +15 -27
- folio_migration_tools/transaction_migration/legacy_request.py +1 -1
- folio_migration_tools/translations/en.json +0 -7
- {folio_migration_tools-1.9.10.dist-info → folio_migration_tools-1.10.0.dist-info}/METADATA +19 -28
- folio_migration_tools-1.10.0.dist-info/RECORD +63 -0
- folio_migration_tools-1.10.0.dist-info/WHEEL +4 -0
- folio_migration_tools-1.10.0.dist-info/entry_points.txt +3 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +0 -241
- folio_migration_tools/migration_tasks/authority_transformer.py +0 -119
- folio_migration_tools/test_infrastructure/__init__.py +0 -0
- folio_migration_tools/test_infrastructure/mocked_classes.py +0 -406
- folio_migration_tools-1.9.10.dist-info/RECORD +0 -67
- folio_migration_tools-1.9.10.dist-info/WHEEL +0 -4
- folio_migration_tools-1.9.10.dist-info/entry_points.txt +0 -3
- folio_migration_tools-1.9.10.dist-info/licenses/LICENSE +0 -21
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import re
|
|
3
3
|
import traceback
|
|
4
|
-
from typing import Union
|
|
5
4
|
|
|
6
5
|
import i18n
|
|
7
6
|
from folioclient import FolioClient
|
|
@@ -48,8 +47,6 @@ class Conditions:
|
|
|
48
47
|
if object_type == "bibs":
|
|
49
48
|
self.setup_reference_data_for_all()
|
|
50
49
|
self.setup_reference_data_for_bibs()
|
|
51
|
-
elif object_type == "auth":
|
|
52
|
-
self.setup_reference_data_for_auth()
|
|
53
50
|
else:
|
|
54
51
|
self.setup_reference_data_for_all()
|
|
55
52
|
self.setup_reference_data_for_items_and_holdings(default_call_number_type_name)
|
|
@@ -58,11 +55,11 @@ class Conditions:
|
|
|
58
55
|
|
|
59
56
|
def setup_reference_data_for_bibs(self):
|
|
60
57
|
logging.info("Setting up reference data for bib transformation")
|
|
61
|
-
logging.info("%s\tcontrib_name_types", len(self.folio.contrib_name_types))
|
|
62
|
-
logging.info("%s\tcontributor_types", len(self.folio.contributor_types))
|
|
63
|
-
logging.info("%s\talt_title_types", len(self.folio.alt_title_types))
|
|
64
|
-
logging.info("%s\tidentifier_types", len(self.folio.identifier_types))
|
|
65
|
-
logging.info("%s\tsubject_types", len(self.folio.subject_types))
|
|
58
|
+
logging.info("%s\tcontrib_name_types", len(self.folio.contrib_name_types)) # type: ignore
|
|
59
|
+
logging.info("%s\tcontributor_types", len(self.folio.contributor_types)) # type: ignore
|
|
60
|
+
logging.info("%s\talt_title_types", len(self.folio.alt_title_types)) # type: ignore
|
|
61
|
+
logging.info("%s\tidentifier_types", len(self.folio.identifier_types)) # type: ignore
|
|
62
|
+
logging.info("%s\tsubject_types", len(self.folio.subject_types)) # type: ignore
|
|
66
63
|
# Raise for empty settings
|
|
67
64
|
if not self.folio.contributor_types:
|
|
68
65
|
raise TransformationProcessError("", "No contributor_types in FOLIO")
|
|
@@ -77,18 +74,20 @@ class Conditions:
|
|
|
77
74
|
|
|
78
75
|
# Set defaults
|
|
79
76
|
logging.info("Setting defaults")
|
|
80
|
-
self.default_contributor_name_type: str = self.folio.contrib_name_types[0]["id"]
|
|
77
|
+
self.default_contributor_name_type: str = self.folio.contrib_name_types[0]["id"] # type: ignore
|
|
81
78
|
logging.info("Contributor name type:\t%s", self.default_contributor_name_type)
|
|
82
79
|
self.default_contributor_type = next(
|
|
83
|
-
ct
|
|
80
|
+
ct
|
|
81
|
+
for ct in self.folio.contributor_types
|
|
82
|
+
if ct["code"] == "ctb" # type: ignore
|
|
84
83
|
)
|
|
85
84
|
logging.info("Contributor type:\t%s", self.default_contributor_type["id"])
|
|
86
85
|
|
|
87
86
|
def setup_reference_data_for_items_and_holdings(self, default_call_number_type_name):
|
|
88
|
-
logging.info(f"{len(self.folio.locations)}\tlocations")
|
|
87
|
+
logging.info(f"{len(self.folio.locations)}\tlocations") # type: ignore
|
|
89
88
|
self.default_call_number_type = {}
|
|
90
|
-
logging.info("%s\tholding_note_types", len(self.folio.holding_note_types))
|
|
91
|
-
logging.info("%s\tcall_number_types", len(self.folio.call_number_types))
|
|
89
|
+
logging.info("%s\tholding_note_types", len(self.folio.holding_note_types)) # type: ignore
|
|
90
|
+
logging.info("%s\tcall_number_types", len(self.folio.call_number_types)) # type: ignore
|
|
92
91
|
self.setup_and_validate_holdings_types()
|
|
93
92
|
self.ill_policies = self.folio.folio_get_all("/ill-policies", "illPolicies")
|
|
94
93
|
# Raise for empty settings
|
|
@@ -104,7 +103,7 @@ class Conditions:
|
|
|
104
103
|
self.default_call_number_type: dict = next(
|
|
105
104
|
(
|
|
106
105
|
ct
|
|
107
|
-
for ct in self.folio.call_number_types
|
|
106
|
+
for ct in self.folio.call_number_types # type: ignore
|
|
108
107
|
if ct["name"] == default_call_number_type_name
|
|
109
108
|
),
|
|
110
109
|
None,
|
|
@@ -127,7 +126,7 @@ class Conditions:
|
|
|
127
126
|
missing_holdings_types = [
|
|
128
127
|
ht
|
|
129
128
|
for ht in self.holdings_type_map.values()
|
|
130
|
-
if ht not in [ht_ref["name"] for ht_ref in self.holdings_types]
|
|
129
|
+
if ht not in [ht_ref["name"] for ht_ref in self.holdings_types] # type: ignore
|
|
131
130
|
]
|
|
132
131
|
if any(missing_holdings_types):
|
|
133
132
|
raise TransformationProcessError(
|
|
@@ -135,34 +134,25 @@ class Conditions:
|
|
|
135
134
|
"Holdings types are missing from the tenant. Please set them up",
|
|
136
135
|
missing_holdings_types,
|
|
137
136
|
)
|
|
138
|
-
logging.info("%s\tholdings types", len(self.holdings_types))
|
|
137
|
+
logging.info("%s\tholdings types", len(self.holdings_types)) # type: ignore
|
|
139
138
|
|
|
140
139
|
def setup_reference_data_for_all(self):
|
|
141
|
-
logging.info(f"{len(self.folio.class_types)}\tclass_types")
|
|
140
|
+
logging.info(f"{len(self.folio.class_types)}\tclass_types") # type: ignore
|
|
142
141
|
logging.info(
|
|
143
|
-
f"{len(self.folio.electronic_access_relationships)}\telectronic_access_relationships"
|
|
142
|
+
f"{len(self.folio.electronic_access_relationships)}\telectronic_access_relationships" # type: ignore
|
|
144
143
|
)
|
|
145
144
|
self.statistical_codes = self.folio.statistical_codes
|
|
146
|
-
logging.info(f"{len(self.statistical_codes)} \tstatistical_codes")
|
|
145
|
+
logging.info(f"{len(self.statistical_codes)} \tstatistical_codes") # type: ignore
|
|
147
146
|
|
|
148
147
|
# Raise for empty settings
|
|
149
148
|
if not self.folio.class_types:
|
|
150
149
|
raise TransformationProcessError("", "No class_types in FOLIO")
|
|
151
150
|
|
|
152
|
-
def setup_reference_data_for_auth(self):
|
|
153
|
-
self.authority_note_types = list(
|
|
154
|
-
self.folio.folio_get_all(
|
|
155
|
-
"/authority-note-types", "authorityNoteTypes", self.folio.cql_all, 1000
|
|
156
|
-
)
|
|
157
|
-
)
|
|
158
|
-
logging.info(f"{len(self.authority_note_types)} \tAuthority note types")
|
|
159
|
-
logging.info(f"{len(self.folio.identifier_types)} \tidentifier types") # type: ignore
|
|
160
|
-
|
|
161
151
|
def get_condition(
|
|
162
|
-
self, name, legacy_id, value, parameter=None, marc_field:
|
|
152
|
+
self, name, legacy_id, value, parameter=None, marc_field: field.Field | None = None
|
|
163
153
|
):
|
|
164
154
|
try:
|
|
165
|
-
return self.condition_cache.get(name)(legacy_id, value, parameter, marc_field)
|
|
155
|
+
return self.condition_cache.get(name)(legacy_id, value, parameter, marc_field) # type: ignore
|
|
166
156
|
# Exception should only handle the missing condition from the cache.
|
|
167
157
|
# All other exceptions should propagate up
|
|
168
158
|
except Exception:
|
|
@@ -381,7 +371,7 @@ class Conditions:
|
|
|
381
371
|
identifier_type: dict = next(
|
|
382
372
|
(
|
|
383
373
|
f
|
|
384
|
-
for f in self.folio.identifier_types
|
|
374
|
+
for f in self.folio.identifier_types # type: ignore
|
|
385
375
|
if (
|
|
386
376
|
f["name"] in parameter.get("names", "non existant")
|
|
387
377
|
or f["name"] in parameter.get("name", "non existant")
|
|
@@ -424,25 +414,7 @@ class Conditions:
|
|
|
424
414
|
logging.error(ee)
|
|
425
415
|
raise TransformationRecordFailedError(
|
|
426
416
|
legacy_id,
|
|
427
|
-
f
|
|
428
|
-
f"MARC Field: {marc_field}. Is mapping rules and ref data aligned?",
|
|
429
|
-
parameter.get("name", ""),
|
|
430
|
-
) from ee
|
|
431
|
-
|
|
432
|
-
def condition_set_authority_note_type_id(
|
|
433
|
-
self, legacy_id, _, parameter, marc_field: field.Field
|
|
434
|
-
):
|
|
435
|
-
try:
|
|
436
|
-
t = self.get_ref_data_tuple_by_name(
|
|
437
|
-
self.authority_note_types, "authority_note_types", parameter["name"]
|
|
438
|
-
)
|
|
439
|
-
self.mapper.migration_report.add("MappedNoteTypes", t[1])
|
|
440
|
-
return t[0]
|
|
441
|
-
except Exception as ee:
|
|
442
|
-
logging.error(ee)
|
|
443
|
-
raise TransformationProcessError(
|
|
444
|
-
legacy_id,
|
|
445
|
-
f'Authority note type mapping error.\tParameter: {parameter.get("name", "")}\t'
|
|
417
|
+
f"Holdings note type mapping error.\tParameter: {parameter.get('name', '')}\t"
|
|
446
418
|
f"MARC Field: {marc_field}. Is mapping rules and ref data aligned?",
|
|
447
419
|
parameter.get("name", ""),
|
|
448
420
|
) from ee
|
|
@@ -456,18 +428,17 @@ class Conditions:
|
|
|
456
428
|
)
|
|
457
429
|
self.mapper.migration_report.add("MappedClassificationTypes", t[1])
|
|
458
430
|
return t[0]
|
|
459
|
-
except Exception:
|
|
431
|
+
except Exception as e:
|
|
460
432
|
raise TransformationRecordFailedError(
|
|
461
433
|
legacy_id,
|
|
462
434
|
f'Classification mapping error.\tParameter: "{parameter.get("name", "")}"\t'
|
|
463
435
|
f"MARC Field: {marc_field}. Is mapping rules and ref data aligned?",
|
|
464
436
|
parameter.get("name", ""),
|
|
465
|
-
)
|
|
437
|
+
) from e
|
|
466
438
|
|
|
467
439
|
def condition_char_select(self, legacy_id, value, parameter, marc_field: field.Field):
|
|
468
440
|
return value[parameter["from"] : parameter["to"]]
|
|
469
441
|
|
|
470
|
-
|
|
471
442
|
def condition_set_identifier_type_id_by_name(
|
|
472
443
|
self, legacy_id, value, parameter, marc_field: field.Field
|
|
473
444
|
):
|
|
@@ -513,8 +484,8 @@ class Conditions:
|
|
|
513
484
|
f"{marc_field.tag} ({parameter.get('name', '')}) -> {t[1]}",
|
|
514
485
|
)
|
|
515
486
|
return t[0]
|
|
516
|
-
except Exception:
|
|
517
|
-
raise ValueError(f"Instance note type not found for {marc_field} {parameter}")
|
|
487
|
+
except Exception as e:
|
|
488
|
+
raise ValueError(f"Instance note type not found for {marc_field} {parameter}") from e
|
|
518
489
|
|
|
519
490
|
def condition_set_contributor_type_id(
|
|
520
491
|
self, legacy_id, value, parameter, marc_field: field.Field
|
|
@@ -562,7 +533,8 @@ class Conditions:
|
|
|
562
533
|
self.mapper.migration_report.add(
|
|
563
534
|
"ContributorTypeMapping",
|
|
564
535
|
i18n.t(
|
|
565
|
-
'Mapping failed for %{tag} "%{subfield}"
|
|
536
|
+
'Mapping failed for %{tag} "%{subfield}" '
|
|
537
|
+
"(Normalized: %{normalized_subfield})",
|
|
566
538
|
tag=f"{marc_field.tag} $e",
|
|
567
539
|
subfield=subfield,
|
|
568
540
|
normalized_subfield=normalized_subfield,
|
|
@@ -631,7 +603,8 @@ class Conditions:
|
|
|
631
603
|
"CallNumberTypeMapping",
|
|
632
604
|
(
|
|
633
605
|
i18n.t(
|
|
634
|
-
'Unhandled call number type in ind1: "%{ind1}".\n Returning default
|
|
606
|
+
'Unhandled call number type in ind1: "%{ind1}".\n Returning default '
|
|
607
|
+
"Callnumber type: %{type}",
|
|
635
608
|
ind1=marc_field.indicator1,
|
|
636
609
|
type=self.default_call_number_type["name"],
|
|
637
610
|
)
|
|
@@ -652,7 +625,7 @@ class Conditions:
|
|
|
652
625
|
"CallNumberTypeMapping",
|
|
653
626
|
(
|
|
654
627
|
"Mapping failed. Setting default CallNumber type: "
|
|
655
|
-
f
|
|
628
|
+
f"{self.default_call_number_type['name']}"
|
|
656
629
|
),
|
|
657
630
|
)
|
|
658
631
|
|
|
@@ -663,7 +636,7 @@ class Conditions:
|
|
|
663
636
|
):
|
|
664
637
|
for subfield in marc_field.get_subfields("4", "e"):
|
|
665
638
|
normalized_subfield = re.sub(r"[^A-Za-z0-9 ]+", "", subfield.strip())
|
|
666
|
-
for cont_type in self.folio.contributor_types:
|
|
639
|
+
for cont_type in self.folio.contributor_types: # type: ignore
|
|
667
640
|
if normalized_subfield in [cont_type["code"], cont_type["name"]]:
|
|
668
641
|
return cont_type["name"]
|
|
669
642
|
try:
|
|
@@ -679,11 +652,11 @@ class Conditions:
|
|
|
679
652
|
)
|
|
680
653
|
self.mapper.migration_report.add("MappedAlternativeTitleTypes", t[1])
|
|
681
654
|
return t[0]
|
|
682
|
-
except Exception:
|
|
655
|
+
except Exception as e:
|
|
683
656
|
raise TransformationProcessError(
|
|
684
657
|
legacy_id,
|
|
685
658
|
f"Alternative title type not found for {parameter['name']} {marc_field}",
|
|
686
|
-
)
|
|
659
|
+
) from e
|
|
687
660
|
|
|
688
661
|
def condition_set_location_id_by_code(
|
|
689
662
|
self, legacy_id, value, parameter, marc_field: field.Field
|
|
@@ -702,7 +675,7 @@ class Conditions:
|
|
|
702
675
|
):
|
|
703
676
|
if "legacy_locations" not in self.ref_data_dicts:
|
|
704
677
|
try:
|
|
705
|
-
d = {lm["legacy_code"]: lm["folio_code"] for lm in self.mapper.location_map}
|
|
678
|
+
d = {lm["legacy_code"]: lm["folio_code"] for lm in self.mapper.location_map} # type: ignore
|
|
706
679
|
self.ref_data_dicts["legacy_locations"] = d
|
|
707
680
|
for folio_code in d.values():
|
|
708
681
|
t = self.get_ref_data_tuple_by_code(
|
|
@@ -802,14 +775,14 @@ class Conditions:
|
|
|
802
775
|
)
|
|
803
776
|
self.mapper.migration_report.add("MappedElectronicRelationshipTypes", t[1])
|
|
804
777
|
return t[0]
|
|
805
|
-
except Exception:
|
|
778
|
+
except Exception as e:
|
|
806
779
|
raise TransformationProcessError(
|
|
807
780
|
legacy_id,
|
|
808
|
-
f"Electronic access relationship not found for {parameter['name']} {marc_field}",
|
|
809
|
-
)
|
|
781
|
+
f"Electronic access relationship not found for {parameter['name']} {marc_field}", # noqa: E501
|
|
782
|
+
) from e
|
|
810
783
|
return self._extracted_from_condition_set_electronic_access_relations_id_2("3", marc_field)
|
|
811
784
|
|
|
812
|
-
# TODO Rename this here and in `condition_set_url_relationship` and `condition_set_electronic_access_relations_id`
|
|
785
|
+
# TODO Rename this here and in `condition_set_url_relationship` and `condition_set_electronic_access_relations_id` # noqa: E501
|
|
813
786
|
def _extracted_from_condition_set_electronic_access_relations_id_2(self, arg0, marc_field):
|
|
814
787
|
enum = {
|
|
815
788
|
"0": "resource",
|
|
@@ -855,44 +828,50 @@ class Conditions:
|
|
|
855
828
|
)
|
|
856
829
|
self.mapper.migration_report.add("MappedSubjectTypes", t[1])
|
|
857
830
|
return t[0]
|
|
858
|
-
except Exception:
|
|
831
|
+
except Exception as e:
|
|
859
832
|
raise TransformationProcessError(
|
|
860
833
|
legacy_id,
|
|
861
834
|
f"Subject type not found for {parameter['name']} {marc_field}",
|
|
862
|
-
)
|
|
835
|
+
) from e
|
|
863
836
|
|
|
864
|
-
def condition_set_subject_source_id(
|
|
837
|
+
def condition_set_subject_source_id(
|
|
838
|
+
self, legacy_id, value, parameter, marc_field: field.Field
|
|
839
|
+
):
|
|
865
840
|
try:
|
|
866
841
|
t = self.get_ref_data_tuple_by_name(
|
|
867
|
-
self.folio.folio_get_all("/subject-sources", "subjectSources"),
|
|
842
|
+
self.folio.folio_get_all("/subject-sources", "subjectSources"),
|
|
843
|
+
"subject_sources",
|
|
844
|
+
parameter["name"],
|
|
868
845
|
)
|
|
869
846
|
self.mapper.migration_report.add("MappedSubjectSources", t[1])
|
|
870
847
|
return t[0]
|
|
871
|
-
except Exception:
|
|
848
|
+
except Exception as e:
|
|
872
849
|
raise TransformationProcessError(
|
|
873
850
|
legacy_id,
|
|
874
851
|
f"Subject source not found for {parameter['name']} {marc_field}",
|
|
875
|
-
)
|
|
852
|
+
) from e
|
|
876
853
|
|
|
877
|
-
def condition_set_subject_source_id_by_code(
|
|
854
|
+
def condition_set_subject_source_id_by_code(
|
|
855
|
+
self, legacy_id, value, parameter, marc_field: field.Field
|
|
856
|
+
):
|
|
878
857
|
try:
|
|
879
858
|
t = self.get_ref_data_tuple_by_code(
|
|
880
|
-
self.folio.folio_get_all("/subject-sources", "subjectSources"),
|
|
859
|
+
self.folio.folio_get_all("/subject-sources", "subjectSources"),
|
|
860
|
+
"subject_sources",
|
|
861
|
+
value,
|
|
881
862
|
)
|
|
882
863
|
self.mapper.migration_report.add("MappedSubjectSources", t[1])
|
|
883
864
|
return t[0]
|
|
884
|
-
except Exception:
|
|
865
|
+
except Exception as e:
|
|
885
866
|
raise TransformationProcessError(
|
|
886
867
|
legacy_id,
|
|
887
868
|
f"Subject source not found for {value} {marc_field}",
|
|
888
|
-
)
|
|
869
|
+
) from e
|
|
889
870
|
|
|
890
|
-
def condition_set_receipt_status(
|
|
891
|
-
self, legacy_id, value, parameter, marc_field: field.Field
|
|
892
|
-
):
|
|
871
|
+
def condition_set_receipt_status(self, legacy_id, value, parameter, marc_field: field.Field):
|
|
893
872
|
"""
|
|
894
873
|
This method maps the receipt status based on the 008 field.
|
|
895
|
-
This condition is not available in FOLIO's MARC mapping engine and
|
|
874
|
+
This condition is not available in FOLIO's MARC mapping engine and
|
|
896
875
|
will require use of a supplemental mapping rules file in the
|
|
897
876
|
HoldingsMarcTransformer task definition.
|
|
898
877
|
"""
|
|
@@ -901,7 +880,7 @@ class Conditions:
|
|
|
901
880
|
"ReceiptStatusMapping", i18n.t("008 is too short") + f": {value}"
|
|
902
881
|
)
|
|
903
882
|
return ""
|
|
904
|
-
|
|
883
|
+
|
|
905
884
|
status_map = {
|
|
906
885
|
"0": "Unknown",
|
|
907
886
|
"1": "Other receipt or acquisition status",
|
|
@@ -934,7 +913,7 @@ class Conditions:
|
|
|
934
913
|
):
|
|
935
914
|
"""
|
|
936
915
|
This method maps the acquisition method based on the 008 field.
|
|
937
|
-
This condition is not available in FOLIO's MARC mapping engine and
|
|
916
|
+
This condition is not available in FOLIO's MARC mapping engine and
|
|
938
917
|
will require use of a supplemental mapping rules file in the
|
|
939
918
|
HoldingsMarcTransformer task definition.
|
|
940
919
|
"""
|
|
@@ -973,9 +952,7 @@ class Conditions:
|
|
|
973
952
|
)
|
|
974
953
|
return ""
|
|
975
954
|
|
|
976
|
-
def condition_set_retention_policy(
|
|
977
|
-
self, legacy_id, value, parameter, marc_field: field.Field
|
|
978
|
-
):
|
|
955
|
+
def condition_set_retention_policy(self, legacy_id, value, parameter, marc_field: field.Field):
|
|
979
956
|
"""
|
|
980
957
|
This method maps the retention policy based on the 008 field.
|
|
981
958
|
This condition is not available in FOLIO's MARC mapping engine and
|
|
@@ -987,7 +964,9 @@ class Conditions:
|
|
|
987
964
|
"RetentionPolicyMapping", i18n.t("008 is too short") + f": {value}"
|
|
988
965
|
)
|
|
989
966
|
return ""
|
|
990
|
-
value = value.replace("|", " ").replace(
|
|
967
|
+
value = value.replace("|", " ").replace(
|
|
968
|
+
"#", " "
|
|
969
|
+
) # Replace pipe with space for mapping consistency
|
|
991
970
|
try:
|
|
992
971
|
retention_policies = {
|
|
993
972
|
"0": "Unknown",
|
|
@@ -1020,21 +999,22 @@ class Conditions:
|
|
|
1020
999
|
"y": "Year",
|
|
1021
1000
|
"e": "Edition",
|
|
1022
1001
|
"i": "Issue",
|
|
1023
|
-
"s": "Supplement"
|
|
1002
|
+
"s": "Supplement",
|
|
1024
1003
|
}
|
|
1025
1004
|
try:
|
|
1026
1005
|
specific_retention_policy = ""
|
|
1027
1006
|
if value[13].strip() or value[15].strip():
|
|
1028
1007
|
if value[14].strip() and int(value[14]) > 1:
|
|
1029
|
-
specific_retention_policy = f"{policy_types.get(value[13], '')} {value[14]} {unit_types.get(value[15], '')}s retained".strip()
|
|
1008
|
+
specific_retention_policy = f"{policy_types.get(value[13], '')} {value[14]} {unit_types.get(value[15], '')}s retained".strip() # noqa: E501
|
|
1030
1009
|
else:
|
|
1031
|
-
specific_retention_policy = f"{policy_types.get(value[13], '')} {unit_types.get(value[15], '')} retained".strip()
|
|
1010
|
+
specific_retention_policy = f"{policy_types.get(value[13], '')} {unit_types.get(value[15], '')} retained".strip() # noqa: E501
|
|
1032
1011
|
if specific_retention_policy:
|
|
1033
1012
|
self.mapper.migration_report.add(
|
|
1034
1013
|
"RetentionPolicyMapping",
|
|
1035
1014
|
i18n.t(
|
|
1036
|
-
"Retention policy 6 indicates a limited period. Specific
|
|
1037
|
-
|
|
1015
|
+
"Retention policy 6 indicates a limited period. Specific "
|
|
1016
|
+
"retention period will be mapped from 008/13-15",
|
|
1017
|
+
),
|
|
1038
1018
|
)
|
|
1039
1019
|
return specific_retention_policy
|
|
1040
1020
|
else:
|
|
@@ -1044,7 +1024,10 @@ class Conditions:
|
|
|
1044
1024
|
except ValueError:
|
|
1045
1025
|
self.mapper.migration_report.add(
|
|
1046
1026
|
"RetentionPolicyMapping",
|
|
1047
|
-
i18n.t(
|
|
1027
|
+
i18n.t(
|
|
1028
|
+
"Invalid specific retention policy in 008/13-15: %{value}",
|
|
1029
|
+
value=value[13:16],
|
|
1030
|
+
),
|
|
1048
1031
|
)
|
|
1049
1032
|
return mapped_value
|
|
1050
1033
|
except Exception:
|
|
@@ -1053,9 +1036,7 @@ class Conditions:
|
|
|
1053
1036
|
)
|
|
1054
1037
|
return ""
|
|
1055
1038
|
|
|
1056
|
-
def condition_set_ill_policy(
|
|
1057
|
-
self, legacy_id, value, parameter, marc_field: field.Field
|
|
1058
|
-
):
|
|
1039
|
+
def condition_set_ill_policy(self, legacy_id, value, parameter, marc_field: field.Field):
|
|
1059
1040
|
"""
|
|
1060
1041
|
This method maps the ILL policy based on the 008 field.
|
|
1061
1042
|
This condition is not available in FOLIO's MARC mapping engine and
|
|
@@ -1077,7 +1058,11 @@ class Conditions:
|
|
|
1077
1058
|
mapped_value = ill_policies[value[20]]
|
|
1078
1059
|
self.mapper.migration_report.add(
|
|
1079
1060
|
"ILLPolicyMapping",
|
|
1080
|
-
i18n.t(
|
|
1061
|
+
i18n.t(
|
|
1062
|
+
"%{value} mapped to %{mapped_value}",
|
|
1063
|
+
value=value[20],
|
|
1064
|
+
mapped_value=mapped_value,
|
|
1065
|
+
),
|
|
1081
1066
|
)
|
|
1082
1067
|
ill_policy_id = self.get_ref_data_tuple_by_name(
|
|
1083
1068
|
self.ill_policies, "ill_policies", mapped_value
|
|
@@ -1123,4 +1108,4 @@ class Conditions:
|
|
|
1123
1108
|
self.mapper.migration_report.add(
|
|
1124
1109
|
"DigitizationPolicyMapping", i18n.t("%{value} not found in map.", value=value[21])
|
|
1125
1110
|
)
|
|
1126
|
-
return ""
|
|
1111
|
+
return ""
|
|
@@ -35,7 +35,7 @@ class HoldingsStatementsParser:
|
|
|
35
35
|
|
|
36
36
|
Returns:
|
|
37
37
|
dict: A dictionary containing parsed holdings statements and related information.
|
|
38
|
-
"""
|
|
38
|
+
""" # noqa: E501
|
|
39
39
|
|
|
40
40
|
# Textual holdings statements
|
|
41
41
|
return_dict: dict = {"statements": [], "migration_report": [], "hlm_stmts": []}
|
|
@@ -67,14 +67,20 @@ class HoldingsStatementsParser:
|
|
|
67
67
|
raise TransformationFieldMappingError(
|
|
68
68
|
legacy_ids,
|
|
69
69
|
i18n.t(
|
|
70
|
-
"subfield present in %{linked_value_tag} but not
|
|
70
|
+
"subfield present in %{linked_value_tag} but not "
|
|
71
|
+
"in %{pattern_field}",
|
|
71
72
|
pattern_field=pattern_field,
|
|
72
73
|
linked_value_tag=linked_value_field,
|
|
73
74
|
),
|
|
74
75
|
pattern_field,
|
|
75
76
|
) from e
|
|
76
77
|
HoldingsStatementsParser.prepare_return_dict(
|
|
77
|
-
pattern_tag,
|
|
78
|
+
pattern_tag,
|
|
79
|
+
legacy_ids,
|
|
80
|
+
return_dict,
|
|
81
|
+
pattern_field,
|
|
82
|
+
linked_value_field,
|
|
83
|
+
parsed_dict,
|
|
78
84
|
)
|
|
79
85
|
|
|
80
86
|
if dedupe_results:
|
|
@@ -296,8 +302,10 @@ class HoldingsStatementsParser:
|
|
|
296
302
|
elif cron_to.strip() and val:
|
|
297
303
|
val_rest = val
|
|
298
304
|
if year:
|
|
299
|
-
cron_from, cron_to =
|
|
300
|
-
|
|
305
|
+
cron_from, cron_to = (
|
|
306
|
+
HoldingsStatementsParser.format_year_cron_from_cron_to(
|
|
307
|
+
cron_from, cron_to, hlm_stmt, val, val_rest
|
|
308
|
+
)
|
|
301
309
|
)
|
|
302
310
|
|
|
303
311
|
else:
|
|
@@ -114,7 +114,7 @@ class HRIDHandler:
|
|
|
114
114
|
f_001 = marc_record["001"].value()
|
|
115
115
|
f_003 = marc_record["003"].value().strip() if "003" in marc_record else ""
|
|
116
116
|
migration_report.add(
|
|
117
|
-
"HridHandling", i18n.t("Values in %{field}", field="003") + f
|
|
117
|
+
"HridHandling", i18n.t("Values in %{field}", field="003") + f": {f_003 or 'Empty'}"
|
|
118
118
|
)
|
|
119
119
|
|
|
120
120
|
if deactivate035_from001:
|
|
@@ -214,7 +214,8 @@ class HRIDHandler:
|
|
|
214
214
|
self.migration_report.add(
|
|
215
215
|
"HridHandling",
|
|
216
216
|
i18n.t(
|
|
217
|
-
"Duplicate 001. Creating HRID instead.\n Previous 001 will be stored in a new
|
|
217
|
+
"Duplicate 001. Creating HRID instead.\n Previous 001 will be stored in a new "
|
|
218
|
+
"035 field"
|
|
218
219
|
),
|
|
219
220
|
)
|
|
220
221
|
self.handle_035_generation(
|
|
@@ -24,7 +24,10 @@ from folio_migration_tools.migration_report import MigrationReport
|
|
|
24
24
|
|
|
25
25
|
class MarcFileProcessor:
|
|
26
26
|
def __init__(
|
|
27
|
-
self,
|
|
27
|
+
self,
|
|
28
|
+
mapper: RulesMapperBase,
|
|
29
|
+
folder_structure: FolderStructure,
|
|
30
|
+
created_objects_file: TextIO,
|
|
28
31
|
):
|
|
29
32
|
self.object_type: FOLIONamespaces = folder_structure.object_type
|
|
30
33
|
self.folder_structure: FolderStructure = folder_structure
|
|
@@ -35,16 +38,15 @@ class MarcFileProcessor:
|
|
|
35
38
|
):
|
|
36
39
|
self.srs_records_file: TextIO = open(self.folder_structure.srs_records_path, "w+")
|
|
37
40
|
if getattr(mapper.task_configuration, "data_import_marc", False):
|
|
38
|
-
self.data_import_marc_file: BinaryIO = open(
|
|
41
|
+
self.data_import_marc_file: BinaryIO = open(
|
|
42
|
+
self.folder_structure.data_import_marc_path, "wb+"
|
|
43
|
+
)
|
|
39
44
|
self.unique_001s: Set[str] = set()
|
|
40
45
|
self.failed_records_count: int = 0
|
|
41
46
|
self.records_count: int = 0
|
|
42
47
|
self.start: float = time.time()
|
|
43
48
|
self.legacy_ids: Set[str] = set()
|
|
44
|
-
if
|
|
45
|
-
self.object_type == FOLIONamespaces.holdings
|
|
46
|
-
and self.mapper.create_source_records
|
|
47
|
-
):
|
|
49
|
+
if self.object_type == FOLIONamespaces.holdings and self.mapper.create_source_records:
|
|
48
50
|
logging.info("Loading Parent HRID map for SRS creation")
|
|
49
51
|
self.parent_hrids = {entity[1]: entity[2] for entity in mapper.parent_id_map.values()}
|
|
50
52
|
|
|
@@ -83,10 +85,7 @@ class MarcFileProcessor:
|
|
|
83
85
|
)
|
|
84
86
|
self.add_legacy_ids_to_map(folio_rec, filtered_legacy_ids)
|
|
85
87
|
|
|
86
|
-
if
|
|
87
|
-
file_def.create_source_records
|
|
88
|
-
and self.mapper.create_source_records
|
|
89
|
-
):
|
|
88
|
+
if file_def.create_source_records and self.mapper.create_source_records:
|
|
90
89
|
self.save_srs_record(
|
|
91
90
|
marc_record,
|
|
92
91
|
file_def,
|
|
@@ -94,12 +93,8 @@ class MarcFileProcessor:
|
|
|
94
93
|
legacy_ids,
|
|
95
94
|
self.object_type,
|
|
96
95
|
)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
marc_record,
|
|
100
|
-
folio_rec,
|
|
101
|
-
self.object_type
|
|
102
|
-
)
|
|
96
|
+
|
|
97
|
+
self.save_marc_record(marc_record, file_def, folio_rec, self.object_type)
|
|
103
98
|
Helper.write_to_file(self.created_objects_file, folio_rec)
|
|
104
99
|
self.mapper.migration_report.add_general_statistics(
|
|
105
100
|
i18n.t("Inventory records written to disk")
|
|
@@ -139,15 +134,20 @@ class MarcFileProcessor:
|
|
|
139
134
|
def save_marc_record(
|
|
140
135
|
self,
|
|
141
136
|
marc_record: Record,
|
|
137
|
+
file_def: FileDefinition,
|
|
142
138
|
folio_rec: Dict,
|
|
143
|
-
object_type: FOLIONamespaces
|
|
139
|
+
object_type: FOLIONamespaces,
|
|
144
140
|
):
|
|
145
|
-
|
|
146
|
-
self.
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
141
|
+
if (
|
|
142
|
+
getattr(self.mapper.task_configuration, "data_import_marc", False)
|
|
143
|
+
and file_def.data_import_marc
|
|
144
|
+
):
|
|
145
|
+
self.mapper.save_data_import_marc_record(
|
|
146
|
+
self.data_import_marc_file,
|
|
147
|
+
object_type,
|
|
148
|
+
marc_record,
|
|
149
|
+
folio_rec,
|
|
150
|
+
)
|
|
151
151
|
|
|
152
152
|
def save_srs_record(
|
|
153
153
|
self,
|
|
@@ -228,7 +228,9 @@ class MarcFileProcessor:
|
|
|
228
228
|
|
|
229
229
|
@staticmethod
|
|
230
230
|
def get_valid_folio_record_ids(
|
|
231
|
-
legacy_ids: List[str],
|
|
231
|
+
legacy_ids: List[str],
|
|
232
|
+
folio_record_identifiers: Set[str],
|
|
233
|
+
migration_report: MigrationReport,
|
|
232
234
|
) -> List[str]:
|
|
233
235
|
new_ids: Set[str] = set()
|
|
234
236
|
for legacy_id in legacy_ids:
|