folio-migration-tools 1.9.10__py3-none-any.whl → 1.10.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__init__.py +3 -4
- folio_migration_tools/__main__.py +44 -31
- folio_migration_tools/circulation_helper.py +114 -105
- folio_migration_tools/custom_dict.py +2 -2
- folio_migration_tools/custom_exceptions.py +4 -5
- folio_migration_tools/folder_structure.py +1 -1
- folio_migration_tools/helper.py +1 -1
- folio_migration_tools/library_configuration.py +65 -37
- folio_migration_tools/mapper_base.py +38 -25
- folio_migration_tools/mapping_file_transformation/courses_mapper.py +1 -1
- folio_migration_tools/mapping_file_transformation/holdings_mapper.py +7 -3
- folio_migration_tools/mapping_file_transformation/item_mapper.py +13 -26
- folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py +1 -2
- folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +13 -11
- folio_migration_tools/mapping_file_transformation/order_mapper.py +6 -5
- folio_migration_tools/mapping_file_transformation/organization_mapper.py +3 -3
- folio_migration_tools/mapping_file_transformation/user_mapper.py +43 -28
- folio_migration_tools/marc_rules_transformation/conditions.py +84 -70
- folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +13 -5
- folio_migration_tools/marc_rules_transformation/hrid_handler.py +3 -2
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +14 -22
- folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +1 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +46 -36
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +25 -15
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +62 -32
- folio_migration_tools/migration_report.py +1 -1
- folio_migration_tools/migration_tasks/authority_transformer.py +1 -2
- folio_migration_tools/migration_tasks/batch_poster.py +78 -68
- folio_migration_tools/migration_tasks/bibs_transformer.py +12 -7
- folio_migration_tools/migration_tasks/courses_migrator.py +2 -3
- folio_migration_tools/migration_tasks/holdings_csv_transformer.py +14 -15
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +11 -21
- folio_migration_tools/migration_tasks/items_transformer.py +17 -30
- folio_migration_tools/migration_tasks/loans_migrator.py +53 -131
- folio_migration_tools/migration_tasks/migration_task_base.py +33 -55
- folio_migration_tools/migration_tasks/orders_transformer.py +21 -39
- folio_migration_tools/migration_tasks/organization_transformer.py +9 -18
- folio_migration_tools/migration_tasks/requests_migrator.py +11 -15
- folio_migration_tools/migration_tasks/reserves_migrator.py +1 -1
- folio_migration_tools/migration_tasks/user_transformer.py +10 -15
- folio_migration_tools/task_configuration.py +6 -7
- folio_migration_tools/transaction_migration/legacy_loan.py +15 -27
- folio_migration_tools/transaction_migration/legacy_request.py +1 -1
- {folio_migration_tools-1.9.10.dist-info → folio_migration_tools-1.10.0b1.dist-info}/METADATA +18 -28
- {folio_migration_tools-1.9.10.dist-info → folio_migration_tools-1.10.0b1.dist-info}/RECORD +47 -50
- folio_migration_tools-1.10.0b1.dist-info/WHEEL +4 -0
- folio_migration_tools-1.10.0b1.dist-info/entry_points.txt +3 -0
- folio_migration_tools/test_infrastructure/__init__.py +0 -0
- folio_migration_tools/test_infrastructure/mocked_classes.py +0 -406
- folio_migration_tools-1.9.10.dist-info/WHEEL +0 -4
- folio_migration_tools-1.9.10.dist-info/entry_points.txt +0 -3
- folio_migration_tools-1.9.10.dist-info/licenses/LICENSE +0 -21
|
@@ -47,7 +47,7 @@ class UserMapper(MappingFileMapperBase):
|
|
|
47
47
|
None,
|
|
48
48
|
FOLIONamespaces.users,
|
|
49
49
|
library_config,
|
|
50
|
-
task_config
|
|
50
|
+
task_config,
|
|
51
51
|
)
|
|
52
52
|
self.task_config = self.task_configuration
|
|
53
53
|
self.notes_mapper: NotesMapper = NotesMapper(
|
|
@@ -157,18 +157,25 @@ class UserMapper(MappingFileMapperBase):
|
|
|
157
157
|
"No Departments mapping set up. Set up a departments mapping file "
|
|
158
158
|
" or remove the mapping of the Departments field",
|
|
159
159
|
)
|
|
160
|
-
if len(
|
|
161
|
-
|
|
162
|
-
|
|
160
|
+
if len(
|
|
161
|
+
self.departments_mapping.mapped_legacy_keys
|
|
162
|
+
) == 1 and self.library_configuration.multi_field_delimiter in legacy_user.get(
|
|
163
|
+
self.departments_mapping.mapped_legacy_keys[0], ""
|
|
164
|
+
):
|
|
165
|
+
split_departments = legacy_user.get(
|
|
166
|
+
self.departments_mapping.mapped_legacy_keys[0], ""
|
|
167
|
+
).split(self.library_configuration.multi_field_delimiter)
|
|
168
|
+
return self.library_configuration.multi_field_delimiter.join(
|
|
169
|
+
[
|
|
170
|
+
self.get_mapped_name(
|
|
171
|
+
self.departments_mapping,
|
|
172
|
+
{self.departments_mapping.mapped_legacy_keys[0]: dept},
|
|
173
|
+
index_or_id,
|
|
174
|
+
True,
|
|
175
|
+
)
|
|
176
|
+
for dept in split_departments
|
|
177
|
+
]
|
|
163
178
|
)
|
|
164
|
-
return self.library_configuration.multi_field_delimiter.join([
|
|
165
|
-
self.get_mapped_name(
|
|
166
|
-
self.departments_mapping,
|
|
167
|
-
{self.departments_mapping.mapped_legacy_keys[0]: dept},
|
|
168
|
-
index_or_id,
|
|
169
|
-
True,
|
|
170
|
-
) for dept in split_departments
|
|
171
|
-
])
|
|
172
179
|
else:
|
|
173
180
|
return self.get_mapped_name(
|
|
174
181
|
self.departments_mapping,
|
|
@@ -198,21 +205,29 @@ class UserMapper(MappingFileMapperBase):
|
|
|
198
205
|
return ""
|
|
199
206
|
|
|
200
207
|
def setup_groups_mapping(self, groups_map):
|
|
201
|
-
return
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
208
|
+
return (
|
|
209
|
+
RefDataMapping(
|
|
210
|
+
self.folio_client,
|
|
211
|
+
"/groups",
|
|
212
|
+
"usergroups",
|
|
213
|
+
groups_map,
|
|
214
|
+
"group",
|
|
215
|
+
"UserGroupMapping",
|
|
216
|
+
)
|
|
217
|
+
if groups_map
|
|
218
|
+
else None
|
|
219
|
+
)
|
|
209
220
|
|
|
210
221
|
def setup_departments_mapping(self, departments_mapping):
|
|
211
|
-
return
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
222
|
+
return (
|
|
223
|
+
RefDataMapping(
|
|
224
|
+
self.folio_client,
|
|
225
|
+
"/departments",
|
|
226
|
+
"departments",
|
|
227
|
+
departments_mapping,
|
|
228
|
+
"name",
|
|
229
|
+
"DepartmentsMapping",
|
|
230
|
+
)
|
|
231
|
+
if departments_mapping
|
|
232
|
+
else None
|
|
233
|
+
)
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import re
|
|
3
3
|
import traceback
|
|
4
|
-
from typing import Union
|
|
5
4
|
|
|
6
5
|
import i18n
|
|
7
6
|
from folioclient import FolioClient
|
|
@@ -58,11 +57,11 @@ class Conditions:
|
|
|
58
57
|
|
|
59
58
|
def setup_reference_data_for_bibs(self):
|
|
60
59
|
logging.info("Setting up reference data for bib transformation")
|
|
61
|
-
logging.info("%s\tcontrib_name_types", len(self.folio.contrib_name_types))
|
|
62
|
-
logging.info("%s\tcontributor_types", len(self.folio.contributor_types))
|
|
63
|
-
logging.info("%s\talt_title_types", len(self.folio.alt_title_types))
|
|
64
|
-
logging.info("%s\tidentifier_types", len(self.folio.identifier_types))
|
|
65
|
-
logging.info("%s\tsubject_types", len(self.folio.subject_types))
|
|
60
|
+
logging.info("%s\tcontrib_name_types", len(self.folio.contrib_name_types)) # type: ignore
|
|
61
|
+
logging.info("%s\tcontributor_types", len(self.folio.contributor_types)) # type: ignore
|
|
62
|
+
logging.info("%s\talt_title_types", len(self.folio.alt_title_types)) # type: ignore
|
|
63
|
+
logging.info("%s\tidentifier_types", len(self.folio.identifier_types)) # type: ignore
|
|
64
|
+
logging.info("%s\tsubject_types", len(self.folio.subject_types)) # type: ignore
|
|
66
65
|
# Raise for empty settings
|
|
67
66
|
if not self.folio.contributor_types:
|
|
68
67
|
raise TransformationProcessError("", "No contributor_types in FOLIO")
|
|
@@ -77,18 +76,20 @@ class Conditions:
|
|
|
77
76
|
|
|
78
77
|
# Set defaults
|
|
79
78
|
logging.info("Setting defaults")
|
|
80
|
-
self.default_contributor_name_type: str = self.folio.contrib_name_types[0]["id"]
|
|
79
|
+
self.default_contributor_name_type: str = self.folio.contrib_name_types[0]["id"] # type: ignore
|
|
81
80
|
logging.info("Contributor name type:\t%s", self.default_contributor_name_type)
|
|
82
81
|
self.default_contributor_type = next(
|
|
83
|
-
ct
|
|
82
|
+
ct
|
|
83
|
+
for ct in self.folio.contributor_types
|
|
84
|
+
if ct["code"] == "ctb" # type: ignore
|
|
84
85
|
)
|
|
85
86
|
logging.info("Contributor type:\t%s", self.default_contributor_type["id"])
|
|
86
87
|
|
|
87
88
|
def setup_reference_data_for_items_and_holdings(self, default_call_number_type_name):
|
|
88
|
-
logging.info(f"{len(self.folio.locations)}\tlocations")
|
|
89
|
+
logging.info(f"{len(self.folio.locations)}\tlocations") # type: ignore
|
|
89
90
|
self.default_call_number_type = {}
|
|
90
|
-
logging.info("%s\tholding_note_types", len(self.folio.holding_note_types))
|
|
91
|
-
logging.info("%s\tcall_number_types", len(self.folio.call_number_types))
|
|
91
|
+
logging.info("%s\tholding_note_types", len(self.folio.holding_note_types)) # type: ignore
|
|
92
|
+
logging.info("%s\tcall_number_types", len(self.folio.call_number_types)) # type: ignore
|
|
92
93
|
self.setup_and_validate_holdings_types()
|
|
93
94
|
self.ill_policies = self.folio.folio_get_all("/ill-policies", "illPolicies")
|
|
94
95
|
# Raise for empty settings
|
|
@@ -104,7 +105,7 @@ class Conditions:
|
|
|
104
105
|
self.default_call_number_type: dict = next(
|
|
105
106
|
(
|
|
106
107
|
ct
|
|
107
|
-
for ct in self.folio.call_number_types
|
|
108
|
+
for ct in self.folio.call_number_types # type: ignore
|
|
108
109
|
if ct["name"] == default_call_number_type_name
|
|
109
110
|
),
|
|
110
111
|
None,
|
|
@@ -127,7 +128,7 @@ class Conditions:
|
|
|
127
128
|
missing_holdings_types = [
|
|
128
129
|
ht
|
|
129
130
|
for ht in self.holdings_type_map.values()
|
|
130
|
-
if ht not in [ht_ref["name"] for ht_ref in self.holdings_types]
|
|
131
|
+
if ht not in [ht_ref["name"] for ht_ref in self.holdings_types] # type: ignore
|
|
131
132
|
]
|
|
132
133
|
if any(missing_holdings_types):
|
|
133
134
|
raise TransformationProcessError(
|
|
@@ -135,15 +136,15 @@ class Conditions:
|
|
|
135
136
|
"Holdings types are missing from the tenant. Please set them up",
|
|
136
137
|
missing_holdings_types,
|
|
137
138
|
)
|
|
138
|
-
logging.info("%s\tholdings types", len(self.holdings_types))
|
|
139
|
+
logging.info("%s\tholdings types", len(self.holdings_types)) # type: ignore
|
|
139
140
|
|
|
140
141
|
def setup_reference_data_for_all(self):
|
|
141
|
-
logging.info(f"{len(self.folio.class_types)}\tclass_types")
|
|
142
|
+
logging.info(f"{len(self.folio.class_types)}\tclass_types") # type: ignore
|
|
142
143
|
logging.info(
|
|
143
|
-
f"{len(self.folio.electronic_access_relationships)}\telectronic_access_relationships"
|
|
144
|
+
f"{len(self.folio.electronic_access_relationships)}\telectronic_access_relationships" # type: ignore
|
|
144
145
|
)
|
|
145
146
|
self.statistical_codes = self.folio.statistical_codes
|
|
146
|
-
logging.info(f"{len(self.statistical_codes)} \tstatistical_codes")
|
|
147
|
+
logging.info(f"{len(self.statistical_codes)} \tstatistical_codes") # type: ignore
|
|
147
148
|
|
|
148
149
|
# Raise for empty settings
|
|
149
150
|
if not self.folio.class_types:
|
|
@@ -156,13 +157,13 @@ class Conditions:
|
|
|
156
157
|
)
|
|
157
158
|
)
|
|
158
159
|
logging.info(f"{len(self.authority_note_types)} \tAuthority note types")
|
|
159
|
-
logging.info(f"{len(self.folio.identifier_types)} \tidentifier types")
|
|
160
|
+
logging.info(f"{len(self.folio.identifier_types)} \tidentifier types") # type: ignore
|
|
160
161
|
|
|
161
162
|
def get_condition(
|
|
162
|
-
self, name, legacy_id, value, parameter=None, marc_field:
|
|
163
|
+
self, name, legacy_id, value, parameter=None, marc_field: field.Field | None = None
|
|
163
164
|
):
|
|
164
165
|
try:
|
|
165
|
-
return self.condition_cache.get(name)(legacy_id, value, parameter, marc_field)
|
|
166
|
+
return self.condition_cache.get(name)(legacy_id, value, parameter, marc_field) # type: ignore
|
|
166
167
|
# Exception should only handle the missing condition from the cache.
|
|
167
168
|
# All other exceptions should propagate up
|
|
168
169
|
except Exception:
|
|
@@ -381,7 +382,7 @@ class Conditions:
|
|
|
381
382
|
identifier_type: dict = next(
|
|
382
383
|
(
|
|
383
384
|
f
|
|
384
|
-
for f in self.folio.identifier_types
|
|
385
|
+
for f in self.folio.identifier_types # type: ignore
|
|
385
386
|
if (
|
|
386
387
|
f["name"] in parameter.get("names", "non existant")
|
|
387
388
|
or f["name"] in parameter.get("name", "non existant")
|
|
@@ -424,7 +425,7 @@ class Conditions:
|
|
|
424
425
|
logging.error(ee)
|
|
425
426
|
raise TransformationRecordFailedError(
|
|
426
427
|
legacy_id,
|
|
427
|
-
f
|
|
428
|
+
f"Holdings note type mapping error.\tParameter: {parameter.get('name', '')}\t"
|
|
428
429
|
f"MARC Field: {marc_field}. Is mapping rules and ref data aligned?",
|
|
429
430
|
parameter.get("name", ""),
|
|
430
431
|
) from ee
|
|
@@ -442,7 +443,7 @@ class Conditions:
|
|
|
442
443
|
logging.error(ee)
|
|
443
444
|
raise TransformationProcessError(
|
|
444
445
|
legacy_id,
|
|
445
|
-
f
|
|
446
|
+
f"Authority note type mapping error.\tParameter: {parameter.get('name', '')}\t"
|
|
446
447
|
f"MARC Field: {marc_field}. Is mapping rules and ref data aligned?",
|
|
447
448
|
parameter.get("name", ""),
|
|
448
449
|
) from ee
|
|
@@ -456,18 +457,17 @@ class Conditions:
|
|
|
456
457
|
)
|
|
457
458
|
self.mapper.migration_report.add("MappedClassificationTypes", t[1])
|
|
458
459
|
return t[0]
|
|
459
|
-
except Exception:
|
|
460
|
+
except Exception as e:
|
|
460
461
|
raise TransformationRecordFailedError(
|
|
461
462
|
legacy_id,
|
|
462
463
|
f'Classification mapping error.\tParameter: "{parameter.get("name", "")}"\t'
|
|
463
464
|
f"MARC Field: {marc_field}. Is mapping rules and ref data aligned?",
|
|
464
465
|
parameter.get("name", ""),
|
|
465
|
-
)
|
|
466
|
+
) from e
|
|
466
467
|
|
|
467
468
|
def condition_char_select(self, legacy_id, value, parameter, marc_field: field.Field):
|
|
468
469
|
return value[parameter["from"] : parameter["to"]]
|
|
469
470
|
|
|
470
|
-
|
|
471
471
|
def condition_set_identifier_type_id_by_name(
|
|
472
472
|
self, legacy_id, value, parameter, marc_field: field.Field
|
|
473
473
|
):
|
|
@@ -513,8 +513,8 @@ class Conditions:
|
|
|
513
513
|
f"{marc_field.tag} ({parameter.get('name', '')}) -> {t[1]}",
|
|
514
514
|
)
|
|
515
515
|
return t[0]
|
|
516
|
-
except Exception:
|
|
517
|
-
raise ValueError(f"Instance note type not found for {marc_field} {parameter}")
|
|
516
|
+
except Exception as e:
|
|
517
|
+
raise ValueError(f"Instance note type not found for {marc_field} {parameter}") from e
|
|
518
518
|
|
|
519
519
|
def condition_set_contributor_type_id(
|
|
520
520
|
self, legacy_id, value, parameter, marc_field: field.Field
|
|
@@ -562,7 +562,8 @@ class Conditions:
|
|
|
562
562
|
self.mapper.migration_report.add(
|
|
563
563
|
"ContributorTypeMapping",
|
|
564
564
|
i18n.t(
|
|
565
|
-
'Mapping failed for %{tag} "%{subfield}"
|
|
565
|
+
'Mapping failed for %{tag} "%{subfield}" '
|
|
566
|
+
"(Normalized: %{normalized_subfield})",
|
|
566
567
|
tag=f"{marc_field.tag} $e",
|
|
567
568
|
subfield=subfield,
|
|
568
569
|
normalized_subfield=normalized_subfield,
|
|
@@ -631,7 +632,8 @@ class Conditions:
|
|
|
631
632
|
"CallNumberTypeMapping",
|
|
632
633
|
(
|
|
633
634
|
i18n.t(
|
|
634
|
-
'Unhandled call number type in ind1: "%{ind1}".\n Returning default
|
|
635
|
+
'Unhandled call number type in ind1: "%{ind1}".\n Returning default '
|
|
636
|
+
"Callnumber type: %{type}",
|
|
635
637
|
ind1=marc_field.indicator1,
|
|
636
638
|
type=self.default_call_number_type["name"],
|
|
637
639
|
)
|
|
@@ -652,7 +654,7 @@ class Conditions:
|
|
|
652
654
|
"CallNumberTypeMapping",
|
|
653
655
|
(
|
|
654
656
|
"Mapping failed. Setting default CallNumber type: "
|
|
655
|
-
f
|
|
657
|
+
f"{self.default_call_number_type['name']}"
|
|
656
658
|
),
|
|
657
659
|
)
|
|
658
660
|
|
|
@@ -663,7 +665,7 @@ class Conditions:
|
|
|
663
665
|
):
|
|
664
666
|
for subfield in marc_field.get_subfields("4", "e"):
|
|
665
667
|
normalized_subfield = re.sub(r"[^A-Za-z0-9 ]+", "", subfield.strip())
|
|
666
|
-
for cont_type in self.folio.contributor_types:
|
|
668
|
+
for cont_type in self.folio.contributor_types: # type: ignore
|
|
667
669
|
if normalized_subfield in [cont_type["code"], cont_type["name"]]:
|
|
668
670
|
return cont_type["name"]
|
|
669
671
|
try:
|
|
@@ -679,11 +681,11 @@ class Conditions:
|
|
|
679
681
|
)
|
|
680
682
|
self.mapper.migration_report.add("MappedAlternativeTitleTypes", t[1])
|
|
681
683
|
return t[0]
|
|
682
|
-
except Exception:
|
|
684
|
+
except Exception as e:
|
|
683
685
|
raise TransformationProcessError(
|
|
684
686
|
legacy_id,
|
|
685
687
|
f"Alternative title type not found for {parameter['name']} {marc_field}",
|
|
686
|
-
)
|
|
688
|
+
) from e
|
|
687
689
|
|
|
688
690
|
def condition_set_location_id_by_code(
|
|
689
691
|
self, legacy_id, value, parameter, marc_field: field.Field
|
|
@@ -702,7 +704,7 @@ class Conditions:
|
|
|
702
704
|
):
|
|
703
705
|
if "legacy_locations" not in self.ref_data_dicts:
|
|
704
706
|
try:
|
|
705
|
-
d = {lm["legacy_code"]: lm["folio_code"] for lm in self.mapper.location_map}
|
|
707
|
+
d = {lm["legacy_code"]: lm["folio_code"] for lm in self.mapper.location_map} # type: ignore
|
|
706
708
|
self.ref_data_dicts["legacy_locations"] = d
|
|
707
709
|
for folio_code in d.values():
|
|
708
710
|
t = self.get_ref_data_tuple_by_code(
|
|
@@ -802,14 +804,14 @@ class Conditions:
|
|
|
802
804
|
)
|
|
803
805
|
self.mapper.migration_report.add("MappedElectronicRelationshipTypes", t[1])
|
|
804
806
|
return t[0]
|
|
805
|
-
except Exception:
|
|
807
|
+
except Exception as e:
|
|
806
808
|
raise TransformationProcessError(
|
|
807
809
|
legacy_id,
|
|
808
|
-
f"Electronic access relationship not found for {parameter['name']} {marc_field}",
|
|
809
|
-
)
|
|
810
|
+
f"Electronic access relationship not found for {parameter['name']} {marc_field}", # noqa: E501
|
|
811
|
+
) from e
|
|
810
812
|
return self._extracted_from_condition_set_electronic_access_relations_id_2("3", marc_field)
|
|
811
813
|
|
|
812
|
-
# TODO Rename this here and in `condition_set_url_relationship` and `condition_set_electronic_access_relations_id`
|
|
814
|
+
# TODO Rename this here and in `condition_set_url_relationship` and `condition_set_electronic_access_relations_id` # noqa: E501
|
|
813
815
|
def _extracted_from_condition_set_electronic_access_relations_id_2(self, arg0, marc_field):
|
|
814
816
|
enum = {
|
|
815
817
|
"0": "resource",
|
|
@@ -855,44 +857,50 @@ class Conditions:
|
|
|
855
857
|
)
|
|
856
858
|
self.mapper.migration_report.add("MappedSubjectTypes", t[1])
|
|
857
859
|
return t[0]
|
|
858
|
-
except Exception:
|
|
860
|
+
except Exception as e:
|
|
859
861
|
raise TransformationProcessError(
|
|
860
862
|
legacy_id,
|
|
861
863
|
f"Subject type not found for {parameter['name']} {marc_field}",
|
|
862
|
-
)
|
|
864
|
+
) from e
|
|
863
865
|
|
|
864
|
-
def condition_set_subject_source_id(
|
|
866
|
+
def condition_set_subject_source_id(
|
|
867
|
+
self, legacy_id, value, parameter, marc_field: field.Field
|
|
868
|
+
):
|
|
865
869
|
try:
|
|
866
870
|
t = self.get_ref_data_tuple_by_name(
|
|
867
|
-
self.folio.folio_get_all("/subject-sources", "subjectSources"),
|
|
871
|
+
self.folio.folio_get_all("/subject-sources", "subjectSources"),
|
|
872
|
+
"subject_sources",
|
|
873
|
+
parameter["name"],
|
|
868
874
|
)
|
|
869
875
|
self.mapper.migration_report.add("MappedSubjectSources", t[1])
|
|
870
876
|
return t[0]
|
|
871
|
-
except Exception:
|
|
877
|
+
except Exception as e:
|
|
872
878
|
raise TransformationProcessError(
|
|
873
879
|
legacy_id,
|
|
874
880
|
f"Subject source not found for {parameter['name']} {marc_field}",
|
|
875
|
-
)
|
|
881
|
+
) from e
|
|
876
882
|
|
|
877
|
-
def condition_set_subject_source_id_by_code(
|
|
883
|
+
def condition_set_subject_source_id_by_code(
|
|
884
|
+
self, legacy_id, value, parameter, marc_field: field.Field
|
|
885
|
+
):
|
|
878
886
|
try:
|
|
879
887
|
t = self.get_ref_data_tuple_by_code(
|
|
880
|
-
self.folio.folio_get_all("/subject-sources", "subjectSources"),
|
|
888
|
+
self.folio.folio_get_all("/subject-sources", "subjectSources"),
|
|
889
|
+
"subject_sources",
|
|
890
|
+
value,
|
|
881
891
|
)
|
|
882
892
|
self.mapper.migration_report.add("MappedSubjectSources", t[1])
|
|
883
893
|
return t[0]
|
|
884
|
-
except Exception:
|
|
894
|
+
except Exception as e:
|
|
885
895
|
raise TransformationProcessError(
|
|
886
896
|
legacy_id,
|
|
887
897
|
f"Subject source not found for {value} {marc_field}",
|
|
888
|
-
)
|
|
898
|
+
) from e
|
|
889
899
|
|
|
890
|
-
def condition_set_receipt_status(
|
|
891
|
-
self, legacy_id, value, parameter, marc_field: field.Field
|
|
892
|
-
):
|
|
900
|
+
def condition_set_receipt_status(self, legacy_id, value, parameter, marc_field: field.Field):
|
|
893
901
|
"""
|
|
894
902
|
This method maps the receipt status based on the 008 field.
|
|
895
|
-
This condition is not available in FOLIO's MARC mapping engine and
|
|
903
|
+
This condition is not available in FOLIO's MARC mapping engine and
|
|
896
904
|
will require use of a supplemental mapping rules file in the
|
|
897
905
|
HoldingsMarcTransformer task definition.
|
|
898
906
|
"""
|
|
@@ -901,7 +909,7 @@ class Conditions:
|
|
|
901
909
|
"ReceiptStatusMapping", i18n.t("008 is too short") + f": {value}"
|
|
902
910
|
)
|
|
903
911
|
return ""
|
|
904
|
-
|
|
912
|
+
|
|
905
913
|
status_map = {
|
|
906
914
|
"0": "Unknown",
|
|
907
915
|
"1": "Other receipt or acquisition status",
|
|
@@ -934,7 +942,7 @@ class Conditions:
|
|
|
934
942
|
):
|
|
935
943
|
"""
|
|
936
944
|
This method maps the acquisition method based on the 008 field.
|
|
937
|
-
This condition is not available in FOLIO's MARC mapping engine and
|
|
945
|
+
This condition is not available in FOLIO's MARC mapping engine and
|
|
938
946
|
will require use of a supplemental mapping rules file in the
|
|
939
947
|
HoldingsMarcTransformer task definition.
|
|
940
948
|
"""
|
|
@@ -973,9 +981,7 @@ class Conditions:
|
|
|
973
981
|
)
|
|
974
982
|
return ""
|
|
975
983
|
|
|
976
|
-
def condition_set_retention_policy(
|
|
977
|
-
self, legacy_id, value, parameter, marc_field: field.Field
|
|
978
|
-
):
|
|
984
|
+
def condition_set_retention_policy(self, legacy_id, value, parameter, marc_field: field.Field):
|
|
979
985
|
"""
|
|
980
986
|
This method maps the retention policy based on the 008 field.
|
|
981
987
|
This condition is not available in FOLIO's MARC mapping engine and
|
|
@@ -987,7 +993,9 @@ class Conditions:
|
|
|
987
993
|
"RetentionPolicyMapping", i18n.t("008 is too short") + f": {value}"
|
|
988
994
|
)
|
|
989
995
|
return ""
|
|
990
|
-
value = value.replace("|", " ").replace(
|
|
996
|
+
value = value.replace("|", " ").replace(
|
|
997
|
+
"#", " "
|
|
998
|
+
) # Replace pipe with space for mapping consistency
|
|
991
999
|
try:
|
|
992
1000
|
retention_policies = {
|
|
993
1001
|
"0": "Unknown",
|
|
@@ -1020,21 +1028,22 @@ class Conditions:
|
|
|
1020
1028
|
"y": "Year",
|
|
1021
1029
|
"e": "Edition",
|
|
1022
1030
|
"i": "Issue",
|
|
1023
|
-
"s": "Supplement"
|
|
1031
|
+
"s": "Supplement",
|
|
1024
1032
|
}
|
|
1025
1033
|
try:
|
|
1026
1034
|
specific_retention_policy = ""
|
|
1027
1035
|
if value[13].strip() or value[15].strip():
|
|
1028
1036
|
if value[14].strip() and int(value[14]) > 1:
|
|
1029
|
-
specific_retention_policy = f"{policy_types.get(value[13], '')} {value[14]} {unit_types.get(value[15], '')}s retained".strip()
|
|
1037
|
+
specific_retention_policy = f"{policy_types.get(value[13], '')} {value[14]} {unit_types.get(value[15], '')}s retained".strip() # noqa: E501
|
|
1030
1038
|
else:
|
|
1031
|
-
specific_retention_policy = f"{policy_types.get(value[13], '')} {unit_types.get(value[15], '')} retained".strip()
|
|
1039
|
+
specific_retention_policy = f"{policy_types.get(value[13], '')} {unit_types.get(value[15], '')} retained".strip() # noqa: E501
|
|
1032
1040
|
if specific_retention_policy:
|
|
1033
1041
|
self.mapper.migration_report.add(
|
|
1034
1042
|
"RetentionPolicyMapping",
|
|
1035
1043
|
i18n.t(
|
|
1036
|
-
"Retention policy 6 indicates a limited period. Specific
|
|
1037
|
-
|
|
1044
|
+
"Retention policy 6 indicates a limited period. Specific "
|
|
1045
|
+
"retention period will be mapped from 008/13-15",
|
|
1046
|
+
),
|
|
1038
1047
|
)
|
|
1039
1048
|
return specific_retention_policy
|
|
1040
1049
|
else:
|
|
@@ -1044,7 +1053,10 @@ class Conditions:
|
|
|
1044
1053
|
except ValueError:
|
|
1045
1054
|
self.mapper.migration_report.add(
|
|
1046
1055
|
"RetentionPolicyMapping",
|
|
1047
|
-
i18n.t(
|
|
1056
|
+
i18n.t(
|
|
1057
|
+
"Invalid specific retention policy in 008/13-15: %{value}",
|
|
1058
|
+
value=value[13:16],
|
|
1059
|
+
),
|
|
1048
1060
|
)
|
|
1049
1061
|
return mapped_value
|
|
1050
1062
|
except Exception:
|
|
@@ -1053,9 +1065,7 @@ class Conditions:
|
|
|
1053
1065
|
)
|
|
1054
1066
|
return ""
|
|
1055
1067
|
|
|
1056
|
-
def condition_set_ill_policy(
|
|
1057
|
-
self, legacy_id, value, parameter, marc_field: field.Field
|
|
1058
|
-
):
|
|
1068
|
+
def condition_set_ill_policy(self, legacy_id, value, parameter, marc_field: field.Field):
|
|
1059
1069
|
"""
|
|
1060
1070
|
This method maps the ILL policy based on the 008 field.
|
|
1061
1071
|
This condition is not available in FOLIO's MARC mapping engine and
|
|
@@ -1077,7 +1087,11 @@ class Conditions:
|
|
|
1077
1087
|
mapped_value = ill_policies[value[20]]
|
|
1078
1088
|
self.mapper.migration_report.add(
|
|
1079
1089
|
"ILLPolicyMapping",
|
|
1080
|
-
i18n.t(
|
|
1090
|
+
i18n.t(
|
|
1091
|
+
"%{value} mapped to %{mapped_value}",
|
|
1092
|
+
value=value[20],
|
|
1093
|
+
mapped_value=mapped_value,
|
|
1094
|
+
),
|
|
1081
1095
|
)
|
|
1082
1096
|
ill_policy_id = self.get_ref_data_tuple_by_name(
|
|
1083
1097
|
self.ill_policies, "ill_policies", mapped_value
|
|
@@ -1123,4 +1137,4 @@ class Conditions:
|
|
|
1123
1137
|
self.mapper.migration_report.add(
|
|
1124
1138
|
"DigitizationPolicyMapping", i18n.t("%{value} not found in map.", value=value[21])
|
|
1125
1139
|
)
|
|
1126
|
-
return ""
|
|
1140
|
+
return ""
|
|
@@ -35,7 +35,7 @@ class HoldingsStatementsParser:
|
|
|
35
35
|
|
|
36
36
|
Returns:
|
|
37
37
|
dict: A dictionary containing parsed holdings statements and related information.
|
|
38
|
-
"""
|
|
38
|
+
""" # noqa: E501
|
|
39
39
|
|
|
40
40
|
# Textual holdings statements
|
|
41
41
|
return_dict: dict = {"statements": [], "migration_report": [], "hlm_stmts": []}
|
|
@@ -67,14 +67,20 @@ class HoldingsStatementsParser:
|
|
|
67
67
|
raise TransformationFieldMappingError(
|
|
68
68
|
legacy_ids,
|
|
69
69
|
i18n.t(
|
|
70
|
-
"subfield present in %{linked_value_tag} but not
|
|
70
|
+
"subfield present in %{linked_value_tag} but not "
|
|
71
|
+
"in %{pattern_field}",
|
|
71
72
|
pattern_field=pattern_field,
|
|
72
73
|
linked_value_tag=linked_value_field,
|
|
73
74
|
),
|
|
74
75
|
pattern_field,
|
|
75
76
|
) from e
|
|
76
77
|
HoldingsStatementsParser.prepare_return_dict(
|
|
77
|
-
pattern_tag,
|
|
78
|
+
pattern_tag,
|
|
79
|
+
legacy_ids,
|
|
80
|
+
return_dict,
|
|
81
|
+
pattern_field,
|
|
82
|
+
linked_value_field,
|
|
83
|
+
parsed_dict,
|
|
78
84
|
)
|
|
79
85
|
|
|
80
86
|
if dedupe_results:
|
|
@@ -296,8 +302,10 @@ class HoldingsStatementsParser:
|
|
|
296
302
|
elif cron_to.strip() and val:
|
|
297
303
|
val_rest = val
|
|
298
304
|
if year:
|
|
299
|
-
cron_from, cron_to =
|
|
300
|
-
|
|
305
|
+
cron_from, cron_to = (
|
|
306
|
+
HoldingsStatementsParser.format_year_cron_from_cron_to(
|
|
307
|
+
cron_from, cron_to, hlm_stmt, val, val_rest
|
|
308
|
+
)
|
|
301
309
|
)
|
|
302
310
|
|
|
303
311
|
else:
|
|
@@ -114,7 +114,7 @@ class HRIDHandler:
|
|
|
114
114
|
f_001 = marc_record["001"].value()
|
|
115
115
|
f_003 = marc_record["003"].value().strip() if "003" in marc_record else ""
|
|
116
116
|
migration_report.add(
|
|
117
|
-
"HridHandling", i18n.t("Values in %{field}", field="003") + f
|
|
117
|
+
"HridHandling", i18n.t("Values in %{field}", field="003") + f": {f_003 or 'Empty'}"
|
|
118
118
|
)
|
|
119
119
|
|
|
120
120
|
if deactivate035_from001:
|
|
@@ -214,7 +214,8 @@ class HRIDHandler:
|
|
|
214
214
|
self.migration_report.add(
|
|
215
215
|
"HridHandling",
|
|
216
216
|
i18n.t(
|
|
217
|
-
"Duplicate 001. Creating HRID instead.\n Previous 001 will be stored in a new
|
|
217
|
+
"Duplicate 001. Creating HRID instead.\n Previous 001 will be stored in a new "
|
|
218
|
+
"035 field"
|
|
218
219
|
),
|
|
219
220
|
)
|
|
220
221
|
self.handle_035_generation(
|
|
@@ -24,7 +24,10 @@ from folio_migration_tools.migration_report import MigrationReport
|
|
|
24
24
|
|
|
25
25
|
class MarcFileProcessor:
|
|
26
26
|
def __init__(
|
|
27
|
-
self,
|
|
27
|
+
self,
|
|
28
|
+
mapper: RulesMapperBase,
|
|
29
|
+
folder_structure: FolderStructure,
|
|
30
|
+
created_objects_file: TextIO,
|
|
28
31
|
):
|
|
29
32
|
self.object_type: FOLIONamespaces = folder_structure.object_type
|
|
30
33
|
self.folder_structure: FolderStructure = folder_structure
|
|
@@ -35,16 +38,15 @@ class MarcFileProcessor:
|
|
|
35
38
|
):
|
|
36
39
|
self.srs_records_file: TextIO = open(self.folder_structure.srs_records_path, "w+")
|
|
37
40
|
if getattr(mapper.task_configuration, "data_import_marc", False):
|
|
38
|
-
self.data_import_marc_file: BinaryIO = open(
|
|
41
|
+
self.data_import_marc_file: BinaryIO = open(
|
|
42
|
+
self.folder_structure.data_import_marc_path, "wb+"
|
|
43
|
+
)
|
|
39
44
|
self.unique_001s: Set[str] = set()
|
|
40
45
|
self.failed_records_count: int = 0
|
|
41
46
|
self.records_count: int = 0
|
|
42
47
|
self.start: float = time.time()
|
|
43
48
|
self.legacy_ids: Set[str] = set()
|
|
44
|
-
if
|
|
45
|
-
self.object_type == FOLIONamespaces.holdings
|
|
46
|
-
and self.mapper.create_source_records
|
|
47
|
-
):
|
|
49
|
+
if self.object_type == FOLIONamespaces.holdings and self.mapper.create_source_records:
|
|
48
50
|
logging.info("Loading Parent HRID map for SRS creation")
|
|
49
51
|
self.parent_hrids = {entity[1]: entity[2] for entity in mapper.parent_id_map.values()}
|
|
50
52
|
|
|
@@ -83,10 +85,7 @@ class MarcFileProcessor:
|
|
|
83
85
|
)
|
|
84
86
|
self.add_legacy_ids_to_map(folio_rec, filtered_legacy_ids)
|
|
85
87
|
|
|
86
|
-
if
|
|
87
|
-
file_def.create_source_records
|
|
88
|
-
and self.mapper.create_source_records
|
|
89
|
-
):
|
|
88
|
+
if file_def.create_source_records and self.mapper.create_source_records:
|
|
90
89
|
self.save_srs_record(
|
|
91
90
|
marc_record,
|
|
92
91
|
file_def,
|
|
@@ -95,11 +94,7 @@ class MarcFileProcessor:
|
|
|
95
94
|
self.object_type,
|
|
96
95
|
)
|
|
97
96
|
if getattr(self.mapper.task_configuration, "data_import_marc", False):
|
|
98
|
-
self.save_marc_record(
|
|
99
|
-
marc_record,
|
|
100
|
-
folio_rec,
|
|
101
|
-
self.object_type
|
|
102
|
-
)
|
|
97
|
+
self.save_marc_record(marc_record, folio_rec, self.object_type)
|
|
103
98
|
Helper.write_to_file(self.created_objects_file, folio_rec)
|
|
104
99
|
self.mapper.migration_report.add_general_statistics(
|
|
105
100
|
i18n.t("Inventory records written to disk")
|
|
@@ -136,12 +131,7 @@ class MarcFileProcessor:
|
|
|
136
131
|
):
|
|
137
132
|
self.mapper.remove_from_id_map(folio_rec.get("formerIds", []))
|
|
138
133
|
|
|
139
|
-
def save_marc_record(
|
|
140
|
-
self,
|
|
141
|
-
marc_record: Record,
|
|
142
|
-
folio_rec: Dict,
|
|
143
|
-
object_type: FOLIONamespaces
|
|
144
|
-
):
|
|
134
|
+
def save_marc_record(self, marc_record: Record, folio_rec: Dict, object_type: FOLIONamespaces):
|
|
145
135
|
self.mapper.save_data_import_marc_record(
|
|
146
136
|
self.data_import_marc_file,
|
|
147
137
|
object_type,
|
|
@@ -228,7 +218,9 @@ class MarcFileProcessor:
|
|
|
228
218
|
|
|
229
219
|
@staticmethod
|
|
230
220
|
def get_valid_folio_record_ids(
|
|
231
|
-
legacy_ids: List[str],
|
|
221
|
+
legacy_ids: List[str],
|
|
222
|
+
folio_record_identifiers: Set[str],
|
|
223
|
+
migration_report: MigrationReport,
|
|
232
224
|
) -> List[str]:
|
|
233
225
|
new_ids: Set[str] = set()
|
|
234
226
|
for legacy_id in legacy_ids:
|