folio-migration-tools 1.10.2__py3-none-any.whl → 1.10.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__init__.py +10 -2
- folio_migration_tools/__main__.py +7 -0
- folio_migration_tools/circulation_helper.py +23 -8
- folio_migration_tools/colors.py +7 -0
- folio_migration_tools/config_file_load.py +7 -0
- folio_migration_tools/custom_dict.py +17 -0
- folio_migration_tools/custom_exceptions.py +40 -4
- folio_migration_tools/extradata_writer.py +12 -0
- folio_migration_tools/folder_structure.py +16 -0
- folio_migration_tools/helper.py +7 -0
- folio_migration_tools/holdings_helper.py +11 -5
- folio_migration_tools/i18n_config.py +6 -0
- folio_migration_tools/library_configuration.py +19 -5
- folio_migration_tools/mapper_base.py +15 -0
- folio_migration_tools/mapping_file_transformation/__init__.py +1 -0
- folio_migration_tools/mapping_file_transformation/courses_mapper.py +17 -0
- folio_migration_tools/mapping_file_transformation/holdings_mapper.py +19 -0
- folio_migration_tools/mapping_file_transformation/item_mapper.py +24 -0
- folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py +18 -0
- folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +26 -9
- folio_migration_tools/mapping_file_transformation/notes_mapper.py +16 -0
- folio_migration_tools/mapping_file_transformation/order_mapper.py +40 -27
- folio_migration_tools/mapping_file_transformation/organization_mapper.py +40 -33
- folio_migration_tools/mapping_file_transformation/ref_data_mapping.py +17 -0
- folio_migration_tools/mapping_file_transformation/user_mapper.py +16 -0
- folio_migration_tools/marc_rules_transformation/__init__.py +1 -0
- folio_migration_tools/marc_rules_transformation/conditions.py +49 -36
- folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +9 -3
- folio_migration_tools/marc_rules_transformation/hrid_handler.py +16 -1
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +15 -1
- folio_migration_tools/marc_rules_transformation/marc_reader_wrapper.py +7 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +35 -29
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +23 -18
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +46 -27
- folio_migration_tools/migration_report.py +14 -6
- folio_migration_tools/migration_tasks/__init__.py +2 -0
- folio_migration_tools/migration_tasks/batch_poster.py +34 -18
- folio_migration_tools/migration_tasks/bibs_transformer.py +16 -0
- folio_migration_tools/migration_tasks/courses_migrator.py +15 -0
- folio_migration_tools/migration_tasks/holdings_csv_transformer.py +18 -3
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +17 -0
- folio_migration_tools/migration_tasks/inventory_batch_poster.py +424 -0
- folio_migration_tools/migration_tasks/items_transformer.py +16 -0
- folio_migration_tools/migration_tasks/loans_migrator.py +17 -2
- folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py +16 -0
- folio_migration_tools/migration_tasks/marc_import.py +407 -0
- folio_migration_tools/migration_tasks/migration_task_base.py +49 -17
- folio_migration_tools/migration_tasks/orders_transformer.py +16 -0
- folio_migration_tools/migration_tasks/organization_transformer.py +17 -2
- folio_migration_tools/migration_tasks/requests_migrator.py +15 -0
- folio_migration_tools/migration_tasks/reserves_migrator.py +15 -0
- folio_migration_tools/migration_tasks/user_importer.py +347 -0
- folio_migration_tools/migration_tasks/user_transformer.py +16 -0
- folio_migration_tools/task_configuration.py +7 -0
- folio_migration_tools/transaction_migration/__init__.py +1 -0
- folio_migration_tools/transaction_migration/legacy_loan.py +16 -0
- folio_migration_tools/transaction_migration/legacy_request.py +14 -0
- folio_migration_tools/transaction_migration/legacy_reserve.py +14 -0
- folio_migration_tools/transaction_migration/transaction_result.py +16 -0
- {folio_migration_tools-1.10.2.dist-info → folio_migration_tools-1.10.3.dist-info}/METADATA +1 -1
- folio_migration_tools-1.10.3.dist-info/RECORD +66 -0
- folio_migration_tools-1.10.2.dist-info/RECORD +0 -63
- {folio_migration_tools-1.10.2.dist-info → folio_migration_tools-1.10.3.dist-info}/WHEEL +0 -0
- {folio_migration_tools-1.10.2.dist-info → folio_migration_tools-1.10.3.dist-info}/entry_points.txt +0 -0
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
"""Base class for MARC rules-based transformations.
|
|
2
|
+
|
|
3
|
+
Provides the abstract RulesMapperBase class that all MARC rules mappers inherit from.
|
|
4
|
+
Handles loading transformation rules from JSON, applying rules with conditions,
|
|
5
|
+
and managing the transformation workflow for MARC-to-FOLIO conversions.
|
|
6
|
+
"""
|
|
7
|
+
|
|
1
8
|
import datetime
|
|
2
9
|
import json
|
|
3
10
|
import logging
|
|
@@ -41,6 +48,17 @@ class RulesMapperBase(MapperBase):
|
|
|
41
48
|
conditions=None,
|
|
42
49
|
parent_id_map: dict[str, tuple] = None,
|
|
43
50
|
):
|
|
51
|
+
"""Initialize base mapper for MARC rules-based transformations.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
folio_client (FolioClient): FOLIO API client.
|
|
55
|
+
library_configuration (LibraryConfiguration): Library configuration.
|
|
56
|
+
task_configuration: Task configuration for MARC transformation.
|
|
57
|
+
statistical_codes_map (Optional[Dict]): Mapping for statistical codes.
|
|
58
|
+
schema (dict): JSON schema for validation.
|
|
59
|
+
conditions: Conditions processor for rules evaluation.
|
|
60
|
+
parent_id_map (dict[str, tuple]): Optional mapping of parent IDs.
|
|
61
|
+
"""
|
|
44
62
|
super().__init__(library_configuration, task_configuration, folio_client, parent_id_map)
|
|
45
63
|
self.parsed_records = 0
|
|
46
64
|
self.id_map: dict[str, tuple] = {}
|
|
@@ -538,7 +556,7 @@ class RulesMapperBase(MapperBase):
|
|
|
538
556
|
)
|
|
539
557
|
|
|
540
558
|
def remove_from_id_map(self, former_ids: List[str]):
|
|
541
|
-
"""
|
|
559
|
+
"""Removes the ID from the map in case parsing failed.
|
|
542
560
|
|
|
543
561
|
Args:
|
|
544
562
|
former_ids (_type_): _description_
|
|
@@ -739,9 +757,7 @@ class RulesMapperBase(MapperBase):
|
|
|
739
757
|
|
|
740
758
|
@staticmethod
|
|
741
759
|
def grouped(marc_field: Field):
|
|
742
|
-
"""
|
|
743
|
-
s -> (s0,s1,s2,...sn-1), (sn,sn+1,sn+2,...s2n-1), (s2n,s2n+1,s2n+2,...s3n-1), ...
|
|
744
|
-
|
|
760
|
+
"""Group subfields into tuples by repeated subfield occurrences.
|
|
745
761
|
|
|
746
762
|
Args:
|
|
747
763
|
marc_field (Field): _description_
|
|
@@ -778,8 +794,7 @@ class RulesMapperBase(MapperBase):
|
|
|
778
794
|
|
|
779
795
|
@staticmethod
|
|
780
796
|
def remove_repeated_subfields(marc_field: Field):
|
|
781
|
-
"""
|
|
782
|
-
s -> (s0,s1,s2,...sn-1), (sn,sn+1,sn+2,...s2n-1), (s2n,s2n+1,s2n+2,...s3n-1), ...
|
|
797
|
+
"""Remove repeated subfields, keeping only the first occurrence of each.
|
|
783
798
|
|
|
784
799
|
Args:
|
|
785
800
|
marc_field (Field): _description_
|
|
@@ -803,16 +818,13 @@ class RulesMapperBase(MapperBase):
|
|
|
803
818
|
marc_record: Record,
|
|
804
819
|
folio_record,
|
|
805
820
|
):
|
|
806
|
-
"""
|
|
821
|
+
"""Save the source MARC record to a file for Data Import loading.
|
|
807
822
|
|
|
808
823
|
Args:
|
|
809
|
-
|
|
810
|
-
record_type (FOLIONamespaces):
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
folio_record (_type_): _description_
|
|
814
|
-
legacy_ids (List[str]): _description_
|
|
815
|
-
suppress (bool): _description_
|
|
824
|
+
data_import_marc_file: File handle for writing MARC records.
|
|
825
|
+
record_type (FOLIONamespaces): Type of record being saved.
|
|
826
|
+
marc_record (Record): The MARC record to save.
|
|
827
|
+
folio_record: The corresponding FOLIO record.
|
|
816
828
|
"""
|
|
817
829
|
marc_record.add_ordered_field(
|
|
818
830
|
Field(
|
|
@@ -838,7 +850,7 @@ class RulesMapperBase(MapperBase):
|
|
|
838
850
|
file_def: FileDefinition,
|
|
839
851
|
marc_record: Record,
|
|
840
852
|
):
|
|
841
|
-
"""Map statistical codes to FOLIO instance
|
|
853
|
+
"""Map statistical codes to FOLIO instance.
|
|
842
854
|
|
|
843
855
|
This method first calls the base class method to map statistical codes
|
|
844
856
|
from the file_def. Then, it checks to see if there are any MARC field
|
|
@@ -920,7 +932,7 @@ class RulesMapperBase(MapperBase):
|
|
|
920
932
|
legacy_ids: List[str],
|
|
921
933
|
suppress: bool,
|
|
922
934
|
):
|
|
923
|
-
"""Saves the source Marc_record to the Source record Storage module
|
|
935
|
+
"""Saves the source Marc_record to the Source record Storage module.
|
|
924
936
|
|
|
925
937
|
Args:
|
|
926
938
|
srs_records_file (_type_): _description_
|
|
@@ -1075,18 +1087,12 @@ def is_array_of_objects(schema_property):
|
|
|
1075
1087
|
|
|
1076
1088
|
|
|
1077
1089
|
def entity_indicators_match(entity_mapping, marc_field):
|
|
1078
|
-
"""
|
|
1079
|
-
|
|
1080
|
-
Entity mappings can limit the fields they are applied to by specifying indicator
|
|
1081
|
-
must match the provided MARC field's indicators. If the entity mapping
|
|
1082
|
-
indicator values, it is assumed to match all MARC fields.
|
|
1083
|
-
specific value or a wildcard "*"
|
|
1084
|
-
|
|
1085
|
-
This function compares the indicators of the entity mapping with the indicators of the MARC field.
|
|
1086
|
-
If the entity does not specify any indicator values, the function returns True. If the entity does
|
|
1087
|
-
specify indicator values, the function checks if the MARC field's indicators match the specified
|
|
1088
|
-
values or if the specified values are wildcards. If both indicators match, the function returns True;
|
|
1089
|
-
otherwise, it returns False.
|
|
1090
|
+
"""Check if entity mapping indicators match the MARC field indicators.
|
|
1091
|
+
|
|
1092
|
+
Entity mappings can limit the fields they are applied to by specifying indicator
|
|
1093
|
+
values that must match the provided MARC field's indicators. If the entity mapping
|
|
1094
|
+
does not specify any indicator values, it is assumed to match all MARC fields.
|
|
1095
|
+
Entity indicator values can be a specific value or a wildcard "*".
|
|
1090
1096
|
|
|
1091
1097
|
Args:
|
|
1092
1098
|
entity_mapping (dict): _description_
|
|
@@ -1094,7 +1100,7 @@ def entity_indicators_match(entity_mapping, marc_field):
|
|
|
1094
1100
|
|
|
1095
1101
|
Returns:
|
|
1096
1102
|
bool: True if the indicators match, False otherwise.
|
|
1097
|
-
"""
|
|
1103
|
+
"""
|
|
1098
1104
|
if indicator_rule := [x["indicators"] for x in entity_mapping if "indicators" in x]:
|
|
1099
1105
|
return all(
|
|
1100
1106
|
[
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
FOLIO community specifications"""
|
|
1
|
+
"""MARC21 to FOLIO Instance mapper using community specifications."""
|
|
3
2
|
|
|
4
3
|
import logging
|
|
5
4
|
import sys
|
|
@@ -37,8 +36,7 @@ from folio_migration_tools.migration_tasks.migration_task_base import MarcTaskCo
|
|
|
37
36
|
|
|
38
37
|
|
|
39
38
|
class BibsRulesMapper(RulesMapperBase):
|
|
40
|
-
"""
|
|
41
|
-
the FOLIO community convention"""
|
|
39
|
+
"""Map MARC records to FOLIO inventory instance format."""
|
|
42
40
|
|
|
43
41
|
def __init__(
|
|
44
42
|
self,
|
|
@@ -47,6 +45,14 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
47
45
|
task_configuration: MarcTaskConfigurationBase,
|
|
48
46
|
statistical_codes_map: Dict[str, str] = None,
|
|
49
47
|
):
|
|
48
|
+
"""Initialize mapper for bibliographic record transformations.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
folio_client (FolioClient): FOLIO API client.
|
|
52
|
+
library_configuration (LibraryConfiguration): Library configuration.
|
|
53
|
+
task_configuration (MarcTaskConfigurationBase): Bibs transformation configuration.
|
|
54
|
+
statistical_codes_map (Dict[str, str]): Mapping for statistical codes.
|
|
55
|
+
"""
|
|
50
56
|
super().__init__(
|
|
51
57
|
folio_client,
|
|
52
58
|
library_configuration,
|
|
@@ -110,9 +116,9 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
110
116
|
def parse_record(
|
|
111
117
|
self, marc_record: Record, file_def: FileDefinition, legacy_ids: List[str]
|
|
112
118
|
) -> list[dict]:
|
|
113
|
-
"""
|
|
119
|
+
"""Parse a MARC bib record into a FOLIO Inventory instance object.
|
|
120
|
+
|
|
114
121
|
Community mapping suggestion: https://bit.ly/2S7Gyp3
|
|
115
|
-
This is the main function
|
|
116
122
|
|
|
117
123
|
Args:
|
|
118
124
|
marc_record (Record): _description_
|
|
@@ -155,17 +161,16 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
155
161
|
ignored_subsequent_fields: set,
|
|
156
162
|
legacy_ids: List[str],
|
|
157
163
|
):
|
|
158
|
-
"""
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
than creating SRS records during transformation.
|
|
164
|
+
"""Apply simplified MARC-to-instance mapping for Data Import flow.
|
|
165
|
+
|
|
166
|
+
Creates a minimal FOLIO Instance record to be used with a Data Import
|
|
167
|
+
based MARC loading flow, rather than creating SRS records during transformation.
|
|
163
168
|
|
|
164
169
|
Args:
|
|
165
|
-
folio_instance (dict):
|
|
166
|
-
marc_record (Record):
|
|
167
|
-
|
|
168
|
-
|
|
170
|
+
folio_instance (dict): The FOLIO instance record to populate.
|
|
171
|
+
marc_record (Record): The source MARC record.
|
|
172
|
+
ignored_subsequent_fields (set): Fields to skip during processing.
|
|
173
|
+
legacy_ids (List[str]): Legacy identifiers for the record.
|
|
169
174
|
"""
|
|
170
175
|
main_entry_field_tags = ["100", "110", "111", "130"]
|
|
171
176
|
main_entry_fields = marc_record.get_fields(*main_entry_field_tags)
|
|
@@ -200,7 +205,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
200
205
|
legacy_ids: List[str],
|
|
201
206
|
file_def: FileDefinition,
|
|
202
207
|
) -> None:
|
|
203
|
-
"""Do stuff not easily captured by the mapping rules
|
|
208
|
+
"""Do stuff not easily captured by the mapping rules.
|
|
204
209
|
|
|
205
210
|
Args:
|
|
206
211
|
folio_instance (dict): _description_
|
|
@@ -575,7 +580,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
575
580
|
return languages
|
|
576
581
|
|
|
577
582
|
def get_languages(self, marc_record: Record, legacy_id: List[str]) -> List[str]:
|
|
578
|
-
"""Get languages and tranforms them to correct codes
|
|
583
|
+
"""Get languages and tranforms them to correct codes.
|
|
579
584
|
|
|
580
585
|
Args:
|
|
581
586
|
marc_record (Record): A pymarc Record object
|
|
@@ -591,7 +596,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
591
596
|
return list(languages)
|
|
592
597
|
|
|
593
598
|
def fetch_language_codes(self) -> Generator[str, None, None]:
|
|
594
|
-
"""Loads the list of standardized language codes from LoC
|
|
599
|
+
"""Loads the list of standardized language codes from LoC.
|
|
595
600
|
|
|
596
601
|
Yields:
|
|
597
602
|
Generator[str, None, None]: _description_
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
"""MARC holdings records rules-based transformation.
|
|
2
|
+
|
|
3
|
+
Implements transformation of MARC21 holdings (MFHD) records to FOLIO Holdings using
|
|
4
|
+
rules-based mapping. Handles holdings-specific fields including locations, call numbers,
|
|
5
|
+
holdings statements, and notes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
1
8
|
import copy
|
|
2
9
|
import json
|
|
3
10
|
import logging
|
|
@@ -45,6 +52,17 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
45
52
|
boundwith_relationship_map_rows: List[Dict],
|
|
46
53
|
statistical_codes_map: Optional[Dict] = None,
|
|
47
54
|
):
|
|
55
|
+
"""Initialize mapper for holdings record transformations.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
folio_client (FolioClient): FOLIO API client.
|
|
59
|
+
location_map: Mapping of legacy to FOLIO locations.
|
|
60
|
+
task_configuration: Holdings transformation configuration.
|
|
61
|
+
library_configuration (LibraryConfiguration): Library configuration.
|
|
62
|
+
parent_id_map (dict): Mapping of parent instance IDs.
|
|
63
|
+
boundwith_relationship_map_rows (List[Dict]): Bound-with relationship mappings.
|
|
64
|
+
statistical_codes_map (Optional[Dict]): Mapping for statistical codes.
|
|
65
|
+
"""
|
|
48
66
|
self.conditions = Conditions(
|
|
49
67
|
folio_client,
|
|
50
68
|
self,
|
|
@@ -116,9 +134,9 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
116
134
|
def parse_record(
|
|
117
135
|
self, marc_record: Record, file_def: FileDefinition, legacy_ids: List[str]
|
|
118
136
|
) -> list[dict]:
|
|
119
|
-
"""
|
|
137
|
+
"""Parse a MFHD record into a FOLIO Inventory holdings object.
|
|
138
|
+
|
|
120
139
|
Community mapping suggestion: https://tinyurl.com/3rh52e2x
|
|
121
|
-
This is the main function
|
|
122
140
|
|
|
123
141
|
Args:
|
|
124
142
|
marc_record (Record): _description_
|
|
@@ -131,7 +149,6 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
131
149
|
Returns:
|
|
132
150
|
dict: _description_
|
|
133
151
|
"""
|
|
134
|
-
|
|
135
152
|
self.print_progress()
|
|
136
153
|
folio_holding = self.perform_initial_preparation(marc_record, legacy_ids)
|
|
137
154
|
self.prep_852_notes(marc_record)
|
|
@@ -246,7 +263,7 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
246
263
|
ignored_subsequent_fields: Set,
|
|
247
264
|
index_or_legacy_ids: List[str],
|
|
248
265
|
):
|
|
249
|
-
"""This overwrites the implementation for Auth and instances
|
|
266
|
+
"""This overwrites the implementation for Auth and instances.
|
|
250
267
|
|
|
251
268
|
Args:
|
|
252
269
|
folio_holding (dict): _description_
|
|
@@ -273,7 +290,7 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
273
290
|
legacy_ids: List[str],
|
|
274
291
|
file_def: FileDefinition,
|
|
275
292
|
):
|
|
276
|
-
"""_summary_
|
|
293
|
+
"""_summary_.
|
|
277
294
|
|
|
278
295
|
Args:
|
|
279
296
|
marc_record (Record): _description_
|
|
@@ -367,15 +384,15 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
367
384
|
self.collect_mrk_statement_notes(marc_record, folio_holding, legacy_ids)
|
|
368
385
|
|
|
369
386
|
def collect_mrk_statement_notes(self, marc_record, folio_holding, legacy_ids):
|
|
370
|
-
"""
|
|
371
|
-
and adds them to the FOLIO holdings record.
|
|
387
|
+
"""Collect MFHD holdings statements as MARC Maker field strings in a note.
|
|
372
388
|
|
|
373
|
-
|
|
389
|
+
Preserves the MARC record information for future reference by adding
|
|
390
|
+
the statements to the FOLIO holdings record.
|
|
374
391
|
|
|
375
392
|
Args:
|
|
376
|
-
marc_record (Record): PyMARC record
|
|
377
|
-
folio_holding (Dict): FOLIO holdings record
|
|
378
|
-
|
|
393
|
+
marc_record (Record): PyMARC record.
|
|
394
|
+
folio_holding (Dict): FOLIO holdings record.
|
|
395
|
+
legacy_ids: Legacy identifiers for the record.
|
|
379
396
|
"""
|
|
380
397
|
if self.task_configuration.include_mrk_statements:
|
|
381
398
|
mrk_statement_notes = []
|
|
@@ -389,13 +406,14 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
389
406
|
) + self.add_mrk_statements_note(mrk_statement_notes, legacy_ids)
|
|
390
407
|
|
|
391
408
|
def add_mrk_statements_note(self, mrk_statement_notes: List[str], legacy_ids) -> List[Dict]:
|
|
392
|
-
"""
|
|
409
|
+
"""Create a note from the MRK statements.
|
|
393
410
|
|
|
394
411
|
Args:
|
|
395
|
-
mrk_statement_notes (List[str]): A list of MFHD holdings statements as MRK strings
|
|
412
|
+
mrk_statement_notes (List[str]): A list of MFHD holdings statements as MRK strings.
|
|
413
|
+
legacy_ids: Legacy identifiers for error reporting.
|
|
396
414
|
|
|
397
415
|
Returns:
|
|
398
|
-
List: A list containing the FOLIO holdings note object (Dict)
|
|
416
|
+
List: A list containing the FOLIO holdings note object (Dict).
|
|
399
417
|
"""
|
|
400
418
|
holdings_note_type_tuple = self.conditions.get_ref_data_tuple_by_name(
|
|
401
419
|
self.folio.holding_note_types,
|
|
@@ -442,13 +460,14 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
442
460
|
def add_mfhd_as_mrk_note(
|
|
443
461
|
self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]
|
|
444
462
|
):
|
|
445
|
-
"""
|
|
463
|
+
"""Add the MFHD as a note to the holdings record.
|
|
446
464
|
|
|
447
|
-
|
|
465
|
+
Preserves the MARC record information for future reference.
|
|
448
466
|
|
|
449
467
|
Args:
|
|
450
|
-
marc_record (Record): PyMARC record
|
|
451
|
-
folio_holding (Dict): FOLIO holdings record
|
|
468
|
+
marc_record (Record): PyMARC record.
|
|
469
|
+
folio_holding (Dict): FOLIO holdings record.
|
|
470
|
+
legacy_ids (List[str]): Legacy identifiers for error reporting.
|
|
452
471
|
"""
|
|
453
472
|
if self.task_configuration.include_mfhd_mrk_as_note:
|
|
454
473
|
holdings_note_type_tuple = self.conditions.get_ref_data_tuple_by_name(
|
|
@@ -498,13 +517,14 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
498
517
|
def add_mfhd_as_mrc_note(
|
|
499
518
|
self, marc_record: Record, folio_holding: Dict, legacy_ids: List[str]
|
|
500
519
|
):
|
|
501
|
-
"""
|
|
520
|
+
"""Add the MFHD as a note to the holdings record.
|
|
502
521
|
|
|
503
|
-
|
|
522
|
+
Preserves the MARC record information for future reference.
|
|
504
523
|
|
|
505
524
|
Args:
|
|
506
|
-
marc_record (Record): PyMARC record
|
|
507
|
-
folio_holding (Dict): FOLIO holdings record
|
|
525
|
+
marc_record (Record): PyMARC record.
|
|
526
|
+
folio_holding (Dict): FOLIO holdings record.
|
|
527
|
+
legacy_ids (List[str]): Legacy identifiers for error reporting.
|
|
508
528
|
"""
|
|
509
529
|
if self.task_configuration.include_mfhd_mrc_as_note:
|
|
510
530
|
holdings_note_type_tuple = self.conditions.get_ref_data_tuple_by_name(
|
|
@@ -660,17 +680,16 @@ class RulesMapperHoldings(RulesMapperBase):
|
|
|
660
680
|
)
|
|
661
681
|
|
|
662
682
|
def setup_boundwith_relationship_map(self, boundwith_relationship_map_list: List[Dict]):
|
|
663
|
-
"""
|
|
664
|
-
Creates a map of MFHD_ID to BIB_ID for boundwith relationships.
|
|
683
|
+
"""Create a map of MFHD_ID to BIB_ID for boundwith relationships.
|
|
665
684
|
|
|
666
|
-
|
|
667
|
-
|
|
685
|
+
Args:
|
|
686
|
+
boundwith_relationship_map_list: A list of dicts containing MFHD_ID and BIB_ID.
|
|
668
687
|
|
|
669
688
|
Returns:
|
|
670
689
|
A dictionary mapping MFHD_ID to a list of BIB_IDs.
|
|
671
690
|
|
|
672
691
|
Raises:
|
|
673
|
-
TransformationProcessError: If MFHD_ID or BIB_ID is missing from the entry or
|
|
692
|
+
TransformationProcessError: If MFHD_ID or BIB_ID is missing from the entry or instance not in parent_id_map.
|
|
674
693
|
TransformationRecordFailedError: If BIB_ID is not in the instance id map.
|
|
675
694
|
""" # noqa: E501
|
|
676
695
|
new_map = {}
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
"""Migration reporting and statistics tracking.
|
|
2
|
+
|
|
3
|
+
Provides the MigrationReport class for tracking migration statistics, errors,
|
|
4
|
+
and warnings during transformation and loading tasks. Generates markdown and
|
|
5
|
+
JSON formatted reports with categorized statistics.
|
|
6
|
+
"""
|
|
7
|
+
|
|
1
8
|
import logging
|
|
2
9
|
import json
|
|
3
10
|
import i18n
|
|
@@ -8,9 +15,10 @@ from folio_migration_tools.i18n_cache import i18n_t
|
|
|
8
15
|
|
|
9
16
|
|
|
10
17
|
class MigrationReport:
|
|
11
|
-
"""Class responsible for handling the migration report"""
|
|
18
|
+
"""Class responsible for handling the migration report."""
|
|
12
19
|
|
|
13
20
|
def __init__(self):
|
|
21
|
+
"""Initialize a new migration report for tracking statistics and issues."""
|
|
14
22
|
self.report = {}
|
|
15
23
|
self.stats = {}
|
|
16
24
|
|
|
@@ -31,19 +39,19 @@ class MigrationReport:
|
|
|
31
39
|
self.report[blurb_id][measure_to_add] = number
|
|
32
40
|
|
|
33
41
|
def set(self, blurb_id, measure_to_add: str, number: int):
|
|
34
|
-
"""Set a section value
|
|
42
|
+
"""Set a section value to a specific number.
|
|
35
43
|
|
|
36
44
|
Args:
|
|
37
|
-
|
|
38
|
-
measure_to_add (str):
|
|
39
|
-
number (int):
|
|
45
|
+
blurb_id: The report section identifier.
|
|
46
|
+
measure_to_add (str): The measure name to set.
|
|
47
|
+
number (int): The value to set.
|
|
40
48
|
"""
|
|
41
49
|
if blurb_id not in self.report:
|
|
42
50
|
self.report[blurb_id] = {}
|
|
43
51
|
self.report[blurb_id][measure_to_add] = number
|
|
44
52
|
|
|
45
53
|
def add_general_statistics(self, measure_to_add: str):
|
|
46
|
-
"""Shortcut for adding to the first breakdown
|
|
54
|
+
"""Shortcut for adding to the first breakdown.
|
|
47
55
|
|
|
48
56
|
Args:
|
|
49
57
|
measure_to_add (str): _description_
|
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
"""Legacy BatchPoster task for posting transformed records to FOLIO.
|
|
2
|
+
|
|
3
|
+
This module provides the BatchPoster migration task which posts transformed FOLIO
|
|
4
|
+
records via batch API endpoints. It supports batch processing, automatic retry of
|
|
5
|
+
failed records, and version management for upsert operations.
|
|
6
|
+
|
|
7
|
+
Note: This is the legacy implementation. For new migrations, consider using
|
|
8
|
+
InventoryBatchPoster or other specialized batch posters instead.
|
|
9
|
+
"""
|
|
10
|
+
|
|
1
11
|
import asyncio
|
|
2
12
|
import copy
|
|
3
13
|
import json
|
|
@@ -37,7 +47,7 @@ def write_failed_batch_to_file(batch, file):
|
|
|
37
47
|
|
|
38
48
|
|
|
39
49
|
class BatchPoster(MigrationTaskBase):
|
|
40
|
-
"""BatchPoster
|
|
50
|
+
"""BatchPoster.
|
|
41
51
|
|
|
42
52
|
Parents:
|
|
43
53
|
MigrationTaskBase (_type_): _description_
|
|
@@ -55,6 +65,8 @@ class BatchPoster(MigrationTaskBase):
|
|
|
55
65
|
"""
|
|
56
66
|
|
|
57
67
|
class TaskConfiguration(AbstractTaskConfiguration):
|
|
68
|
+
"""Task configuration for BatchPoster."""
|
|
69
|
+
|
|
58
70
|
name: Annotated[
|
|
59
71
|
str,
|
|
60
72
|
Field(
|
|
@@ -220,6 +232,14 @@ class BatchPoster(MigrationTaskBase):
|
|
|
220
232
|
folio_client,
|
|
221
233
|
use_logging: bool = True,
|
|
222
234
|
):
|
|
235
|
+
"""Initialize BatchPoster for posting transformed records to FOLIO.
|
|
236
|
+
|
|
237
|
+
Args:
|
|
238
|
+
task_config (TaskConfiguration): Batch posting configuration.
|
|
239
|
+
library_config (LibraryConfiguration): Library configuration.
|
|
240
|
+
folio_client: FOLIO API client.
|
|
241
|
+
use_logging (bool): Whether to set up task logging.
|
|
242
|
+
"""
|
|
223
243
|
super().__init__(library_config, task_config, folio_client, use_logging)
|
|
224
244
|
self.migration_report = MigrationReport()
|
|
225
245
|
self.performing_rerun = False
|
|
@@ -335,9 +355,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
335
355
|
json_rec["source"] = "CONSORTIUM-FOLIO"
|
|
336
356
|
|
|
337
357
|
def set_version(self, batch, query_api, object_type) -> None:
|
|
338
|
-
"""
|
|
339
|
-
Synchronous wrapper for set_version_async
|
|
340
|
-
"""
|
|
358
|
+
"""Synchronous wrapper for set_version_async."""
|
|
341
359
|
try:
|
|
342
360
|
loop = asyncio.get_running_loop()
|
|
343
361
|
except RuntimeError:
|
|
@@ -349,8 +367,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
349
367
|
loop.run_until_complete(self.set_version_async(batch, query_api, object_type))
|
|
350
368
|
|
|
351
369
|
async def set_version_async(self, batch, query_api, object_type) -> None:
|
|
352
|
-
"""
|
|
353
|
-
Fetches the current version of the records in the batch if the record exists in FOLIO
|
|
370
|
+
"""Fetches the current version of the records in the batch if the record exists in FOLIO.
|
|
354
371
|
|
|
355
372
|
Args:
|
|
356
373
|
batch (list): List of records to fetch versions for
|
|
@@ -388,8 +405,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
388
405
|
self.prepare_record_for_upsert(record, existing_records[record["id"]])
|
|
389
406
|
|
|
390
407
|
def patch_record(self, new_record: dict, existing_record: dict, patch_paths: List[str]):
|
|
391
|
-
"""
|
|
392
|
-
Updates new_record with values from existing_record according to patch_paths.
|
|
408
|
+
"""Updates new_record with values from existing_record according to patch_paths.
|
|
393
409
|
|
|
394
410
|
Args:
|
|
395
411
|
new_record (dict): The new record to be updated.
|
|
@@ -420,8 +436,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
420
436
|
def collect_existing_records_for_upsert(
|
|
421
437
|
object_type: str, response_json: dict, existing_records: dict
|
|
422
438
|
):
|
|
423
|
-
"""
|
|
424
|
-
Collects existing records from API response into existing_records dict.
|
|
439
|
+
"""Collects existing records from API response into existing_records dict.
|
|
425
440
|
|
|
426
441
|
Args:
|
|
427
442
|
object_type: The key in response containing the records array
|
|
@@ -508,8 +523,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
508
523
|
new_record.update(updates)
|
|
509
524
|
|
|
510
525
|
async def get_with_retry(self, url: str, params=None):
|
|
511
|
-
"""
|
|
512
|
-
Wrapper around folio_get_async with selective retry logic.
|
|
526
|
+
"""Wrapper around folio_get_async with selective retry logic.
|
|
513
527
|
|
|
514
528
|
Retries on:
|
|
515
529
|
- Connection errors (FolioConnectionError): Always retry
|
|
@@ -723,6 +737,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
723
737
|
len(batch),
|
|
724
738
|
get_req_size(response),
|
|
725
739
|
)
|
|
740
|
+
self.num_posted += len(batch)
|
|
726
741
|
elif response.status_code == 200:
|
|
727
742
|
json_report = json.loads(response.text)
|
|
728
743
|
self.users_created += json_report.get("createdRecords", 0)
|
|
@@ -926,7 +941,10 @@ class BatchPoster(MigrationTaskBase):
|
|
|
926
941
|
temp_start = self.start_datetime
|
|
927
942
|
self.task_configuration.rerun_failed_records = False
|
|
928
943
|
self.__init__(
|
|
929
|
-
self.task_configuration,
|
|
944
|
+
self.task_configuration,
|
|
945
|
+
self.library_configuration,
|
|
946
|
+
self.folio_client,
|
|
947
|
+
use_logging=False,
|
|
930
948
|
)
|
|
931
949
|
self.performing_rerun = True
|
|
932
950
|
self.migration_report = temp_report
|
|
@@ -1062,9 +1080,7 @@ def get_req_size(response: "Response"):
|
|
|
1062
1080
|
|
|
1063
1081
|
|
|
1064
1082
|
def parse_path(path):
|
|
1065
|
-
"""
|
|
1066
|
-
Parses a path like 'foo.bar[0].baz' into ['foo', 'bar', 0, 'baz']
|
|
1067
|
-
"""
|
|
1083
|
+
"""Parses a path like 'foo.bar[0].baz' into ['foo', 'bar', 0, 'baz']."""
|
|
1068
1084
|
tokens = []
|
|
1069
1085
|
# Split by dot, then extract indices
|
|
1070
1086
|
for part in path.split("."):
|
|
@@ -1118,8 +1134,8 @@ def extract_paths(data, paths):
|
|
|
1118
1134
|
|
|
1119
1135
|
|
|
1120
1136
|
def deep_update(target, patch):
|
|
1121
|
-
"""
|
|
1122
|
-
|
|
1137
|
+
"""Recursively update target dict/list with values from patch.
|
|
1138
|
+
|
|
1123
1139
|
For lists, only non-None values in patch are merged into target.
|
|
1124
1140
|
"""
|
|
1125
1141
|
if isinstance(patch, dict):
|
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
"""Bibliographic records transformation task.
|
|
2
|
+
|
|
3
|
+
Transforms MARC21 bibliographic records to FOLIO Inventory Instances using
|
|
4
|
+
rules-based mapping. Supports various ILS flavors and custom field mappings.
|
|
5
|
+
"""
|
|
6
|
+
|
|
1
7
|
import logging
|
|
2
8
|
from typing import Annotated, List
|
|
3
9
|
|
|
@@ -24,6 +30,8 @@ from folio_migration_tools.migration_tasks.migration_task_base import (
|
|
|
24
30
|
|
|
25
31
|
class BibsTransformer(MigrationTaskBase):
|
|
26
32
|
class TaskConfiguration(MarcTaskConfigurationBase):
|
|
33
|
+
"""Task configuration for BibsTransformer."""
|
|
34
|
+
|
|
27
35
|
ils_flavour: Annotated[
|
|
28
36
|
IlsFlavour,
|
|
29
37
|
Field(
|
|
@@ -116,6 +124,14 @@ class BibsTransformer(MigrationTaskBase):
|
|
|
116
124
|
folio_client,
|
|
117
125
|
use_logging: bool = True,
|
|
118
126
|
):
|
|
127
|
+
"""Initialize BibsTransformer for transforming bibliographic records.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
task_config (TaskConfiguration): MARC transformation configuration.
|
|
131
|
+
library_config (LibraryConfiguration): Library configuration.
|
|
132
|
+
folio_client: FOLIO API client.
|
|
133
|
+
use_logging (bool): Whether to set up task logging.
|
|
134
|
+
"""
|
|
119
135
|
super().__init__(library_config, task_config, folio_client, use_logging)
|
|
120
136
|
self.task_config = task_config
|
|
121
137
|
self.task_configuration = self.task_config
|
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
"""Course records migration task.
|
|
2
|
+
|
|
3
|
+
Migrates course information from CSV files to FOLIO Course Reserves module.
|
|
4
|
+
Transforms and validates course data including departments and terms.
|
|
5
|
+
"""
|
|
6
|
+
|
|
1
7
|
import csv
|
|
2
8
|
import json
|
|
3
9
|
import logging
|
|
@@ -30,6 +36,8 @@ from folio_migration_tools.task_configuration import AbstractTaskConfiguration
|
|
|
30
36
|
|
|
31
37
|
class CoursesMigrator(MigrationTaskBase):
|
|
32
38
|
class TaskConfiguration(AbstractTaskConfiguration):
|
|
39
|
+
"""Task configuration for CoursesMigrator."""
|
|
40
|
+
|
|
33
41
|
name: Annotated[
|
|
34
42
|
str,
|
|
35
43
|
Field(
|
|
@@ -92,6 +100,13 @@ class CoursesMigrator(MigrationTaskBase):
|
|
|
92
100
|
library_config: LibraryConfiguration,
|
|
93
101
|
folio_client,
|
|
94
102
|
):
|
|
103
|
+
"""Initialize CoursesMigrator for migrating course reserves.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
task_configuration (TaskConfiguration): Courses migration configuration.
|
|
107
|
+
library_config (LibraryConfiguration): Library configuration.
|
|
108
|
+
folio_client: FOLIO API client.
|
|
109
|
+
"""
|
|
95
110
|
csv.register_dialect("tsv", delimiter="\t")
|
|
96
111
|
self.task_configuration = task_configuration
|
|
97
112
|
super().__init__(library_config, task_configuration, folio_client)
|