folio-migration-tools 1.2.1__py3-none-any.whl → 1.9.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__init__.py +11 -0
- folio_migration_tools/__main__.py +169 -85
- folio_migration_tools/circulation_helper.py +96 -59
- folio_migration_tools/config_file_load.py +66 -0
- folio_migration_tools/custom_dict.py +6 -4
- folio_migration_tools/custom_exceptions.py +21 -19
- folio_migration_tools/extradata_writer.py +46 -0
- folio_migration_tools/folder_structure.py +63 -66
- folio_migration_tools/helper.py +29 -21
- folio_migration_tools/holdings_helper.py +57 -34
- folio_migration_tools/i18n_config.py +9 -0
- folio_migration_tools/library_configuration.py +173 -13
- folio_migration_tools/mapper_base.py +317 -106
- folio_migration_tools/mapping_file_transformation/courses_mapper.py +203 -0
- folio_migration_tools/mapping_file_transformation/holdings_mapper.py +83 -69
- folio_migration_tools/mapping_file_transformation/item_mapper.py +98 -94
- folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py +352 -0
- folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +702 -223
- folio_migration_tools/mapping_file_transformation/notes_mapper.py +90 -0
- folio_migration_tools/mapping_file_transformation/order_mapper.py +492 -0
- folio_migration_tools/mapping_file_transformation/organization_mapper.py +389 -0
- folio_migration_tools/mapping_file_transformation/ref_data_mapping.py +38 -27
- folio_migration_tools/mapping_file_transformation/user_mapper.py +149 -361
- folio_migration_tools/marc_rules_transformation/conditions.py +650 -246
- folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +292 -130
- folio_migration_tools/marc_rules_transformation/hrid_handler.py +244 -0
- folio_migration_tools/marc_rules_transformation/loc_language_codes.xml +20846 -0
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +300 -0
- folio_migration_tools/marc_rules_transformation/marc_reader_wrapper.py +136 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +241 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +681 -201
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +395 -429
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +531 -100
- folio_migration_tools/migration_report.py +85 -38
- folio_migration_tools/migration_tasks/__init__.py +1 -3
- folio_migration_tools/migration_tasks/authority_transformer.py +119 -0
- folio_migration_tools/migration_tasks/batch_poster.py +911 -198
- folio_migration_tools/migration_tasks/bibs_transformer.py +121 -116
- folio_migration_tools/migration_tasks/courses_migrator.py +192 -0
- folio_migration_tools/migration_tasks/holdings_csv_transformer.py +252 -247
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +321 -115
- folio_migration_tools/migration_tasks/items_transformer.py +264 -84
- folio_migration_tools/migration_tasks/loans_migrator.py +506 -195
- folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py +187 -0
- folio_migration_tools/migration_tasks/migration_task_base.py +364 -74
- folio_migration_tools/migration_tasks/orders_transformer.py +373 -0
- folio_migration_tools/migration_tasks/organization_transformer.py +451 -0
- folio_migration_tools/migration_tasks/requests_migrator.py +130 -62
- folio_migration_tools/migration_tasks/reserves_migrator.py +253 -0
- folio_migration_tools/migration_tasks/user_transformer.py +180 -139
- folio_migration_tools/task_configuration.py +46 -0
- folio_migration_tools/test_infrastructure/__init__.py +0 -0
- folio_migration_tools/test_infrastructure/mocked_classes.py +406 -0
- folio_migration_tools/transaction_migration/legacy_loan.py +148 -34
- folio_migration_tools/transaction_migration/legacy_request.py +65 -25
- folio_migration_tools/transaction_migration/legacy_reserve.py +47 -0
- folio_migration_tools/transaction_migration/transaction_result.py +12 -1
- folio_migration_tools/translations/en.json +476 -0
- folio_migration_tools-1.9.10.dist-info/METADATA +169 -0
- folio_migration_tools-1.9.10.dist-info/RECORD +67 -0
- {folio_migration_tools-1.2.1.dist-info → folio_migration_tools-1.9.10.dist-info}/WHEEL +1 -2
- folio_migration_tools-1.9.10.dist-info/entry_points.txt +3 -0
- folio_migration_tools/generate_schemas.py +0 -46
- folio_migration_tools/mapping_file_transformation/mapping_file_mapping_base_impl.py +0 -44
- folio_migration_tools/mapping_file_transformation/user_mapper_base.py +0 -212
- folio_migration_tools/marc_rules_transformation/bibs_processor.py +0 -163
- folio_migration_tools/marc_rules_transformation/holdings_processor.py +0 -284
- folio_migration_tools/report_blurbs.py +0 -219
- folio_migration_tools/transaction_migration/legacy_fee_fine.py +0 -36
- folio_migration_tools-1.2.1.dist-info/METADATA +0 -134
- folio_migration_tools-1.2.1.dist-info/RECORD +0 -50
- folio_migration_tools-1.2.1.dist-info/top_level.txt +0 -1
- {folio_migration_tools-1.2.1.dist-info → folio_migration_tools-1.9.10.dist-info/licenses}/LICENSE +0 -0
|
@@ -1,22 +1,30 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
3
|
import sys
|
|
4
|
-
from datetime import datetime
|
|
5
|
-
from typing import Dict
|
|
4
|
+
from datetime import datetime, timezone
|
|
5
|
+
from typing import Dict, List, Set, Union
|
|
6
6
|
from uuid import uuid4
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
import i18n
|
|
9
9
|
from folio_uuid.folio_uuid import FOLIONamespaces
|
|
10
|
-
from
|
|
11
|
-
|
|
10
|
+
from folioclient import FolioClient
|
|
11
|
+
|
|
12
|
+
from folio_migration_tools.custom_exceptions import (
|
|
13
|
+
TransformationProcessError,
|
|
14
|
+
TransformationRecordFailedError,
|
|
15
|
+
)
|
|
16
|
+
from folio_migration_tools.helper import Helper
|
|
17
|
+
from folio_migration_tools.library_configuration import (
|
|
18
|
+
FileDefinition,
|
|
19
|
+
LibraryConfiguration,
|
|
20
|
+
)
|
|
12
21
|
from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
|
|
13
22
|
MappingFileMapperBase,
|
|
14
23
|
)
|
|
15
|
-
from folio_migration_tools.helper import Helper
|
|
16
24
|
from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
|
|
17
25
|
RefDataMapping,
|
|
18
26
|
)
|
|
19
|
-
from folio_migration_tools.
|
|
27
|
+
from folio_migration_tools.task_configuration import AbstractTaskConfiguration
|
|
20
28
|
|
|
21
29
|
|
|
22
30
|
class ItemMapper(MappingFileMapperBase):
|
|
@@ -34,6 +42,7 @@ class ItemMapper(MappingFileMapperBase):
|
|
|
34
42
|
temporary_loan_type_mapping,
|
|
35
43
|
temporary_location_mapping,
|
|
36
44
|
library_configuration: LibraryConfiguration,
|
|
45
|
+
task_configuration: AbstractTaskConfiguration,
|
|
37
46
|
):
|
|
38
47
|
item_schema = folio_client.get_item_schema()
|
|
39
48
|
super().__init__(
|
|
@@ -43,14 +52,13 @@ class ItemMapper(MappingFileMapperBase):
|
|
|
43
52
|
statistical_codes_map,
|
|
44
53
|
FOLIONamespaces.items,
|
|
45
54
|
library_configuration,
|
|
55
|
+
task_configuration,
|
|
46
56
|
)
|
|
47
57
|
self.item_schema = self.folio_client.get_item_schema()
|
|
48
58
|
self.items_map = items_map
|
|
49
59
|
self.holdings_id_map = holdings_id_map
|
|
50
|
-
self.unique_barcodes = set()
|
|
51
|
-
self.
|
|
52
|
-
self.use_map = True
|
|
53
|
-
self.status_mapping = {}
|
|
60
|
+
self.unique_barcodes: Set[str] = set()
|
|
61
|
+
self.status_mapping: dict = {}
|
|
54
62
|
if temporary_loan_type_mapping:
|
|
55
63
|
self.temp_loan_type_mapping = RefDataMapping(
|
|
56
64
|
self.folio_client,
|
|
@@ -58,8 +66,9 @@ class ItemMapper(MappingFileMapperBase):
|
|
|
58
66
|
"loantypes",
|
|
59
67
|
temporary_loan_type_mapping,
|
|
60
68
|
"name",
|
|
61
|
-
|
|
69
|
+
"TemporaryLoanTypeMapping",
|
|
62
70
|
)
|
|
71
|
+
self.temp_location_mapping = None
|
|
63
72
|
if temporary_location_mapping:
|
|
64
73
|
self.temp_location_mapping = RefDataMapping(
|
|
65
74
|
self.folio_client,
|
|
@@ -67,7 +76,7 @@ class ItemMapper(MappingFileMapperBase):
|
|
|
67
76
|
"locations",
|
|
68
77
|
temporary_location_mapping,
|
|
69
78
|
"code",
|
|
70
|
-
|
|
79
|
+
"TemporaryLocationMapping",
|
|
71
80
|
)
|
|
72
81
|
|
|
73
82
|
if item_statuses_map:
|
|
@@ -79,7 +88,7 @@ class ItemMapper(MappingFileMapperBase):
|
|
|
79
88
|
"callNumberTypes",
|
|
80
89
|
call_number_type_map,
|
|
81
90
|
"name",
|
|
82
|
-
|
|
91
|
+
"CallNumberTypeMapping",
|
|
83
92
|
)
|
|
84
93
|
self.loan_type_mapping = RefDataMapping(
|
|
85
94
|
self.folio_client,
|
|
@@ -87,7 +96,7 @@ class ItemMapper(MappingFileMapperBase):
|
|
|
87
96
|
"loantypes",
|
|
88
97
|
loan_type_map,
|
|
89
98
|
"name",
|
|
90
|
-
|
|
99
|
+
"PermanentLoanTypeMapping",
|
|
91
100
|
)
|
|
92
101
|
|
|
93
102
|
self.material_type_mapping = RefDataMapping(
|
|
@@ -96,7 +105,7 @@ class ItemMapper(MappingFileMapperBase):
|
|
|
96
105
|
"mtypes",
|
|
97
106
|
material_type_map,
|
|
98
107
|
"name",
|
|
99
|
-
|
|
108
|
+
"MaterialTypeMapping",
|
|
100
109
|
)
|
|
101
110
|
|
|
102
111
|
self.location_mapping = RefDataMapping(
|
|
@@ -105,11 +114,21 @@ class ItemMapper(MappingFileMapperBase):
|
|
|
105
114
|
"locations",
|
|
106
115
|
location_map,
|
|
107
116
|
"code",
|
|
108
|
-
|
|
117
|
+
"LocationMapping",
|
|
109
118
|
)
|
|
110
119
|
|
|
111
|
-
def perform_additional_mappings(self):
|
|
112
|
-
|
|
120
|
+
def perform_additional_mappings(self, legacy_ids: Union[str, List[str]], folio_rec: Dict, file_def: FileDefinition):
|
|
121
|
+
self.handle_suppression(folio_rec, file_def)
|
|
122
|
+
self.map_statistical_codes(folio_rec, file_def)
|
|
123
|
+
self.map_statistical_code_ids(legacy_ids, folio_rec)
|
|
124
|
+
|
|
125
|
+
def handle_suppression(self, folio_record: Dict, file_def: FileDefinition):
|
|
126
|
+
folio_record["discoverySuppress"] = file_def.discovery_suppressed
|
|
127
|
+
self.migration_report.add(
|
|
128
|
+
"Suppression",
|
|
129
|
+
i18n.t("Suppressed from discovery")
|
|
130
|
+
+ f" = {folio_record['discoverySuppress']}",
|
|
131
|
+
)
|
|
113
132
|
|
|
114
133
|
def setup_status_mapping(self, item_statuses_map):
|
|
115
134
|
statuses = self.item_schema["properties"]["status"]["properties"]["name"][
|
|
@@ -149,126 +168,111 @@ class ItemMapper(MappingFileMapperBase):
|
|
|
149
168
|
}
|
|
150
169
|
logging.info(json.dumps(statuses, indent=True))
|
|
151
170
|
|
|
152
|
-
def get_prop(self, legacy_item, folio_prop_name, index_or_id):
|
|
153
|
-
value_tuple = (legacy_item, folio_prop_name, index_or_id)
|
|
154
|
-
|
|
155
|
-
# Legacy contstruct
|
|
156
|
-
if not self.use_map:
|
|
157
|
-
return legacy_item[folio_prop_name]
|
|
158
|
-
|
|
159
|
-
legacy_item_keys = self.mapped_from_legacy_data.get(folio_prop_name, [])
|
|
160
|
-
|
|
161
|
-
# IF there is a value mapped, return that one
|
|
162
|
-
if len(legacy_item_keys) == 1 and folio_prop_name in self.mapped_from_values:
|
|
163
|
-
value = self.mapped_from_values.get(folio_prop_name, "")
|
|
164
|
-
self.migration_report.add(
|
|
165
|
-
Blurbs.DefaultValuesAdded, f"{value} added to {folio_prop_name}"
|
|
166
|
-
)
|
|
167
|
-
return value
|
|
168
|
-
|
|
169
|
-
legacy_values = MappingFileMapperBase.get_legacy_vals(
|
|
170
|
-
legacy_item, legacy_item_keys
|
|
171
|
-
)
|
|
172
|
-
legacy_value = " ".join(legacy_values).strip()
|
|
173
|
-
|
|
171
|
+
def get_prop(self, legacy_item, folio_prop_name, index_or_id, schema_default_value):
|
|
174
172
|
if folio_prop_name == "permanentLocationId":
|
|
175
|
-
return self.
|
|
173
|
+
return self.get_mapped_ref_data_value(
|
|
176
174
|
self.location_mapping,
|
|
177
|
-
|
|
178
|
-
|
|
175
|
+
legacy_item,
|
|
176
|
+
folio_prop_name,
|
|
177
|
+
index_or_id,
|
|
178
|
+
self.task_configuration.prevent_permanent_location_map_default,
|
|
179
179
|
)
|
|
180
180
|
elif folio_prop_name == "temporaryLocationId":
|
|
181
|
-
|
|
181
|
+
if not self.temp_location_mapping:
|
|
182
|
+
raise TransformationProcessError(
|
|
183
|
+
"Temporary location is mapped, but there is no "
|
|
184
|
+
"temporary location mapping file referenced in configuration"
|
|
185
|
+
)
|
|
186
|
+
temp_loc = self.get_mapped_ref_data_value(
|
|
182
187
|
self.temp_location_mapping,
|
|
183
|
-
|
|
188
|
+
legacy_item,
|
|
189
|
+
folio_prop_name,
|
|
190
|
+
index_or_id,
|
|
184
191
|
True,
|
|
185
192
|
)
|
|
186
|
-
self.migration_report.add(
|
|
193
|
+
self.migration_report.add("TemporaryLocationMapping", f"{temp_loc}")
|
|
187
194
|
return temp_loc
|
|
188
195
|
elif folio_prop_name == "materialTypeId":
|
|
189
|
-
return self.
|
|
196
|
+
return self.get_mapped_ref_data_value(
|
|
190
197
|
self.material_type_mapping,
|
|
191
|
-
|
|
198
|
+
legacy_item,
|
|
199
|
+
folio_prop_name,
|
|
200
|
+
index_or_id,
|
|
192
201
|
)
|
|
193
202
|
elif folio_prop_name == "itemLevelCallNumberTypeId":
|
|
194
203
|
return self.get_item_level_call_number_type_id(
|
|
195
204
|
legacy_item, folio_prop_name, index_or_id
|
|
196
205
|
)
|
|
197
|
-
elif folio_prop_name == "status.name":
|
|
198
|
-
return self.transform_status(legacy_value)
|
|
199
|
-
elif folio_prop_name == "barcode":
|
|
200
|
-
barcode = next((v for v in legacy_values if v), "")
|
|
201
|
-
if barcode.strip() and barcode in self.unique_barcodes:
|
|
202
|
-
Helper.log_data_issue(
|
|
203
|
-
index_or_id, "Duplicate barcode", "-".join(legacy_values)
|
|
204
|
-
)
|
|
205
|
-
self.migration_report.add_general_statistics("Duplicate barcodes")
|
|
206
|
-
return f"{barcode}-{uuid4()}"
|
|
207
|
-
else:
|
|
208
|
-
if barcode.strip():
|
|
209
|
-
self.unique_barcodes.add(barcode)
|
|
210
|
-
return barcode
|
|
211
|
-
|
|
212
206
|
elif folio_prop_name == "status.date":
|
|
213
|
-
return datetime.
|
|
207
|
+
return datetime.now(timezone.utc).isoformat()
|
|
214
208
|
elif folio_prop_name == "temporaryLoanTypeId":
|
|
215
|
-
ltid = self.
|
|
209
|
+
ltid = self.get_mapped_ref_data_value(
|
|
216
210
|
self.temp_loan_type_mapping,
|
|
217
|
-
|
|
211
|
+
legacy_item,
|
|
212
|
+
folio_prop_name,
|
|
213
|
+
index_or_id,
|
|
218
214
|
True,
|
|
219
215
|
)
|
|
220
216
|
self.migration_report.add(
|
|
221
|
-
|
|
217
|
+
"TemporaryLoanTypeMapping", f"{folio_prop_name} -> {ltid}"
|
|
222
218
|
)
|
|
223
219
|
return ltid
|
|
224
220
|
elif folio_prop_name == "permanentLoanTypeId":
|
|
225
|
-
return self.
|
|
226
|
-
|
|
227
|
-
statistical_code_id = self.get_statistical_codes(
|
|
228
|
-
legacy_item, folio_prop_name, index_or_id
|
|
221
|
+
return self.get_mapped_ref_data_value(
|
|
222
|
+
self.loan_type_mapping, legacy_item, folio_prop_name, index_or_id
|
|
229
223
|
)
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
224
|
+
|
|
225
|
+
mapped_value = super().get_prop(
|
|
226
|
+
legacy_item, folio_prop_name, index_or_id, schema_default_value
|
|
227
|
+
)
|
|
228
|
+
if folio_prop_name == "status.name":
|
|
229
|
+
return self.transform_status(mapped_value)
|
|
230
|
+
elif folio_prop_name == "barcode":
|
|
231
|
+
barcode = mapped_value
|
|
232
|
+
normalized_barcode = barcode.strip().lower()
|
|
233
|
+
if normalized_barcode and normalized_barcode in self.unique_barcodes:
|
|
234
|
+
Helper.log_data_issue(index_or_id, "Duplicate barcode", mapped_value)
|
|
235
|
+
self.migration_report.add_general_statistics(
|
|
236
|
+
i18n.t("Duplicate barcodes")
|
|
237
|
+
)
|
|
238
|
+
return f"{barcode}-{uuid4()}"
|
|
239
|
+
else:
|
|
240
|
+
if normalized_barcode:
|
|
241
|
+
self.unique_barcodes.add(normalized_barcode)
|
|
242
|
+
return barcode
|
|
235
243
|
elif folio_prop_name == "holdingsRecordId":
|
|
236
|
-
if
|
|
237
|
-
return self.holdings_id_map[
|
|
244
|
+
if mapped_value in self.holdings_id_map:
|
|
245
|
+
return self.holdings_id_map[mapped_value][1]
|
|
246
|
+
elif f"{self.bib_id_template}{mapped_value}" in self.holdings_id_map:
|
|
247
|
+
return self.holdings_id_map[f"{self.bib_id_template}{mapped_value}"][1]
|
|
238
248
|
self.migration_report.add_general_statistics(
|
|
239
|
-
"Records failed because of failed holdings",
|
|
249
|
+
i18n.t("Records failed because of failed holdings"),
|
|
240
250
|
)
|
|
241
251
|
s = (
|
|
242
252
|
"Holdings id referenced in legacy item "
|
|
243
253
|
"was not found amongst transformed Holdings records"
|
|
244
254
|
)
|
|
245
|
-
raise TransformationRecordFailedError(index_or_id, s,
|
|
246
|
-
elif
|
|
247
|
-
|
|
248
|
-
self.migration_report.add(
|
|
249
|
-
Blurbs.Details, f"{legacy_item_keys} were concatenated"
|
|
250
|
-
)
|
|
251
|
-
return legacy_value
|
|
255
|
+
raise TransformationRecordFailedError(index_or_id, s, mapped_value)
|
|
256
|
+
elif mapped_value:
|
|
257
|
+
return mapped_value
|
|
252
258
|
else:
|
|
253
|
-
self.migration_report.add(
|
|
254
|
-
Blurbs.UnmappedProperties, f"{folio_prop_name} {legacy_item_keys}"
|
|
255
|
-
)
|
|
259
|
+
self.migration_report.add("UnmappedProperties", f"{folio_prop_name}")
|
|
256
260
|
return ""
|
|
257
261
|
|
|
258
262
|
def get_item_level_call_number_type_id(
|
|
259
263
|
self, legacy_item, folio_prop_name: str, index_or_id
|
|
260
264
|
):
|
|
261
265
|
if self.call_number_mapping:
|
|
262
|
-
return self.
|
|
266
|
+
return self.get_mapped_ref_data_value(
|
|
263
267
|
self.call_number_mapping, legacy_item, index_or_id, folio_prop_name
|
|
264
268
|
)
|
|
265
269
|
self.migration_report.add(
|
|
266
|
-
|
|
267
|
-
"Mapping not setup",
|
|
270
|
+
"CallNumberTypeMapping",
|
|
271
|
+
i18n.t("Mapping not setup"),
|
|
268
272
|
)
|
|
269
273
|
return ""
|
|
270
274
|
|
|
271
275
|
def transform_status(self, legacy_value):
|
|
272
276
|
status = self.status_mapping.get(legacy_value, "Available")
|
|
273
|
-
self.migration_report.add(
|
|
277
|
+
self.migration_report.add("StatusMapping", f"'{legacy_value}' -> {status}")
|
|
274
278
|
return status
|
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
import uuid
|
|
4
|
+
import i18n
|
|
5
|
+
from typing import Any
|
|
6
|
+
from typing import Dict
|
|
7
|
+
from zoneinfo import ZoneInfo
|
|
8
|
+
|
|
9
|
+
from dateutil import parser as dateutil_parser
|
|
10
|
+
from dateutil import tz
|
|
11
|
+
from folio_uuid.folio_uuid import FOLIONamespaces
|
|
12
|
+
from folioclient import FolioClient
|
|
13
|
+
|
|
14
|
+
from folio_migration_tools.custom_exceptions import TransformationProcessError
|
|
15
|
+
from folio_migration_tools.custom_exceptions import TransformationRecordFailedError
|
|
16
|
+
from folio_migration_tools.library_configuration import LibraryConfiguration
|
|
17
|
+
from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
|
|
18
|
+
MappingFileMapperBase,
|
|
19
|
+
)
|
|
20
|
+
from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
|
|
21
|
+
RefDataMapping,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ManualFeeFinesMapper(MappingFileMapperBase):
|
|
26
|
+
def __init__(
|
|
27
|
+
self,
|
|
28
|
+
folio_client: FolioClient,
|
|
29
|
+
library_configuration: LibraryConfiguration,
|
|
30
|
+
task_configuration,
|
|
31
|
+
feefines_map,
|
|
32
|
+
feefines_owner_map,
|
|
33
|
+
feefines_type_map,
|
|
34
|
+
service_point_map,
|
|
35
|
+
ignore_legacy_identifier: bool = True,
|
|
36
|
+
):
|
|
37
|
+
self.folio_client: FolioClient = folio_client
|
|
38
|
+
self.composite_feefine_schema = self.get_composite_feefine_schema()
|
|
39
|
+
self.task_configuration = task_configuration
|
|
40
|
+
self.tenant_timezone = self.get_tenant_timezone()
|
|
41
|
+
|
|
42
|
+
super().__init__(
|
|
43
|
+
folio_client,
|
|
44
|
+
self.composite_feefine_schema,
|
|
45
|
+
feefines_map,
|
|
46
|
+
None,
|
|
47
|
+
FOLIONamespaces.fees_fines,
|
|
48
|
+
library_configuration,
|
|
49
|
+
task_configuration,
|
|
50
|
+
ignore_legacy_identifier,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
self.feefines_map = feefines_map
|
|
54
|
+
self.user_cache: dict = {}
|
|
55
|
+
self.item_cache: dict = {}
|
|
56
|
+
|
|
57
|
+
if feefines_owner_map:
|
|
58
|
+
self.feefines_owner_map = RefDataMapping(
|
|
59
|
+
self.folio_client,
|
|
60
|
+
"/owners",
|
|
61
|
+
"owners",
|
|
62
|
+
feefines_owner_map,
|
|
63
|
+
"owner",
|
|
64
|
+
"FeeFineOnwerMapping",
|
|
65
|
+
)
|
|
66
|
+
else:
|
|
67
|
+
self.feefines_owner_map = None
|
|
68
|
+
|
|
69
|
+
if feefines_type_map:
|
|
70
|
+
self.feefines_type_map = RefDataMapping(
|
|
71
|
+
self.folio_client,
|
|
72
|
+
"/feefines",
|
|
73
|
+
"feefines",
|
|
74
|
+
feefines_type_map,
|
|
75
|
+
"feeFineType",
|
|
76
|
+
"FeeFineTypesMapping",
|
|
77
|
+
)
|
|
78
|
+
else:
|
|
79
|
+
self.feefines_type_map = None
|
|
80
|
+
|
|
81
|
+
if service_point_map:
|
|
82
|
+
self.service_point_map = RefDataMapping(
|
|
83
|
+
self.folio_client,
|
|
84
|
+
"/service-points",
|
|
85
|
+
"servicepoints",
|
|
86
|
+
service_point_map,
|
|
87
|
+
"name",
|
|
88
|
+
"FeeFineServicePointTypesMapping",
|
|
89
|
+
)
|
|
90
|
+
else:
|
|
91
|
+
self.service_point_map = None
|
|
92
|
+
|
|
93
|
+
def store_objects(self, composite_feefine):
|
|
94
|
+
try:
|
|
95
|
+
self.extradata_writer.write("account", composite_feefine["account"])
|
|
96
|
+
self.migration_report.add_general_statistics(i18n.t("TOTAL Accounts created"))
|
|
97
|
+
self.extradata_writer.write("feefineaction", composite_feefine["feefineaction"])
|
|
98
|
+
self.migration_report.add_general_statistics(i18n.t("TOTAL Feefineactions created"))
|
|
99
|
+
|
|
100
|
+
except Exception as ee:
|
|
101
|
+
raise TransformationRecordFailedError(
|
|
102
|
+
composite_feefine, "Failed when storing", ee
|
|
103
|
+
) from ee
|
|
104
|
+
|
|
105
|
+
def get_prop(self, legacy_object, folio_prop_name, index_or_id, schema_default_value):
|
|
106
|
+
if folio_prop_name == "account.ownerId" and self.feefines_owner_map:
|
|
107
|
+
return self.get_mapped_ref_data_value(
|
|
108
|
+
self.feefines_owner_map, legacy_object, index_or_id, folio_prop_name, False
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
elif folio_prop_name == "account.feeFineId" and self.feefines_type_map:
|
|
112
|
+
return self.get_mapped_ref_data_value(
|
|
113
|
+
self.feefines_type_map, legacy_object, index_or_id, folio_prop_name, False
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
elif folio_prop_name == "feefineaction.createdAt" and self.service_point_map:
|
|
117
|
+
return self.get_mapped_ref_data_value(
|
|
118
|
+
self.service_point_map, legacy_object, index_or_id, folio_prop_name, False
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
elif folio_prop_name == "account.amount" or folio_prop_name == "account.remaining":
|
|
122
|
+
return self.parse_sum_as_float(
|
|
123
|
+
index_or_id,
|
|
124
|
+
super().get_prop(
|
|
125
|
+
legacy_object, folio_prop_name, index_or_id, schema_default_value
|
|
126
|
+
),
|
|
127
|
+
folio_prop_name,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
elif folio_prop_name == "feefineaction.dateAction":
|
|
131
|
+
return self.parse_date_with_tenant_timezone(
|
|
132
|
+
"feefineaction.dateAction",
|
|
133
|
+
index_or_id,
|
|
134
|
+
super().get_prop(
|
|
135
|
+
legacy_object, folio_prop_name, index_or_id, schema_default_value
|
|
136
|
+
),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
elif folio_prop_name == "feefineaction.id":
|
|
140
|
+
return str(uuid.uuid4())
|
|
141
|
+
|
|
142
|
+
elif mapped_value := super().get_prop(
|
|
143
|
+
legacy_object, folio_prop_name, index_or_id, schema_default_value
|
|
144
|
+
):
|
|
145
|
+
return mapped_value
|
|
146
|
+
|
|
147
|
+
else:
|
|
148
|
+
return ""
|
|
149
|
+
|
|
150
|
+
def get_composite_feefine_schema(self) -> Dict[str, Any]:
|
|
151
|
+
return {
|
|
152
|
+
"properties": {
|
|
153
|
+
"account": FolioClient.get_latest_from_github(
|
|
154
|
+
"folio-org", "mod-feesfines", "/ramls/accountdata.json"
|
|
155
|
+
),
|
|
156
|
+
"feefineaction": FolioClient.get_latest_from_github(
|
|
157
|
+
"folio-org", "mod-feesfines", "/ramls/feefineactiondata.json"
|
|
158
|
+
),
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
def get_tenant_timezone(self):
|
|
163
|
+
config_path = (
|
|
164
|
+
"/configurations/entries?query=(module==ORG%20and%20configName==localeSettings)"
|
|
165
|
+
)
|
|
166
|
+
try:
|
|
167
|
+
tenant_timezone_str = json.loads(
|
|
168
|
+
self.folio_client.folio_get_single_object(config_path)["configs"][0]["value"]
|
|
169
|
+
)["timezone"]
|
|
170
|
+
logging.info("Tenant timezone is: %s", tenant_timezone_str)
|
|
171
|
+
return ZoneInfo(tenant_timezone_str)
|
|
172
|
+
except TypeError as te:
|
|
173
|
+
raise TransformationProcessError(
|
|
174
|
+
"",
|
|
175
|
+
"Failed to fetch Tenant Locale Settings. "
|
|
176
|
+
"Is your library configuration correct?",
|
|
177
|
+
) from te
|
|
178
|
+
except KeyError as ke:
|
|
179
|
+
raise TransformationProcessError(
|
|
180
|
+
"",
|
|
181
|
+
"Failed to parse Tenant Locale Settings. "
|
|
182
|
+
"Is the Tenant Locale config correctly formatted?",
|
|
183
|
+
) from ke
|
|
184
|
+
|
|
185
|
+
def parse_date_with_tenant_timezone(self, folio_prop_name: str, index_or_id, mapped_value):
|
|
186
|
+
try:
|
|
187
|
+
format_date = dateutil_parser.parse(mapped_value, fuzzy=True)
|
|
188
|
+
if format_date.tzinfo != tz.UTC:
|
|
189
|
+
format_date = format_date.replace(tzinfo=self.tenant_timezone)
|
|
190
|
+
return format_date.isoformat()
|
|
191
|
+
except Exception:
|
|
192
|
+
self.migration_report.add(
|
|
193
|
+
"GeneralStatistics",
|
|
194
|
+
i18n.t("DATA ISSUE Invalid dates"),
|
|
195
|
+
)
|
|
196
|
+
logging.log(
|
|
197
|
+
26,
|
|
198
|
+
"DATA ISSUE\t%s\t%s\t%s",
|
|
199
|
+
index_or_id,
|
|
200
|
+
f"{folio_prop_name} Not a valid date.",
|
|
201
|
+
mapped_value,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
def parse_sum_as_float(self, index_or_id, legacy_sum, folio_prop_name):
|
|
205
|
+
try:
|
|
206
|
+
return float(legacy_sum)
|
|
207
|
+
except Exception as ee:
|
|
208
|
+
self.migration_report.add(
|
|
209
|
+
"GeneralStatistics",
|
|
210
|
+
i18n.t("DATA ISSUE Invalid sum"),
|
|
211
|
+
)
|
|
212
|
+
raise TransformationRecordFailedError(
|
|
213
|
+
index_or_id,
|
|
214
|
+
f"{folio_prop_name} Value must only contain numbers/decimals.",
|
|
215
|
+
legacy_sum,
|
|
216
|
+
) from ee
|
|
217
|
+
|
|
218
|
+
def get_matching_record_from_folio(
|
|
219
|
+
self,
|
|
220
|
+
index_or_id,
|
|
221
|
+
cache: dict,
|
|
222
|
+
path: str,
|
|
223
|
+
match_property: str,
|
|
224
|
+
match_value: str,
|
|
225
|
+
result_type: str,
|
|
226
|
+
):
|
|
227
|
+
if match_value in cache:
|
|
228
|
+
return cache[match_value]
|
|
229
|
+
else:
|
|
230
|
+
query = f'?query=({match_property}=="{match_value}")'
|
|
231
|
+
if matching_record := next(
|
|
232
|
+
self.folio_client.folio_get_all(path, result_type, query), None
|
|
233
|
+
):
|
|
234
|
+
cache[match_value] = matching_record
|
|
235
|
+
return matching_record
|
|
236
|
+
|
|
237
|
+
def get_folio_user_uuid(self, index_or_id, user_barcode):
|
|
238
|
+
if matching_user := self.get_matching_record_from_folio(
|
|
239
|
+
index_or_id, self.user_cache, "/users", "barcode", user_barcode, "users"
|
|
240
|
+
):
|
|
241
|
+
return matching_user["id"]
|
|
242
|
+
else:
|
|
243
|
+
self.migration_report.add(
|
|
244
|
+
"GeneralStatistics",
|
|
245
|
+
i18n.t("DATA ISSUE Users not in FOLIO"),
|
|
246
|
+
)
|
|
247
|
+
raise TransformationRecordFailedError(
|
|
248
|
+
index_or_id,
|
|
249
|
+
"No matching user in FOLIO for barcode",
|
|
250
|
+
user_barcode,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
def perform_additional_mapping(self, index_or_id, composite_feefine, legacy_object):
|
|
254
|
+
# Generate account ID
|
|
255
|
+
composite_feefine["account"]["id"] = composite_feefine["id"]
|
|
256
|
+
composite_feefine["feefineaction"]["accountId"] = composite_feefine["id"]
|
|
257
|
+
|
|
258
|
+
# Link to FOLIO user
|
|
259
|
+
composite_feefine["account"]["userId"] = self.get_folio_user_uuid(
|
|
260
|
+
index_or_id,
|
|
261
|
+
composite_feefine["account"]["userId"],
|
|
262
|
+
)
|
|
263
|
+
composite_feefine["feefineaction"]["userId"] = composite_feefine["account"]["userId"]
|
|
264
|
+
|
|
265
|
+
# Add item data from FOLIO if available
|
|
266
|
+
if item_barcode := composite_feefine["account"].get("itemId"):
|
|
267
|
+
self.enrich_with_folio_item_data(index_or_id, composite_feefine, item_barcode)
|
|
268
|
+
|
|
269
|
+
self.add_additional_fields_and_values(composite_feefine, legacy_object)
|
|
270
|
+
|
|
271
|
+
return composite_feefine
|
|
272
|
+
|
|
273
|
+
def stringify_legacy_object(self, legacy_object):
|
|
274
|
+
legacy_string = (
|
|
275
|
+
"MIGRATION NOTE : This fee/fine was migrated to FOLIO from a previous "
|
|
276
|
+
"library management system. The following is the original data: "
|
|
277
|
+
)
|
|
278
|
+
for key, value in legacy_object.items():
|
|
279
|
+
legacy_string += f"{key.title()}: {value}; "
|
|
280
|
+
return legacy_string.strip().strip(";")
|
|
281
|
+
|
|
282
|
+
def enrich_with_folio_item_data(self, index_or_id, feefine, item_barcode):
|
|
283
|
+
if folio_item := self.get_matching_record_from_folio(
|
|
284
|
+
index_or_id,
|
|
285
|
+
self.item_cache,
|
|
286
|
+
"/inventory/items",
|
|
287
|
+
"barcode",
|
|
288
|
+
item_barcode,
|
|
289
|
+
"items",
|
|
290
|
+
):
|
|
291
|
+
feefine["account"]["itemId"] = folio_item.get("id", "")
|
|
292
|
+
feefine["account"]["title"] = folio_item.get("title", "")
|
|
293
|
+
feefine["account"]["barcode"] = folio_item.get("barcode", "")
|
|
294
|
+
feefine["account"]["callNumber"] = folio_item.get("callNumber", "")
|
|
295
|
+
feefine["account"]["materialType"] = folio_item.get("materialType", {}).get("name")
|
|
296
|
+
feefine["account"]["materialTypeId"] = folio_item.get("materialType", {}).get("id")
|
|
297
|
+
feefine["account"]["location"] = folio_item.get("effectiveLocation", {}).get("name")
|
|
298
|
+
else:
|
|
299
|
+
feefine["account"].pop("itemId")
|
|
300
|
+
self.migration_report.add(
|
|
301
|
+
"GeneralStatistics",
|
|
302
|
+
i18n.t("DATA ISSUE Items not in FOLIO"),
|
|
303
|
+
)
|
|
304
|
+
logging.log(
|
|
305
|
+
26,
|
|
306
|
+
"DATA ISSUE\t%s\t%s\t%s",
|
|
307
|
+
index_or_id,
|
|
308
|
+
"No matching item in FOLIO for barcode",
|
|
309
|
+
item_barcode,
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
def add_additional_fields_and_values(self, feefine, legacy_object):
|
|
313
|
+
# Add standard values
|
|
314
|
+
feefine["feefineaction"]["source"] = self.folio_client.username
|
|
315
|
+
feefine["feefineaction"]["notify"] = False
|
|
316
|
+
feefine["feefineaction"]["amountAction"] = feefine["account"]["amount"]
|
|
317
|
+
feefine["feefineaction"]["balance"] = feefine["account"]["remaining"]
|
|
318
|
+
|
|
319
|
+
# Set the account status to Open/Closed based on remainign amount
|
|
320
|
+
if feefine["account"]["remaining"] > 0:
|
|
321
|
+
feefine["account"]["status"] = {"name": "Open"}
|
|
322
|
+
else:
|
|
323
|
+
feefine["account"]["status"] = {"name": "Closed"}
|
|
324
|
+
|
|
325
|
+
# Add the full legacy item dict to the comment field
|
|
326
|
+
if feefine["feefineaction"].get("comments"):
|
|
327
|
+
feefine["feefineaction"]["comments"] = (
|
|
328
|
+
("STAFF : " + feefine["feefineaction"]["comments"])
|
|
329
|
+
+ " "
|
|
330
|
+
+ self.stringify_legacy_object(legacy_object)
|
|
331
|
+
)
|
|
332
|
+
else:
|
|
333
|
+
feefine["feefineaction"]["comments"] = self.stringify_legacy_object(legacy_object)
|
|
334
|
+
|
|
335
|
+
# Add name values from reference data mapping
|
|
336
|
+
if self.feefines_owner_map:
|
|
337
|
+
feefine["account"]["feeFineOwner"] = [
|
|
338
|
+
owner["owner"]
|
|
339
|
+
for owner in self.feefines_owner_map.ref_data
|
|
340
|
+
if owner["id"] == feefine["account"]["ownerId"]
|
|
341
|
+
][0]
|
|
342
|
+
|
|
343
|
+
if self.feefines_type_map:
|
|
344
|
+
type_name = [
|
|
345
|
+
type["feeFineType"]
|
|
346
|
+
for type in self.feefines_type_map.ref_data
|
|
347
|
+
if type["id"] == feefine["account"]["feeFineId"]
|
|
348
|
+
][0]
|
|
349
|
+
feefine["account"]["feeFineType"] = type_name
|
|
350
|
+
feefine["feefineaction"]["typeAction"] = type_name
|
|
351
|
+
|
|
352
|
+
return feefine
|