folio-migration-tools 1.9.9__py3-none-any.whl → 1.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__init__.py +3 -4
- folio_migration_tools/__main__.py +53 -31
- folio_migration_tools/circulation_helper.py +118 -108
- folio_migration_tools/custom_dict.py +2 -2
- folio_migration_tools/custom_exceptions.py +4 -5
- folio_migration_tools/folder_structure.py +17 -7
- folio_migration_tools/helper.py +8 -7
- folio_migration_tools/holdings_helper.py +4 -3
- folio_migration_tools/i18n_cache.py +79 -0
- folio_migration_tools/library_configuration.py +77 -37
- folio_migration_tools/mapper_base.py +45 -31
- folio_migration_tools/mapping_file_transformation/courses_mapper.py +1 -1
- folio_migration_tools/mapping_file_transformation/holdings_mapper.py +7 -3
- folio_migration_tools/mapping_file_transformation/item_mapper.py +13 -26
- folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py +1 -2
- folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +13 -11
- folio_migration_tools/mapping_file_transformation/order_mapper.py +23 -5
- folio_migration_tools/mapping_file_transformation/organization_mapper.py +3 -3
- folio_migration_tools/mapping_file_transformation/ref_data_mapping.py +3 -0
- folio_migration_tools/mapping_file_transformation/user_mapper.py +47 -28
- folio_migration_tools/marc_rules_transformation/conditions.py +82 -97
- folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +13 -5
- folio_migration_tools/marc_rules_transformation/hrid_handler.py +3 -2
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +26 -24
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +56 -51
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +28 -17
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +68 -37
- folio_migration_tools/migration_report.py +18 -7
- folio_migration_tools/migration_tasks/batch_poster.py +285 -354
- folio_migration_tools/migration_tasks/bibs_transformer.py +14 -9
- folio_migration_tools/migration_tasks/courses_migrator.py +2 -3
- folio_migration_tools/migration_tasks/holdings_csv_transformer.py +23 -24
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +14 -24
- folio_migration_tools/migration_tasks/items_transformer.py +23 -34
- folio_migration_tools/migration_tasks/loans_migrator.py +67 -144
- folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py +3 -3
- folio_migration_tools/migration_tasks/migration_task_base.py +43 -52
- folio_migration_tools/migration_tasks/orders_transformer.py +25 -41
- folio_migration_tools/migration_tasks/organization_transformer.py +9 -18
- folio_migration_tools/migration_tasks/requests_migrator.py +21 -24
- folio_migration_tools/migration_tasks/reserves_migrator.py +6 -5
- folio_migration_tools/migration_tasks/user_transformer.py +25 -20
- folio_migration_tools/task_configuration.py +6 -7
- folio_migration_tools/transaction_migration/legacy_loan.py +15 -27
- folio_migration_tools/transaction_migration/legacy_request.py +1 -1
- folio_migration_tools/translations/en.json +3 -8
- {folio_migration_tools-1.9.9.dist-info → folio_migration_tools-1.10.0.dist-info}/METADATA +19 -28
- folio_migration_tools-1.10.0.dist-info/RECORD +63 -0
- folio_migration_tools-1.10.0.dist-info/WHEEL +4 -0
- folio_migration_tools-1.10.0.dist-info/entry_points.txt +3 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +0 -241
- folio_migration_tools/migration_tasks/authority_transformer.py +0 -119
- folio_migration_tools/test_infrastructure/__init__.py +0 -0
- folio_migration_tools/test_infrastructure/mocked_classes.py +0 -406
- folio_migration_tools-1.9.9.dist-info/RECORD +0 -67
- folio_migration_tools-1.9.9.dist-info/WHEEL +0 -4
- folio_migration_tools-1.9.9.dist-info/entry_points.txt +0 -3
- folio_migration_tools-1.9.9.dist-info/licenses/LICENSE +0 -21
|
@@ -47,15 +47,14 @@ class MigrationTaskBase:
|
|
|
47
47
|
logging.info("MigrationTaskBase init")
|
|
48
48
|
self.start_datetime = datetime.now(timezone.utc)
|
|
49
49
|
self.task_configuration = task_configuration
|
|
50
|
-
logging.info(self.task_configuration.
|
|
50
|
+
logging.info(self.task_configuration.model_dump_json(indent=4))
|
|
51
51
|
self.folio_client: FolioClient = folio_client
|
|
52
52
|
self.ecs_tenant_id = (
|
|
53
53
|
task_configuration.ecs_tenant_id or library_configuration.ecs_tenant_id
|
|
54
54
|
)
|
|
55
|
-
self.
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
self.folio_client.okapi_headers.update(self.ecs_tenant_header)
|
|
55
|
+
if self.ecs_tenant_id:
|
|
56
|
+
self.folio_client.tenant_id = self.ecs_tenant_id
|
|
57
|
+
|
|
59
58
|
self.central_folder_structure: Optional[FolderStructure] = None
|
|
60
59
|
if library_configuration.is_ecs and library_configuration.ecs_central_iteration_identifier:
|
|
61
60
|
self.central_folder_structure = FolderStructure(
|
|
@@ -100,6 +99,7 @@ class MigrationTaskBase:
|
|
|
100
99
|
raise NotImplementedError()
|
|
101
100
|
|
|
102
101
|
def clean_out_empty_logs(self):
|
|
102
|
+
_close_handler(self.data_issue_file_handler)
|
|
103
103
|
if (
|
|
104
104
|
self.folder_structure.data_issue_file_path.is_file()
|
|
105
105
|
and os.stat(self.folder_structure.data_issue_file_path).st_size == 0
|
|
@@ -133,11 +133,7 @@ class MigrationTaskBase:
|
|
|
133
133
|
TransformationProcessError: _description_
|
|
134
134
|
|
|
135
135
|
"""
|
|
136
|
-
files = [
|
|
137
|
-
source_path / f.file_name
|
|
138
|
-
for f in file_defs
|
|
139
|
-
if isfile(source_path / f.file_name)
|
|
140
|
-
]
|
|
136
|
+
files = [source_path / f.file_name for f in file_defs if isfile(source_path / f.file_name)]
|
|
141
137
|
ret_str = ", ".join(f.file_name for f in file_defs)
|
|
142
138
|
|
|
143
139
|
if files and len(files) < len(file_defs):
|
|
@@ -162,12 +158,13 @@ class MigrationTaskBase:
|
|
|
162
158
|
This is in the base class because multiple tasks need it. It exists because instances in an ECS environment
|
|
163
159
|
are transformed for the central and data tenants separately, but the data tenants need to know about
|
|
164
160
|
the central tenant instance ids. This is a bit of a hack, but it works for now.
|
|
165
|
-
"""
|
|
161
|
+
""" # noqa: E501
|
|
166
162
|
map_files = []
|
|
167
163
|
instance_id_map = {}
|
|
168
164
|
if self.library_configuration.is_ecs and self.central_folder_structure:
|
|
169
165
|
logging.info(
|
|
170
|
-
"Loading ECS central tenant instance id map from %s",
|
|
166
|
+
"Loading ECS central tenant instance id map from %s",
|
|
167
|
+
self.central_folder_structure.instance_id_map_path,
|
|
171
168
|
)
|
|
172
169
|
instance_id_map = self.load_id_map(
|
|
173
170
|
self.central_folder_structure.instance_id_map_path,
|
|
@@ -176,7 +173,7 @@ class MigrationTaskBase:
|
|
|
176
173
|
map_files.append(str(self.central_folder_structure.instance_id_map_path))
|
|
177
174
|
logging.info(
|
|
178
175
|
"Loading member tenant isntance id map from %s",
|
|
179
|
-
self.folder_structure.instance_id_map_path
|
|
176
|
+
self.folder_structure.instance_id_map_path,
|
|
180
177
|
)
|
|
181
178
|
instance_id_map = self.load_id_map(
|
|
182
179
|
self.folder_structure.instance_id_map_path,
|
|
@@ -190,13 +187,11 @@ class MigrationTaskBase:
|
|
|
190
187
|
return instance_id_map
|
|
191
188
|
|
|
192
189
|
@staticmethod
|
|
193
|
-
def load_id_map(map_path, raise_if_empty=False, existing_id_map=
|
|
190
|
+
def load_id_map(map_path, raise_if_empty=False, existing_id_map=None):
|
|
194
191
|
if not isfile(map_path):
|
|
195
|
-
logging.warning(
|
|
196
|
-
"No legacy id map found at %s. Will build one from scratch", map_path
|
|
197
|
-
)
|
|
192
|
+
logging.warning("No legacy id map found at %s. Will build one from scratch", map_path)
|
|
198
193
|
return {}
|
|
199
|
-
id_map = existing_id_map
|
|
194
|
+
id_map = existing_id_map or {}
|
|
200
195
|
loaded_rows = len(id_map)
|
|
201
196
|
with open(map_path) as id_map_file:
|
|
202
197
|
for index, json_string in enumerate(id_map_file, start=1):
|
|
@@ -247,9 +242,7 @@ class MigrationTaskBase:
|
|
|
247
242
|
else:
|
|
248
243
|
logger.setLevel(logging.INFO)
|
|
249
244
|
stream_handler.setLevel(logging.INFO)
|
|
250
|
-
stream_handler.addFilter(
|
|
251
|
-
ExcludeLevelFilter(30)
|
|
252
|
-
) # Exclude warnings from pymarc
|
|
245
|
+
stream_handler.addFilter(ExcludeLevelFilter(30)) # Exclude warnings from pymarc
|
|
253
246
|
stream_handler.setFormatter(formatter)
|
|
254
247
|
logger.addHandler(stream_handler)
|
|
255
248
|
|
|
@@ -268,34 +261,28 @@ class MigrationTaskBase:
|
|
|
268
261
|
|
|
269
262
|
# Data issue file formatter
|
|
270
263
|
data_issue_file_formatter = logging.Formatter("%(message)s")
|
|
271
|
-
data_issue_file_handler = logging.FileHandler(
|
|
264
|
+
self.data_issue_file_handler = logging.FileHandler(
|
|
272
265
|
filename=str(self.folder_structure.data_issue_file_path), mode="w"
|
|
273
266
|
)
|
|
274
|
-
data_issue_file_handler.addFilter(LevelFilter(26))
|
|
275
|
-
data_issue_file_handler.setFormatter(data_issue_file_formatter)
|
|
276
|
-
data_issue_file_handler.setLevel(26)
|
|
277
|
-
logging.getLogger().addHandler(data_issue_file_handler)
|
|
267
|
+
self.data_issue_file_handler.addFilter(LevelFilter(26))
|
|
268
|
+
self.data_issue_file_handler.setFormatter(data_issue_file_formatter)
|
|
269
|
+
self.data_issue_file_handler.setLevel(26)
|
|
270
|
+
logging.getLogger().addHandler(self.data_issue_file_handler)
|
|
278
271
|
logger.info("Logging set up")
|
|
279
272
|
|
|
280
273
|
def setup_records_map(self, mapping_file_path):
|
|
281
274
|
with open(mapping_file_path) as mapping_file:
|
|
282
275
|
field_map = json.load(mapping_file)
|
|
283
|
-
logging.info(
|
|
284
|
-
"%s fields present in record mapping file", len(field_map["data"])
|
|
285
|
-
)
|
|
276
|
+
logging.info("%s fields present in record mapping file", len(field_map["data"]))
|
|
286
277
|
mapped_fields = (
|
|
287
278
|
f
|
|
288
279
|
for f in field_map["data"]
|
|
289
280
|
if f["legacy_field"] and f["legacy_field"] != "Not mapped"
|
|
290
281
|
)
|
|
291
|
-
logging.info(
|
|
292
|
-
"%s fields mapped in record mapping file", len(list(mapped_fields))
|
|
293
|
-
)
|
|
282
|
+
logging.info("%s fields mapped in record mapping file", len(list(mapped_fields)))
|
|
294
283
|
return field_map
|
|
295
284
|
|
|
296
|
-
def log_and_exit_if_too_many_errors(
|
|
297
|
-
self, error: TransformationRecordFailedError, idx
|
|
298
|
-
):
|
|
285
|
+
def log_and_exit_if_too_many_errors(self, error: TransformationRecordFailedError, idx):
|
|
299
286
|
self.num_exeptions += 1
|
|
300
287
|
error.log_it()
|
|
301
288
|
if self.num_exeptions / (1 + idx) > 0.2 and self.num_exeptions > 5000:
|
|
@@ -311,9 +298,7 @@ class MigrationTaskBase:
|
|
|
311
298
|
if num_processed > 1 and num_processed % 10000 == 0:
|
|
312
299
|
elapsed = num_processed / (time.time() - start_time)
|
|
313
300
|
elapsed_formatted = "{0:.4g}".format(elapsed)
|
|
314
|
-
logging.info(
|
|
315
|
-
f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} "
|
|
316
|
-
)
|
|
301
|
+
logging.info(f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} ")
|
|
317
302
|
|
|
318
303
|
def do_work_marc_transformer(
|
|
319
304
|
self,
|
|
@@ -322,9 +307,7 @@ class MigrationTaskBase:
|
|
|
322
307
|
if self.folder_structure.failed_marc_recs_file.is_file():
|
|
323
308
|
os.remove(self.folder_structure.failed_marc_recs_file)
|
|
324
309
|
logging.info("Removed failed marc records file to prevent duplicating data")
|
|
325
|
-
with open(
|
|
326
|
-
self.folder_structure.created_objects_path, "w+"
|
|
327
|
-
) as created_records_file:
|
|
310
|
+
with open(self.folder_structure.created_objects_path, "w+") as created_records_file:
|
|
328
311
|
self.processor = MarcFileProcessor(
|
|
329
312
|
self.mapper, self.folder_structure, created_records_file
|
|
330
313
|
)
|
|
@@ -377,7 +360,7 @@ class MigrationTaskBase:
|
|
|
377
360
|
|
|
378
361
|
Returns:
|
|
379
362
|
None
|
|
380
|
-
"""
|
|
363
|
+
""" # noqa: E501
|
|
381
364
|
current_pos = map_file.tell()
|
|
382
365
|
try:
|
|
383
366
|
map_file.seek(0)
|
|
@@ -391,13 +374,12 @@ class MigrationTaskBase:
|
|
|
391
374
|
"",
|
|
392
375
|
(
|
|
393
376
|
f"Mapping file {map_file.name} has rows with different number "
|
|
394
|
-
f"of columns ({'Row' if len(invalid_lines) == 1 else 'Rows'}
|
|
377
|
+
f"of columns ({'Row' if len(invalid_lines) == 1 else 'Rows'} "
|
|
378
|
+
f"{', '.join(invalid_lines)})"
|
|
395
379
|
),
|
|
396
380
|
)
|
|
397
381
|
if not valid_lines:
|
|
398
|
-
raise TransformationProcessError(
|
|
399
|
-
"", f"Map has no rows: {map_file.name}"
|
|
400
|
-
)
|
|
382
|
+
raise TransformationProcessError("", f"Map has no rows: {map_file.name}")
|
|
401
383
|
finally:
|
|
402
384
|
map_file.seek(current_pos)
|
|
403
385
|
|
|
@@ -422,7 +404,8 @@ class MigrationTaskBase:
|
|
|
422
404
|
or required
|
|
423
405
|
or folio_property_name.startswith("statisticalCodeIds")
|
|
424
406
|
or folio_property_name.startswith("locationMap")
|
|
425
|
-
|
|
407
|
+
or folio_property_name.startswith("fundsMap")
|
|
408
|
+
) and map_file_path.is_file():
|
|
426
409
|
try:
|
|
427
410
|
with open(map_file_path) as map_file:
|
|
428
411
|
# Validate the structure of the mapping file
|
|
@@ -482,7 +465,7 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
|
|
|
482
465
|
deactivate035_from001 (bool):
|
|
483
466
|
Disables the default FOLIO functionality of moving the previous 001 field into a 035 field, prefixed with the value from 003.
|
|
484
467
|
Default is False, meaning the functionality remains active.
|
|
485
|
-
"""
|
|
468
|
+
""" # noqa: E501
|
|
486
469
|
|
|
487
470
|
files: Annotated[
|
|
488
471
|
List[library_configuration.FileDefinition],
|
|
@@ -496,8 +479,7 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
|
|
|
496
479
|
Field(
|
|
497
480
|
title="Create source records",
|
|
498
481
|
description=(
|
|
499
|
-
"Controls whether or not to retain the MARC records in "
|
|
500
|
-
"Source Record Storage."
|
|
482
|
+
"Controls whether or not to retain the MARC records in Source Record Storage."
|
|
501
483
|
),
|
|
502
484
|
),
|
|
503
485
|
] = False
|
|
@@ -538,12 +520,14 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
|
|
|
538
520
|
title="Statistical code mapping fields",
|
|
539
521
|
description=(
|
|
540
522
|
"List of fields + subfields to be used for mapping statistical codes. "
|
|
541
|
-
|
|
542
|
-
"will be treated as unique values. Multiple subfields will be concatenated
|
|
523
|
+
'Subfields should be delimited by a "$" (eg. 907$a). Single repeating subfields '
|
|
524
|
+
"will be treated as unique values. Multiple subfields will be concatenated "
|
|
525
|
+
"together with a space."
|
|
543
526
|
),
|
|
544
527
|
),
|
|
545
528
|
] = []
|
|
546
529
|
|
|
530
|
+
|
|
547
531
|
class ExcludeLevelFilter(logging.Filter):
|
|
548
532
|
def __init__(self, level):
|
|
549
533
|
super().__init__()
|
|
@@ -570,3 +554,10 @@ class LevelFilter(logging.Filter):
|
|
|
570
554
|
|
|
571
555
|
def filter(self, record):
|
|
572
556
|
return record.levelno == self.level
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _close_handler(handler: logging.Handler | None):
|
|
560
|
+
if handler is None:
|
|
561
|
+
return
|
|
562
|
+
handler.flush()
|
|
563
|
+
handler.close()
|
|
@@ -4,7 +4,6 @@ import json
|
|
|
4
4
|
import logging
|
|
5
5
|
import sys
|
|
6
6
|
import time
|
|
7
|
-
from os.path import isfile
|
|
8
7
|
from typing import List, Optional, Annotated
|
|
9
8
|
from pydantic import Field
|
|
10
9
|
|
|
@@ -17,6 +16,7 @@ from folio_migration_tools.custom_exceptions import (
|
|
|
17
16
|
TransformationRecordFailedError,
|
|
18
17
|
)
|
|
19
18
|
from folio_migration_tools.helper import Helper
|
|
19
|
+
from folio_migration_tools.i18n_cache import i18n_t
|
|
20
20
|
from folio_migration_tools.library_configuration import (
|
|
21
21
|
FileDefinition,
|
|
22
22
|
LibraryConfiguration,
|
|
@@ -84,50 +84,35 @@ class OrdersTransformer(MigrationTaskBase):
|
|
|
84
84
|
Optional[str],
|
|
85
85
|
Field(
|
|
86
86
|
title="Payment Status Map File Name",
|
|
87
|
-
description=(
|
|
88
|
-
"File name for payment status mapping. "
|
|
89
|
-
"By default is empty string."
|
|
90
|
-
),
|
|
87
|
+
description=("File name for payment status mapping. By default is empty string."),
|
|
91
88
|
),
|
|
92
89
|
] = ""
|
|
93
90
|
receipt_status_map_file_name: Annotated[
|
|
94
91
|
Optional[str],
|
|
95
92
|
Field(
|
|
96
93
|
title="Receipt Status Map File Name",
|
|
97
|
-
description=(
|
|
98
|
-
"File name for receipt status mapping. "
|
|
99
|
-
"By default is empty string."
|
|
100
|
-
),
|
|
94
|
+
description=("File name for receipt status mapping. By default is empty string."),
|
|
101
95
|
),
|
|
102
96
|
] = ""
|
|
103
97
|
workflow_status_map_file_name: Annotated[
|
|
104
98
|
Optional[str],
|
|
105
99
|
Field(
|
|
106
100
|
title="Workflow Status Map File Name",
|
|
107
|
-
description=(
|
|
108
|
-
"File name for workflow status mapping. "
|
|
109
|
-
"By default is empty string."
|
|
110
|
-
),
|
|
101
|
+
description=("File name for workflow status mapping. By default is empty string."),
|
|
111
102
|
),
|
|
112
103
|
] = ""
|
|
113
104
|
location_map_file_name: Annotated[
|
|
114
105
|
Optional[str],
|
|
115
106
|
Field(
|
|
116
107
|
title="Location Map File Name",
|
|
117
|
-
description=(
|
|
118
|
-
"File name for location mapping. "
|
|
119
|
-
"By default is empty string."
|
|
120
|
-
),
|
|
108
|
+
description=("File name for location mapping. By default is empty string."),
|
|
121
109
|
),
|
|
122
110
|
] = ""
|
|
123
111
|
funds_map_file_name: Annotated[
|
|
124
112
|
Optional[str],
|
|
125
113
|
Field(
|
|
126
114
|
title="Funds Map File Name",
|
|
127
|
-
description=(
|
|
128
|
-
"File name for funds mapping. "
|
|
129
|
-
"By default is empty string."
|
|
130
|
-
),
|
|
115
|
+
description=("File name for funds mapping. By default is empty string."),
|
|
131
116
|
),
|
|
132
117
|
] = ""
|
|
133
118
|
funds_expense_class_map_file_name: Annotated[
|
|
@@ -135,8 +120,7 @@ class OrdersTransformer(MigrationTaskBase):
|
|
|
135
120
|
Field(
|
|
136
121
|
title="Funds Expense Class Map File Name",
|
|
137
122
|
description=(
|
|
138
|
-
"File name for funds expense class mapping. "
|
|
139
|
-
"By default is empty string."
|
|
123
|
+
"File name for funds expense class mapping. By default is empty string."
|
|
140
124
|
),
|
|
141
125
|
),
|
|
142
126
|
] = ""
|
|
@@ -218,7 +202,7 @@ class OrdersTransformer(MigrationTaskBase):
|
|
|
218
202
|
"fundsMap",
|
|
219
203
|
self.folder_structure.mapping_files_folder / self.task_config.funds_map_file_name,
|
|
220
204
|
self.folio_keys,
|
|
221
|
-
|
|
205
|
+
True,
|
|
222
206
|
),
|
|
223
207
|
self.load_ref_data_mapping_file( # Todo: The property in the schema has no type
|
|
224
208
|
"fundsExpenseClassMap",
|
|
@@ -230,28 +214,28 @@ class OrdersTransformer(MigrationTaskBase):
|
|
|
230
214
|
)
|
|
231
215
|
|
|
232
216
|
def list_source_files(self):
|
|
233
|
-
files = [
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
)
|
|
217
|
+
files = []
|
|
218
|
+
for f in self.task_config.files:
|
|
219
|
+
file_path = self.folder_structure.data_folder / self.object_type_name / f.file_name
|
|
220
|
+
|
|
221
|
+
if not file_path.is_file():
|
|
222
|
+
print(f"\n\nERROR: File defined in task not found - {f.file_name}")
|
|
223
|
+
raise TransformationProcessError(
|
|
224
|
+
f"\n\nERROR: File defined in task not found - {f.file_name}"
|
|
225
|
+
)
|
|
226
|
+
files.append(file_path)
|
|
244
227
|
logging.info("Files to process:")
|
|
245
228
|
for filename in files:
|
|
246
229
|
logging.info("\t%s", filename)
|
|
247
230
|
return files
|
|
248
231
|
|
|
249
232
|
def process_single_file(self, filename):
|
|
250
|
-
with
|
|
251
|
-
|
|
252
|
-
|
|
233
|
+
with (
|
|
234
|
+
open(filename, encoding="utf-8-sig") as records_file,
|
|
235
|
+
open(self.folder_structure.created_objects_path, "w+") as results_file,
|
|
236
|
+
):
|
|
253
237
|
self.mapper.migration_report.add_general_statistics(
|
|
254
|
-
|
|
238
|
+
i18n_t("Number of files processed")
|
|
255
239
|
)
|
|
256
240
|
start = time.time()
|
|
257
241
|
records_processed = 0
|
|
@@ -270,7 +254,7 @@ class OrdersTransformer(MigrationTaskBase):
|
|
|
270
254
|
self.mapper.perform_additional_mapping(legacy_id, folio_rec)
|
|
271
255
|
|
|
272
256
|
self.mapper.migration_report.add_general_statistics(
|
|
273
|
-
|
|
257
|
+
i18n_t("TOTAL Purchase Order Lines created")
|
|
274
258
|
)
|
|
275
259
|
self.mapper.report_folio_mapping(folio_rec, self.mapper.composite_order_schema)
|
|
276
260
|
self.mapper.notes_mapper.map_notes(
|
|
@@ -333,7 +317,7 @@ class OrdersTransformer(MigrationTaskBase):
|
|
|
333
317
|
self.folder_structure.migration_reports_file,
|
|
334
318
|
)
|
|
335
319
|
self.mapper.migration_report.write_migration_report(
|
|
336
|
-
|
|
320
|
+
i18n_t("Pruchase Orders and Purchase Order Lines Transformation Report"),
|
|
337
321
|
migration_report_file,
|
|
338
322
|
self.start_datetime,
|
|
339
323
|
)
|
|
@@ -50,27 +50,21 @@ class OrganizationTransformer(MigrationTaskBase):
|
|
|
50
50
|
str,
|
|
51
51
|
Field(
|
|
52
52
|
title="Migration task type",
|
|
53
|
-
description=(
|
|
54
|
-
"The type of migration task you want to perform"
|
|
55
|
-
),
|
|
53
|
+
description=("The type of migration task you want to perform"),
|
|
56
54
|
),
|
|
57
55
|
]
|
|
58
56
|
files: Annotated[
|
|
59
57
|
List[FileDefinition],
|
|
60
58
|
Field(
|
|
61
59
|
title="Source files",
|
|
62
|
-
description=(
|
|
63
|
-
"List of MARC21 files with holdings records"
|
|
64
|
-
),
|
|
60
|
+
description=("List of MARC21 files with holdings records"),
|
|
65
61
|
),
|
|
66
62
|
]
|
|
67
63
|
organization_map_path: Annotated[
|
|
68
64
|
str,
|
|
69
65
|
Field(
|
|
70
66
|
title="Organization map path",
|
|
71
|
-
description=(
|
|
72
|
-
"Path to the organization map file"
|
|
73
|
-
),
|
|
67
|
+
description=("Path to the organization map file"),
|
|
74
68
|
),
|
|
75
69
|
]
|
|
76
70
|
organization_types_map_path: Annotated[
|
|
@@ -95,18 +89,14 @@ class OrganizationTransformer(MigrationTaskBase):
|
|
|
95
89
|
Optional[str],
|
|
96
90
|
Field(
|
|
97
91
|
title="Email categories map path",
|
|
98
|
-
description=(
|
|
99
|
-
"Path to the email categories map file. By default is empty string"
|
|
100
|
-
),
|
|
92
|
+
description=("Path to the email categories map file. By default is empty string"),
|
|
101
93
|
),
|
|
102
94
|
] = ""
|
|
103
95
|
phone_categories_map_path: Annotated[
|
|
104
96
|
Optional[str],
|
|
105
97
|
Field(
|
|
106
98
|
title="Phone categories map path",
|
|
107
|
-
description=(
|
|
108
|
-
"Path to the phone categories map file. By default is empty string"
|
|
109
|
-
),
|
|
99
|
+
description=("Path to the phone categories map file. By default is empty string"),
|
|
110
100
|
),
|
|
111
101
|
] = ""
|
|
112
102
|
|
|
@@ -201,9 +191,10 @@ class OrganizationTransformer(MigrationTaskBase):
|
|
|
201
191
|
return files
|
|
202
192
|
|
|
203
193
|
def process_single_file(self, filename):
|
|
204
|
-
with
|
|
205
|
-
|
|
206
|
-
|
|
194
|
+
with (
|
|
195
|
+
open(filename, encoding="utf-8-sig") as records_file,
|
|
196
|
+
open(self.folder_structure.created_objects_path, "w+") as results_file,
|
|
197
|
+
):
|
|
207
198
|
self.mapper.migration_report.add_general_statistics(
|
|
208
199
|
i18n.t("Number of files processed")
|
|
209
200
|
)
|
|
@@ -13,6 +13,7 @@ from zoneinfo import ZoneInfo
|
|
|
13
13
|
from folio_migration_tools.circulation_helper import CirculationHelper
|
|
14
14
|
from folio_migration_tools.custom_dict import InsensitiveDictReader
|
|
15
15
|
from folio_migration_tools.helper import Helper
|
|
16
|
+
from folio_migration_tools.i18n_cache import i18n_t
|
|
16
17
|
from folio_migration_tools.library_configuration import (
|
|
17
18
|
FileDefinition,
|
|
18
19
|
LibraryConfiguration,
|
|
@@ -32,7 +33,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
32
33
|
description=(
|
|
33
34
|
"Name of this migration task. The name is being used to call "
|
|
34
35
|
"the specific task, and to distinguish tasks of similar types"
|
|
35
|
-
)
|
|
36
|
+
),
|
|
36
37
|
),
|
|
37
38
|
]
|
|
38
39
|
migration_task_type: Annotated[
|
|
@@ -54,8 +55,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
54
55
|
Field(
|
|
55
56
|
title="Starting row",
|
|
56
57
|
description=(
|
|
57
|
-
"Row number to start processing data from. "
|
|
58
|
-
"Optional, by default is first row"
|
|
58
|
+
"Row number to start processing data from. Optional, by default is first row"
|
|
59
59
|
),
|
|
60
60
|
),
|
|
61
61
|
] = 1
|
|
@@ -64,8 +64,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
64
64
|
Field(
|
|
65
65
|
title="Item files",
|
|
66
66
|
description=(
|
|
67
|
-
"List of files containing item data. "
|
|
68
|
-
"Optional, by default is empty list"
|
|
67
|
+
"List of files containing item data. Optional, by default is empty list"
|
|
69
68
|
),
|
|
70
69
|
),
|
|
71
70
|
] = []
|
|
@@ -74,8 +73,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
74
73
|
Field(
|
|
75
74
|
title="Patron files",
|
|
76
75
|
description=(
|
|
77
|
-
"List of files containing patron data. "
|
|
78
|
-
"Optional, by default is empty list"
|
|
76
|
+
"List of files containing patron data. Optional, by default is empty list"
|
|
79
77
|
),
|
|
80
78
|
),
|
|
81
79
|
] = []
|
|
@@ -88,7 +86,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
88
86
|
self,
|
|
89
87
|
task_configuration: TaskConfiguration,
|
|
90
88
|
library_config: LibraryConfiguration,
|
|
91
|
-
folio_client
|
|
89
|
+
folio_client,
|
|
92
90
|
):
|
|
93
91
|
csv.register_dialect("tsv", delimiter="\t")
|
|
94
92
|
self.migration_report = MigrationReport()
|
|
@@ -135,8 +133,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
135
133
|
)
|
|
136
134
|
else:
|
|
137
135
|
logging.info(
|
|
138
|
-
"No item or user files supplied. Not validating
|
|
139
|
-
"previously migrated objects"
|
|
136
|
+
"No item or user files supplied. Not validating againstpreviously migrated objects"
|
|
140
137
|
)
|
|
141
138
|
self.valid_legacy_requests = self.semi_valid_legacy_requests
|
|
142
139
|
|
|
@@ -151,7 +148,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
151
148
|
|
|
152
149
|
def prepare_legacy_request(self, legacy_request: LegacyRequest):
|
|
153
150
|
patron = self.circulation_helper.get_user_by_barcode(legacy_request.patron_barcode)
|
|
154
|
-
self.migration_report.add_general_statistics(
|
|
151
|
+
self.migration_report.add_general_statistics(i18n_t("Patron lookups performed"))
|
|
155
152
|
|
|
156
153
|
if not patron:
|
|
157
154
|
logging.error(f"No user with barcode {legacy_request.patron_barcode} found in FOLIO")
|
|
@@ -161,18 +158,18 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
161
158
|
f"{legacy_request.patron_barcode}",
|
|
162
159
|
)
|
|
163
160
|
self.migration_report.add_general_statistics(
|
|
164
|
-
|
|
161
|
+
i18n_t("No user with barcode found in FOLIO")
|
|
165
162
|
)
|
|
166
163
|
self.failed_requests.add(legacy_request)
|
|
167
164
|
return False, legacy_request
|
|
168
165
|
legacy_request.patron_id = patron.get("id")
|
|
169
166
|
|
|
170
167
|
item = self.circulation_helper.get_item_by_barcode(legacy_request.item_barcode)
|
|
171
|
-
self.migration_report.add_general_statistics(
|
|
168
|
+
self.migration_report.add_general_statistics(i18n_t("Item lookups performed"))
|
|
172
169
|
if not item:
|
|
173
170
|
logging.error(f"No item with barcode {legacy_request.item_barcode} found in FOLIO")
|
|
174
171
|
self.migration_report.add_general_statistics(
|
|
175
|
-
|
|
172
|
+
i18n_t("No item with barcode found in FOLIO")
|
|
176
173
|
)
|
|
177
174
|
Helper.log_data_issue(
|
|
178
175
|
f"{legacy_request.item_barcode}",
|
|
@@ -182,22 +179,22 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
182
179
|
self.failed_requests.add(legacy_request)
|
|
183
180
|
return False, legacy_request
|
|
184
181
|
holding = self.circulation_helper.get_holding_by_uuid(item.get("holdingsRecordId"))
|
|
185
|
-
self.migration_report.add_general_statistics(
|
|
182
|
+
self.migration_report.add_general_statistics(i18n_t("Holdings lookups performed"))
|
|
186
183
|
legacy_request.item_id = item.get("id")
|
|
187
184
|
legacy_request.holdings_record_id = item.get("holdingsRecordId")
|
|
188
185
|
legacy_request.instance_id = holding.get("instanceId")
|
|
189
186
|
if item["status"]["name"] in ["Available"]:
|
|
190
187
|
legacy_request.request_type = "Page"
|
|
191
|
-
logging.info(f
|
|
188
|
+
logging.info(f"Setting request to Page, since the status is {item['status']['name']}")
|
|
192
189
|
self.migration_report.add_general_statistics(
|
|
193
|
-
|
|
190
|
+
i18n_t("Valid, prepared requests, ready for posting")
|
|
194
191
|
)
|
|
195
192
|
return True, legacy_request
|
|
196
193
|
|
|
197
194
|
def do_work(self):
|
|
198
195
|
logging.info("Starting")
|
|
199
196
|
if self.task_configuration.starting_row > 1:
|
|
200
|
-
logging.info(f"Skipping {(self.task_configuration.starting_row-1)} records")
|
|
197
|
+
logging.info(f"Skipping {(self.task_configuration.starting_row - 1)} records")
|
|
201
198
|
for num_requests, legacy_request in enumerate(
|
|
202
199
|
self.valid_legacy_requests[self.task_configuration.starting_row - 1 :],
|
|
203
200
|
start=1,
|
|
@@ -210,11 +207,11 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
210
207
|
self.folio_client, legacy_request, self.migration_report
|
|
211
208
|
):
|
|
212
209
|
self.migration_report.add_general_statistics(
|
|
213
|
-
|
|
210
|
+
i18n_t("Successfully migrated requests")
|
|
214
211
|
)
|
|
215
212
|
else:
|
|
216
213
|
self.migration_report.add_general_statistics(
|
|
217
|
-
|
|
214
|
+
i18n_t("Unsuccessfully migrated requests")
|
|
218
215
|
)
|
|
219
216
|
self.failed_requests.add(legacy_request)
|
|
220
217
|
if num_requests == 1:
|
|
@@ -237,7 +234,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
237
234
|
|
|
238
235
|
with open(self.folder_structure.migration_reports_file, "w+") as report_file:
|
|
239
236
|
self.migration_report.write_migration_report(
|
|
240
|
-
|
|
237
|
+
i18n_t("Requests migration report"), report_file, self.start_datetime
|
|
241
238
|
)
|
|
242
239
|
self.clean_out_empty_logs()
|
|
243
240
|
|
|
@@ -281,7 +278,8 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
281
278
|
self.migration_report.add(
|
|
282
279
|
"DiscardedLoans",
|
|
283
280
|
i18n.t(
|
|
284
|
-
"Requests discarded. Had migrated item barcode: %{item_barcode}.\n
|
|
281
|
+
"Requests discarded. Had migrated item barcode: %{item_barcode}.\n "
|
|
282
|
+
"Had migrated user barcode: %{patron_barcode}",
|
|
285
283
|
item_barcode=has_item_barcode,
|
|
286
284
|
patron_barcode=has_patron_barcode,
|
|
287
285
|
),
|
|
@@ -333,8 +331,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
333
331
|
except ValueError as ve:
|
|
334
332
|
logging.exception(ve)
|
|
335
333
|
logging.info(
|
|
336
|
-
f"Done validating {legacy_reques_count} "
|
|
337
|
-
f"legacy requests with {num_bad} rotten apples"
|
|
334
|
+
f"Done validating {legacy_reques_count} legacy requests with {num_bad} rotten apples"
|
|
338
335
|
)
|
|
339
336
|
if num_bad > 0 and (num_bad / legacy_reques_count) > 0.5:
|
|
340
337
|
q = num_bad / legacy_reques_count
|
|
@@ -14,6 +14,7 @@ from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
|
14
14
|
|
|
15
15
|
from folio_migration_tools.custom_dict import InsensitiveDictReader
|
|
16
16
|
from folio_migration_tools.custom_exceptions import TransformationProcessError
|
|
17
|
+
from folio_migration_tools.i18n_cache import i18n_t
|
|
17
18
|
from folio_migration_tools.library_configuration import (
|
|
18
19
|
FileDefinition,
|
|
19
20
|
LibraryConfiguration,
|
|
@@ -59,7 +60,7 @@ class ReservesMigrator(MigrationTaskBase):
|
|
|
59
60
|
self,
|
|
60
61
|
task_configuration: TaskConfiguration,
|
|
61
62
|
library_config: LibraryConfiguration,
|
|
62
|
-
folio_client
|
|
63
|
+
folio_client,
|
|
63
64
|
):
|
|
64
65
|
csv.register_dialect("tsv", delimiter="\t")
|
|
65
66
|
self.migration_report = MigrationReport()
|
|
@@ -90,7 +91,7 @@ class ReservesMigrator(MigrationTaskBase):
|
|
|
90
91
|
logging.info("Starting")
|
|
91
92
|
for num_reserves, legacy_reserve in enumerate(self.valid_reserves, start=1):
|
|
92
93
|
t0_migration = time.time()
|
|
93
|
-
self.migration_report.add_general_statistics(
|
|
94
|
+
self.migration_report.add_general_statistics(i18n_t("Processed reserves"))
|
|
94
95
|
try:
|
|
95
96
|
self.post_single_reserve(legacy_reserve)
|
|
96
97
|
except Exception as ee:
|
|
@@ -107,10 +108,10 @@ class ReservesMigrator(MigrationTaskBase):
|
|
|
107
108
|
path, legacy_reserve.to_dict(), "POST", i18n.t("Posted reserves")
|
|
108
109
|
):
|
|
109
110
|
self.migration_report.add_general_statistics(
|
|
110
|
-
|
|
111
|
+
i18n_t("Successfully posted reserves")
|
|
111
112
|
)
|
|
112
113
|
else:
|
|
113
|
-
self.migration_report.add_general_statistics(
|
|
114
|
+
self.migration_report.add_general_statistics(i18n_t("Failure to post reserve"))
|
|
114
115
|
except Exception as ee:
|
|
115
116
|
logging.error(ee)
|
|
116
117
|
|
|
@@ -123,7 +124,7 @@ class ReservesMigrator(MigrationTaskBase):
|
|
|
123
124
|
|
|
124
125
|
with open(self.folder_structure.migration_reports_file, "w+") as report_file:
|
|
125
126
|
self.migration_report.write_migration_report(
|
|
126
|
-
|
|
127
|
+
i18n_t("Reserves migration report"), report_file, self.start_datetime
|
|
127
128
|
)
|
|
128
129
|
self.clean_out_empty_logs()
|
|
129
130
|
|