folio-migration-tools 1.10.0__py3-none-any.whl → 1.10.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__main__.py +0 -9
- folio_migration_tools/circulation_helper.py +5 -6
- folio_migration_tools/folder_structure.py +6 -16
- folio_migration_tools/helper.py +6 -7
- folio_migration_tools/holdings_helper.py +3 -4
- folio_migration_tools/library_configuration.py +0 -12
- folio_migration_tools/mapper_base.py +6 -7
- folio_migration_tools/mapping_file_transformation/user_mapper.py +0 -4
- folio_migration_tools/marc_rules_transformation/conditions.py +29 -0
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +9 -19
- folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +242 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +15 -10
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +2 -3
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +5 -6
- folio_migration_tools/migration_report.py +6 -17
- folio_migration_tools/migration_tasks/authority_transformer.py +118 -0
- folio_migration_tools/migration_tasks/batch_poster.py +298 -219
- folio_migration_tools/migration_tasks/bibs_transformer.py +2 -2
- folio_migration_tools/migration_tasks/holdings_csv_transformer.py +9 -9
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +3 -3
- folio_migration_tools/migration_tasks/items_transformer.py +4 -6
- folio_migration_tools/migration_tasks/loans_migrator.py +18 -19
- folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py +3 -3
- folio_migration_tools/migration_tasks/migration_task_base.py +6 -15
- folio_migration_tools/migration_tasks/orders_transformer.py +3 -4
- folio_migration_tools/migration_tasks/requests_migrator.py +9 -10
- folio_migration_tools/migration_tasks/reserves_migrator.py +4 -5
- folio_migration_tools/migration_tasks/user_transformer.py +5 -15
- folio_migration_tools/translations/en.json +7 -0
- {folio_migration_tools-1.10.0.dist-info → folio_migration_tools-1.10.0b1.dist-info}/METADATA +2 -3
- {folio_migration_tools-1.10.0.dist-info → folio_migration_tools-1.10.0b1.dist-info}/RECORD +33 -32
- folio_migration_tools/i18n_cache.py +0 -79
- {folio_migration_tools-1.10.0.dist-info → folio_migration_tools-1.10.0b1.dist-info}/WHEEL +0 -0
- {folio_migration_tools-1.10.0.dist-info → folio_migration_tools-1.10.0b1.dist-info}/entry_points.txt +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from typing import Annotated, List
|
|
3
3
|
|
|
4
|
+
import i18n
|
|
4
5
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
5
6
|
from pydantic import Field
|
|
6
7
|
|
|
7
8
|
from folio_migration_tools.helper import Helper
|
|
8
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
9
9
|
from folio_migration_tools.library_configuration import (
|
|
10
10
|
IlsFlavour,
|
|
11
11
|
LibraryConfiguration,
|
|
@@ -153,7 +153,7 @@ class BibsTransformer(MigrationTaskBase):
|
|
|
153
153
|
self.processor.wrap_up()
|
|
154
154
|
with open(self.folder_structure.migration_reports_file, "w+") as report_file:
|
|
155
155
|
self.mapper.migration_report.write_migration_report(
|
|
156
|
-
|
|
156
|
+
i18n.t("Bibliographic records transformation report"),
|
|
157
157
|
report_file,
|
|
158
158
|
self.start_datetime,
|
|
159
159
|
)
|
|
@@ -7,6 +7,7 @@ import time
|
|
|
7
7
|
import traceback
|
|
8
8
|
from typing import Annotated, List, Optional
|
|
9
9
|
|
|
10
|
+
import i18n
|
|
10
11
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
11
12
|
from httpx import HTTPError
|
|
12
13
|
from pydantic import Field
|
|
@@ -17,7 +18,6 @@ from folio_migration_tools.custom_exceptions import (
|
|
|
17
18
|
)
|
|
18
19
|
from folio_migration_tools.helper import Helper
|
|
19
20
|
from folio_migration_tools.holdings_helper import HoldingsHelper
|
|
20
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
21
21
|
from folio_migration_tools.library_configuration import (
|
|
22
22
|
FileDefinition,
|
|
23
23
|
HridHandling,
|
|
@@ -351,14 +351,14 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
351
351
|
)
|
|
352
352
|
Helper.write_to_file(holdings_file, holding)
|
|
353
353
|
self.mapper.migration_report.add_general_statistics(
|
|
354
|
-
|
|
354
|
+
i18n.t("Holdings Records Written to disk")
|
|
355
355
|
)
|
|
356
356
|
self.mapper.save_id_map_file(
|
|
357
357
|
self.folder_structure.holdings_id_map_path, self.holdings_id_map
|
|
358
358
|
)
|
|
359
359
|
with open(self.folder_structure.migration_reports_file, "w") as migration_report_file:
|
|
360
360
|
self.mapper.migration_report.write_migration_report(
|
|
361
|
-
|
|
361
|
+
i18n.t("Holdings transformation report"),
|
|
362
362
|
migration_report_file,
|
|
363
363
|
self.mapper.start_datetime,
|
|
364
364
|
)
|
|
@@ -393,7 +393,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
393
393
|
full_path = self.folder_structure.data_folder / "items" / file_def.file_name
|
|
394
394
|
with open(full_path, encoding="utf-8-sig") as records_file:
|
|
395
395
|
self.mapper.migration_report.add_general_statistics(
|
|
396
|
-
|
|
396
|
+
i18n.t("Number of files processed")
|
|
397
397
|
)
|
|
398
398
|
start = time.time()
|
|
399
399
|
records_processed = 0
|
|
@@ -412,7 +412,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
412
412
|
except Exception as excepion:
|
|
413
413
|
self.mapper.handle_generic_exception(idx, excepion)
|
|
414
414
|
self.mapper.migration_report.add_general_statistics(
|
|
415
|
-
|
|
415
|
+
i18n.t("Number of Legacy items in file")
|
|
416
416
|
)
|
|
417
417
|
if idx > 1 and idx % 10000 == 0:
|
|
418
418
|
elapsed = idx / (time.time() - start)
|
|
@@ -481,7 +481,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
481
481
|
self.holdings[bw_key] = incoming_holding
|
|
482
482
|
self.mapper.create_and_write_boundwith_part(legacy_item_id, incoming_holding["id"])
|
|
483
483
|
self.mapper.migration_report.add_general_statistics(
|
|
484
|
-
|
|
484
|
+
i18n.t("Unique BW Holdings created from Items")
|
|
485
485
|
)
|
|
486
486
|
else:
|
|
487
487
|
self.merge_holding(bw_key, incoming_holding)
|
|
@@ -492,7 +492,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
492
492
|
legacy_item_id, self.holdings[bw_key], self.object_type
|
|
493
493
|
)
|
|
494
494
|
self.mapper.migration_report.add_general_statistics(
|
|
495
|
-
|
|
495
|
+
i18n.t("BW Items found tied to previously created BW Holdings")
|
|
496
496
|
)
|
|
497
497
|
else:
|
|
498
498
|
# Regular holding. Merge according to criteria
|
|
@@ -504,12 +504,12 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
504
504
|
)
|
|
505
505
|
if self.holdings.get(new_holding_key, None):
|
|
506
506
|
self.mapper.migration_report.add_general_statistics(
|
|
507
|
-
|
|
507
|
+
i18n.t("Holdings already created from Item")
|
|
508
508
|
)
|
|
509
509
|
self.merge_holding(new_holding_key, incoming_holding)
|
|
510
510
|
else:
|
|
511
511
|
self.mapper.migration_report.add_general_statistics(
|
|
512
|
-
|
|
512
|
+
i18n.t("Unique Holdings created from Items")
|
|
513
513
|
)
|
|
514
514
|
self.holdings[new_holding_key] = incoming_holding
|
|
515
515
|
|
|
@@ -3,12 +3,12 @@ import json
|
|
|
3
3
|
import logging
|
|
4
4
|
from typing import Annotated, List
|
|
5
5
|
|
|
6
|
+
import i18n
|
|
6
7
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
7
8
|
from pydantic import Field
|
|
8
9
|
|
|
9
10
|
from folio_migration_tools.custom_exceptions import TransformationProcessError
|
|
10
11
|
from folio_migration_tools.helper import Helper
|
|
11
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
12
12
|
from folio_migration_tools.library_configuration import (
|
|
13
13
|
FileDefinition,
|
|
14
14
|
HridHandling,
|
|
@@ -269,7 +269,7 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
269
269
|
except FileNotFoundError as fnfe:
|
|
270
270
|
raise TransformationProcessError(
|
|
271
271
|
"",
|
|
272
|
-
|
|
272
|
+
i18n.t("Provided boundwith relationship file not found"),
|
|
273
273
|
self.task_configuration.boundwith_relationship_file_path,
|
|
274
274
|
) from fnfe
|
|
275
275
|
|
|
@@ -350,7 +350,7 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
350
350
|
|
|
351
351
|
with open(self.folder_structure.migration_reports_file, "w+") as report_file:
|
|
352
352
|
self.mapper.migration_report.write_migration_report(
|
|
353
|
-
|
|
353
|
+
i18n.t("Bibliographic records transformation report"),
|
|
354
354
|
report_file,
|
|
355
355
|
self.start_datetime,
|
|
356
356
|
)
|
|
@@ -12,8 +12,6 @@ import i18n
|
|
|
12
12
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
13
13
|
from pydantic import Field
|
|
14
14
|
|
|
15
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
16
|
-
|
|
17
15
|
from folio_migration_tools.custom_exceptions import (
|
|
18
16
|
TransformationProcessError,
|
|
19
17
|
TransformationRecordFailedError,
|
|
@@ -337,7 +335,7 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
337
335
|
records_in_file = 0
|
|
338
336
|
with open(full_path, encoding="utf-8-sig") as records_file:
|
|
339
337
|
self.mapper.migration_report.add_general_statistics(
|
|
340
|
-
|
|
338
|
+
i18n.t("Number of files processed")
|
|
341
339
|
)
|
|
342
340
|
start = time.time()
|
|
343
341
|
for idx, record in enumerate(self.mapper.get_objects(records_file, full_path)):
|
|
@@ -370,7 +368,7 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
370
368
|
# TODO: turn this into a asynchronous task
|
|
371
369
|
Helper.write_to_file(results_file, folio_rec)
|
|
372
370
|
self.mapper.migration_report.add_general_statistics(
|
|
373
|
-
|
|
371
|
+
i18n.t("Number of records written to disk")
|
|
374
372
|
)
|
|
375
373
|
self.mapper.report_folio_mapping(folio_rec, self.mapper.schema)
|
|
376
374
|
except TransformationProcessError as process_error:
|
|
@@ -389,7 +387,7 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
389
387
|
i18n.t("Number of Legacy items in %{container}", container=file_def),
|
|
390
388
|
)
|
|
391
389
|
self.mapper.migration_report.add_general_statistics(
|
|
392
|
-
|
|
390
|
+
i18n.t("Number of Legacy items in total")
|
|
393
391
|
)
|
|
394
392
|
self.print_progress(idx, start)
|
|
395
393
|
records_in_file = idx + 1
|
|
@@ -471,7 +469,7 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
471
469
|
self.extradata_writer.flush()
|
|
472
470
|
with open(self.folder_structure.migration_reports_file, "w") as migration_report_file:
|
|
473
471
|
self.mapper.migration_report.write_migration_report(
|
|
474
|
-
|
|
472
|
+
i18n.t("Item transformation report"),
|
|
475
473
|
migration_report_file,
|
|
476
474
|
self.mapper.start_datetime,
|
|
477
475
|
)
|
|
@@ -19,7 +19,6 @@ from art import tprint
|
|
|
19
19
|
from folio_migration_tools.circulation_helper import CirculationHelper
|
|
20
20
|
from folio_migration_tools.custom_exceptions import TransformationRecordFailedError
|
|
21
21
|
from folio_migration_tools.helper import Helper
|
|
22
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
23
22
|
from folio_migration_tools.library_configuration import (
|
|
24
23
|
FileDefinition,
|
|
25
24
|
LibraryConfiguration,
|
|
@@ -206,7 +205,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
206
205
|
):
|
|
207
206
|
t0_migration = time.time()
|
|
208
207
|
self.migration_report.add_general_statistics(
|
|
209
|
-
|
|
208
|
+
i18n.t("Processed pre-validated loans")
|
|
210
209
|
)
|
|
211
210
|
try:
|
|
212
211
|
self.checkout_single_loan(legacy_loan)
|
|
@@ -234,15 +233,15 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
234
233
|
res_checkout = self.circulation_helper.check_out_by_barcode(legacy_loan)
|
|
235
234
|
|
|
236
235
|
if res_checkout.was_successful:
|
|
237
|
-
self.migration_report.add("Details",
|
|
238
|
-
self.migration_report.add_general_statistics(
|
|
236
|
+
self.migration_report.add("Details", i18n.t("Checked out on first try"))
|
|
237
|
+
self.migration_report.add_general_statistics(i18n.t("Successfully checked out"))
|
|
239
238
|
self.set_renewal_count(legacy_loan, res_checkout)
|
|
240
239
|
self.set_new_status(legacy_loan, res_checkout)
|
|
241
240
|
elif res_checkout.should_be_retried:
|
|
242
241
|
res_checkout2 = self.handle_checkout_failure(legacy_loan, res_checkout)
|
|
243
242
|
if res_checkout2.was_successful and res_checkout2.folio_loan:
|
|
244
|
-
self.migration_report.add("Details",
|
|
245
|
-
self.migration_report.add_general_statistics(
|
|
243
|
+
self.migration_report.add("Details", i18n.t("Checked out on second try"))
|
|
244
|
+
self.migration_report.add_general_statistics(i18n.t("Successfully checked out"))
|
|
246
245
|
logging.info("Checked out on second try")
|
|
247
246
|
self.set_renewal_count(legacy_loan, res_checkout2)
|
|
248
247
|
self.set_new_status(legacy_loan, res_checkout2)
|
|
@@ -257,7 +256,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
257
256
|
)
|
|
258
257
|
else:
|
|
259
258
|
self.failed[legacy_loan.item_barcode] = legacy_loan
|
|
260
|
-
self.migration_report.add_general_statistics(
|
|
259
|
+
self.migration_report.add_general_statistics(i18n.t("Failed loans"))
|
|
261
260
|
logging.error("Failed on second try: %s", res_checkout2.error_message)
|
|
262
261
|
self.migration_report.add(
|
|
263
262
|
"Details",
|
|
@@ -265,21 +264,21 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
265
264
|
)
|
|
266
265
|
raise TransformationRecordFailedError(
|
|
267
266
|
f"Row {legacy_loan.row}",
|
|
268
|
-
|
|
267
|
+
i18n.t("Loans failing during checkout, second try"),
|
|
269
268
|
json.dumps(legacy_loan.to_dict()),
|
|
270
269
|
)
|
|
271
270
|
elif not res_checkout.should_be_retried:
|
|
272
271
|
logging.error("Failed first time. No retries: %s", res_checkout.error_message)
|
|
273
|
-
self.migration_report.add_general_statistics(
|
|
272
|
+
self.migration_report.add_general_statistics(i18n.t("Failed loans"))
|
|
274
273
|
self.migration_report.add(
|
|
275
274
|
"Details",
|
|
276
|
-
|
|
275
|
+
i18n.t("Failed 1st time. No retries")
|
|
277
276
|
+ f": {res_checkout.migration_report_message}",
|
|
278
277
|
)
|
|
279
278
|
self.failed[legacy_loan.item_barcode] = legacy_loan
|
|
280
279
|
raise TransformationRecordFailedError(
|
|
281
280
|
f"Row {legacy_loan.row}",
|
|
282
|
-
|
|
281
|
+
i18n.t("Loans failing during checkout"),
|
|
283
282
|
json.dumps(legacy_loan.to_dict()),
|
|
284
283
|
)
|
|
285
284
|
|
|
@@ -301,7 +300,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
301
300
|
def set_renewal_count(self, legacy_loan: LegacyLoan, res_checkout: TransactionResult):
|
|
302
301
|
if legacy_loan.renewal_count > 0:
|
|
303
302
|
self.update_open_loan(res_checkout.folio_loan, legacy_loan)
|
|
304
|
-
self.migration_report.add_general_statistics(
|
|
303
|
+
self.migration_report.add_general_statistics(i18n.t("Updated renewal count for loan"))
|
|
305
304
|
|
|
306
305
|
def wrap_up(self):
|
|
307
306
|
for k, v in self.failed.items():
|
|
@@ -312,7 +311,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
312
311
|
|
|
313
312
|
with open(self.folder_structure.migration_reports_file, "w+") as report_file:
|
|
314
313
|
self.migration_report.write_migration_report(
|
|
315
|
-
|
|
314
|
+
i18n.t("Loans migration report"), report_file, self.start_datetime
|
|
316
315
|
)
|
|
317
316
|
self.clean_out_empty_logs()
|
|
318
317
|
|
|
@@ -352,7 +351,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
352
351
|
)
|
|
353
352
|
if has_item_barcode and has_patron_barcode and has_proxy_barcode:
|
|
354
353
|
self.migration_report.add_general_statistics(
|
|
355
|
-
|
|
354
|
+
i18n.t("Loans verified against migrated user and item")
|
|
356
355
|
)
|
|
357
356
|
yield loan
|
|
358
357
|
else:
|
|
@@ -361,9 +360,9 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
361
360
|
self.migration_report.add_general_statistics(i18n.t("Failed loans"))
|
|
362
361
|
self.migration_report.add(
|
|
363
362
|
"DiscardedLoans",
|
|
364
|
-
|
|
363
|
+
i18n.t("Loans discarded. Had migrated item barcode")
|
|
365
364
|
+ f": {has_item_barcode}. "
|
|
366
|
-
+
|
|
365
|
+
+ i18n.t("Had migrated user barcode")
|
|
367
366
|
+ f": {has_patron_barcode}"
|
|
368
367
|
+ f": {has_proxy_barcode}",
|
|
369
368
|
)
|
|
@@ -400,9 +399,9 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
400
399
|
if any(legacy_loan.errors):
|
|
401
400
|
num_bad += 1
|
|
402
401
|
self.migration_report.add_general_statistics(
|
|
403
|
-
|
|
402
|
+
i18n.t("Loans failed pre-validation")
|
|
404
403
|
)
|
|
405
|
-
self.migration_report.add_general_statistics(
|
|
404
|
+
self.migration_report.add_general_statistics(i18n.t("Failed loans"))
|
|
406
405
|
for error in legacy_loan.errors:
|
|
407
406
|
self.migration_report.add("DiscardedLoans", f"{error[0]} - {error[1]}")
|
|
408
407
|
# Add this loan to failed loans for later correction and re-run.
|
|
@@ -413,7 +412,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
413
412
|
results.append(legacy_loan)
|
|
414
413
|
except TransformationRecordFailedError as trfe:
|
|
415
414
|
num_bad += 1
|
|
416
|
-
self.migration_report.add_general_statistics(
|
|
415
|
+
self.migration_report.add_general_statistics(i18n.t("Loans failed pre-validation"))
|
|
417
416
|
self.migration_report.add(
|
|
418
417
|
"DiscardedLoans",
|
|
419
418
|
f"{trfe.message} - see data issues log",
|
|
@@ -6,6 +6,7 @@ import time
|
|
|
6
6
|
import traceback
|
|
7
7
|
from typing import List, Optional
|
|
8
8
|
|
|
9
|
+
import i18n
|
|
9
10
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
10
11
|
|
|
11
12
|
from folio_migration_tools.custom_exceptions import (
|
|
@@ -14,7 +15,6 @@ from folio_migration_tools.custom_exceptions import (
|
|
|
14
15
|
TransformationRecordFailedError,
|
|
15
16
|
)
|
|
16
17
|
from folio_migration_tools.helper import Helper
|
|
17
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
18
18
|
from folio_migration_tools.library_configuration import (
|
|
19
19
|
FileDefinition,
|
|
20
20
|
LibraryConfiguration,
|
|
@@ -118,7 +118,7 @@ class ManualFeeFinesTransformer(MigrationTaskBase):
|
|
|
118
118
|
full_path = self.folder_structure.legacy_records_folder / file_def.file_name
|
|
119
119
|
with open(full_path, encoding="utf-8-sig") as records_file:
|
|
120
120
|
self.mapper.migration_report.add_general_statistics(
|
|
121
|
-
|
|
121
|
+
i18n.t("Number of files processed")
|
|
122
122
|
)
|
|
123
123
|
start = time.time()
|
|
124
124
|
|
|
@@ -172,7 +172,7 @@ class ManualFeeFinesTransformer(MigrationTaskBase):
|
|
|
172
172
|
self.folder_structure.migration_reports_file,
|
|
173
173
|
)
|
|
174
174
|
self.mapper.migration_report.write_migration_report(
|
|
175
|
-
|
|
175
|
+
i18n.t("Manual fee/fine transformation report"),
|
|
176
176
|
migration_report_file,
|
|
177
177
|
self.start_datetime,
|
|
178
178
|
)
|
|
@@ -52,8 +52,7 @@ class MigrationTaskBase:
|
|
|
52
52
|
self.ecs_tenant_id = (
|
|
53
53
|
task_configuration.ecs_tenant_id or library_configuration.ecs_tenant_id
|
|
54
54
|
)
|
|
55
|
-
|
|
56
|
-
self.folio_client.tenant_id = self.ecs_tenant_id
|
|
55
|
+
self.folio_client.tenant_id = self.ecs_tenant_id
|
|
57
56
|
|
|
58
57
|
self.central_folder_structure: Optional[FolderStructure] = None
|
|
59
58
|
if library_configuration.is_ecs and library_configuration.ecs_central_iteration_identifier:
|
|
@@ -99,7 +98,6 @@ class MigrationTaskBase:
|
|
|
99
98
|
raise NotImplementedError()
|
|
100
99
|
|
|
101
100
|
def clean_out_empty_logs(self):
|
|
102
|
-
_close_handler(self.data_issue_file_handler)
|
|
103
101
|
if (
|
|
104
102
|
self.folder_structure.data_issue_file_path.is_file()
|
|
105
103
|
and os.stat(self.folder_structure.data_issue_file_path).st_size == 0
|
|
@@ -261,13 +259,13 @@ class MigrationTaskBase:
|
|
|
261
259
|
|
|
262
260
|
# Data issue file formatter
|
|
263
261
|
data_issue_file_formatter = logging.Formatter("%(message)s")
|
|
264
|
-
|
|
262
|
+
data_issue_file_handler = logging.FileHandler(
|
|
265
263
|
filename=str(self.folder_structure.data_issue_file_path), mode="w"
|
|
266
264
|
)
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
logging.getLogger().addHandler(
|
|
265
|
+
data_issue_file_handler.addFilter(LevelFilter(26))
|
|
266
|
+
data_issue_file_handler.setFormatter(data_issue_file_formatter)
|
|
267
|
+
data_issue_file_handler.setLevel(26)
|
|
268
|
+
logging.getLogger().addHandler(data_issue_file_handler)
|
|
271
269
|
logger.info("Logging set up")
|
|
272
270
|
|
|
273
271
|
def setup_records_map(self, mapping_file_path):
|
|
@@ -554,10 +552,3 @@ class LevelFilter(logging.Filter):
|
|
|
554
552
|
|
|
555
553
|
def filter(self, record):
|
|
556
554
|
return record.levelno == self.level
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
def _close_handler(handler: logging.Handler | None):
|
|
560
|
-
if handler is None:
|
|
561
|
-
return
|
|
562
|
-
handler.flush()
|
|
563
|
-
handler.close()
|
|
@@ -16,7 +16,6 @@ from folio_migration_tools.custom_exceptions import (
|
|
|
16
16
|
TransformationRecordFailedError,
|
|
17
17
|
)
|
|
18
18
|
from folio_migration_tools.helper import Helper
|
|
19
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
20
19
|
from folio_migration_tools.library_configuration import (
|
|
21
20
|
FileDefinition,
|
|
22
21
|
LibraryConfiguration,
|
|
@@ -235,7 +234,7 @@ class OrdersTransformer(MigrationTaskBase):
|
|
|
235
234
|
open(self.folder_structure.created_objects_path, "w+") as results_file,
|
|
236
235
|
):
|
|
237
236
|
self.mapper.migration_report.add_general_statistics(
|
|
238
|
-
|
|
237
|
+
i18n.t("Number of files processed")
|
|
239
238
|
)
|
|
240
239
|
start = time.time()
|
|
241
240
|
records_processed = 0
|
|
@@ -254,7 +253,7 @@ class OrdersTransformer(MigrationTaskBase):
|
|
|
254
253
|
self.mapper.perform_additional_mapping(legacy_id, folio_rec)
|
|
255
254
|
|
|
256
255
|
self.mapper.migration_report.add_general_statistics(
|
|
257
|
-
|
|
256
|
+
i18n.t("TOTAL Purchase Order Lines created")
|
|
258
257
|
)
|
|
259
258
|
self.mapper.report_folio_mapping(folio_rec, self.mapper.composite_order_schema)
|
|
260
259
|
self.mapper.notes_mapper.map_notes(
|
|
@@ -317,7 +316,7 @@ class OrdersTransformer(MigrationTaskBase):
|
|
|
317
316
|
self.folder_structure.migration_reports_file,
|
|
318
317
|
)
|
|
319
318
|
self.mapper.migration_report.write_migration_report(
|
|
320
|
-
|
|
319
|
+
i18n.t("Pruchase Orders and Purchase Order Lines Transformation Report"),
|
|
321
320
|
migration_report_file,
|
|
322
321
|
self.start_datetime,
|
|
323
322
|
)
|
|
@@ -13,7 +13,6 @@ from zoneinfo import ZoneInfo
|
|
|
13
13
|
from folio_migration_tools.circulation_helper import CirculationHelper
|
|
14
14
|
from folio_migration_tools.custom_dict import InsensitiveDictReader
|
|
15
15
|
from folio_migration_tools.helper import Helper
|
|
16
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
17
16
|
from folio_migration_tools.library_configuration import (
|
|
18
17
|
FileDefinition,
|
|
19
18
|
LibraryConfiguration,
|
|
@@ -148,7 +147,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
148
147
|
|
|
149
148
|
def prepare_legacy_request(self, legacy_request: LegacyRequest):
|
|
150
149
|
patron = self.circulation_helper.get_user_by_barcode(legacy_request.patron_barcode)
|
|
151
|
-
self.migration_report.add_general_statistics(
|
|
150
|
+
self.migration_report.add_general_statistics(i18n.t("Patron lookups performed"))
|
|
152
151
|
|
|
153
152
|
if not patron:
|
|
154
153
|
logging.error(f"No user with barcode {legacy_request.patron_barcode} found in FOLIO")
|
|
@@ -158,18 +157,18 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
158
157
|
f"{legacy_request.patron_barcode}",
|
|
159
158
|
)
|
|
160
159
|
self.migration_report.add_general_statistics(
|
|
161
|
-
|
|
160
|
+
i18n.t("No user with barcode found in FOLIO")
|
|
162
161
|
)
|
|
163
162
|
self.failed_requests.add(legacy_request)
|
|
164
163
|
return False, legacy_request
|
|
165
164
|
legacy_request.patron_id = patron.get("id")
|
|
166
165
|
|
|
167
166
|
item = self.circulation_helper.get_item_by_barcode(legacy_request.item_barcode)
|
|
168
|
-
self.migration_report.add_general_statistics(
|
|
167
|
+
self.migration_report.add_general_statistics(i18n.t("Item lookups performed"))
|
|
169
168
|
if not item:
|
|
170
169
|
logging.error(f"No item with barcode {legacy_request.item_barcode} found in FOLIO")
|
|
171
170
|
self.migration_report.add_general_statistics(
|
|
172
|
-
|
|
171
|
+
i18n.t("No item with barcode found in FOLIO")
|
|
173
172
|
)
|
|
174
173
|
Helper.log_data_issue(
|
|
175
174
|
f"{legacy_request.item_barcode}",
|
|
@@ -179,7 +178,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
179
178
|
self.failed_requests.add(legacy_request)
|
|
180
179
|
return False, legacy_request
|
|
181
180
|
holding = self.circulation_helper.get_holding_by_uuid(item.get("holdingsRecordId"))
|
|
182
|
-
self.migration_report.add_general_statistics(
|
|
181
|
+
self.migration_report.add_general_statistics(i18n.t("Holdings lookups performed"))
|
|
183
182
|
legacy_request.item_id = item.get("id")
|
|
184
183
|
legacy_request.holdings_record_id = item.get("holdingsRecordId")
|
|
185
184
|
legacy_request.instance_id = holding.get("instanceId")
|
|
@@ -187,7 +186,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
187
186
|
legacy_request.request_type = "Page"
|
|
188
187
|
logging.info(f"Setting request to Page, since the status is {item['status']['name']}")
|
|
189
188
|
self.migration_report.add_general_statistics(
|
|
190
|
-
|
|
189
|
+
i18n.t("Valid, prepared requests, ready for posting")
|
|
191
190
|
)
|
|
192
191
|
return True, legacy_request
|
|
193
192
|
|
|
@@ -207,11 +206,11 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
207
206
|
self.folio_client, legacy_request, self.migration_report
|
|
208
207
|
):
|
|
209
208
|
self.migration_report.add_general_statistics(
|
|
210
|
-
|
|
209
|
+
i18n.t("Successfully migrated requests")
|
|
211
210
|
)
|
|
212
211
|
else:
|
|
213
212
|
self.migration_report.add_general_statistics(
|
|
214
|
-
|
|
213
|
+
i18n.t("Unsuccessfully migrated requests")
|
|
215
214
|
)
|
|
216
215
|
self.failed_requests.add(legacy_request)
|
|
217
216
|
if num_requests == 1:
|
|
@@ -234,7 +233,7 @@ class RequestsMigrator(MigrationTaskBase):
|
|
|
234
233
|
|
|
235
234
|
with open(self.folder_structure.migration_reports_file, "w+") as report_file:
|
|
236
235
|
self.migration_report.write_migration_report(
|
|
237
|
-
|
|
236
|
+
i18n.t("Requests migration report"), report_file, self.start_datetime
|
|
238
237
|
)
|
|
239
238
|
self.clean_out_empty_logs()
|
|
240
239
|
|
|
@@ -14,7 +14,6 @@ from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
|
14
14
|
|
|
15
15
|
from folio_migration_tools.custom_dict import InsensitiveDictReader
|
|
16
16
|
from folio_migration_tools.custom_exceptions import TransformationProcessError
|
|
17
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
18
17
|
from folio_migration_tools.library_configuration import (
|
|
19
18
|
FileDefinition,
|
|
20
19
|
LibraryConfiguration,
|
|
@@ -91,7 +90,7 @@ class ReservesMigrator(MigrationTaskBase):
|
|
|
91
90
|
logging.info("Starting")
|
|
92
91
|
for num_reserves, legacy_reserve in enumerate(self.valid_reserves, start=1):
|
|
93
92
|
t0_migration = time.time()
|
|
94
|
-
self.migration_report.add_general_statistics(
|
|
93
|
+
self.migration_report.add_general_statistics(i18n.t("Processed reserves"))
|
|
95
94
|
try:
|
|
96
95
|
self.post_single_reserve(legacy_reserve)
|
|
97
96
|
except Exception as ee:
|
|
@@ -108,10 +107,10 @@ class ReservesMigrator(MigrationTaskBase):
|
|
|
108
107
|
path, legacy_reserve.to_dict(), "POST", i18n.t("Posted reserves")
|
|
109
108
|
):
|
|
110
109
|
self.migration_report.add_general_statistics(
|
|
111
|
-
|
|
110
|
+
i18n.t("Successfully posted reserves")
|
|
112
111
|
)
|
|
113
112
|
else:
|
|
114
|
-
self.migration_report.add_general_statistics(
|
|
113
|
+
self.migration_report.add_general_statistics(i18n.t("Failure to post reserve"))
|
|
115
114
|
except Exception as ee:
|
|
116
115
|
logging.error(ee)
|
|
117
116
|
|
|
@@ -124,7 +123,7 @@ class ReservesMigrator(MigrationTaskBase):
|
|
|
124
123
|
|
|
125
124
|
with open(self.folder_structure.migration_reports_file, "w+") as report_file:
|
|
126
125
|
self.migration_report.write_migration_report(
|
|
127
|
-
|
|
126
|
+
i18n.t("Reserves migration report"), report_file, self.start_datetime
|
|
128
127
|
)
|
|
129
128
|
self.clean_out_empty_logs()
|
|
130
129
|
|
|
@@ -4,6 +4,7 @@ import sys
|
|
|
4
4
|
from typing import Optional, Annotated
|
|
5
5
|
from pydantic import Field
|
|
6
6
|
|
|
7
|
+
import i18n
|
|
7
8
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
8
9
|
from art import tprint
|
|
9
10
|
|
|
@@ -12,7 +13,6 @@ from folio_migration_tools.custom_exceptions import (
|
|
|
12
13
|
TransformationRecordFailedError,
|
|
13
14
|
)
|
|
14
15
|
from folio_migration_tools.helper import Helper
|
|
15
|
-
from folio_migration_tools.i18n_cache import i18n_t
|
|
16
16
|
from folio_migration_tools.library_configuration import (
|
|
17
17
|
FileDefinition,
|
|
18
18
|
LibraryConfiguration,
|
|
@@ -101,16 +101,6 @@ class UserTransformer(MigrationTaskBase):
|
|
|
101
101
|
),
|
|
102
102
|
),
|
|
103
103
|
] = False
|
|
104
|
-
remove_username: Annotated[
|
|
105
|
-
Optional[bool],
|
|
106
|
-
Field(
|
|
107
|
-
title="Remove username",
|
|
108
|
-
description=(
|
|
109
|
-
"Specify whether to remove username. Resulting objects are not compatible with"
|
|
110
|
-
" the mod-user-import. Optional, by default is False"
|
|
111
|
-
),
|
|
112
|
-
),
|
|
113
|
-
] = False
|
|
114
104
|
|
|
115
105
|
@staticmethod
|
|
116
106
|
def get_object_type() -> FOLIONamespaces:
|
|
@@ -218,13 +208,13 @@ class UserTransformer(MigrationTaskBase):
|
|
|
218
208
|
logging.info("## First FOLIO user")
|
|
219
209
|
logging.info(json.dumps(folio_user, indent=4, sort_keys=True))
|
|
220
210
|
self.mapper.migration_report.add_general_statistics(
|
|
221
|
-
|
|
211
|
+
i18n.t("Successful user transformations")
|
|
222
212
|
)
|
|
223
213
|
if num_users % 1000 == 0:
|
|
224
214
|
logging.info(f"{num_users} users processed.")
|
|
225
215
|
except TransformationRecordFailedError as tre:
|
|
226
216
|
self.mapper.migration_report.add_general_statistics(
|
|
227
|
-
|
|
217
|
+
i18n.t("Records failed")
|
|
228
218
|
)
|
|
229
219
|
Helper.log_data_issue(tre.index_or_id, tre.message, tre.data_value)
|
|
230
220
|
logging.error(tre)
|
|
@@ -241,7 +231,7 @@ class UserTransformer(MigrationTaskBase):
|
|
|
241
231
|
logging.error(num_users)
|
|
242
232
|
logging.error(json.dumps(legacy_user))
|
|
243
233
|
self.mapper.migration_report.add_general_statistics(
|
|
244
|
-
|
|
234
|
+
i18n.t("Failed user transformations")
|
|
245
235
|
)
|
|
246
236
|
logging.error(ee, exc_info=True)
|
|
247
237
|
|
|
@@ -255,7 +245,7 @@ class UserTransformer(MigrationTaskBase):
|
|
|
255
245
|
self.extradata_writer.flush()
|
|
256
246
|
with open(self.folder_structure.migration_reports_file, "w") as migration_report_file:
|
|
257
247
|
self.mapper.migration_report.write_migration_report(
|
|
258
|
-
|
|
248
|
+
i18n.t("Users transformation report"),
|
|
259
249
|
migration_report_file,
|
|
260
250
|
self.mapper.start_datetime,
|
|
261
251
|
)
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
"Aged to lost and checked out": "Aged to lost and checked out",
|
|
25
25
|
"Already set to %{value}. %{leader_key} was %{leader}": "Already set to %{value}. %{leader_key} was %{leader}",
|
|
26
26
|
"An Unmapped": "An Unmapped",
|
|
27
|
+
"Authority records transformation report": "Authority records transformation report",
|
|
27
28
|
"BW Items found tied to previously created BW Holdings": "BW Items found tied to previously created BW Holdings",
|
|
28
29
|
"Bib identifier not in instances_id_map, no instance linked": "Bib identifier not in instances_id_map, no instance linked",
|
|
29
30
|
"Bib ids referenced in bound-with items": "Bib ids referenced in bound-with items",
|
|
@@ -263,6 +264,12 @@
|
|
|
263
264
|
"blurbs.AcquisitionMethodMapping.title": "POL Acquisition Method Mapping",
|
|
264
265
|
"blurbs.AddedValueFromParameter.description": "",
|
|
265
266
|
"blurbs.AddedValueFromParameter.title": "Added value from parameter since value is empty",
|
|
267
|
+
"blurbs.AuthorityEncodingLevel.description": "Library action: **All values that are not n or o will be set to n. If this is not what you want, you need to correct these values in your system. **<br/>An overview of the Encoding levels (Leader position 17) present in your source data. Allowed values according to the MARC standard are n or o",
|
|
268
|
+
"blurbs.AuthorityEncodingLevel.title": "Encoding level (leader pos 17)",
|
|
269
|
+
"blurbs.AuthoritySourceFileMapping.description": "Mappings based on FOLIO authority `naturalId` alpha prefix",
|
|
270
|
+
"blurbs.AuthoritySourceFileMapping.title": "Authority Source File Mapping Results",
|
|
271
|
+
"blurbs.AuthoritySources.description": "",
|
|
272
|
+
"blurbs.AuthoritySources.title": "Authorization sources and related information",
|
|
266
273
|
"blurbs.BoundWithMappings.description": "",
|
|
267
274
|
"blurbs.BoundWithMappings.title": "Bound-with mapping",
|
|
268
275
|
"blurbs.CallNumberTypeMapping.description": "Call number types in MFHDs are mapped from 852, Indicator 1 according to a certain scheme. (LOC documentation)[https://www.loc.gov/marc/holdings/hd852.html]",
|