folio-migration-tools 1.9.9__py3-none-any.whl → 1.10.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. folio_migration_tools/__init__.py +3 -4
  2. folio_migration_tools/__main__.py +53 -31
  3. folio_migration_tools/circulation_helper.py +118 -108
  4. folio_migration_tools/custom_dict.py +2 -2
  5. folio_migration_tools/custom_exceptions.py +4 -5
  6. folio_migration_tools/folder_structure.py +17 -7
  7. folio_migration_tools/helper.py +8 -7
  8. folio_migration_tools/holdings_helper.py +4 -3
  9. folio_migration_tools/i18n_cache.py +79 -0
  10. folio_migration_tools/library_configuration.py +77 -37
  11. folio_migration_tools/mapper_base.py +45 -31
  12. folio_migration_tools/mapping_file_transformation/courses_mapper.py +1 -1
  13. folio_migration_tools/mapping_file_transformation/holdings_mapper.py +7 -3
  14. folio_migration_tools/mapping_file_transformation/item_mapper.py +13 -26
  15. folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py +1 -2
  16. folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +13 -11
  17. folio_migration_tools/mapping_file_transformation/order_mapper.py +23 -5
  18. folio_migration_tools/mapping_file_transformation/organization_mapper.py +3 -3
  19. folio_migration_tools/mapping_file_transformation/ref_data_mapping.py +3 -0
  20. folio_migration_tools/mapping_file_transformation/user_mapper.py +47 -28
  21. folio_migration_tools/marc_rules_transformation/conditions.py +82 -97
  22. folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +13 -5
  23. folio_migration_tools/marc_rules_transformation/hrid_handler.py +3 -2
  24. folio_migration_tools/marc_rules_transformation/marc_file_processor.py +26 -24
  25. folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +56 -51
  26. folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +28 -17
  27. folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +68 -37
  28. folio_migration_tools/migration_report.py +18 -7
  29. folio_migration_tools/migration_tasks/batch_poster.py +285 -354
  30. folio_migration_tools/migration_tasks/bibs_transformer.py +14 -9
  31. folio_migration_tools/migration_tasks/courses_migrator.py +2 -3
  32. folio_migration_tools/migration_tasks/holdings_csv_transformer.py +23 -24
  33. folio_migration_tools/migration_tasks/holdings_marc_transformer.py +14 -24
  34. folio_migration_tools/migration_tasks/items_transformer.py +23 -34
  35. folio_migration_tools/migration_tasks/loans_migrator.py +67 -144
  36. folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py +3 -3
  37. folio_migration_tools/migration_tasks/migration_task_base.py +43 -52
  38. folio_migration_tools/migration_tasks/orders_transformer.py +25 -41
  39. folio_migration_tools/migration_tasks/organization_transformer.py +9 -18
  40. folio_migration_tools/migration_tasks/requests_migrator.py +21 -24
  41. folio_migration_tools/migration_tasks/reserves_migrator.py +6 -5
  42. folio_migration_tools/migration_tasks/user_transformer.py +25 -20
  43. folio_migration_tools/task_configuration.py +6 -7
  44. folio_migration_tools/transaction_migration/legacy_loan.py +15 -27
  45. folio_migration_tools/transaction_migration/legacy_request.py +1 -1
  46. folio_migration_tools/translations/en.json +3 -8
  47. {folio_migration_tools-1.9.9.dist-info → folio_migration_tools-1.10.0.dist-info}/METADATA +19 -28
  48. folio_migration_tools-1.10.0.dist-info/RECORD +63 -0
  49. folio_migration_tools-1.10.0.dist-info/WHEEL +4 -0
  50. folio_migration_tools-1.10.0.dist-info/entry_points.txt +3 -0
  51. folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +0 -241
  52. folio_migration_tools/migration_tasks/authority_transformer.py +0 -119
  53. folio_migration_tools/test_infrastructure/__init__.py +0 -0
  54. folio_migration_tools/test_infrastructure/mocked_classes.py +0 -406
  55. folio_migration_tools-1.9.9.dist-info/RECORD +0 -67
  56. folio_migration_tools-1.9.9.dist-info/WHEEL +0 -4
  57. folio_migration_tools-1.9.9.dist-info/entry_points.txt +0 -3
  58. folio_migration_tools-1.9.9.dist-info/licenses/LICENSE +0 -21
@@ -47,15 +47,14 @@ class MigrationTaskBase:
47
47
  logging.info("MigrationTaskBase init")
48
48
  self.start_datetime = datetime.now(timezone.utc)
49
49
  self.task_configuration = task_configuration
50
- logging.info(self.task_configuration.json(indent=4))
50
+ logging.info(self.task_configuration.model_dump_json(indent=4))
51
51
  self.folio_client: FolioClient = folio_client
52
52
  self.ecs_tenant_id = (
53
53
  task_configuration.ecs_tenant_id or library_configuration.ecs_tenant_id
54
54
  )
55
- self.ecs_tenant_header = (
56
- {"x-okapi-tenant": self.ecs_tenant_id} if self.ecs_tenant_id else {}
57
- )
58
- self.folio_client.okapi_headers.update(self.ecs_tenant_header)
55
+ if self.ecs_tenant_id:
56
+ self.folio_client.tenant_id = self.ecs_tenant_id
57
+
59
58
  self.central_folder_structure: Optional[FolderStructure] = None
60
59
  if library_configuration.is_ecs and library_configuration.ecs_central_iteration_identifier:
61
60
  self.central_folder_structure = FolderStructure(
@@ -100,6 +99,7 @@ class MigrationTaskBase:
100
99
  raise NotImplementedError()
101
100
 
102
101
  def clean_out_empty_logs(self):
102
+ _close_handler(self.data_issue_file_handler)
103
103
  if (
104
104
  self.folder_structure.data_issue_file_path.is_file()
105
105
  and os.stat(self.folder_structure.data_issue_file_path).st_size == 0
@@ -133,11 +133,7 @@ class MigrationTaskBase:
133
133
  TransformationProcessError: _description_
134
134
 
135
135
  """
136
- files = [
137
- source_path / f.file_name
138
- for f in file_defs
139
- if isfile(source_path / f.file_name)
140
- ]
136
+ files = [source_path / f.file_name for f in file_defs if isfile(source_path / f.file_name)]
141
137
  ret_str = ", ".join(f.file_name for f in file_defs)
142
138
 
143
139
  if files and len(files) < len(file_defs):
@@ -162,12 +158,13 @@ class MigrationTaskBase:
162
158
  This is in the base class because multiple tasks need it. It exists because instances in an ECS environment
163
159
  are transformed for the central and data tenants separately, but the data tenants need to know about
164
160
  the central tenant instance ids. This is a bit of a hack, but it works for now.
165
- """
161
+ """ # noqa: E501
166
162
  map_files = []
167
163
  instance_id_map = {}
168
164
  if self.library_configuration.is_ecs and self.central_folder_structure:
169
165
  logging.info(
170
- "Loading ECS central tenant instance id map from %s", self.central_folder_structure.instance_id_map_path
166
+ "Loading ECS central tenant instance id map from %s",
167
+ self.central_folder_structure.instance_id_map_path,
171
168
  )
172
169
  instance_id_map = self.load_id_map(
173
170
  self.central_folder_structure.instance_id_map_path,
@@ -176,7 +173,7 @@ class MigrationTaskBase:
176
173
  map_files.append(str(self.central_folder_structure.instance_id_map_path))
177
174
  logging.info(
178
175
  "Loading member tenant isntance id map from %s",
179
- self.folder_structure.instance_id_map_path
176
+ self.folder_structure.instance_id_map_path,
180
177
  )
181
178
  instance_id_map = self.load_id_map(
182
179
  self.folder_structure.instance_id_map_path,
@@ -190,13 +187,11 @@ class MigrationTaskBase:
190
187
  return instance_id_map
191
188
 
192
189
  @staticmethod
193
- def load_id_map(map_path, raise_if_empty=False, existing_id_map={}):
190
+ def load_id_map(map_path, raise_if_empty=False, existing_id_map=None):
194
191
  if not isfile(map_path):
195
- logging.warning(
196
- "No legacy id map found at %s. Will build one from scratch", map_path
197
- )
192
+ logging.warning("No legacy id map found at %s. Will build one from scratch", map_path)
198
193
  return {}
199
- id_map = existing_id_map
194
+ id_map = existing_id_map or {}
200
195
  loaded_rows = len(id_map)
201
196
  with open(map_path) as id_map_file:
202
197
  for index, json_string in enumerate(id_map_file, start=1):
@@ -247,9 +242,7 @@ class MigrationTaskBase:
247
242
  else:
248
243
  logger.setLevel(logging.INFO)
249
244
  stream_handler.setLevel(logging.INFO)
250
- stream_handler.addFilter(
251
- ExcludeLevelFilter(30)
252
- ) # Exclude warnings from pymarc
245
+ stream_handler.addFilter(ExcludeLevelFilter(30)) # Exclude warnings from pymarc
253
246
  stream_handler.setFormatter(formatter)
254
247
  logger.addHandler(stream_handler)
255
248
 
@@ -268,34 +261,28 @@ class MigrationTaskBase:
268
261
 
269
262
  # Data issue file formatter
270
263
  data_issue_file_formatter = logging.Formatter("%(message)s")
271
- data_issue_file_handler = logging.FileHandler(
264
+ self.data_issue_file_handler = logging.FileHandler(
272
265
  filename=str(self.folder_structure.data_issue_file_path), mode="w"
273
266
  )
274
- data_issue_file_handler.addFilter(LevelFilter(26))
275
- data_issue_file_handler.setFormatter(data_issue_file_formatter)
276
- data_issue_file_handler.setLevel(26)
277
- logging.getLogger().addHandler(data_issue_file_handler)
267
+ self.data_issue_file_handler.addFilter(LevelFilter(26))
268
+ self.data_issue_file_handler.setFormatter(data_issue_file_formatter)
269
+ self.data_issue_file_handler.setLevel(26)
270
+ logging.getLogger().addHandler(self.data_issue_file_handler)
278
271
  logger.info("Logging set up")
279
272
 
280
273
  def setup_records_map(self, mapping_file_path):
281
274
  with open(mapping_file_path) as mapping_file:
282
275
  field_map = json.load(mapping_file)
283
- logging.info(
284
- "%s fields present in record mapping file", len(field_map["data"])
285
- )
276
+ logging.info("%s fields present in record mapping file", len(field_map["data"]))
286
277
  mapped_fields = (
287
278
  f
288
279
  for f in field_map["data"]
289
280
  if f["legacy_field"] and f["legacy_field"] != "Not mapped"
290
281
  )
291
- logging.info(
292
- "%s fields mapped in record mapping file", len(list(mapped_fields))
293
- )
282
+ logging.info("%s fields mapped in record mapping file", len(list(mapped_fields)))
294
283
  return field_map
295
284
 
296
- def log_and_exit_if_too_many_errors(
297
- self, error: TransformationRecordFailedError, idx
298
- ):
285
+ def log_and_exit_if_too_many_errors(self, error: TransformationRecordFailedError, idx):
299
286
  self.num_exeptions += 1
300
287
  error.log_it()
301
288
  if self.num_exeptions / (1 + idx) > 0.2 and self.num_exeptions > 5000:
@@ -311,9 +298,7 @@ class MigrationTaskBase:
311
298
  if num_processed > 1 and num_processed % 10000 == 0:
312
299
  elapsed = num_processed / (time.time() - start_time)
313
300
  elapsed_formatted = "{0:.4g}".format(elapsed)
314
- logging.info(
315
- f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} "
316
- )
301
+ logging.info(f"{num_processed:,} records processed. Recs/sec: {elapsed_formatted} ")
317
302
 
318
303
  def do_work_marc_transformer(
319
304
  self,
@@ -322,9 +307,7 @@ class MigrationTaskBase:
322
307
  if self.folder_structure.failed_marc_recs_file.is_file():
323
308
  os.remove(self.folder_structure.failed_marc_recs_file)
324
309
  logging.info("Removed failed marc records file to prevent duplicating data")
325
- with open(
326
- self.folder_structure.created_objects_path, "w+"
327
- ) as created_records_file:
310
+ with open(self.folder_structure.created_objects_path, "w+") as created_records_file:
328
311
  self.processor = MarcFileProcessor(
329
312
  self.mapper, self.folder_structure, created_records_file
330
313
  )
@@ -377,7 +360,7 @@ class MigrationTaskBase:
377
360
 
378
361
  Returns:
379
362
  None
380
- """
363
+ """ # noqa: E501
381
364
  current_pos = map_file.tell()
382
365
  try:
383
366
  map_file.seek(0)
@@ -391,13 +374,12 @@ class MigrationTaskBase:
391
374
  "",
392
375
  (
393
376
  f"Mapping file {map_file.name} has rows with different number "
394
- f"of columns ({'Row' if len(invalid_lines) == 1 else 'Rows'} {', '.join(invalid_lines)})"
377
+ f"of columns ({'Row' if len(invalid_lines) == 1 else 'Rows'} "
378
+ f"{', '.join(invalid_lines)})"
395
379
  ),
396
380
  )
397
381
  if not valid_lines:
398
- raise TransformationProcessError(
399
- "", f"Map has no rows: {map_file.name}"
400
- )
382
+ raise TransformationProcessError("", f"Map has no rows: {map_file.name}")
401
383
  finally:
402
384
  map_file.seek(current_pos)
403
385
 
@@ -422,7 +404,8 @@ class MigrationTaskBase:
422
404
  or required
423
405
  or folio_property_name.startswith("statisticalCodeIds")
424
406
  or folio_property_name.startswith("locationMap")
425
- ):
407
+ or folio_property_name.startswith("fundsMap")
408
+ ) and map_file_path.is_file():
426
409
  try:
427
410
  with open(map_file_path) as map_file:
428
411
  # Validate the structure of the mapping file
@@ -482,7 +465,7 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
482
465
  deactivate035_from001 (bool):
483
466
  Disables the default FOLIO functionality of moving the previous 001 field into a 035 field, prefixed with the value from 003.
484
467
  Default is False, meaning the functionality remains active.
485
- """
468
+ """ # noqa: E501
486
469
 
487
470
  files: Annotated[
488
471
  List[library_configuration.FileDefinition],
@@ -496,8 +479,7 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
496
479
  Field(
497
480
  title="Create source records",
498
481
  description=(
499
- "Controls whether or not to retain the MARC records in "
500
- "Source Record Storage."
482
+ "Controls whether or not to retain the MARC records in Source Record Storage."
501
483
  ),
502
484
  ),
503
485
  ] = False
@@ -538,12 +520,14 @@ class MarcTaskConfigurationBase(task_configuration.AbstractTaskConfiguration):
538
520
  title="Statistical code mapping fields",
539
521
  description=(
540
522
  "List of fields + subfields to be used for mapping statistical codes. "
541
- "Subfields should be delimited by a \"$\" (eg. 907$a). Single repeating subfields "
542
- "will be treated as unique values. Multiple subfields will be concatenated together with a space."
523
+ 'Subfields should be delimited by a "$" (eg. 907$a). Single repeating subfields '
524
+ "will be treated as unique values. Multiple subfields will be concatenated "
525
+ "together with a space."
543
526
  ),
544
527
  ),
545
528
  ] = []
546
529
 
530
+
547
531
  class ExcludeLevelFilter(logging.Filter):
548
532
  def __init__(self, level):
549
533
  super().__init__()
@@ -570,3 +554,10 @@ class LevelFilter(logging.Filter):
570
554
 
571
555
  def filter(self, record):
572
556
  return record.levelno == self.level
557
+
558
+
559
+ def _close_handler(handler: logging.Handler | None):
560
+ if handler is None:
561
+ return
562
+ handler.flush()
563
+ handler.close()
@@ -4,7 +4,6 @@ import json
4
4
  import logging
5
5
  import sys
6
6
  import time
7
- from os.path import isfile
8
7
  from typing import List, Optional, Annotated
9
8
  from pydantic import Field
10
9
 
@@ -17,6 +16,7 @@ from folio_migration_tools.custom_exceptions import (
17
16
  TransformationRecordFailedError,
18
17
  )
19
18
  from folio_migration_tools.helper import Helper
19
+ from folio_migration_tools.i18n_cache import i18n_t
20
20
  from folio_migration_tools.library_configuration import (
21
21
  FileDefinition,
22
22
  LibraryConfiguration,
@@ -84,50 +84,35 @@ class OrdersTransformer(MigrationTaskBase):
84
84
  Optional[str],
85
85
  Field(
86
86
  title="Payment Status Map File Name",
87
- description=(
88
- "File name for payment status mapping. "
89
- "By default is empty string."
90
- ),
87
+ description=("File name for payment status mapping. By default is empty string."),
91
88
  ),
92
89
  ] = ""
93
90
  receipt_status_map_file_name: Annotated[
94
91
  Optional[str],
95
92
  Field(
96
93
  title="Receipt Status Map File Name",
97
- description=(
98
- "File name for receipt status mapping. "
99
- "By default is empty string."
100
- ),
94
+ description=("File name for receipt status mapping. By default is empty string."),
101
95
  ),
102
96
  ] = ""
103
97
  workflow_status_map_file_name: Annotated[
104
98
  Optional[str],
105
99
  Field(
106
100
  title="Workflow Status Map File Name",
107
- description=(
108
- "File name for workflow status mapping. "
109
- "By default is empty string."
110
- ),
101
+ description=("File name for workflow status mapping. By default is empty string."),
111
102
  ),
112
103
  ] = ""
113
104
  location_map_file_name: Annotated[
114
105
  Optional[str],
115
106
  Field(
116
107
  title="Location Map File Name",
117
- description=(
118
- "File name for location mapping. "
119
- "By default is empty string."
120
- ),
108
+ description=("File name for location mapping. By default is empty string."),
121
109
  ),
122
110
  ] = ""
123
111
  funds_map_file_name: Annotated[
124
112
  Optional[str],
125
113
  Field(
126
114
  title="Funds Map File Name",
127
- description=(
128
- "File name for funds mapping. "
129
- "By default is empty string."
130
- ),
115
+ description=("File name for funds mapping. By default is empty string."),
131
116
  ),
132
117
  ] = ""
133
118
  funds_expense_class_map_file_name: Annotated[
@@ -135,8 +120,7 @@ class OrdersTransformer(MigrationTaskBase):
135
120
  Field(
136
121
  title="Funds Expense Class Map File Name",
137
122
  description=(
138
- "File name for funds expense class mapping. "
139
- "By default is empty string."
123
+ "File name for funds expense class mapping. By default is empty string."
140
124
  ),
141
125
  ),
142
126
  ] = ""
@@ -218,7 +202,7 @@ class OrdersTransformer(MigrationTaskBase):
218
202
  "fundsMap",
219
203
  self.folder_structure.mapping_files_folder / self.task_config.funds_map_file_name,
220
204
  self.folio_keys,
221
- False,
205
+ True,
222
206
  ),
223
207
  self.load_ref_data_mapping_file( # Todo: The property in the schema has no type
224
208
  "fundsExpenseClassMap",
@@ -230,28 +214,28 @@ class OrdersTransformer(MigrationTaskBase):
230
214
  )
231
215
 
232
216
  def list_source_files(self):
233
- files = [
234
- self.folder_structure.data_folder / self.object_type_name / f.file_name
235
- for f in self.task_config.files
236
- if isfile(self.folder_structure.data_folder / self.object_type_name / f.file_name)
237
- ]
238
- if not any(files):
239
- ret_str = ",".join(f.file_name for f in self.task_config.files)
240
- raise TransformationProcessError(
241
- f"Files {ret_str} not found in"
242
- "{self.folder_structure.data_folder} / {self.object_type_name}"
243
- )
217
+ files = []
218
+ for f in self.task_config.files:
219
+ file_path = self.folder_structure.data_folder / self.object_type_name / f.file_name
220
+
221
+ if not file_path.is_file():
222
+ print(f"\n\nERROR: File defined in task not found - {f.file_name}")
223
+ raise TransformationProcessError(
224
+ f"\n\nERROR: File defined in task not found - {f.file_name}"
225
+ )
226
+ files.append(file_path)
244
227
  logging.info("Files to process:")
245
228
  for filename in files:
246
229
  logging.info("\t%s", filename)
247
230
  return files
248
231
 
249
232
  def process_single_file(self, filename):
250
- with open(filename, encoding="utf-8-sig") as records_file, open(
251
- self.folder_structure.created_objects_path, "w+"
252
- ) as results_file:
233
+ with (
234
+ open(filename, encoding="utf-8-sig") as records_file,
235
+ open(self.folder_structure.created_objects_path, "w+") as results_file,
236
+ ):
253
237
  self.mapper.migration_report.add_general_statistics(
254
- i18n.t("Number of files processed")
238
+ i18n_t("Number of files processed")
255
239
  )
256
240
  start = time.time()
257
241
  records_processed = 0
@@ -270,7 +254,7 @@ class OrdersTransformer(MigrationTaskBase):
270
254
  self.mapper.perform_additional_mapping(legacy_id, folio_rec)
271
255
 
272
256
  self.mapper.migration_report.add_general_statistics(
273
- i18n.t("TOTAL Purchase Order Lines created")
257
+ i18n_t("TOTAL Purchase Order Lines created")
274
258
  )
275
259
  self.mapper.report_folio_mapping(folio_rec, self.mapper.composite_order_schema)
276
260
  self.mapper.notes_mapper.map_notes(
@@ -333,7 +317,7 @@ class OrdersTransformer(MigrationTaskBase):
333
317
  self.folder_structure.migration_reports_file,
334
318
  )
335
319
  self.mapper.migration_report.write_migration_report(
336
- i18n.t("Pruchase Orders and Purchase Order Lines Transformation Report"),
320
+ i18n_t("Pruchase Orders and Purchase Order Lines Transformation Report"),
337
321
  migration_report_file,
338
322
  self.start_datetime,
339
323
  )
@@ -50,27 +50,21 @@ class OrganizationTransformer(MigrationTaskBase):
50
50
  str,
51
51
  Field(
52
52
  title="Migration task type",
53
- description=(
54
- "The type of migration task you want to perform"
55
- ),
53
+ description=("The type of migration task you want to perform"),
56
54
  ),
57
55
  ]
58
56
  files: Annotated[
59
57
  List[FileDefinition],
60
58
  Field(
61
59
  title="Source files",
62
- description=(
63
- "List of MARC21 files with holdings records"
64
- ),
60
+ description=("List of MARC21 files with holdings records"),
65
61
  ),
66
62
  ]
67
63
  organization_map_path: Annotated[
68
64
  str,
69
65
  Field(
70
66
  title="Organization map path",
71
- description=(
72
- "Path to the organization map file"
73
- ),
67
+ description=("Path to the organization map file"),
74
68
  ),
75
69
  ]
76
70
  organization_types_map_path: Annotated[
@@ -95,18 +89,14 @@ class OrganizationTransformer(MigrationTaskBase):
95
89
  Optional[str],
96
90
  Field(
97
91
  title="Email categories map path",
98
- description=(
99
- "Path to the email categories map file. By default is empty string"
100
- ),
92
+ description=("Path to the email categories map file. By default is empty string"),
101
93
  ),
102
94
  ] = ""
103
95
  phone_categories_map_path: Annotated[
104
96
  Optional[str],
105
97
  Field(
106
98
  title="Phone categories map path",
107
- description=(
108
- "Path to the phone categories map file. By default is empty string"
109
- ),
99
+ description=("Path to the phone categories map file. By default is empty string"),
110
100
  ),
111
101
  ] = ""
112
102
 
@@ -201,9 +191,10 @@ class OrganizationTransformer(MigrationTaskBase):
201
191
  return files
202
192
 
203
193
  def process_single_file(self, filename):
204
- with open(filename, encoding="utf-8-sig") as records_file, open(
205
- self.folder_structure.created_objects_path, "w+"
206
- ) as results_file:
194
+ with (
195
+ open(filename, encoding="utf-8-sig") as records_file,
196
+ open(self.folder_structure.created_objects_path, "w+") as results_file,
197
+ ):
207
198
  self.mapper.migration_report.add_general_statistics(
208
199
  i18n.t("Number of files processed")
209
200
  )
@@ -13,6 +13,7 @@ from zoneinfo import ZoneInfo
13
13
  from folio_migration_tools.circulation_helper import CirculationHelper
14
14
  from folio_migration_tools.custom_dict import InsensitiveDictReader
15
15
  from folio_migration_tools.helper import Helper
16
+ from folio_migration_tools.i18n_cache import i18n_t
16
17
  from folio_migration_tools.library_configuration import (
17
18
  FileDefinition,
18
19
  LibraryConfiguration,
@@ -32,7 +33,7 @@ class RequestsMigrator(MigrationTaskBase):
32
33
  description=(
33
34
  "Name of this migration task. The name is being used to call "
34
35
  "the specific task, and to distinguish tasks of similar types"
35
- )
36
+ ),
36
37
  ),
37
38
  ]
38
39
  migration_task_type: Annotated[
@@ -54,8 +55,7 @@ class RequestsMigrator(MigrationTaskBase):
54
55
  Field(
55
56
  title="Starting row",
56
57
  description=(
57
- "Row number to start processing data from. "
58
- "Optional, by default is first row"
58
+ "Row number to start processing data from. Optional, by default is first row"
59
59
  ),
60
60
  ),
61
61
  ] = 1
@@ -64,8 +64,7 @@ class RequestsMigrator(MigrationTaskBase):
64
64
  Field(
65
65
  title="Item files",
66
66
  description=(
67
- "List of files containing item data. "
68
- "Optional, by default is empty list"
67
+ "List of files containing item data. Optional, by default is empty list"
69
68
  ),
70
69
  ),
71
70
  ] = []
@@ -74,8 +73,7 @@ class RequestsMigrator(MigrationTaskBase):
74
73
  Field(
75
74
  title="Patron files",
76
75
  description=(
77
- "List of files containing patron data. "
78
- "Optional, by default is empty list"
76
+ "List of files containing patron data. Optional, by default is empty list"
79
77
  ),
80
78
  ),
81
79
  ] = []
@@ -88,7 +86,7 @@ class RequestsMigrator(MigrationTaskBase):
88
86
  self,
89
87
  task_configuration: TaskConfiguration,
90
88
  library_config: LibraryConfiguration,
91
- folio_client
89
+ folio_client,
92
90
  ):
93
91
  csv.register_dialect("tsv", delimiter="\t")
94
92
  self.migration_report = MigrationReport()
@@ -135,8 +133,7 @@ class RequestsMigrator(MigrationTaskBase):
135
133
  )
136
134
  else:
137
135
  logging.info(
138
- "No item or user files supplied. Not validating against"
139
- "previously migrated objects"
136
+ "No item or user files supplied. Not validating againstpreviously migrated objects"
140
137
  )
141
138
  self.valid_legacy_requests = self.semi_valid_legacy_requests
142
139
 
@@ -151,7 +148,7 @@ class RequestsMigrator(MigrationTaskBase):
151
148
 
152
149
  def prepare_legacy_request(self, legacy_request: LegacyRequest):
153
150
  patron = self.circulation_helper.get_user_by_barcode(legacy_request.patron_barcode)
154
- self.migration_report.add_general_statistics(i18n.t("Patron lookups performed"))
151
+ self.migration_report.add_general_statistics(i18n_t("Patron lookups performed"))
155
152
 
156
153
  if not patron:
157
154
  logging.error(f"No user with barcode {legacy_request.patron_barcode} found in FOLIO")
@@ -161,18 +158,18 @@ class RequestsMigrator(MigrationTaskBase):
161
158
  f"{legacy_request.patron_barcode}",
162
159
  )
163
160
  self.migration_report.add_general_statistics(
164
- i18n.t("No user with barcode found in FOLIO")
161
+ i18n_t("No user with barcode found in FOLIO")
165
162
  )
166
163
  self.failed_requests.add(legacy_request)
167
164
  return False, legacy_request
168
165
  legacy_request.patron_id = patron.get("id")
169
166
 
170
167
  item = self.circulation_helper.get_item_by_barcode(legacy_request.item_barcode)
171
- self.migration_report.add_general_statistics(i18n.t("Item lookups performed"))
168
+ self.migration_report.add_general_statistics(i18n_t("Item lookups performed"))
172
169
  if not item:
173
170
  logging.error(f"No item with barcode {legacy_request.item_barcode} found in FOLIO")
174
171
  self.migration_report.add_general_statistics(
175
- i18n.t("No item with barcode found in FOLIO")
172
+ i18n_t("No item with barcode found in FOLIO")
176
173
  )
177
174
  Helper.log_data_issue(
178
175
  f"{legacy_request.item_barcode}",
@@ -182,22 +179,22 @@ class RequestsMigrator(MigrationTaskBase):
182
179
  self.failed_requests.add(legacy_request)
183
180
  return False, legacy_request
184
181
  holding = self.circulation_helper.get_holding_by_uuid(item.get("holdingsRecordId"))
185
- self.migration_report.add_general_statistics(i18n.t("Holdings lookups performed"))
182
+ self.migration_report.add_general_statistics(i18n_t("Holdings lookups performed"))
186
183
  legacy_request.item_id = item.get("id")
187
184
  legacy_request.holdings_record_id = item.get("holdingsRecordId")
188
185
  legacy_request.instance_id = holding.get("instanceId")
189
186
  if item["status"]["name"] in ["Available"]:
190
187
  legacy_request.request_type = "Page"
191
- logging.info(f'Setting request to Page, since the status is {item["status"]["name"]}')
188
+ logging.info(f"Setting request to Page, since the status is {item['status']['name']}")
192
189
  self.migration_report.add_general_statistics(
193
- i18n.t("Valid, prepared requests, ready for posting")
190
+ i18n_t("Valid, prepared requests, ready for posting")
194
191
  )
195
192
  return True, legacy_request
196
193
 
197
194
  def do_work(self):
198
195
  logging.info("Starting")
199
196
  if self.task_configuration.starting_row > 1:
200
- logging.info(f"Skipping {(self.task_configuration.starting_row-1)} records")
197
+ logging.info(f"Skipping {(self.task_configuration.starting_row - 1)} records")
201
198
  for num_requests, legacy_request in enumerate(
202
199
  self.valid_legacy_requests[self.task_configuration.starting_row - 1 :],
203
200
  start=1,
@@ -210,11 +207,11 @@ class RequestsMigrator(MigrationTaskBase):
210
207
  self.folio_client, legacy_request, self.migration_report
211
208
  ):
212
209
  self.migration_report.add_general_statistics(
213
- i18n.t("Successfully migrated requests")
210
+ i18n_t("Successfully migrated requests")
214
211
  )
215
212
  else:
216
213
  self.migration_report.add_general_statistics(
217
- i18n.t("Unsuccessfully migrated requests")
214
+ i18n_t("Unsuccessfully migrated requests")
218
215
  )
219
216
  self.failed_requests.add(legacy_request)
220
217
  if num_requests == 1:
@@ -237,7 +234,7 @@ class RequestsMigrator(MigrationTaskBase):
237
234
 
238
235
  with open(self.folder_structure.migration_reports_file, "w+") as report_file:
239
236
  self.migration_report.write_migration_report(
240
- i18n.t("Requests migration report"), report_file, self.start_datetime
237
+ i18n_t("Requests migration report"), report_file, self.start_datetime
241
238
  )
242
239
  self.clean_out_empty_logs()
243
240
 
@@ -281,7 +278,8 @@ class RequestsMigrator(MigrationTaskBase):
281
278
  self.migration_report.add(
282
279
  "DiscardedLoans",
283
280
  i18n.t(
284
- "Requests discarded. Had migrated item barcode: %{item_barcode}.\n Had migrated user barcode: %{patron_barcode}",
281
+ "Requests discarded. Had migrated item barcode: %{item_barcode}.\n "
282
+ "Had migrated user barcode: %{patron_barcode}",
285
283
  item_barcode=has_item_barcode,
286
284
  patron_barcode=has_patron_barcode,
287
285
  ),
@@ -333,8 +331,7 @@ class RequestsMigrator(MigrationTaskBase):
333
331
  except ValueError as ve:
334
332
  logging.exception(ve)
335
333
  logging.info(
336
- f"Done validating {legacy_reques_count} "
337
- f"legacy requests with {num_bad} rotten apples"
334
+ f"Done validating {legacy_reques_count} legacy requests with {num_bad} rotten apples"
338
335
  )
339
336
  if num_bad > 0 and (num_bad / legacy_reques_count) > 0.5:
340
337
  q = num_bad / legacy_reques_count
@@ -14,6 +14,7 @@ from folio_uuid.folio_namespaces import FOLIONamespaces
14
14
 
15
15
  from folio_migration_tools.custom_dict import InsensitiveDictReader
16
16
  from folio_migration_tools.custom_exceptions import TransformationProcessError
17
+ from folio_migration_tools.i18n_cache import i18n_t
17
18
  from folio_migration_tools.library_configuration import (
18
19
  FileDefinition,
19
20
  LibraryConfiguration,
@@ -59,7 +60,7 @@ class ReservesMigrator(MigrationTaskBase):
59
60
  self,
60
61
  task_configuration: TaskConfiguration,
61
62
  library_config: LibraryConfiguration,
62
- folio_client
63
+ folio_client,
63
64
  ):
64
65
  csv.register_dialect("tsv", delimiter="\t")
65
66
  self.migration_report = MigrationReport()
@@ -90,7 +91,7 @@ class ReservesMigrator(MigrationTaskBase):
90
91
  logging.info("Starting")
91
92
  for num_reserves, legacy_reserve in enumerate(self.valid_reserves, start=1):
92
93
  t0_migration = time.time()
93
- self.migration_report.add_general_statistics(i18n.t("Processed reserves"))
94
+ self.migration_report.add_general_statistics(i18n_t("Processed reserves"))
94
95
  try:
95
96
  self.post_single_reserve(legacy_reserve)
96
97
  except Exception as ee:
@@ -107,10 +108,10 @@ class ReservesMigrator(MigrationTaskBase):
107
108
  path, legacy_reserve.to_dict(), "POST", i18n.t("Posted reserves")
108
109
  ):
109
110
  self.migration_report.add_general_statistics(
110
- i18n.t("Successfully posted reserves")
111
+ i18n_t("Successfully posted reserves")
111
112
  )
112
113
  else:
113
- self.migration_report.add_general_statistics(i18n.t("Failure to post reserve"))
114
+ self.migration_report.add_general_statistics(i18n_t("Failure to post reserve"))
114
115
  except Exception as ee:
115
116
  logging.error(ee)
116
117
 
@@ -123,7 +124,7 @@ class ReservesMigrator(MigrationTaskBase):
123
124
 
124
125
  with open(self.folder_structure.migration_reports_file, "w+") as report_file:
125
126
  self.migration_report.write_migration_report(
126
- i18n.t("Reserves migration report"), report_file, self.start_datetime
127
+ i18n_t("Reserves migration report"), report_file, self.start_datetime
127
128
  )
128
129
  self.clean_out_empty_logs()
129
130