folio-migration-tools 1.9.9__py3-none-any.whl → 1.10.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. folio_migration_tools/__init__.py +3 -4
  2. folio_migration_tools/__main__.py +53 -31
  3. folio_migration_tools/circulation_helper.py +118 -108
  4. folio_migration_tools/custom_dict.py +2 -2
  5. folio_migration_tools/custom_exceptions.py +4 -5
  6. folio_migration_tools/folder_structure.py +17 -7
  7. folio_migration_tools/helper.py +8 -7
  8. folio_migration_tools/holdings_helper.py +4 -3
  9. folio_migration_tools/i18n_cache.py +79 -0
  10. folio_migration_tools/library_configuration.py +77 -37
  11. folio_migration_tools/mapper_base.py +45 -31
  12. folio_migration_tools/mapping_file_transformation/courses_mapper.py +1 -1
  13. folio_migration_tools/mapping_file_transformation/holdings_mapper.py +7 -3
  14. folio_migration_tools/mapping_file_transformation/item_mapper.py +13 -26
  15. folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py +1 -2
  16. folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +13 -11
  17. folio_migration_tools/mapping_file_transformation/order_mapper.py +23 -5
  18. folio_migration_tools/mapping_file_transformation/organization_mapper.py +3 -3
  19. folio_migration_tools/mapping_file_transformation/ref_data_mapping.py +3 -0
  20. folio_migration_tools/mapping_file_transformation/user_mapper.py +47 -28
  21. folio_migration_tools/marc_rules_transformation/conditions.py +82 -97
  22. folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +13 -5
  23. folio_migration_tools/marc_rules_transformation/hrid_handler.py +3 -2
  24. folio_migration_tools/marc_rules_transformation/marc_file_processor.py +26 -24
  25. folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +56 -51
  26. folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +28 -17
  27. folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +68 -37
  28. folio_migration_tools/migration_report.py +18 -7
  29. folio_migration_tools/migration_tasks/batch_poster.py +285 -354
  30. folio_migration_tools/migration_tasks/bibs_transformer.py +14 -9
  31. folio_migration_tools/migration_tasks/courses_migrator.py +2 -3
  32. folio_migration_tools/migration_tasks/holdings_csv_transformer.py +23 -24
  33. folio_migration_tools/migration_tasks/holdings_marc_transformer.py +14 -24
  34. folio_migration_tools/migration_tasks/items_transformer.py +23 -34
  35. folio_migration_tools/migration_tasks/loans_migrator.py +67 -144
  36. folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py +3 -3
  37. folio_migration_tools/migration_tasks/migration_task_base.py +43 -52
  38. folio_migration_tools/migration_tasks/orders_transformer.py +25 -41
  39. folio_migration_tools/migration_tasks/organization_transformer.py +9 -18
  40. folio_migration_tools/migration_tasks/requests_migrator.py +21 -24
  41. folio_migration_tools/migration_tasks/reserves_migrator.py +6 -5
  42. folio_migration_tools/migration_tasks/user_transformer.py +25 -20
  43. folio_migration_tools/task_configuration.py +6 -7
  44. folio_migration_tools/transaction_migration/legacy_loan.py +15 -27
  45. folio_migration_tools/transaction_migration/legacy_request.py +1 -1
  46. folio_migration_tools/translations/en.json +3 -8
  47. {folio_migration_tools-1.9.9.dist-info → folio_migration_tools-1.10.0.dist-info}/METADATA +19 -28
  48. folio_migration_tools-1.10.0.dist-info/RECORD +63 -0
  49. folio_migration_tools-1.10.0.dist-info/WHEEL +4 -0
  50. folio_migration_tools-1.10.0.dist-info/entry_points.txt +3 -0
  51. folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +0 -241
  52. folio_migration_tools/migration_tasks/authority_transformer.py +0 -119
  53. folio_migration_tools/test_infrastructure/__init__.py +0 -0
  54. folio_migration_tools/test_infrastructure/mocked_classes.py +0 -406
  55. folio_migration_tools-1.9.9.dist-info/RECORD +0 -67
  56. folio_migration_tools-1.9.9.dist-info/WHEEL +0 -4
  57. folio_migration_tools-1.9.9.dist-info/entry_points.txt +0 -3
  58. folio_migration_tools-1.9.9.dist-info/licenses/LICENSE +0 -21
@@ -3,9 +3,8 @@ from typing import Protocol
3
3
 
4
4
  __version__ = importlib.metadata.version("folio_migration_tools")
5
5
 
6
+
6
7
  class StrCoercible(Protocol):
7
- def __repr__(self) -> str:
8
- ...
8
+ def __repr__(self) -> str: ...
9
9
 
10
- def __str__(self) -> str:
11
- ...
10
+ def __str__(self) -> str: ...
@@ -4,6 +4,7 @@ import logging
4
4
  import sys
5
5
  from os import environ
6
6
  from pathlib import Path
7
+ from warnings import warn
7
8
 
8
9
  import httpx
9
10
  import humps
@@ -33,23 +34,25 @@ def parse_args(args):
33
34
 
34
35
  parser.add_argument(
35
36
  "task_name",
36
- help=("Task name. Use one of: " f"{tasks_string}"),
37
+ help=(f"Task name. Use one of: {tasks_string}"),
37
38
  nargs="?" if "FOLIO_MIGRATION_TOOLS_TASK_NAME" in environ else None,
38
39
  prompt="FOLIO_MIGRATION_TOOLS_TASK_NAME" not in environ,
39
40
  default=environ.get("FOLIO_MIGRATION_TOOLS_TASK_NAME"),
40
41
  )
41
42
  parser.add_argument(
42
- "--folio_password", "--okapi_password",
43
+ "--folio_password",
44
+ "--okapi_password",
43
45
  help="password for the tenant in the configuration file",
44
46
  prompt="FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD" not in environ,
45
- default=environ.get("FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD"),
47
+ default=environ.get(
48
+ "FOLIO_MIGRATION_TOOLS_FOLIO_PASSWORD",
49
+ environ.get("FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD"),
50
+ ),
46
51
  secure=True,
47
52
  )
48
53
  parser.add_argument(
49
54
  "--base_folder_path",
50
- help=(
51
- "path to the base folder for this library. Built on migration_repo_template"
52
- ),
55
+ help=("path to the base folder for this library. Built on migration_repo_template"),
53
56
  prompt="FOLIO_MIGRATION_TOOLS_BASE_FOLDER_PATH" not in environ,
54
57
  default=environ.get("FOLIO_MIGRATION_TOOLS_BASE_FOLDER_PATH"),
55
58
  )
@@ -62,34 +65,43 @@ def parse_args(args):
62
65
  prompt=False,
63
66
  )
64
67
  parser.add_argument(
65
- "--version", "-V",
68
+ "--version",
69
+ "-V",
66
70
  help="Show the version of the FOLIO Migration Tools",
67
71
  action="store_true",
68
72
  prompt=False,
69
73
  )
70
74
  return parser.parse_args(args)
71
75
 
76
+
72
77
  def prep_library_config(args):
73
- config_file_humped = merge_load(args.configuration_path)
74
- config_file_humped["libraryInformation"]["okapiPassword"] = args.folio_password
75
- config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
76
- config_file = humps.decamelize(config_file_humped)
77
- library_config = LibraryConfiguration(**config_file["library_information"])
78
- if library_config.ecs_tenant_id:
79
- library_config.is_ecs = True
80
- if library_config.ecs_tenant_id and not library_config.ecs_central_iteration_identifier:
81
- print(
82
- "ECS tenant ID is set, but no central iteration identifier is provided. "
83
- "Please provide the central iteration identifier in the configuration file."
84
- )
85
- sys.exit("ECS Central Iteration Identifier Not Found")
86
- return config_file, library_config
78
+ config_file_humped = merge_load(args.configuration_path)
87
79
 
88
- def print_version(args):
89
- if "-V" in args or "--version" in args:
80
+ # Only set folioPassword if neither folioPassword nor okapiPassword exist in config
81
+ # The Pydantic validator will handle backward compatibility for existing okapiPassword
82
+ if (
83
+ "folioPassword" not in config_file_humped["libraryInformation"]
84
+ and "okapiPassword" not in config_file_humped["libraryInformation"]
85
+ ):
86
+ config_file_humped["libraryInformation"]["folioPassword"] = args.folio_password
87
+
88
+ config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
89
+ config_file = humps.decamelize(config_file_humped)
90
+ library_config = LibraryConfiguration(**config_file["library_information"])
91
+ if library_config.ecs_tenant_id:
92
+ library_config.is_ecs = True
93
+ if library_config.ecs_tenant_id and not library_config.ecs_central_iteration_identifier:
90
94
  print(
91
- f"FOLIO Migration Tools: {metadata.version('folio_migration_tools')}"
95
+ "ECS tenant ID is set, but no central iteration identifier is provided. "
96
+ "Please provide the central iteration identifier in the configuration file."
92
97
  )
98
+ sys.exit("ECS Central Iteration Identifier Not Found")
99
+ return config_file, library_config
100
+
101
+
102
+ def print_version(args):
103
+ if "-V" in args or "--version" in args:
104
+ print(f"FOLIO Migration Tools: {metadata.version('folio_migration_tools')}")
93
105
  sys.exit(0)
94
106
  return None
95
107
 
@@ -115,6 +127,14 @@ def main():
115
127
  i18n.set("locale", args.report_language)
116
128
  config_file, library_config = prep_library_config(args)
117
129
  try:
130
+ if args.task_name == "AuthorityTransformer":
131
+ warn(
132
+ "The AuthorityTransformer has been removed."
133
+ " Please update your configuration accordingly."
134
+ " Use Data Import to load authority records.",
135
+ DeprecationWarning,
136
+ stacklevel=2,
137
+ )
118
138
  migration_task_config = next(
119
139
  t for t in config_file["migration_tasks"] if t["name"] == args.task_name
120
140
  )
@@ -122,7 +142,7 @@ def main():
122
142
  task_names = [t.get("name", "") for t in config_file["migration_tasks"]]
123
143
  print(
124
144
  f"Referenced task name {args.task_name} not found in the "
125
- f'configuration file. Use one of {", ".join(task_names)}'
145
+ f"configuration file. Use one of {', '.join(task_names)}"
126
146
  "\nHalting..."
127
147
  )
128
148
  sys.exit("Task Name Not Found")
@@ -134,13 +154,15 @@ def main():
134
154
  )
135
155
  except StopIteration:
136
156
  print(
137
- f'Referenced task {migration_task_config["migration_task_type"]} '
157
+ f"Referenced task {migration_task_config['migration_task_type']} "
138
158
  "is not a valid option. Update your task to incorporate "
139
159
  f"one of {json.dumps([tc.__name__ for tc in task_classes], indent=4)}"
140
160
  )
141
161
  sys.exit("Task Type Not Found")
142
162
  try:
143
- logging.getLogger("httpx").setLevel(logging.WARNING) # Exclude info messages from httpx
163
+ logging.getLogger("httpx").setLevel(
164
+ logging.WARNING
165
+ ) # Exclude info messages from httpx
144
166
  with FolioClient(
145
167
  library_config.gateway_url,
146
168
  library_config.tenant_id,
@@ -161,16 +183,15 @@ def main():
161
183
  logging.critical(json_error)
162
184
  print(json_error.doc)
163
185
  print(
164
- f"\n{json_error}"
165
- f"\nError parsing the above JSON mapping or configruation file. Halting."
186
+ f"\n{json_error}\nError parsing the above JSON mapping or configruation file. Halting."
166
187
  )
167
188
  sys.exit("Invalid JSON")
168
189
  except ValidationError as e:
169
- print(e.json())
190
+ print(json.dumps(e.errors(), indent=2))
170
191
  print("Validation errors in configuration file:")
171
192
  print("==========================================")
172
193
 
173
- for validation_message in json.loads(e.json()):
194
+ for validation_message in e.errors():
174
195
  print(
175
196
  f"{validation_message['msg']}\t"
176
197
  f"{', '.join(humps.camelize(str(x)) for x in validation_message['loc'])}"
@@ -199,6 +220,7 @@ def main():
199
220
  sys.exit(ee.__class__.__name__)
200
221
  sys.exit(0)
201
222
 
223
+
202
224
  def inheritors(base_class):
203
225
  subclasses = set()
204
226
  work = [base_class]
@@ -3,14 +3,15 @@ import json
3
3
  import logging
4
4
  import re
5
5
  import time
6
+ from http import HTTPStatus
6
7
  from typing import Set
7
8
 
8
9
  import httpx
9
10
  import i18n
10
- from folioclient import FolioClient
11
- from httpx import HTTPError
11
+ from folioclient import FolioClient, FolioClientError, FolioConnectionError, FolioValidationError
12
12
 
13
13
  from folio_migration_tools.helper import Helper
14
+ from folio_migration_tools.i18n_cache import i18n_t
14
15
  from folio_migration_tools.migration_report import MigrationReport
15
16
  from folio_migration_tools.transaction_migration.legacy_loan import LegacyLoan
16
17
  from folio_migration_tools.transaction_migration.legacy_request import LegacyRequest
@@ -37,7 +38,7 @@ class CirculationHelper:
37
38
  def get_user_by_barcode(self, user_barcode):
38
39
  if user_barcode in self.missing_patron_barcodes:
39
40
  self.migration_report.add_general_statistics(
40
- i18n.t("Users already detected as missing")
41
+ i18n_t("Users already detected as missing")
41
42
  )
42
43
  logging.info("User is already detected as missing")
43
44
  return {}
@@ -55,7 +56,7 @@ class CirculationHelper:
55
56
  def get_item_by_barcode(self, item_barcode):
56
57
  if item_barcode in self.missing_item_barcodes:
57
58
  self.migration_report.add_general_statistics(
58
- i18n.t("Items already detected as missing")
59
+ i18n_t("Items already detected as missing")
59
60
  )
60
61
  logging.info("Item is already detected as missing")
61
62
  return {}
@@ -138,117 +139,117 @@ class CirculationHelper:
138
139
  if legacy_loan.proxy_patron_barcode:
139
140
  data.update({"proxyUserBarcode": legacy_loan.proxy_patron_barcode})
140
141
  path = "/circulation/check-out-by-barcode"
141
- url = f"{self.folio_client.gateway_url}{path}"
142
142
  try:
143
143
  if legacy_loan.patron_barcode in self.missing_patron_barcodes:
144
- error_message = i18n.t("Patron barcode already detected as missing")
144
+ error_message = i18n_t("Patron barcode already detected as missing")
145
145
  logging.error(
146
146
  f"{error_message} Patron barcode: {legacy_loan.patron_barcode} "
147
147
  f"Item Barcode:{legacy_loan.item_barcode}"
148
148
  )
149
149
  return TransactionResult(False, False, "", error_message, error_message)
150
- req = httpx.post(url, headers=self.folio_client.okapi_headers, json=data, timeout=None)
151
- if req.status_code == 422:
152
- error_message_from_folio = json.loads(req.text)["errors"][0]["message"]
153
- stat_message = error_message_from_folio
154
- error_message = error_message_from_folio
155
- if "has the item status" in error_message_from_folio:
156
- stat_message = re.findall(
157
- r"(?<=has the item status\s).*(?=\sand cannot be checked out)",
158
- error_message_from_folio,
159
- )[0]
160
- error_message = (
161
- f"{stat_message} for item with barcode {legacy_loan.item_barcode}"
162
- )
163
- return TransactionResult(
164
- False,
165
- True,
166
- None,
167
- error_message_from_folio,
168
- stat_message,
169
- )
170
- elif "No item with barcode" in error_message_from_folio:
171
- error_message = f"No item with barcode {legacy_loan.item_barcode} in FOLIO"
172
- stat_message = "Item barcode not in FOLIO"
173
- self.missing_item_barcodes.add(legacy_loan.item_barcode)
174
- return TransactionResult(
175
- False,
176
- False,
177
- None,
178
- error_message_from_folio,
179
- stat_message,
180
- )
181
-
182
- elif "find user with matching barcode" in error_message_from_folio:
183
- self.missing_patron_barcodes.add(legacy_loan.patron_barcode)
184
- error_message = f"No patron with barcode {legacy_loan.patron_barcode} in FOLIO"
185
- stat_message = i18n.t("Patron barcode not in FOLIO")
186
- return TransactionResult(
187
- False,
188
- False,
189
- None,
190
- error_message_from_folio,
191
- stat_message,
192
- )
193
- elif "Cannot check out item that already has an open" in error_message_from_folio:
194
- return TransactionResult(
195
- False,
196
- False,
197
- None,
198
- error_message_from_folio,
199
- error_message_from_folio,
200
- )
201
- elif "Item is already checked out" in error_message_from_folio:
202
- return TransactionResult(
203
- False,
204
- True,
205
- None,
206
- error_message_from_folio,
207
- error_message_from_folio,
208
- )
209
- logging.error(
210
- f"{error_message} "
211
- f"Patron barcode: {legacy_loan.patron_barcode} "
212
- f"Item Barcode:{legacy_loan.item_barcode}"
150
+ loan = self.folio_client.folio_post(path, data)
151
+ stats = "Successfully checked out by barcode"
152
+ logging.debug(
153
+ "%s (item barcode %s}) in %ss",
154
+ stats,
155
+ legacy_loan.item_barcode,
156
+ f"{(time.time() - t0_function):.2f}",
157
+ )
158
+ return TransactionResult(True, False, loan, "", stats)
159
+ except FolioValidationError as fve:
160
+ error_message_from_folio = self.folio_client.handle_json_response(fve.response)[
161
+ "errors"
162
+ ][0]["message"]
163
+ stat_message = error_message_from_folio
164
+ error_message = error_message_from_folio
165
+ if "has the item status" in error_message_from_folio:
166
+ stat_message = re.findall(
167
+ r"(?<=has the item status\s).*(?=\sand cannot be checked out)",
168
+ error_message_from_folio,
169
+ )[0]
170
+ error_message = f"{stat_message} for item with barcode {legacy_loan.item_barcode}"
171
+ return TransactionResult(
172
+ False,
173
+ True,
174
+ None,
175
+ error_message_from_folio,
176
+ stat_message,
213
177
  )
214
- self.migration_report.add("Details", stat_message)
178
+ elif "No item with barcode" in error_message_from_folio:
179
+ error_message = f"No item with barcode {legacy_loan.item_barcode} in FOLIO"
180
+ stat_message = "Item barcode not in FOLIO"
181
+ self.missing_item_barcodes.add(legacy_loan.item_barcode)
215
182
  return TransactionResult(
216
- False, True, None, error_message, f"Check out error: {stat_message}"
183
+ False,
184
+ False,
185
+ None,
186
+ error_message_from_folio,
187
+ stat_message,
217
188
  )
218
- elif req.status_code == 201:
219
- stats = "Successfully checked out by barcode"
220
- logging.debug(
221
- "%s (item barcode %s}) in %ss",
222
- stats,
223
- legacy_loan.item_barcode,
224
- f"{(time.time() - t0_function):.2f}",
189
+
190
+ elif "find user with matching barcode" in error_message_from_folio:
191
+ self.missing_patron_barcodes.add(legacy_loan.patron_barcode)
192
+ error_message = f"No patron with barcode {legacy_loan.patron_barcode} in FOLIO"
193
+ stat_message = i18n_t("Patron barcode not in FOLIO")
194
+ return TransactionResult(
195
+ False,
196
+ False,
197
+ None,
198
+ error_message_from_folio,
199
+ stat_message,
225
200
  )
226
- return TransactionResult(True, False, json.loads(req.text), "", stats)
227
- elif req.status_code == 204:
228
- stats = "Successfully checked out by barcode"
229
- logging.debug(
230
- "%s (item barcode %s) %s",
231
- stats,
232
- legacy_loan.item_barcode,
233
- req.status_code,
201
+ elif "Cannot check out item that already has an open" in error_message_from_folio:
202
+ return TransactionResult(
203
+ False,
204
+ False,
205
+ None,
206
+ error_message_from_folio,
207
+ error_message_from_folio,
234
208
  )
235
- return TransactionResult(True, False, None, "", stats)
236
- else:
237
- req.raise_for_status()
238
- except HTTPError:
209
+ elif "Item is already checked out" in error_message_from_folio:
210
+ return TransactionResult(
211
+ False,
212
+ True,
213
+ None,
214
+ error_message_from_folio,
215
+ error_message_from_folio,
216
+ )
217
+ logging.error(
218
+ f"{error_message} "
219
+ f"Patron barcode: {legacy_loan.patron_barcode} "
220
+ f"Item Barcode:{legacy_loan.item_barcode}"
221
+ )
222
+ self.migration_report.add("Details", stat_message)
223
+ return TransactionResult(
224
+ False, True, None, error_message, f"Check out error: {stat_message}"
225
+ )
226
+ except FolioClientError as fce:
239
227
  logging.exception(
240
228
  "%s\tPOST FAILED %s\n\t%s\n\t%s",
241
- req.status_code,
242
- url,
229
+ fce.response.status_code,
230
+ fce.request.url,
243
231
  json.dumps(data),
244
- req.text,
232
+ fce.response.text,
245
233
  )
246
234
  return TransactionResult(
247
235
  False,
248
236
  False,
249
237
  None,
250
238
  "5XX",
251
- i18n.t("Failed checkout http status %{code}", code=req.status_code),
239
+ i18n.t("Failed checkout http status %{code}", code=fce.response.status_code),
240
+ )
241
+ except FolioConnectionError as fce:
242
+ logging.exception(
243
+ "Connection error\tPOST FAILED %s\n\t%s\n\t%s",
244
+ fce.request.url,
245
+ json.dumps(data),
246
+ )
247
+ return TransactionResult(
248
+ False,
249
+ False,
250
+ None,
251
+ "Connection error",
252
+ i18n_t("Connection error during checkout"),
252
253
  )
253
254
 
254
255
  @staticmethod
@@ -257,7 +258,6 @@ class CirculationHelper:
257
258
  ):
258
259
  try:
259
260
  path = "/circulation/requests"
260
- url = f"{folio_client.gateway_url}{path}"
261
261
  data = legacy_request.serialize()
262
262
  data["requestProcessingParameters"] = {
263
263
  "overrideBlocks": {
@@ -269,21 +269,31 @@ class CirculationHelper:
269
269
  "comment": "Migrated from legacy system",
270
270
  }
271
271
  }
272
- req = httpx.post(url, headers=folio_client.okapi_headers, json=data, timeout=None)
273
- logging.debug(f"POST {req.status_code}\t{url}\t{json.dumps(data)}")
274
- if str(req.status_code) == "422":
275
- message = json.loads(req.text)["errors"][0]["message"]
276
- logging.error(f"{message}")
272
+ _ = folio_client.folio_post(path, data)
273
+ logging.debug(f"POST {path}\t{json.dumps(data)}")
274
+ logging.info(
275
+ "%s Successfully created %s",
276
+ HTTPStatus.OK,
277
+ legacy_request.request_type,
278
+ )
279
+ return True
280
+ except FolioValidationError as fve:
281
+ message = folio_client.handle_json_response(fve.response)["errors"][0]["message"]
282
+ logging.error(message)
283
+ migration_report.add_general_statistics(message)
284
+ return False
285
+ except (FolioConnectionError, FolioClientError) as fce:
286
+ if client_response := getattr(fce, "response", None):
287
+ message = (
288
+ f"HTTP {client_response.status_code} Error creating request: "
289
+ f"{client_response.text}"
290
+ )
291
+ logging.error(message)
277
292
  migration_report.add_general_statistics(message)
278
- return False
279
293
  else:
280
- req.raise_for_status()
281
- logging.info(
282
- "%s Successfully created %s",
283
- req.status_code,
284
- legacy_request.request_type,
285
- )
286
- return True
294
+ logging.error(f"Connection error creating request: {fce}")
295
+ migration_report.add_general_statistics("Connection error creating request")
296
+ return False
287
297
  except Exception as exception:
288
298
  logging.error(exception, exc_info=True)
289
299
  migration_report.add("Details", exception)
@@ -7,10 +7,10 @@ class InsensitiveDictReader(csv.DictReader):
7
7
  # spaces and to lower case.
8
8
  @property
9
9
  def fieldnames(self):
10
- return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)] # type: ignore
10
+ return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)] # type: ignore
11
11
 
12
12
  def next(self):
13
- return InsensitiveDict(csv.DictReader.next(self)) # type: ignore
13
+ return InsensitiveDict(csv.DictReader.next(self)) # type: ignore
14
14
 
15
15
 
16
16
  class InsensitiveDict(dict):
@@ -1,5 +1,4 @@
1
1
  import logging
2
- from typing import Union
3
2
  import i18n
4
3
 
5
4
  from folio_migration_tools import StrCoercible
@@ -13,10 +12,10 @@ class TransformationFieldMappingError(TransformationError):
13
12
  """Raised when the field mapping fails, but the error is not critical.
14
13
  The issue should be logged for the library to act upon it"""
15
14
 
16
- def __init__(self, index_or_id="", message="", data_value: Union[str, StrCoercible]=""):
15
+ def __init__(self, index_or_id="", message="", data_value: str | StrCoercible = ""):
17
16
  self.index_or_id = index_or_id or ""
18
17
  self.message = message
19
- self.data_value: Union[str, StrCoercible] = data_value
18
+ self.data_value: str | StrCoercible = data_value
20
19
  super().__init__(self.message)
21
20
 
22
21
  def __str__(self):
@@ -41,7 +40,7 @@ class TransformationRecordFailedError(TransformationError):
41
40
  def __init__(self, index_or_id, message="", data_value=""):
42
41
  self.index_or_id = index_or_id
43
42
  self.message = message
44
- self.data_value: Union[str, StrCoercible] = data_value
43
+ self.data_value: str | StrCoercible = data_value
45
44
  # logging.log(26, f"RECORD FAILED\t{self.id}\t{self.message}\t{self.data_value}")
46
45
  super().__init__(self.message)
47
46
 
@@ -70,7 +69,7 @@ class TransformationProcessError(TransformationError):
70
69
  index_or_id,
71
70
  message="Critical Process issue. Transformation failed."
72
71
  " Check configuration, mapping files and reference data",
73
- data_value: Union[str, StrCoercible]="",
72
+ data_value: str | StrCoercible = "",
74
73
  ):
75
74
  self.index_or_id = index_or_id
76
75
  self.message = message
@@ -22,6 +22,8 @@ class FolderStructure:
22
22
  self.add_time_stamp_to_file_names = add_time_stamp_to_file_names
23
23
  self.iteration_identifier = iteration_identifier
24
24
  self.base_folder = Path(base_path)
25
+ # Ensure the base folder exists and is a directory. This differs from other folders, which
26
+ # are created if missing.
25
27
  if not self.base_folder.is_dir():
26
28
  logging.critical("Base Folder Path is not a folder. Exiting.")
27
29
  sys.exit(1)
@@ -43,6 +45,10 @@ class FolderStructure:
43
45
  self.reports_folder = self.iteration_folder / "reports"
44
46
  self.verify_folder(self.reports_folder)
45
47
 
48
+ # Raw migration reports directory
49
+ self.raw_reports_folder = self.reports_folder / ".raw"
50
+ self.verify_folder(self.raw_reports_folder)
51
+
46
52
  def log_folder_structure(self):
47
53
  logging.info("Mapping files folder is %s", self.mapping_files_folder)
48
54
  logging.info("Git ignore is set up correctly")
@@ -58,7 +64,7 @@ class FolderStructure:
58
64
  logging.info("Migration report file will be saved at %s", self.migration_reports_file)
59
65
 
60
66
  def setup_migration_file_structure(self, source_file_type: str = ""):
61
- self.time_stamp = f'_{time.strftime("%Y%m%d-%H%M%S")}'
67
+ self.time_stamp = f"_{time.strftime('%Y%m%d-%H%M%S')}"
62
68
  self.time_str = self.time_stamp if self.add_time_stamp_to_file_names else ""
63
69
  self.file_template = f"{self.time_str}_{self.migration_task_name}"
64
70
  object_type_string = str(self.object_type.name).lower()
@@ -98,6 +104,10 @@ class FolderStructure:
98
104
 
99
105
  self.migration_reports_file = self.reports_folder / f"report{self.file_template}.md"
100
106
 
107
+ self.migration_reports_raw_file = (
108
+ self.raw_reports_folder / f"raw_report{self.file_template}.json"
109
+ )
110
+
101
111
  self.srs_records_path = (
102
112
  self.results_folder / f"folio_srs_{object_type_string}{self.file_template}.json"
103
113
  )
@@ -110,9 +120,6 @@ class FolderStructure:
110
120
  self.instance_id_map_path = (
111
121
  self.results_folder / f"{str(FOLIONamespaces.instances.name).lower()}_id_map.json"
112
122
  )
113
- self.auth_id_map_path = (
114
- self.results_folder / f"{str(FOLIONamespaces.authorities.name).lower()}_id_map.json"
115
- )
116
123
 
117
124
  self.holdings_id_map_path = (
118
125
  self.results_folder / f"{str(FOLIONamespaces.holdings.name).lower()}_id_map.json"
@@ -131,10 +138,13 @@ class FolderStructure:
131
138
  self.item_statuses_map_path = self.mapping_files_folder / "item_statuses.tsv"
132
139
 
133
140
  def verify_folder(self, folder_path: Path):
134
- if not folder_path.is_dir():
135
- logging.critical("There is no folder located at %s. Exiting.", folder_path)
136
- logging.critical("Create a folder by calling\n\tmkdir %s", folder_path)
141
+ if folder_path.exists() and not folder_path.is_dir():
142
+ logging.critical("Path exists but is not a directory: %s", folder_path)
137
143
  sys.exit(1)
144
+
145
+ if not folder_path.exists():
146
+ logging.info("Creating missing folder %s", folder_path)
147
+ folder_path.mkdir(parents=True, exist_ok=True)
138
148
  else:
139
149
  logging.info("Located %s", folder_path)
140
150
 
@@ -1,6 +1,7 @@
1
1
  import json
2
2
  import logging
3
- import i18n
3
+
4
+ from folio_migration_tools.i18n_cache import i18n_t
4
5
 
5
6
 
6
7
  class Helper:
@@ -9,15 +10,15 @@ class Helper:
9
10
  report_file, total_records: int, mapped_folio_fields, mapped_legacy_fields
10
11
  ):
11
12
  details_start = (
12
- "<details><summary>" + i18n.t("Click to expand field report") + "</summary>\n\n"
13
+ "<details><summary>" + i18n_t("Click to expand field report") + "</summary>\n\n"
13
14
  )
14
15
  details_end = "</details>\n"
15
- report_file.write("\n## " + i18n.t("Mapped FOLIO fields") + "\n")
16
+ report_file.write("\n## " + i18n_t("Mapped FOLIO fields") + "\n")
16
17
  # report_file.write(f"{blurbs[header]}\n")
17
18
 
18
19
  d_sorted = {k: mapped_folio_fields[k] for k in sorted(mapped_folio_fields)}
19
20
  report_file.write(details_start)
20
- columns = [i18n.t("FOLIO Field"), i18n.t("Mapped"), i18n.t("Unmapped")]
21
+ columns = [i18n_t("FOLIO Field"), i18n_t("Mapped"), i18n_t("Unmapped")]
21
22
  report_file.write(" | ".join(columns) + "\n")
22
23
  report_file.write("|".join(len(columns) * ["---"]) + "\n")
23
24
  for k, v in d_sorted.items():
@@ -32,12 +33,12 @@ class Helper:
32
33
  )
33
34
  report_file.write(details_end)
34
35
 
35
- report_file.write("\n## " + i18n.t("Mapped Legacy fields") + "\n")
36
+ report_file.write("\n## " + i18n_t("Mapped Legacy fields") + "\n")
36
37
  # report_file.write(f"{blurbs[header]}\n")
37
38
 
38
39
  d_sorted = {k: mapped_legacy_fields[k] for k in sorted(mapped_legacy_fields)}
39
40
  report_file.write(details_start)
40
- columns = [i18n.t("Legacy Field"), i18n.t("Present"), i18n.t("Mapped"), i18n.t("Unmapped")]
41
+ columns = [i18n_t("Legacy Field"), i18n_t("Present"), i18n_t("Mapped"), i18n_t("Unmapped")]
41
42
  report_file.write("|".join(columns) + "\n")
42
43
  report_file.write("|".join(len(columns) * ["---"]) + "\n")
43
44
  for k, v in d_sorted.items():
@@ -56,7 +57,7 @@ class Helper:
56
57
  @staticmethod
57
58
  def log_data_issue(index_or_id, message, legacy_value):
58
59
  logging.log(26, "DATA ISSUE\t%s\t%s\t%s", index_or_id, message, legacy_value)
59
-
60
+
60
61
  @staticmethod
61
62
  def log_data_issue_failed(index_or_id, message, legacy_value):
62
63
  logging.log(26, "RECORD FAILED\t%s\t%s\t%s", index_or_id, message, legacy_value)