folio-migration-tools 1.9.10__py3-none-any.whl → 1.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__init__.py +3 -4
- folio_migration_tools/__main__.py +53 -31
- folio_migration_tools/circulation_helper.py +118 -108
- folio_migration_tools/custom_dict.py +2 -2
- folio_migration_tools/custom_exceptions.py +4 -5
- folio_migration_tools/folder_structure.py +17 -7
- folio_migration_tools/helper.py +8 -7
- folio_migration_tools/holdings_helper.py +4 -3
- folio_migration_tools/i18n_cache.py +79 -0
- folio_migration_tools/library_configuration.py +77 -37
- folio_migration_tools/mapper_base.py +45 -31
- folio_migration_tools/mapping_file_transformation/courses_mapper.py +1 -1
- folio_migration_tools/mapping_file_transformation/holdings_mapper.py +7 -3
- folio_migration_tools/mapping_file_transformation/item_mapper.py +13 -26
- folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py +1 -2
- folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +13 -11
- folio_migration_tools/mapping_file_transformation/order_mapper.py +6 -5
- folio_migration_tools/mapping_file_transformation/organization_mapper.py +3 -3
- folio_migration_tools/mapping_file_transformation/user_mapper.py +47 -28
- folio_migration_tools/marc_rules_transformation/conditions.py +82 -97
- folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +13 -5
- folio_migration_tools/marc_rules_transformation/hrid_handler.py +3 -2
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +26 -24
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +56 -51
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +28 -17
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +68 -37
- folio_migration_tools/migration_report.py +18 -7
- folio_migration_tools/migration_tasks/batch_poster.py +285 -354
- folio_migration_tools/migration_tasks/bibs_transformer.py +14 -9
- folio_migration_tools/migration_tasks/courses_migrator.py +2 -3
- folio_migration_tools/migration_tasks/holdings_csv_transformer.py +23 -24
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +14 -24
- folio_migration_tools/migration_tasks/items_transformer.py +23 -34
- folio_migration_tools/migration_tasks/loans_migrator.py +67 -144
- folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py +3 -3
- folio_migration_tools/migration_tasks/migration_task_base.py +47 -60
- folio_migration_tools/migration_tasks/orders_transformer.py +25 -42
- folio_migration_tools/migration_tasks/organization_transformer.py +9 -18
- folio_migration_tools/migration_tasks/requests_migrator.py +21 -24
- folio_migration_tools/migration_tasks/reserves_migrator.py +6 -5
- folio_migration_tools/migration_tasks/user_transformer.py +25 -20
- folio_migration_tools/task_configuration.py +6 -7
- folio_migration_tools/transaction_migration/legacy_loan.py +15 -27
- folio_migration_tools/transaction_migration/legacy_request.py +1 -1
- folio_migration_tools/translations/en.json +0 -7
- {folio_migration_tools-1.9.10.dist-info → folio_migration_tools-1.10.0.dist-info}/METADATA +19 -28
- folio_migration_tools-1.10.0.dist-info/RECORD +63 -0
- folio_migration_tools-1.10.0.dist-info/WHEEL +4 -0
- folio_migration_tools-1.10.0.dist-info/entry_points.txt +3 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +0 -241
- folio_migration_tools/migration_tasks/authority_transformer.py +0 -119
- folio_migration_tools/test_infrastructure/__init__.py +0 -0
- folio_migration_tools/test_infrastructure/mocked_classes.py +0 -406
- folio_migration_tools-1.9.10.dist-info/RECORD +0 -67
- folio_migration_tools-1.9.10.dist-info/WHEEL +0 -4
- folio_migration_tools-1.9.10.dist-info/entry_points.txt +0 -3
- folio_migration_tools-1.9.10.dist-info/licenses/LICENSE +0 -21
|
@@ -3,9 +3,8 @@ from typing import Protocol
|
|
|
3
3
|
|
|
4
4
|
__version__ = importlib.metadata.version("folio_migration_tools")
|
|
5
5
|
|
|
6
|
+
|
|
6
7
|
class StrCoercible(Protocol):
|
|
7
|
-
def __repr__(self) -> str:
|
|
8
|
-
...
|
|
8
|
+
def __repr__(self) -> str: ...
|
|
9
9
|
|
|
10
|
-
def __str__(self) -> str:
|
|
11
|
-
...
|
|
10
|
+
def __str__(self) -> str: ...
|
|
@@ -4,6 +4,7 @@ import logging
|
|
|
4
4
|
import sys
|
|
5
5
|
from os import environ
|
|
6
6
|
from pathlib import Path
|
|
7
|
+
from warnings import warn
|
|
7
8
|
|
|
8
9
|
import httpx
|
|
9
10
|
import humps
|
|
@@ -33,23 +34,25 @@ def parse_args(args):
|
|
|
33
34
|
|
|
34
35
|
parser.add_argument(
|
|
35
36
|
"task_name",
|
|
36
|
-
help=("Task name. Use one of:
|
|
37
|
+
help=(f"Task name. Use one of: {tasks_string}"),
|
|
37
38
|
nargs="?" if "FOLIO_MIGRATION_TOOLS_TASK_NAME" in environ else None,
|
|
38
39
|
prompt="FOLIO_MIGRATION_TOOLS_TASK_NAME" not in environ,
|
|
39
40
|
default=environ.get("FOLIO_MIGRATION_TOOLS_TASK_NAME"),
|
|
40
41
|
)
|
|
41
42
|
parser.add_argument(
|
|
42
|
-
"--folio_password",
|
|
43
|
+
"--folio_password",
|
|
44
|
+
"--okapi_password",
|
|
43
45
|
help="password for the tenant in the configuration file",
|
|
44
46
|
prompt="FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD" not in environ,
|
|
45
|
-
default=environ.get(
|
|
47
|
+
default=environ.get(
|
|
48
|
+
"FOLIO_MIGRATION_TOOLS_FOLIO_PASSWORD",
|
|
49
|
+
environ.get("FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD"),
|
|
50
|
+
),
|
|
46
51
|
secure=True,
|
|
47
52
|
)
|
|
48
53
|
parser.add_argument(
|
|
49
54
|
"--base_folder_path",
|
|
50
|
-
help=(
|
|
51
|
-
"path to the base folder for this library. Built on migration_repo_template"
|
|
52
|
-
),
|
|
55
|
+
help=("path to the base folder for this library. Built on migration_repo_template"),
|
|
53
56
|
prompt="FOLIO_MIGRATION_TOOLS_BASE_FOLDER_PATH" not in environ,
|
|
54
57
|
default=environ.get("FOLIO_MIGRATION_TOOLS_BASE_FOLDER_PATH"),
|
|
55
58
|
)
|
|
@@ -62,34 +65,43 @@ def parse_args(args):
|
|
|
62
65
|
prompt=False,
|
|
63
66
|
)
|
|
64
67
|
parser.add_argument(
|
|
65
|
-
"--version",
|
|
68
|
+
"--version",
|
|
69
|
+
"-V",
|
|
66
70
|
help="Show the version of the FOLIO Migration Tools",
|
|
67
71
|
action="store_true",
|
|
68
72
|
prompt=False,
|
|
69
73
|
)
|
|
70
74
|
return parser.parse_args(args)
|
|
71
75
|
|
|
76
|
+
|
|
72
77
|
def prep_library_config(args):
|
|
73
|
-
|
|
74
|
-
config_file_humped["libraryInformation"]["okapiPassword"] = args.folio_password
|
|
75
|
-
config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
|
|
76
|
-
config_file = humps.decamelize(config_file_humped)
|
|
77
|
-
library_config = LibraryConfiguration(**config_file["library_information"])
|
|
78
|
-
if library_config.ecs_tenant_id:
|
|
79
|
-
library_config.is_ecs = True
|
|
80
|
-
if library_config.ecs_tenant_id and not library_config.ecs_central_iteration_identifier:
|
|
81
|
-
print(
|
|
82
|
-
"ECS tenant ID is set, but no central iteration identifier is provided. "
|
|
83
|
-
"Please provide the central iteration identifier in the configuration file."
|
|
84
|
-
)
|
|
85
|
-
sys.exit("ECS Central Iteration Identifier Not Found")
|
|
86
|
-
return config_file, library_config
|
|
78
|
+
config_file_humped = merge_load(args.configuration_path)
|
|
87
79
|
|
|
88
|
-
|
|
89
|
-
|
|
80
|
+
# Only set folioPassword if neither folioPassword nor okapiPassword exist in config
|
|
81
|
+
# The Pydantic validator will handle backward compatibility for existing okapiPassword
|
|
82
|
+
if (
|
|
83
|
+
"folioPassword" not in config_file_humped["libraryInformation"]
|
|
84
|
+
and "okapiPassword" not in config_file_humped["libraryInformation"]
|
|
85
|
+
):
|
|
86
|
+
config_file_humped["libraryInformation"]["folioPassword"] = args.folio_password
|
|
87
|
+
|
|
88
|
+
config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
|
|
89
|
+
config_file = humps.decamelize(config_file_humped)
|
|
90
|
+
library_config = LibraryConfiguration(**config_file["library_information"])
|
|
91
|
+
if library_config.ecs_tenant_id:
|
|
92
|
+
library_config.is_ecs = True
|
|
93
|
+
if library_config.ecs_tenant_id and not library_config.ecs_central_iteration_identifier:
|
|
90
94
|
print(
|
|
91
|
-
|
|
95
|
+
"ECS tenant ID is set, but no central iteration identifier is provided. "
|
|
96
|
+
"Please provide the central iteration identifier in the configuration file."
|
|
92
97
|
)
|
|
98
|
+
sys.exit("ECS Central Iteration Identifier Not Found")
|
|
99
|
+
return config_file, library_config
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def print_version(args):
|
|
103
|
+
if "-V" in args or "--version" in args:
|
|
104
|
+
print(f"FOLIO Migration Tools: {metadata.version('folio_migration_tools')}")
|
|
93
105
|
sys.exit(0)
|
|
94
106
|
return None
|
|
95
107
|
|
|
@@ -115,6 +127,14 @@ def main():
|
|
|
115
127
|
i18n.set("locale", args.report_language)
|
|
116
128
|
config_file, library_config = prep_library_config(args)
|
|
117
129
|
try:
|
|
130
|
+
if args.task_name == "AuthorityTransformer":
|
|
131
|
+
warn(
|
|
132
|
+
"The AuthorityTransformer has been removed."
|
|
133
|
+
" Please update your configuration accordingly."
|
|
134
|
+
" Use Data Import to load authority records.",
|
|
135
|
+
DeprecationWarning,
|
|
136
|
+
stacklevel=2,
|
|
137
|
+
)
|
|
118
138
|
migration_task_config = next(
|
|
119
139
|
t for t in config_file["migration_tasks"] if t["name"] == args.task_name
|
|
120
140
|
)
|
|
@@ -122,7 +142,7 @@ def main():
|
|
|
122
142
|
task_names = [t.get("name", "") for t in config_file["migration_tasks"]]
|
|
123
143
|
print(
|
|
124
144
|
f"Referenced task name {args.task_name} not found in the "
|
|
125
|
-
f
|
|
145
|
+
f"configuration file. Use one of {', '.join(task_names)}"
|
|
126
146
|
"\nHalting..."
|
|
127
147
|
)
|
|
128
148
|
sys.exit("Task Name Not Found")
|
|
@@ -134,13 +154,15 @@ def main():
|
|
|
134
154
|
)
|
|
135
155
|
except StopIteration:
|
|
136
156
|
print(
|
|
137
|
-
f
|
|
157
|
+
f"Referenced task {migration_task_config['migration_task_type']} "
|
|
138
158
|
"is not a valid option. Update your task to incorporate "
|
|
139
159
|
f"one of {json.dumps([tc.__name__ for tc in task_classes], indent=4)}"
|
|
140
160
|
)
|
|
141
161
|
sys.exit("Task Type Not Found")
|
|
142
162
|
try:
|
|
143
|
-
logging.getLogger("httpx").setLevel(
|
|
163
|
+
logging.getLogger("httpx").setLevel(
|
|
164
|
+
logging.WARNING
|
|
165
|
+
) # Exclude info messages from httpx
|
|
144
166
|
with FolioClient(
|
|
145
167
|
library_config.gateway_url,
|
|
146
168
|
library_config.tenant_id,
|
|
@@ -161,16 +183,15 @@ def main():
|
|
|
161
183
|
logging.critical(json_error)
|
|
162
184
|
print(json_error.doc)
|
|
163
185
|
print(
|
|
164
|
-
f"\n{json_error}"
|
|
165
|
-
f"\nError parsing the above JSON mapping or configruation file. Halting."
|
|
186
|
+
f"\n{json_error}\nError parsing the above JSON mapping or configruation file. Halting."
|
|
166
187
|
)
|
|
167
188
|
sys.exit("Invalid JSON")
|
|
168
189
|
except ValidationError as e:
|
|
169
|
-
print(e.
|
|
190
|
+
print(json.dumps(e.errors(), indent=2))
|
|
170
191
|
print("Validation errors in configuration file:")
|
|
171
192
|
print("==========================================")
|
|
172
193
|
|
|
173
|
-
for validation_message in
|
|
194
|
+
for validation_message in e.errors():
|
|
174
195
|
print(
|
|
175
196
|
f"{validation_message['msg']}\t"
|
|
176
197
|
f"{', '.join(humps.camelize(str(x)) for x in validation_message['loc'])}"
|
|
@@ -199,6 +220,7 @@ def main():
|
|
|
199
220
|
sys.exit(ee.__class__.__name__)
|
|
200
221
|
sys.exit(0)
|
|
201
222
|
|
|
223
|
+
|
|
202
224
|
def inheritors(base_class):
|
|
203
225
|
subclasses = set()
|
|
204
226
|
work = [base_class]
|
|
@@ -3,14 +3,15 @@ import json
|
|
|
3
3
|
import logging
|
|
4
4
|
import re
|
|
5
5
|
import time
|
|
6
|
+
from http import HTTPStatus
|
|
6
7
|
from typing import Set
|
|
7
8
|
|
|
8
9
|
import httpx
|
|
9
10
|
import i18n
|
|
10
|
-
from folioclient import FolioClient
|
|
11
|
-
from httpx import HTTPError
|
|
11
|
+
from folioclient import FolioClient, FolioClientError, FolioConnectionError, FolioValidationError
|
|
12
12
|
|
|
13
13
|
from folio_migration_tools.helper import Helper
|
|
14
|
+
from folio_migration_tools.i18n_cache import i18n_t
|
|
14
15
|
from folio_migration_tools.migration_report import MigrationReport
|
|
15
16
|
from folio_migration_tools.transaction_migration.legacy_loan import LegacyLoan
|
|
16
17
|
from folio_migration_tools.transaction_migration.legacy_request import LegacyRequest
|
|
@@ -37,7 +38,7 @@ class CirculationHelper:
|
|
|
37
38
|
def get_user_by_barcode(self, user_barcode):
|
|
38
39
|
if user_barcode in self.missing_patron_barcodes:
|
|
39
40
|
self.migration_report.add_general_statistics(
|
|
40
|
-
|
|
41
|
+
i18n_t("Users already detected as missing")
|
|
41
42
|
)
|
|
42
43
|
logging.info("User is already detected as missing")
|
|
43
44
|
return {}
|
|
@@ -55,7 +56,7 @@ class CirculationHelper:
|
|
|
55
56
|
def get_item_by_barcode(self, item_barcode):
|
|
56
57
|
if item_barcode in self.missing_item_barcodes:
|
|
57
58
|
self.migration_report.add_general_statistics(
|
|
58
|
-
|
|
59
|
+
i18n_t("Items already detected as missing")
|
|
59
60
|
)
|
|
60
61
|
logging.info("Item is already detected as missing")
|
|
61
62
|
return {}
|
|
@@ -138,117 +139,117 @@ class CirculationHelper:
|
|
|
138
139
|
if legacy_loan.proxy_patron_barcode:
|
|
139
140
|
data.update({"proxyUserBarcode": legacy_loan.proxy_patron_barcode})
|
|
140
141
|
path = "/circulation/check-out-by-barcode"
|
|
141
|
-
url = f"{self.folio_client.gateway_url}{path}"
|
|
142
142
|
try:
|
|
143
143
|
if legacy_loan.patron_barcode in self.missing_patron_barcodes:
|
|
144
|
-
error_message =
|
|
144
|
+
error_message = i18n_t("Patron barcode already detected as missing")
|
|
145
145
|
logging.error(
|
|
146
146
|
f"{error_message} Patron barcode: {legacy_loan.patron_barcode} "
|
|
147
147
|
f"Item Barcode:{legacy_loan.item_barcode}"
|
|
148
148
|
)
|
|
149
149
|
return TransactionResult(False, False, "", error_message, error_message)
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
None,
|
|
178
|
-
error_message_from_folio,
|
|
179
|
-
stat_message,
|
|
180
|
-
)
|
|
181
|
-
|
|
182
|
-
elif "find user with matching barcode" in error_message_from_folio:
|
|
183
|
-
self.missing_patron_barcodes.add(legacy_loan.patron_barcode)
|
|
184
|
-
error_message = f"No patron with barcode {legacy_loan.patron_barcode} in FOLIO"
|
|
185
|
-
stat_message = i18n.t("Patron barcode not in FOLIO")
|
|
186
|
-
return TransactionResult(
|
|
187
|
-
False,
|
|
188
|
-
False,
|
|
189
|
-
None,
|
|
190
|
-
error_message_from_folio,
|
|
191
|
-
stat_message,
|
|
192
|
-
)
|
|
193
|
-
elif "Cannot check out item that already has an open" in error_message_from_folio:
|
|
194
|
-
return TransactionResult(
|
|
195
|
-
False,
|
|
196
|
-
False,
|
|
197
|
-
None,
|
|
198
|
-
error_message_from_folio,
|
|
199
|
-
error_message_from_folio,
|
|
200
|
-
)
|
|
201
|
-
elif "Item is already checked out" in error_message_from_folio:
|
|
202
|
-
return TransactionResult(
|
|
203
|
-
False,
|
|
204
|
-
True,
|
|
205
|
-
None,
|
|
206
|
-
error_message_from_folio,
|
|
207
|
-
error_message_from_folio,
|
|
208
|
-
)
|
|
209
|
-
logging.error(
|
|
210
|
-
f"{error_message} "
|
|
211
|
-
f"Patron barcode: {legacy_loan.patron_barcode} "
|
|
212
|
-
f"Item Barcode:{legacy_loan.item_barcode}"
|
|
150
|
+
loan = self.folio_client.folio_post(path, data)
|
|
151
|
+
stats = "Successfully checked out by barcode"
|
|
152
|
+
logging.debug(
|
|
153
|
+
"%s (item barcode %s}) in %ss",
|
|
154
|
+
stats,
|
|
155
|
+
legacy_loan.item_barcode,
|
|
156
|
+
f"{(time.time() - t0_function):.2f}",
|
|
157
|
+
)
|
|
158
|
+
return TransactionResult(True, False, loan, "", stats)
|
|
159
|
+
except FolioValidationError as fve:
|
|
160
|
+
error_message_from_folio = self.folio_client.handle_json_response(fve.response)[
|
|
161
|
+
"errors"
|
|
162
|
+
][0]["message"]
|
|
163
|
+
stat_message = error_message_from_folio
|
|
164
|
+
error_message = error_message_from_folio
|
|
165
|
+
if "has the item status" in error_message_from_folio:
|
|
166
|
+
stat_message = re.findall(
|
|
167
|
+
r"(?<=has the item status\s).*(?=\sand cannot be checked out)",
|
|
168
|
+
error_message_from_folio,
|
|
169
|
+
)[0]
|
|
170
|
+
error_message = f"{stat_message} for item with barcode {legacy_loan.item_barcode}"
|
|
171
|
+
return TransactionResult(
|
|
172
|
+
False,
|
|
173
|
+
True,
|
|
174
|
+
None,
|
|
175
|
+
error_message_from_folio,
|
|
176
|
+
stat_message,
|
|
213
177
|
)
|
|
214
|
-
|
|
178
|
+
elif "No item with barcode" in error_message_from_folio:
|
|
179
|
+
error_message = f"No item with barcode {legacy_loan.item_barcode} in FOLIO"
|
|
180
|
+
stat_message = "Item barcode not in FOLIO"
|
|
181
|
+
self.missing_item_barcodes.add(legacy_loan.item_barcode)
|
|
215
182
|
return TransactionResult(
|
|
216
|
-
False,
|
|
183
|
+
False,
|
|
184
|
+
False,
|
|
185
|
+
None,
|
|
186
|
+
error_message_from_folio,
|
|
187
|
+
stat_message,
|
|
217
188
|
)
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
189
|
+
|
|
190
|
+
elif "find user with matching barcode" in error_message_from_folio:
|
|
191
|
+
self.missing_patron_barcodes.add(legacy_loan.patron_barcode)
|
|
192
|
+
error_message = f"No patron with barcode {legacy_loan.patron_barcode} in FOLIO"
|
|
193
|
+
stat_message = i18n_t("Patron barcode not in FOLIO")
|
|
194
|
+
return TransactionResult(
|
|
195
|
+
False,
|
|
196
|
+
False,
|
|
197
|
+
None,
|
|
198
|
+
error_message_from_folio,
|
|
199
|
+
stat_message,
|
|
225
200
|
)
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
req.status_code,
|
|
201
|
+
elif "Cannot check out item that already has an open" in error_message_from_folio:
|
|
202
|
+
return TransactionResult(
|
|
203
|
+
False,
|
|
204
|
+
False,
|
|
205
|
+
None,
|
|
206
|
+
error_message_from_folio,
|
|
207
|
+
error_message_from_folio,
|
|
234
208
|
)
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
209
|
+
elif "Item is already checked out" in error_message_from_folio:
|
|
210
|
+
return TransactionResult(
|
|
211
|
+
False,
|
|
212
|
+
True,
|
|
213
|
+
None,
|
|
214
|
+
error_message_from_folio,
|
|
215
|
+
error_message_from_folio,
|
|
216
|
+
)
|
|
217
|
+
logging.error(
|
|
218
|
+
f"{error_message} "
|
|
219
|
+
f"Patron barcode: {legacy_loan.patron_barcode} "
|
|
220
|
+
f"Item Barcode:{legacy_loan.item_barcode}"
|
|
221
|
+
)
|
|
222
|
+
self.migration_report.add("Details", stat_message)
|
|
223
|
+
return TransactionResult(
|
|
224
|
+
False, True, None, error_message, f"Check out error: {stat_message}"
|
|
225
|
+
)
|
|
226
|
+
except FolioClientError as fce:
|
|
239
227
|
logging.exception(
|
|
240
228
|
"%s\tPOST FAILED %s\n\t%s\n\t%s",
|
|
241
|
-
|
|
242
|
-
url,
|
|
229
|
+
fce.response.status_code,
|
|
230
|
+
fce.request.url,
|
|
243
231
|
json.dumps(data),
|
|
244
|
-
|
|
232
|
+
fce.response.text,
|
|
245
233
|
)
|
|
246
234
|
return TransactionResult(
|
|
247
235
|
False,
|
|
248
236
|
False,
|
|
249
237
|
None,
|
|
250
238
|
"5XX",
|
|
251
|
-
i18n.t("Failed checkout http status %{code}", code=
|
|
239
|
+
i18n.t("Failed checkout http status %{code}", code=fce.response.status_code),
|
|
240
|
+
)
|
|
241
|
+
except FolioConnectionError as fce:
|
|
242
|
+
logging.exception(
|
|
243
|
+
"Connection error\tPOST FAILED %s\n\t%s\n\t%s",
|
|
244
|
+
fce.request.url,
|
|
245
|
+
json.dumps(data),
|
|
246
|
+
)
|
|
247
|
+
return TransactionResult(
|
|
248
|
+
False,
|
|
249
|
+
False,
|
|
250
|
+
None,
|
|
251
|
+
"Connection error",
|
|
252
|
+
i18n_t("Connection error during checkout"),
|
|
252
253
|
)
|
|
253
254
|
|
|
254
255
|
@staticmethod
|
|
@@ -257,7 +258,6 @@ class CirculationHelper:
|
|
|
257
258
|
):
|
|
258
259
|
try:
|
|
259
260
|
path = "/circulation/requests"
|
|
260
|
-
url = f"{folio_client.gateway_url}{path}"
|
|
261
261
|
data = legacy_request.serialize()
|
|
262
262
|
data["requestProcessingParameters"] = {
|
|
263
263
|
"overrideBlocks": {
|
|
@@ -269,21 +269,31 @@ class CirculationHelper:
|
|
|
269
269
|
"comment": "Migrated from legacy system",
|
|
270
270
|
}
|
|
271
271
|
}
|
|
272
|
-
|
|
273
|
-
logging.debug(f"POST {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
272
|
+
_ = folio_client.folio_post(path, data)
|
|
273
|
+
logging.debug(f"POST {path}\t{json.dumps(data)}")
|
|
274
|
+
logging.info(
|
|
275
|
+
"%s Successfully created %s",
|
|
276
|
+
HTTPStatus.OK,
|
|
277
|
+
legacy_request.request_type,
|
|
278
|
+
)
|
|
279
|
+
return True
|
|
280
|
+
except FolioValidationError as fve:
|
|
281
|
+
message = folio_client.handle_json_response(fve.response)["errors"][0]["message"]
|
|
282
|
+
logging.error(message)
|
|
283
|
+
migration_report.add_general_statistics(message)
|
|
284
|
+
return False
|
|
285
|
+
except (FolioConnectionError, FolioClientError) as fce:
|
|
286
|
+
if client_response := getattr(fce, "response", None):
|
|
287
|
+
message = (
|
|
288
|
+
f"HTTP {client_response.status_code} Error creating request: "
|
|
289
|
+
f"{client_response.text}"
|
|
290
|
+
)
|
|
291
|
+
logging.error(message)
|
|
277
292
|
migration_report.add_general_statistics(message)
|
|
278
|
-
return False
|
|
279
293
|
else:
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
req.status_code,
|
|
284
|
-
legacy_request.request_type,
|
|
285
|
-
)
|
|
286
|
-
return True
|
|
294
|
+
logging.error(f"Connection error creating request: {fce}")
|
|
295
|
+
migration_report.add_general_statistics("Connection error creating request")
|
|
296
|
+
return False
|
|
287
297
|
except Exception as exception:
|
|
288
298
|
logging.error(exception, exc_info=True)
|
|
289
299
|
migration_report.add("Details", exception)
|
|
@@ -7,10 +7,10 @@ class InsensitiveDictReader(csv.DictReader):
|
|
|
7
7
|
# spaces and to lower case.
|
|
8
8
|
@property
|
|
9
9
|
def fieldnames(self):
|
|
10
|
-
return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)]
|
|
10
|
+
return [field.strip().lower() for field in csv.DictReader.fieldnames.fget(self)] # type: ignore
|
|
11
11
|
|
|
12
12
|
def next(self):
|
|
13
|
-
return InsensitiveDict(csv.DictReader.next(self))
|
|
13
|
+
return InsensitiveDict(csv.DictReader.next(self)) # type: ignore
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class InsensitiveDict(dict):
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import Union
|
|
3
2
|
import i18n
|
|
4
3
|
|
|
5
4
|
from folio_migration_tools import StrCoercible
|
|
@@ -13,10 +12,10 @@ class TransformationFieldMappingError(TransformationError):
|
|
|
13
12
|
"""Raised when the field mapping fails, but the error is not critical.
|
|
14
13
|
The issue should be logged for the library to act upon it"""
|
|
15
14
|
|
|
16
|
-
def __init__(self, index_or_id="", message="", data_value:
|
|
15
|
+
def __init__(self, index_or_id="", message="", data_value: str | StrCoercible = ""):
|
|
17
16
|
self.index_or_id = index_or_id or ""
|
|
18
17
|
self.message = message
|
|
19
|
-
self.data_value:
|
|
18
|
+
self.data_value: str | StrCoercible = data_value
|
|
20
19
|
super().__init__(self.message)
|
|
21
20
|
|
|
22
21
|
def __str__(self):
|
|
@@ -41,7 +40,7 @@ class TransformationRecordFailedError(TransformationError):
|
|
|
41
40
|
def __init__(self, index_or_id, message="", data_value=""):
|
|
42
41
|
self.index_or_id = index_or_id
|
|
43
42
|
self.message = message
|
|
44
|
-
self.data_value:
|
|
43
|
+
self.data_value: str | StrCoercible = data_value
|
|
45
44
|
# logging.log(26, f"RECORD FAILED\t{self.id}\t{self.message}\t{self.data_value}")
|
|
46
45
|
super().__init__(self.message)
|
|
47
46
|
|
|
@@ -70,7 +69,7 @@ class TransformationProcessError(TransformationError):
|
|
|
70
69
|
index_or_id,
|
|
71
70
|
message="Critical Process issue. Transformation failed."
|
|
72
71
|
" Check configuration, mapping files and reference data",
|
|
73
|
-
data_value:
|
|
72
|
+
data_value: str | StrCoercible = "",
|
|
74
73
|
):
|
|
75
74
|
self.index_or_id = index_or_id
|
|
76
75
|
self.message = message
|
|
@@ -22,6 +22,8 @@ class FolderStructure:
|
|
|
22
22
|
self.add_time_stamp_to_file_names = add_time_stamp_to_file_names
|
|
23
23
|
self.iteration_identifier = iteration_identifier
|
|
24
24
|
self.base_folder = Path(base_path)
|
|
25
|
+
# Ensure the base folder exists and is a directory. This differs from other folders, which
|
|
26
|
+
# are created if missing.
|
|
25
27
|
if not self.base_folder.is_dir():
|
|
26
28
|
logging.critical("Base Folder Path is not a folder. Exiting.")
|
|
27
29
|
sys.exit(1)
|
|
@@ -43,6 +45,10 @@ class FolderStructure:
|
|
|
43
45
|
self.reports_folder = self.iteration_folder / "reports"
|
|
44
46
|
self.verify_folder(self.reports_folder)
|
|
45
47
|
|
|
48
|
+
# Raw migration reports directory
|
|
49
|
+
self.raw_reports_folder = self.reports_folder / ".raw"
|
|
50
|
+
self.verify_folder(self.raw_reports_folder)
|
|
51
|
+
|
|
46
52
|
def log_folder_structure(self):
|
|
47
53
|
logging.info("Mapping files folder is %s", self.mapping_files_folder)
|
|
48
54
|
logging.info("Git ignore is set up correctly")
|
|
@@ -58,7 +64,7 @@ class FolderStructure:
|
|
|
58
64
|
logging.info("Migration report file will be saved at %s", self.migration_reports_file)
|
|
59
65
|
|
|
60
66
|
def setup_migration_file_structure(self, source_file_type: str = ""):
|
|
61
|
-
self.time_stamp = f
|
|
67
|
+
self.time_stamp = f"_{time.strftime('%Y%m%d-%H%M%S')}"
|
|
62
68
|
self.time_str = self.time_stamp if self.add_time_stamp_to_file_names else ""
|
|
63
69
|
self.file_template = f"{self.time_str}_{self.migration_task_name}"
|
|
64
70
|
object_type_string = str(self.object_type.name).lower()
|
|
@@ -98,6 +104,10 @@ class FolderStructure:
|
|
|
98
104
|
|
|
99
105
|
self.migration_reports_file = self.reports_folder / f"report{self.file_template}.md"
|
|
100
106
|
|
|
107
|
+
self.migration_reports_raw_file = (
|
|
108
|
+
self.raw_reports_folder / f"raw_report{self.file_template}.json"
|
|
109
|
+
)
|
|
110
|
+
|
|
101
111
|
self.srs_records_path = (
|
|
102
112
|
self.results_folder / f"folio_srs_{object_type_string}{self.file_template}.json"
|
|
103
113
|
)
|
|
@@ -110,9 +120,6 @@ class FolderStructure:
|
|
|
110
120
|
self.instance_id_map_path = (
|
|
111
121
|
self.results_folder / f"{str(FOLIONamespaces.instances.name).lower()}_id_map.json"
|
|
112
122
|
)
|
|
113
|
-
self.auth_id_map_path = (
|
|
114
|
-
self.results_folder / f"{str(FOLIONamespaces.authorities.name).lower()}_id_map.json"
|
|
115
|
-
)
|
|
116
123
|
|
|
117
124
|
self.holdings_id_map_path = (
|
|
118
125
|
self.results_folder / f"{str(FOLIONamespaces.holdings.name).lower()}_id_map.json"
|
|
@@ -131,10 +138,13 @@ class FolderStructure:
|
|
|
131
138
|
self.item_statuses_map_path = self.mapping_files_folder / "item_statuses.tsv"
|
|
132
139
|
|
|
133
140
|
def verify_folder(self, folder_path: Path):
|
|
134
|
-
if not folder_path.is_dir():
|
|
135
|
-
logging.critical("
|
|
136
|
-
logging.critical("Create a folder by calling\n\tmkdir %s", folder_path)
|
|
141
|
+
if folder_path.exists() and not folder_path.is_dir():
|
|
142
|
+
logging.critical("Path exists but is not a directory: %s", folder_path)
|
|
137
143
|
sys.exit(1)
|
|
144
|
+
|
|
145
|
+
if not folder_path.exists():
|
|
146
|
+
logging.info("Creating missing folder %s", folder_path)
|
|
147
|
+
folder_path.mkdir(parents=True, exist_ok=True)
|
|
138
148
|
else:
|
|
139
149
|
logging.info("Located %s", folder_path)
|
|
140
150
|
|
folio_migration_tools/helper.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
|
-
|
|
3
|
+
|
|
4
|
+
from folio_migration_tools.i18n_cache import i18n_t
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
class Helper:
|
|
@@ -9,15 +10,15 @@ class Helper:
|
|
|
9
10
|
report_file, total_records: int, mapped_folio_fields, mapped_legacy_fields
|
|
10
11
|
):
|
|
11
12
|
details_start = (
|
|
12
|
-
"<details><summary>" +
|
|
13
|
+
"<details><summary>" + i18n_t("Click to expand field report") + "</summary>\n\n"
|
|
13
14
|
)
|
|
14
15
|
details_end = "</details>\n"
|
|
15
|
-
report_file.write("\n## " +
|
|
16
|
+
report_file.write("\n## " + i18n_t("Mapped FOLIO fields") + "\n")
|
|
16
17
|
# report_file.write(f"{blurbs[header]}\n")
|
|
17
18
|
|
|
18
19
|
d_sorted = {k: mapped_folio_fields[k] for k in sorted(mapped_folio_fields)}
|
|
19
20
|
report_file.write(details_start)
|
|
20
|
-
columns = [
|
|
21
|
+
columns = [i18n_t("FOLIO Field"), i18n_t("Mapped"), i18n_t("Unmapped")]
|
|
21
22
|
report_file.write(" | ".join(columns) + "\n")
|
|
22
23
|
report_file.write("|".join(len(columns) * ["---"]) + "\n")
|
|
23
24
|
for k, v in d_sorted.items():
|
|
@@ -32,12 +33,12 @@ class Helper:
|
|
|
32
33
|
)
|
|
33
34
|
report_file.write(details_end)
|
|
34
35
|
|
|
35
|
-
report_file.write("\n## " +
|
|
36
|
+
report_file.write("\n## " + i18n_t("Mapped Legacy fields") + "\n")
|
|
36
37
|
# report_file.write(f"{blurbs[header]}\n")
|
|
37
38
|
|
|
38
39
|
d_sorted = {k: mapped_legacy_fields[k] for k in sorted(mapped_legacy_fields)}
|
|
39
40
|
report_file.write(details_start)
|
|
40
|
-
columns = [
|
|
41
|
+
columns = [i18n_t("Legacy Field"), i18n_t("Present"), i18n_t("Mapped"), i18n_t("Unmapped")]
|
|
41
42
|
report_file.write("|".join(columns) + "\n")
|
|
42
43
|
report_file.write("|".join(len(columns) * ["---"]) + "\n")
|
|
43
44
|
for k, v in d_sorted.items():
|
|
@@ -56,7 +57,7 @@ class Helper:
|
|
|
56
57
|
@staticmethod
|
|
57
58
|
def log_data_issue(index_or_id, message, legacy_value):
|
|
58
59
|
logging.log(26, "DATA ISSUE\t%s\t%s\t%s", index_or_id, message, legacy_value)
|
|
59
|
-
|
|
60
|
+
|
|
60
61
|
@staticmethod
|
|
61
62
|
def log_data_issue_failed(index_or_id, message, legacy_value):
|
|
62
63
|
logging.log(26, "RECORD FAILED\t%s\t%s\t%s", index_or_id, message, legacy_value)
|