folio-migration-tools 1.9.0rc9__py3-none-any.whl → 1.9.0rc11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__main__.py +23 -5
- folio_migration_tools/circulation_helper.py +3 -3
- folio_migration_tools/library_configuration.py +54 -6
- folio_migration_tools/mapper_base.py +2 -2
- folio_migration_tools/marc_rules_transformation/hrid_handler.py +1 -1
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +21 -23
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +3 -0
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +52 -37
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +131 -21
- folio_migration_tools/migration_tasks/batch_poster.py +7 -7
- folio_migration_tools/migration_tasks/bibs_transformer.py +6 -59
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +61 -23
- folio_migration_tools/migration_tasks/loans_migrator.py +5 -5
- folio_migration_tools/migration_tasks/migration_task_base.py +64 -1
- folio_migration_tools/migration_tasks/reserves_migrator.py +1 -1
- folio_migration_tools/task_configuration.py +18 -1
- folio_migration_tools/test_infrastructure/mocked_classes.py +94 -0
- folio_migration_tools/transaction_migration/legacy_loan.py +14 -12
- folio_migration_tools/transaction_migration/legacy_reserve.py +1 -1
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/METADATA +2 -2
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/RECORD +24 -24
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/LICENSE +0 -0
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/WHEEL +0 -0
- {folio_migration_tools-1.9.0rc9.dist-info → folio_migration_tools-1.9.0rc11.dist-info}/entry_points.txt +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from importlib import metadata
|
|
1
2
|
import json
|
|
2
3
|
import logging
|
|
3
4
|
import sys
|
|
@@ -38,7 +39,7 @@ def parse_args(args):
|
|
|
38
39
|
default=environ.get("FOLIO_MIGRATION_TOOLS_TASK_NAME"),
|
|
39
40
|
)
|
|
40
41
|
parser.add_argument(
|
|
41
|
-
"--okapi_password",
|
|
42
|
+
"--folio_password", "--okapi_password",
|
|
42
43
|
help="password for the tenant in the configuration file",
|
|
43
44
|
prompt="FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD" not in environ,
|
|
44
45
|
default=environ.get("FOLIO_MIGRATION_TOOLS_OKAPI_PASSWORD"),
|
|
@@ -60,11 +61,17 @@ def parse_args(args):
|
|
|
60
61
|
default=environ.get("FOLIO_MIGRATION_TOOLS_REPORT_LANGUAGE", "en"),
|
|
61
62
|
prompt=False,
|
|
62
63
|
)
|
|
64
|
+
parser.add_argument(
|
|
65
|
+
"--version", "-V",
|
|
66
|
+
help="Show the version of the FOLIO Migration Tools",
|
|
67
|
+
action="store_true",
|
|
68
|
+
prompt=False,
|
|
69
|
+
)
|
|
63
70
|
return parser.parse_args(args)
|
|
64
71
|
|
|
65
72
|
def prep_library_config(args):
|
|
66
73
|
config_file_humped = merge_load(args.configuration_path)
|
|
67
|
-
config_file_humped["libraryInformation"]["okapiPassword"] = args.
|
|
74
|
+
config_file_humped["libraryInformation"]["okapiPassword"] = args.folio_password
|
|
68
75
|
config_file_humped["libraryInformation"]["baseFolder"] = args.base_folder_path
|
|
69
76
|
config_file = humps.decamelize(config_file_humped)
|
|
70
77
|
library_config = LibraryConfiguration(**config_file["library_information"])
|
|
@@ -78,11 +85,22 @@ def prep_library_config(args):
|
|
|
78
85
|
sys.exit("ECS Central Iteration Identifier Not Found")
|
|
79
86
|
return config_file, library_config
|
|
80
87
|
|
|
88
|
+
def print_version(args):
|
|
89
|
+
if "-V" in args or "--version" in args:
|
|
90
|
+
print(
|
|
91
|
+
f"FOLIO Migration Tools: {metadata.version('folio_migration_tools')}"
|
|
92
|
+
)
|
|
93
|
+
sys.exit(0)
|
|
94
|
+
return None
|
|
95
|
+
|
|
81
96
|
|
|
82
97
|
def main():
|
|
83
98
|
try:
|
|
84
99
|
task_classes = list(inheritors(migration_task_base.MigrationTaskBase))
|
|
100
|
+
# Check if the script is run with the --version or -V flag
|
|
101
|
+
print_version(sys.argv)
|
|
85
102
|
|
|
103
|
+
# Parse command line arguments
|
|
86
104
|
args = parse_args(sys.argv[1:])
|
|
87
105
|
try:
|
|
88
106
|
i18n.load_config(
|
|
@@ -124,10 +142,10 @@ def main():
|
|
|
124
142
|
try:
|
|
125
143
|
logging.getLogger("httpx").setLevel(logging.WARNING) # Exclude info messages from httpx
|
|
126
144
|
with FolioClient(
|
|
127
|
-
library_config.
|
|
145
|
+
library_config.gateway_url,
|
|
128
146
|
library_config.tenant_id,
|
|
129
|
-
library_config.
|
|
130
|
-
library_config.
|
|
147
|
+
library_config.folio_username,
|
|
148
|
+
library_config.folio_password,
|
|
131
149
|
) as folio_client:
|
|
132
150
|
task_config = task_class.TaskConfiguration(**migration_task_config)
|
|
133
151
|
task_obj = task_class(task_config, library_config, folio_client)
|
|
@@ -138,7 +138,7 @@ class CirculationHelper:
|
|
|
138
138
|
if legacy_loan.proxy_patron_barcode:
|
|
139
139
|
data.update({"proxyUserBarcode": legacy_loan.proxy_patron_barcode})
|
|
140
140
|
path = "/circulation/check-out-by-barcode"
|
|
141
|
-
url = f"{self.folio_client.
|
|
141
|
+
url = f"{self.folio_client.gateway_url}{path}"
|
|
142
142
|
try:
|
|
143
143
|
if legacy_loan.patron_barcode in self.missing_patron_barcodes:
|
|
144
144
|
error_message = i18n.t("Patron barcode already detected as missing")
|
|
@@ -249,7 +249,7 @@ class CirculationHelper:
|
|
|
249
249
|
):
|
|
250
250
|
try:
|
|
251
251
|
path = "/circulation/requests"
|
|
252
|
-
url = f"{folio_client.
|
|
252
|
+
url = f"{folio_client.gateway_url}{path}"
|
|
253
253
|
data = legacy_request.serialize()
|
|
254
254
|
data["requestProcessingParameters"] = {
|
|
255
255
|
"overrideBlocks": {
|
|
@@ -313,7 +313,7 @@ class CirculationHelper:
|
|
|
313
313
|
del loan_to_put["metadata"]
|
|
314
314
|
loan_to_put["dueDate"] = extension_due_date.isoformat()
|
|
315
315
|
loan_to_put["loanDate"] = extend_out_date.isoformat()
|
|
316
|
-
url = f"{folio_client.
|
|
316
|
+
url = f"{folio_client.gateway_url}/circulation/loans/{loan_to_put['id']}"
|
|
317
317
|
|
|
318
318
|
req = httpx.put(
|
|
319
319
|
url, headers=folio_client.okapi_headers, json=loan_to_put, timeout=None
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from enum import Enum
|
|
2
|
-
from typing import Annotated
|
|
2
|
+
from typing import Annotated
|
|
3
3
|
|
|
4
4
|
from pydantic import BaseModel, Field
|
|
5
5
|
from pydantic.types import DirectoryPath
|
|
@@ -68,8 +68,28 @@ class FolioRelease(str, Enum):
|
|
|
68
68
|
|
|
69
69
|
|
|
70
70
|
class LibraryConfiguration(BaseModel):
|
|
71
|
-
|
|
72
|
-
|
|
71
|
+
gateway_url: Annotated[
|
|
72
|
+
str,
|
|
73
|
+
Field(
|
|
74
|
+
title="FOLIO API Gateway URL",
|
|
75
|
+
description=(
|
|
76
|
+
"The URL of the FOLIO API gateway instance. "
|
|
77
|
+
"You can find this in Settings > Software versions > API gateway services."
|
|
78
|
+
),
|
|
79
|
+
alias="okapi_url"
|
|
80
|
+
),
|
|
81
|
+
]
|
|
82
|
+
tenant_id: Annotated[
|
|
83
|
+
str,
|
|
84
|
+
Field(
|
|
85
|
+
title="FOLIO tenant ID",
|
|
86
|
+
description=(
|
|
87
|
+
"The ID of the FOLIO tenant instance. "
|
|
88
|
+
"You can find this in Settings > Software versions > API gateway services. "
|
|
89
|
+
"In an ECS environment, this is the ID of the central tenant, for all configurations."
|
|
90
|
+
),
|
|
91
|
+
),
|
|
92
|
+
]
|
|
73
93
|
ecs_tenant_id: Annotated[
|
|
74
94
|
str,
|
|
75
95
|
Field(
|
|
@@ -80,15 +100,43 @@ class LibraryConfiguration(BaseModel):
|
|
|
80
100
|
),
|
|
81
101
|
),
|
|
82
102
|
] = ""
|
|
83
|
-
|
|
84
|
-
|
|
103
|
+
folio_username: Annotated[
|
|
104
|
+
str,
|
|
105
|
+
Field(
|
|
106
|
+
title="FOLIO API Gateway username",
|
|
107
|
+
description=(
|
|
108
|
+
"The username for the FOLIO user account performing the migration. "
|
|
109
|
+
"User should have a full admin permissions/roles in FOLIO. "
|
|
110
|
+
),
|
|
111
|
+
alias="okapi_username"
|
|
112
|
+
),
|
|
113
|
+
]
|
|
114
|
+
folio_password: Annotated[
|
|
115
|
+
str,
|
|
116
|
+
Field(
|
|
117
|
+
title="FOLIO API Gateway password",
|
|
118
|
+
description=(
|
|
119
|
+
"The password for the FOLIO user account performing the migration. "
|
|
120
|
+
),
|
|
121
|
+
alias="okapi_password"
|
|
122
|
+
)
|
|
123
|
+
]
|
|
85
124
|
base_folder: DirectoryPath = Field(
|
|
86
125
|
description=(
|
|
87
126
|
"The base folder for migration. "
|
|
88
127
|
"Should ideally be a github clone of the migration_repo_template"
|
|
89
128
|
)
|
|
90
129
|
)
|
|
91
|
-
multi_field_delimiter:
|
|
130
|
+
multi_field_delimiter: Annotated[
|
|
131
|
+
str,
|
|
132
|
+
Field(
|
|
133
|
+
title="Multi field delimiter",
|
|
134
|
+
description=(
|
|
135
|
+
"The delimiter used to separate multiple values in a single field. "
|
|
136
|
+
"This is used for delimited text (CSV/TSV) fields with multiple sub-delimited values."
|
|
137
|
+
),
|
|
138
|
+
),
|
|
139
|
+
] = "<delimiter>"
|
|
92
140
|
failed_records_threshold: Annotated[
|
|
93
141
|
int,
|
|
94
142
|
Field(description=("Number of failed records until the process shuts down")),
|
|
@@ -272,7 +272,7 @@ class MapperBase:
|
|
|
272
272
|
[
|
|
273
273
|
object_type == FOLIONamespaces.instances,
|
|
274
274
|
(not getattr(self.task_configuration, "data_import_marc", False)),
|
|
275
|
-
getattr(self
|
|
275
|
+
getattr(self, "create_source_records", True),
|
|
276
276
|
]
|
|
277
277
|
):
|
|
278
278
|
return (legacy_id, folio_record["id"], folio_record["hrid"])
|
|
@@ -443,7 +443,7 @@ class MapperBase:
|
|
|
443
443
|
@property
|
|
444
444
|
def base_string_for_folio_uuid(self):
|
|
445
445
|
if self.library_configuration.use_gateway_url_for_uuids and not self.library_configuration.is_ecs:
|
|
446
|
-
return str(self.folio_client.
|
|
446
|
+
return str(self.folio_client.gateway_url)
|
|
447
447
|
elif self.library_configuration.ecs_tenant_id:
|
|
448
448
|
return str(self.library_configuration.ecs_tenant_id)
|
|
449
449
|
else:
|
|
@@ -160,7 +160,7 @@ class HRIDHandler:
|
|
|
160
160
|
self.hrid_settings["instances"]["startNumber"] = self.instance_hrid_counter
|
|
161
161
|
self.hrid_settings["holdings"]["startNumber"] = self.holdings_hrid_counter
|
|
162
162
|
self.hrid_settings["items"]["startNumber"] = self.items_hrid_counter
|
|
163
|
-
url = self.folio_client.
|
|
163
|
+
url = self.folio_client.gateway_url + self.hrid_path
|
|
164
164
|
resp = httpx.put(
|
|
165
165
|
url,
|
|
166
166
|
json=self.hrid_settings,
|
|
@@ -3,7 +3,7 @@ import os
|
|
|
3
3
|
import sys
|
|
4
4
|
import time
|
|
5
5
|
import traceback
|
|
6
|
-
from typing import List
|
|
6
|
+
from typing import BinaryIO, Dict, List, Set, TextIO
|
|
7
7
|
|
|
8
8
|
import i18n
|
|
9
9
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
@@ -24,24 +24,26 @@ from folio_migration_tools.migration_report import MigrationReport
|
|
|
24
24
|
|
|
25
25
|
class MarcFileProcessor:
|
|
26
26
|
def __init__(
|
|
27
|
-
self, mapper: RulesMapperBase, folder_structure: FolderStructure, created_objects_file
|
|
27
|
+
self, mapper: RulesMapperBase, folder_structure: FolderStructure, created_objects_file: TextIO
|
|
28
28
|
):
|
|
29
29
|
self.object_type: FOLIONamespaces = folder_structure.object_type
|
|
30
30
|
self.folder_structure: FolderStructure = folder_structure
|
|
31
31
|
self.mapper: RulesMapperBase = mapper
|
|
32
|
-
self.created_objects_file = created_objects_file
|
|
33
|
-
if mapper.
|
|
34
|
-
|
|
32
|
+
self.created_objects_file: TextIO = created_objects_file
|
|
33
|
+
if mapper.create_source_records and any(
|
|
34
|
+
x.create_source_records for x in mapper.task_configuration.files
|
|
35
|
+
):
|
|
36
|
+
self.srs_records_file: TextIO = open(self.folder_structure.srs_records_path, "w+")
|
|
35
37
|
if getattr(mapper.task_configuration, "data_import_marc", False):
|
|
36
|
-
self.data_import_marc_file = open(self.folder_structure.data_import_marc_path, "wb+")
|
|
37
|
-
self.unique_001s:
|
|
38
|
+
self.data_import_marc_file: BinaryIO = open(self.folder_structure.data_import_marc_path, "wb+")
|
|
39
|
+
self.unique_001s: Set[str] = set()
|
|
38
40
|
self.failed_records_count: int = 0
|
|
39
41
|
self.records_count: int = 0
|
|
40
42
|
self.start: float = time.time()
|
|
41
|
-
self.legacy_ids:
|
|
43
|
+
self.legacy_ids: Set[str] = set()
|
|
42
44
|
if (
|
|
43
45
|
self.object_type == FOLIONamespaces.holdings
|
|
44
|
-
and self.mapper.
|
|
46
|
+
and self.mapper.create_source_records
|
|
45
47
|
):
|
|
46
48
|
logging.info("Loading Parent HRID map for SRS creation")
|
|
47
49
|
self.parent_hrids = {entity[1]: entity[2] for entity in mapper.parent_id_map.values()}
|
|
@@ -78,7 +80,7 @@ class MarcFileProcessor:
|
|
|
78
80
|
|
|
79
81
|
if (
|
|
80
82
|
file_def.create_source_records
|
|
81
|
-
and self.mapper.
|
|
83
|
+
and self.mapper.create_source_records
|
|
82
84
|
):
|
|
83
85
|
self.save_srs_record(
|
|
84
86
|
marc_record,
|
|
@@ -132,7 +134,7 @@ class MarcFileProcessor:
|
|
|
132
134
|
def save_marc_record(
|
|
133
135
|
self,
|
|
134
136
|
marc_record: Record,
|
|
135
|
-
folio_rec:
|
|
137
|
+
folio_rec: Dict,
|
|
136
138
|
object_type: FOLIONamespaces
|
|
137
139
|
):
|
|
138
140
|
self.mapper.save_data_import_marc_record(
|
|
@@ -146,14 +148,10 @@ class MarcFileProcessor:
|
|
|
146
148
|
self,
|
|
147
149
|
marc_record: Record,
|
|
148
150
|
file_def: FileDefinition,
|
|
149
|
-
folio_rec,
|
|
151
|
+
folio_rec: Dict,
|
|
150
152
|
legacy_ids: List[str],
|
|
151
153
|
object_type: FOLIONamespaces,
|
|
152
154
|
):
|
|
153
|
-
if not all(
|
|
154
|
-
[file_def.create_source_records, self.mapper.task_configuration.create_source_records]
|
|
155
|
-
):
|
|
156
|
-
return
|
|
157
155
|
if object_type in [FOLIONamespaces.holdings]:
|
|
158
156
|
if "008" in marc_record and len(marc_record["008"].data) > 32:
|
|
159
157
|
remain, rest = (
|
|
@@ -188,7 +186,7 @@ class MarcFileProcessor:
|
|
|
188
186
|
)
|
|
189
187
|
self.mapper.migration_report.add_general_statistics(i18n.t("SRS records written to disk"))
|
|
190
188
|
|
|
191
|
-
def add_mapped_location_code_to_record(self, marc_record, folio_rec):
|
|
189
|
+
def add_mapped_location_code_to_record(self, marc_record: Record, folio_rec: Dict):
|
|
192
190
|
location_code = next(
|
|
193
191
|
(
|
|
194
192
|
location["code"]
|
|
@@ -225,9 +223,9 @@ class MarcFileProcessor:
|
|
|
225
223
|
|
|
226
224
|
@staticmethod
|
|
227
225
|
def get_valid_folio_record_ids(
|
|
228
|
-
legacy_ids, folio_record_identifiers, migration_report: MigrationReport
|
|
229
|
-
):
|
|
230
|
-
new_ids = set()
|
|
226
|
+
legacy_ids: List[str], folio_record_identifiers: Set[str], migration_report: MigrationReport
|
|
227
|
+
) -> List[str]:
|
|
228
|
+
new_ids: Set[str] = set()
|
|
231
229
|
for legacy_id in legacy_ids:
|
|
232
230
|
if legacy_id not in folio_record_identifiers:
|
|
233
231
|
new_ids.add(legacy_id)
|
|
@@ -266,12 +264,12 @@ class MarcFileProcessor:
|
|
|
266
264
|
self.mapper.mapped_folio_fields,
|
|
267
265
|
self.mapper.mapped_legacy_fields,
|
|
268
266
|
)
|
|
269
|
-
if self
|
|
267
|
+
if hasattr(self, "srs_records_file"):
|
|
270
268
|
self.srs_records_file.seek(0)
|
|
271
269
|
if not self.srs_records_file.seek(0):
|
|
272
270
|
os.remove(self.srs_records_file.name)
|
|
273
271
|
self.srs_records_file.close()
|
|
274
|
-
if
|
|
272
|
+
if hasattr(self, "data_import_marc_file"):
|
|
275
273
|
self.data_import_marc_file.seek(0)
|
|
276
274
|
if not self.data_import_marc_file.read(1):
|
|
277
275
|
os.remove(self.data_import_marc_file.name)
|
|
@@ -281,7 +279,7 @@ class MarcFileProcessor:
|
|
|
281
279
|
logging.info("Transformation report written to %s", report_file.name)
|
|
282
280
|
logging.info("Processor is done.")
|
|
283
281
|
|
|
284
|
-
def add_legacy_ids_to_map(self, folio_rec, filtered_legacy_ids):
|
|
282
|
+
def add_legacy_ids_to_map(self, folio_rec: Dict, filtered_legacy_ids: List[str]):
|
|
285
283
|
for legacy_id in filtered_legacy_ids:
|
|
286
284
|
self.legacy_ids.add(legacy_id)
|
|
287
285
|
if legacy_id not in self.mapper.id_map:
|
|
@@ -51,6 +51,9 @@ class RulesMapperBase(MapperBase):
|
|
|
51
51
|
self.item_json_schema = ""
|
|
52
52
|
self.mappings: dict = {}
|
|
53
53
|
self.schema_properties = None
|
|
54
|
+
self.create_source_records = all(
|
|
55
|
+
[self.task_configuration.create_source_records, (not getattr(self.task_configuration, "data_import_marc", False))]
|
|
56
|
+
)
|
|
54
57
|
if hasattr(self.task_configuration, "hrid_handling"):
|
|
55
58
|
self.hrid_handler = HRIDHandler(
|
|
56
59
|
folio_client,
|
|
@@ -7,7 +7,7 @@ import time
|
|
|
7
7
|
import typing
|
|
8
8
|
import uuid
|
|
9
9
|
from pathlib import Path
|
|
10
|
-
from typing import Generator, List
|
|
10
|
+
from typing import Dict, Generator, List
|
|
11
11
|
|
|
12
12
|
import i18n
|
|
13
13
|
import pymarc
|
|
@@ -16,6 +16,7 @@ from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
|
16
16
|
from folio_uuid.folio_uuid import FolioUUID
|
|
17
17
|
from folioclient import FolioClient
|
|
18
18
|
from pymarc.record import Leader, Record
|
|
19
|
+
from pymarc.field import Field
|
|
19
20
|
|
|
20
21
|
from folio_migration_tools.custom_exceptions import (
|
|
21
22
|
TransformationProcessError,
|
|
@@ -32,6 +33,7 @@ from folio_migration_tools.marc_rules_transformation.conditions import Condition
|
|
|
32
33
|
from folio_migration_tools.marc_rules_transformation.rules_mapper_base import (
|
|
33
34
|
RulesMapperBase,
|
|
34
35
|
)
|
|
36
|
+
from folio_migration_tools.migration_tasks.migration_task_base import MarcTaskConfigurationBase
|
|
35
37
|
|
|
36
38
|
|
|
37
39
|
class BibsRulesMapper(RulesMapperBase):
|
|
@@ -40,9 +42,9 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
40
42
|
|
|
41
43
|
def __init__(
|
|
42
44
|
self,
|
|
43
|
-
folio_client,
|
|
45
|
+
folio_client: FolioClient,
|
|
44
46
|
library_configuration: LibraryConfiguration,
|
|
45
|
-
task_configuration,
|
|
47
|
+
task_configuration: MarcTaskConfigurationBase,
|
|
46
48
|
):
|
|
47
49
|
super().__init__(
|
|
48
50
|
folio_client,
|
|
@@ -59,15 +61,12 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
59
61
|
self.instance_relationships: dict = {}
|
|
60
62
|
self.instance_relationship_types: dict = {}
|
|
61
63
|
self.other_mode_of_issuance_id = get_unspecified_mode_of_issuance(self.folio_client)
|
|
62
|
-
self.create_source_records = all(
|
|
63
|
-
[self.task_configuration.create_source_records, (not getattr(self.task_configuration, "data_import_marc", False))]
|
|
64
|
-
)
|
|
65
64
|
self.data_import_marc = self.task_configuration.data_import_marc
|
|
66
65
|
if self.data_import_marc:
|
|
67
66
|
self.hrid_handler.deactivate035_from001 = True
|
|
68
67
|
self.start = time.time()
|
|
69
68
|
|
|
70
|
-
def perform_initial_preparation(self, marc_record:
|
|
69
|
+
def perform_initial_preparation(self, file_def: FileDefinition, marc_record: Record, legacy_ids: List[str]):
|
|
71
70
|
folio_instance = {}
|
|
72
71
|
folio_instance["id"] = str(
|
|
73
72
|
FolioUUID(
|
|
@@ -76,7 +75,10 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
76
75
|
str(legacy_ids[-1]),
|
|
77
76
|
)
|
|
78
77
|
)
|
|
79
|
-
if
|
|
78
|
+
if (
|
|
79
|
+
all([self.create_source_records, file_def.create_source_records])
|
|
80
|
+
or self.hrid_handler.handling == HridHandling.preserve001
|
|
81
|
+
):
|
|
80
82
|
self.hrid_handler.handle_hrid(
|
|
81
83
|
FOLIONamespaces.instances,
|
|
82
84
|
folio_instance,
|
|
@@ -90,7 +92,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
90
92
|
|
|
91
93
|
return folio_instance
|
|
92
94
|
|
|
93
|
-
def handle_leader_05(self, marc_record, legacy_ids):
|
|
95
|
+
def handle_leader_05(self, marc_record: Record, legacy_ids: List[str]):
|
|
94
96
|
leader_05 = marc_record.leader[5] or "Empty"
|
|
95
97
|
self.migration_report.add("RecordStatus", i18n.t("Original value") + f": {leader_05}")
|
|
96
98
|
if leader_05 not in ["a", "c", "d", "n", "p"]:
|
|
@@ -102,14 +104,14 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
102
104
|
Helper.log_data_issue(legacy_ids, "d in leader. Is this correct?", marc_record.leader)
|
|
103
105
|
|
|
104
106
|
def parse_record(
|
|
105
|
-
self, marc_record:
|
|
107
|
+
self, marc_record: Record, file_def: FileDefinition, legacy_ids: List[str]
|
|
106
108
|
) -> list[dict]:
|
|
107
109
|
"""Parses a bib recod into a FOLIO Inventory instance object
|
|
108
110
|
Community mapping suggestion: https://bit.ly/2S7Gyp3
|
|
109
111
|
This is the main function
|
|
110
112
|
|
|
111
113
|
Args:
|
|
112
|
-
marc_record (
|
|
114
|
+
marc_record (Record): _description_
|
|
113
115
|
file_def (FileDefinition): _description_
|
|
114
116
|
legacy_ids (List[str]): List of legacy ids in record
|
|
115
117
|
|
|
@@ -119,7 +121,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
119
121
|
self.print_progress()
|
|
120
122
|
ignored_subsequent_fields: set = set()
|
|
121
123
|
bad_tags = set(self.task_configuration.tags_to_delete) # "907"
|
|
122
|
-
folio_instance = self.perform_initial_preparation(marc_record, legacy_ids)
|
|
124
|
+
folio_instance = self.perform_initial_preparation(file_def, marc_record, legacy_ids)
|
|
123
125
|
if self.data_import_marc:
|
|
124
126
|
self.simple_bib_map(folio_instance, marc_record, ignored_subsequent_fields, legacy_ids)
|
|
125
127
|
else:
|
|
@@ -155,6 +157,19 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
155
157
|
legacy_ids (List[str]): _description_
|
|
156
158
|
file_def (FileDefinition): _description_
|
|
157
159
|
"""
|
|
160
|
+
main_entry_field_tags = ["100", "110", "111", "130"]
|
|
161
|
+
main_entry_fields = marc_record.get_fields(*main_entry_field_tags)
|
|
162
|
+
main_entry_fields.sort(key=lambda x: int(x.tag))
|
|
163
|
+
if len(main_entry_fields) > 1:
|
|
164
|
+
Helper.log_data_issue(
|
|
165
|
+
legacy_ids,
|
|
166
|
+
"Multiple main entry fields in record. Record will fail Data Import. Creating Instance anyway.",
|
|
167
|
+
main_entry_fields
|
|
168
|
+
)
|
|
169
|
+
if not main_entry_fields:
|
|
170
|
+
main_entry_fields += marc_record.get_fields("700", "710", "711", "730")
|
|
171
|
+
main_entry_fields.sort(key=lambda x: int(x.tag))
|
|
172
|
+
self.process_marc_field(folio_instnace, main_entry_fields[0], ignored_subsequent_fields, legacy_ids)
|
|
158
173
|
try:
|
|
159
174
|
self.process_marc_field(folio_instnace, marc_record['245'], ignored_subsequent_fields, legacy_ids)
|
|
160
175
|
except KeyError:
|
|
@@ -178,7 +193,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
178
193
|
legacy_ids (List[str]): _description_
|
|
179
194
|
file_def (FileDefinition): _description_
|
|
180
195
|
"""
|
|
181
|
-
if file_def.create_source_records and self.
|
|
196
|
+
if file_def.create_source_records and self.create_source_records:
|
|
182
197
|
folio_instance["source"] = "MARC"
|
|
183
198
|
else:
|
|
184
199
|
folio_instance["source"] = "FOLIO"
|
|
@@ -198,7 +213,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
198
213
|
del folio_instance["succeedingTitles"]
|
|
199
214
|
self.migration_report.add("PrecedingSuccedingTitles", f"{len(succ_titles)}")
|
|
200
215
|
|
|
201
|
-
def handle_languages(self, folio_instance, marc_record, legacy_ids):
|
|
216
|
+
def handle_languages(self, folio_instance: Dict, marc_record: Record, legacy_ids: List[str]):
|
|
202
217
|
if "languages" in folio_instance:
|
|
203
218
|
orig_languages = {lang: None for lang in folio_instance["languages"]}
|
|
204
219
|
orig_languages.update(
|
|
@@ -255,7 +270,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
255
270
|
if self.task_configuration.update_hrid_settings:
|
|
256
271
|
self.hrid_handler.store_hrid_settings()
|
|
257
272
|
|
|
258
|
-
def get_instance_type_id(self, marc_record,
|
|
273
|
+
def get_instance_type_id(self, marc_record: Record, legacy_ids: List[str]) -> str:
|
|
259
274
|
return_id = ""
|
|
260
275
|
|
|
261
276
|
def get_folio_id_by_name(f336a: str):
|
|
@@ -283,7 +298,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
283
298
|
+ f" ({f336a})",
|
|
284
299
|
)
|
|
285
300
|
Helper.log_data_issue(
|
|
286
|
-
|
|
301
|
+
legacy_ids,
|
|
287
302
|
"instance type name (336$a) -Unsuccessful matching",
|
|
288
303
|
f336a,
|
|
289
304
|
)
|
|
@@ -316,7 +331,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
316
331
|
),
|
|
317
332
|
)
|
|
318
333
|
Helper.log_data_issue(
|
|
319
|
-
|
|
334
|
+
legacy_ids,
|
|
320
335
|
i18n.t("instance type code (%{code}) not found in FOLIO", code="336$b"),
|
|
321
336
|
f336_b,
|
|
322
337
|
)
|
|
@@ -337,7 +352,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
337
352
|
return_id = t[0]
|
|
338
353
|
return return_id
|
|
339
354
|
|
|
340
|
-
def get_instance_format_id_by_code(self,
|
|
355
|
+
def get_instance_format_id_by_code(self, legacy_ids: List[str], code: str):
|
|
341
356
|
try:
|
|
342
357
|
match = next(f for f in self.folio_client.instance_formats if f["code"] == code)
|
|
343
358
|
self.migration_report.add(
|
|
@@ -347,14 +362,14 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
347
362
|
return match["id"]
|
|
348
363
|
except Exception:
|
|
349
364
|
# TODO: Distinguish between generated codes and proper 338bs
|
|
350
|
-
Helper.log_data_issue(
|
|
365
|
+
Helper.log_data_issue(legacy_ids, "Instance format Code not found in FOLIO", code)
|
|
351
366
|
self.migration_report.add(
|
|
352
367
|
"InstanceFormat",
|
|
353
368
|
i18n.t("Code '%{code}' not found in FOLIO", code=code),
|
|
354
369
|
)
|
|
355
370
|
return ""
|
|
356
371
|
|
|
357
|
-
def get_instance_format_id_by_name(self, f337a: str, f338a: str,
|
|
372
|
+
def get_instance_format_id_by_name(self, f337a: str, f338a: str, legacy_ids: List[str]):
|
|
358
373
|
f337a = f337a.lower().strip()
|
|
359
374
|
f338a = f338a.lower().strip()
|
|
360
375
|
match_template = f"{f337a} -- {f338a}"
|
|
@@ -376,7 +391,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
376
391
|
return match["id"]
|
|
377
392
|
except Exception:
|
|
378
393
|
Helper.log_data_issue(
|
|
379
|
-
|
|
394
|
+
legacy_ids,
|
|
380
395
|
"Unsuccessful matching on 337$a and 338$a",
|
|
381
396
|
match_template,
|
|
382
397
|
)
|
|
@@ -391,7 +406,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
391
406
|
)
|
|
392
407
|
return ""
|
|
393
408
|
|
|
394
|
-
def f338_source_is_rda_carrier(self, field:
|
|
409
|
+
def f338_source_is_rda_carrier(self, field: Field):
|
|
395
410
|
if "2" not in field:
|
|
396
411
|
self.migration_report.add(
|
|
397
412
|
"InstanceFormat",
|
|
@@ -407,7 +422,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
407
422
|
return False
|
|
408
423
|
|
|
409
424
|
def get_instance_format_ids_from_a(
|
|
410
|
-
self, field_index, f_338:
|
|
425
|
+
self, field_index: int, f_338: Field, all_337s: List[Field], legacy_id: List[str]
|
|
411
426
|
):
|
|
412
427
|
self.migration_report.add(
|
|
413
428
|
"InstanceFormat",
|
|
@@ -421,7 +436,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
421
436
|
):
|
|
422
437
|
yield fmt_id
|
|
423
438
|
|
|
424
|
-
def get_instance_format_ids(self, marc_record, legacy_id):
|
|
439
|
+
def get_instance_format_ids(self, marc_record: Record, legacy_id: List[str]):
|
|
425
440
|
all_337s = marc_record.get_fields("337")
|
|
426
441
|
all_338s = marc_record.get_fields("338")
|
|
427
442
|
for fidx, f_338 in enumerate(all_338s):
|
|
@@ -466,7 +481,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
466
481
|
legacy_id, combined_code
|
|
467
482
|
)
|
|
468
483
|
|
|
469
|
-
def get_mode_of_issuance_id(self, marc_record: Record,
|
|
484
|
+
def get_mode_of_issuance_id(self, marc_record: Record, legacy_ids: List[str]) -> str:
|
|
470
485
|
level = marc_record.leader[7]
|
|
471
486
|
try:
|
|
472
487
|
name = "unspecified"
|
|
@@ -496,7 +511,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
496
511
|
return ret
|
|
497
512
|
except IndexError:
|
|
498
513
|
self.migration_report.add(
|
|
499
|
-
"PossibleCleaningTasks", i18n.t("No Leader[7] in") + f" {
|
|
514
|
+
"PossibleCleaningTasks", i18n.t("No Leader[7] in") + f" {legacy_ids}"
|
|
500
515
|
)
|
|
501
516
|
|
|
502
517
|
return self.other_mode_of_issuance_id
|
|
@@ -512,7 +527,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
512
527
|
return "".join(marc_record["008"].data[35:38])
|
|
513
528
|
return ""
|
|
514
529
|
|
|
515
|
-
def get_languages_041(self, marc_record, legacy_id):
|
|
530
|
+
def get_languages_041(self, marc_record: Record, legacy_id: List[str]) -> Dict[str, None]:
|
|
516
531
|
languages = dict()
|
|
517
532
|
lang_fields = marc_record.get_fields("041")
|
|
518
533
|
if not any(lang_fields):
|
|
@@ -543,21 +558,21 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
543
558
|
}
|
|
544
559
|
return languages
|
|
545
560
|
|
|
546
|
-
def get_languages(self, marc_record: Record, legacy_id: str) -> List[str]:
|
|
561
|
+
def get_languages(self, marc_record: Record, legacy_id: List[str]) -> List[str]:
|
|
547
562
|
"""Get languages and tranforms them to correct codes
|
|
548
563
|
|
|
549
564
|
Args:
|
|
550
|
-
marc_record (Record):
|
|
551
|
-
legacy_id (str):
|
|
565
|
+
marc_record (Record): A pymarc Record object
|
|
566
|
+
legacy_id (List[str]): A list of legacy ids from the legacy record
|
|
552
567
|
|
|
553
568
|
Returns:
|
|
554
|
-
List[str]:
|
|
569
|
+
List[str]: List of language codes
|
|
555
570
|
"""
|
|
556
571
|
languages = self.get_languages_041(marc_record, legacy_id)
|
|
557
572
|
languages[self.get_languages_008(marc_record)] = None
|
|
558
|
-
for lang in languages
|
|
573
|
+
for lang in languages:
|
|
559
574
|
self.migration_report.add("LanguagesInRecords", lang)
|
|
560
|
-
return list(languages
|
|
575
|
+
return list(languages)
|
|
561
576
|
|
|
562
577
|
def fetch_language_codes(self) -> Generator[str, None, None]:
|
|
563
578
|
"""Loads the list of standardized language codes from LoC
|
|
@@ -575,7 +590,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
575
590
|
yield code.text
|
|
576
591
|
|
|
577
592
|
def filter_langs(
|
|
578
|
-
self, language_values: List[str], marc_record: Record, index_or_legacy_id
|
|
593
|
+
self, language_values: List[str], marc_record: Record, index_or_legacy_id: List[str]
|
|
579
594
|
) -> typing.Generator:
|
|
580
595
|
forbidden_values = ["###", "zxx", "n/a", "N/A", "|||"]
|
|
581
596
|
for language_value in language_values:
|
|
@@ -629,7 +644,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
629
644
|
else:
|
|
630
645
|
raise TransformationProcessError("", f"ILS {ils_flavour} not configured")
|
|
631
646
|
|
|
632
|
-
def get_aleph_bib_id(self, marc_record: Record):
|
|
647
|
+
def get_aleph_bib_id(self, marc_record: Record) -> List[str]:
|
|
633
648
|
res = {f["b"].strip(): None for f in marc_record.get_fields("998") if "b" in f}
|
|
634
649
|
if any(res):
|
|
635
650
|
self.migration_report.add_general_statistics(
|
|
@@ -651,7 +666,7 @@ class BibsRulesMapper(RulesMapperBase):
|
|
|
651
666
|
) from e
|
|
652
667
|
|
|
653
668
|
|
|
654
|
-
def get_unspecified_mode_of_issuance(folio_client: FolioClient):
|
|
669
|
+
def get_unspecified_mode_of_issuance(folio_client: FolioClient) -> str:
|
|
655
670
|
m_o_is = list(folio_client.modes_of_issuance)
|
|
656
671
|
if not any(m_o_is):
|
|
657
672
|
logging.critical("No Modes of issuance set up in tenant. Quitting...")
|
|
@@ -684,7 +699,7 @@ def get_custom_bib_id(marc_record: Record, field_string: str):
|
|
|
684
699
|
)
|
|
685
700
|
|
|
686
701
|
|
|
687
|
-
def get_iii_bib_id(marc_record: Record):
|
|
702
|
+
def get_iii_bib_id(marc_record: Record) -> List[str]:
|
|
688
703
|
try:
|
|
689
704
|
return [marc_record["907"]["a"]]
|
|
690
705
|
except Exception as e:
|