folio-migration-tools 1.9.0a2__py3-none-any.whl → 1.9.0a4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- folio_migration_tools/__init__.py +3 -0
- folio_migration_tools/__main__.py +16 -6
- folio_migration_tools/folder_structure.py +3 -0
- folio_migration_tools/library_configuration.py +8 -7
- folio_migration_tools/mapper_base.py +26 -17
- folio_migration_tools/mapping_file_transformation/holdings_mapper.py +23 -11
- folio_migration_tools/mapping_file_transformation/item_mapper.py +13 -11
- folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +9 -10
- folio_migration_tools/mapping_file_transformation/order_mapper.py +2 -2
- folio_migration_tools/mapping_file_transformation/organization_mapper.py +1 -1
- folio_migration_tools/mapping_file_transformation/user_mapper.py +6 -4
- folio_migration_tools/marc_rules_transformation/conditions.py +23 -7
- folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +21 -11
- folio_migration_tools/marc_rules_transformation/marc_file_processor.py +36 -9
- folio_migration_tools/marc_rules_transformation/marc_reader_wrapper.py +15 -11
- folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +7 -5
- folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +98 -45
- folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +53 -27
- folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +13 -11
- folio_migration_tools/migration_tasks/batch_poster.py +78 -38
- folio_migration_tools/migration_tasks/bibs_transformer.py +21 -8
- folio_migration_tools/migration_tasks/courses_migrator.py +11 -6
- folio_migration_tools/migration_tasks/holdings_csv_transformer.py +22 -16
- folio_migration_tools/migration_tasks/holdings_marc_transformer.py +9 -7
- folio_migration_tools/migration_tasks/items_transformer.py +13 -10
- folio_migration_tools/migration_tasks/loans_migrator.py +10 -9
- folio_migration_tools/migration_tasks/manual_fee_fines_transformer.py +13 -9
- folio_migration_tools/migration_tasks/migration_task_base.py +18 -18
- folio_migration_tools/migration_tasks/orders_transformer.py +14 -10
- folio_migration_tools/migration_tasks/organization_transformer.py +12 -8
- folio_migration_tools/migration_tasks/requests_migrator.py +8 -5
- folio_migration_tools/migration_tasks/reserves_migrator.py +7 -4
- folio_migration_tools/migration_tasks/user_transformer.py +46 -17
- folio_migration_tools/task_configuration.py +3 -3
- folio_migration_tools/translations/en.json +2 -1
- {folio_migration_tools-1.9.0a2.dist-info → folio_migration_tools-1.9.0a4.dist-info}/METADATA +6 -5
- {folio_migration_tools-1.9.0a2.dist-info → folio_migration_tools-1.9.0a4.dist-info}/RECORD +39 -39
- {folio_migration_tools-1.9.0a2.dist-info → folio_migration_tools-1.9.0a4.dist-info}/WHEEL +1 -1
- {folio_migration_tools-1.9.0a2.dist-info → folio_migration_tools-1.9.0a4.dist-info}/LICENSE +0 -0
|
@@ -5,8 +5,7 @@ import sys
|
|
|
5
5
|
import time
|
|
6
6
|
import traceback
|
|
7
7
|
from datetime import datetime
|
|
8
|
-
from typing import Annotated
|
|
9
|
-
from typing import List
|
|
8
|
+
from typing import Annotated, List
|
|
10
9
|
from uuid import uuid4
|
|
11
10
|
|
|
12
11
|
import httpx
|
|
@@ -14,10 +13,14 @@ import i18n
|
|
|
14
13
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
15
14
|
from pydantic import Field
|
|
16
15
|
|
|
17
|
-
from folio_migration_tools.custom_exceptions import
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
from folio_migration_tools.custom_exceptions import (
|
|
17
|
+
TransformationProcessError,
|
|
18
|
+
TransformationRecordFailedError,
|
|
19
|
+
)
|
|
20
|
+
from folio_migration_tools.library_configuration import (
|
|
21
|
+
FileDefinition,
|
|
22
|
+
LibraryConfiguration,
|
|
23
|
+
)
|
|
21
24
|
from folio_migration_tools.migration_report import MigrationReport
|
|
22
25
|
from folio_migration_tools.migration_tasks.migration_task_base import MigrationTaskBase
|
|
23
26
|
from folio_migration_tools.task_configuration import AbstractTaskConfiguration
|
|
@@ -71,6 +74,24 @@ class BatchPoster(MigrationTaskBase):
|
|
|
71
74
|
)
|
|
72
75
|
),
|
|
73
76
|
] = True
|
|
77
|
+
extradata_endpoints: Annotated[
|
|
78
|
+
dict,
|
|
79
|
+
Field(
|
|
80
|
+
description=(
|
|
81
|
+
"A dictionary of extradata endpoints. "
|
|
82
|
+
"The key is the object type and the value is the endpoint"
|
|
83
|
+
)
|
|
84
|
+
),
|
|
85
|
+
] = {}
|
|
86
|
+
upsert: Annotated[
|
|
87
|
+
bool,
|
|
88
|
+
Field(
|
|
89
|
+
description=(
|
|
90
|
+
"Toggles whether or not to use the upsert feature of the Inventory storage "
|
|
91
|
+
"endpoints. Defaults to False"
|
|
92
|
+
)
|
|
93
|
+
),
|
|
94
|
+
] = False
|
|
74
95
|
|
|
75
96
|
@staticmethod
|
|
76
97
|
def get_object_type() -> FOLIONamespaces:
|
|
@@ -80,9 +101,10 @@ class BatchPoster(MigrationTaskBase):
|
|
|
80
101
|
self,
|
|
81
102
|
task_config: TaskConfiguration,
|
|
82
103
|
library_config: LibraryConfiguration,
|
|
104
|
+
folio_client,
|
|
83
105
|
use_logging: bool = True,
|
|
84
106
|
):
|
|
85
|
-
super().__init__(library_config, task_config, use_logging)
|
|
107
|
+
super().__init__(library_config, task_config, folio_client, use_logging)
|
|
86
108
|
self.migration_report = MigrationReport()
|
|
87
109
|
self.performing_rerun = False
|
|
88
110
|
self.failed_ids: list = []
|
|
@@ -91,6 +113,11 @@ class BatchPoster(MigrationTaskBase):
|
|
|
91
113
|
self.task_configuration.object_type,
|
|
92
114
|
self.task_configuration.use_safe_inventory_endpoints,
|
|
93
115
|
)
|
|
116
|
+
self.query_params = {}
|
|
117
|
+
if self.api_info["supports_upsert"]:
|
|
118
|
+
self.query_params["upsert"] = self.task_configuration.upsert
|
|
119
|
+
elif self.task_configuration.upsert and not self.api_info["supports_upsert"]:
|
|
120
|
+
logging.info("Upsert is not supported for this object type. Query parameter will not be set.")
|
|
94
121
|
self.snapshot_id = str(uuid4())
|
|
95
122
|
self.failed_objects: list = []
|
|
96
123
|
self.batch_size = self.task_configuration.batch_size
|
|
@@ -107,7 +134,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
107
134
|
self.http_client = None
|
|
108
135
|
|
|
109
136
|
def do_work(self):
|
|
110
|
-
with
|
|
137
|
+
with self.folio_client.get_folio_http_client() as httpx_client:
|
|
111
138
|
self.http_client = httpx_client
|
|
112
139
|
try:
|
|
113
140
|
batch = []
|
|
@@ -193,7 +220,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
193
220
|
|
|
194
221
|
def post_extra_data(self, row: str, num_records: int, failed_recs_file):
|
|
195
222
|
(object_name, data) = row.split("\t")
|
|
196
|
-
endpoint = get_extradata_endpoint(object_name, data)
|
|
223
|
+
endpoint = self.get_extradata_endpoint(self.task_configuration, object_name, data)
|
|
197
224
|
url = f"{self.folio_client.okapi_url}/{endpoint}"
|
|
198
225
|
body = data
|
|
199
226
|
response = self.post_objects(url, body)
|
|
@@ -216,6 +243,35 @@ class BatchPoster(MigrationTaskBase):
|
|
|
216
243
|
self.num_failures,
|
|
217
244
|
)
|
|
218
245
|
|
|
246
|
+
@staticmethod
|
|
247
|
+
def get_extradata_endpoint(
|
|
248
|
+
task_configuration: TaskConfiguration, object_name: str, string_object: str
|
|
249
|
+
):
|
|
250
|
+
object_types = {
|
|
251
|
+
"precedingSucceedingTitles": "preceding-succeeding-titles",
|
|
252
|
+
"precedingTitles": "preceding-succeeding-titles",
|
|
253
|
+
"succeedingTitles": "preceding-succeeding-titles",
|
|
254
|
+
"boundwithPart": "inventory-storage/bound-with-parts",
|
|
255
|
+
"notes": "notes",
|
|
256
|
+
"course": "coursereserves/courses",
|
|
257
|
+
"courselisting": "coursereserves/courselistings",
|
|
258
|
+
"contacts": "organizations-storage/contacts",
|
|
259
|
+
"interfaces": "organizations-storage/interfaces",
|
|
260
|
+
"account": "accounts",
|
|
261
|
+
"feefineaction": "feefineactions",
|
|
262
|
+
"bankInfo": "organizations/banking-information",
|
|
263
|
+
}
|
|
264
|
+
object_types.update(task_configuration.extradata_endpoints)
|
|
265
|
+
if object_name == "instructor":
|
|
266
|
+
instructor = json.loads(string_object)
|
|
267
|
+
return f'coursereserves/courselistings/{instructor["courseListingId"]}/instructors'
|
|
268
|
+
|
|
269
|
+
if object_name == "interfaceCredential":
|
|
270
|
+
credential = json.loads(string_object)
|
|
271
|
+
return f'organizations-storage/interfaces/{credential["interfaceId"]}/credentials'
|
|
272
|
+
|
|
273
|
+
return object_types[object_name]
|
|
274
|
+
|
|
219
275
|
def post_single_records(self, row: str, num_records: int, failed_recs_file):
|
|
220
276
|
if self.api_info["is_batch"]:
|
|
221
277
|
raise TypeError("This record type supports batch processing, use post_batch method")
|
|
@@ -345,7 +401,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
345
401
|
elif response.status_code == 400:
|
|
346
402
|
# Likely a json parsing error
|
|
347
403
|
logging.error(response.text)
|
|
348
|
-
raise TransformationProcessError("", "HTTP 400.
|
|
404
|
+
raise TransformationProcessError("", "HTTP 400. Something is wrong. Quitting")
|
|
349
405
|
elif self.task_configuration.object_type == "SRS" and response.status_code >= 500:
|
|
350
406
|
logging.info(
|
|
351
407
|
"Post failed. Size: %s Waiting 30s until reposting. Number of tries: %s of 5",
|
|
@@ -398,10 +454,10 @@ class BatchPoster(MigrationTaskBase):
|
|
|
398
454
|
payload = {self.api_info["object_name"]: batch}
|
|
399
455
|
if self.http_client and not self.http_client.is_closed:
|
|
400
456
|
return self.http_client.post(
|
|
401
|
-
url, json=payload, headers=self.folio_client.okapi_headers
|
|
457
|
+
url, json=payload, headers=self.folio_client.okapi_headers, params=self.query_params
|
|
402
458
|
)
|
|
403
459
|
else:
|
|
404
|
-
return httpx.post(url, headers=self.okapi_headers, json=payload, timeout=None)
|
|
460
|
+
return httpx.post(url, headers=self.okapi_headers, json=payload, params=self.query_params, timeout=None)
|
|
405
461
|
|
|
406
462
|
def wrap_up(self):
|
|
407
463
|
logging.info("Done. Wrapping up")
|
|
@@ -448,7 +504,7 @@ class BatchPoster(MigrationTaskBase):
|
|
|
448
504
|
temp_report = copy.deepcopy(self.migration_report)
|
|
449
505
|
temp_start = self.start_datetime
|
|
450
506
|
self.task_configuration.rerun_failed_records = False
|
|
451
|
-
self.__init__(self.task_configuration, self.library_configuration)
|
|
507
|
+
self.__init__(self.task_configuration, self.library_configuration, self.folio_client)
|
|
452
508
|
self.performing_rerun = True
|
|
453
509
|
self.migration_report = temp_report
|
|
454
510
|
self.start_datetime = temp_start
|
|
@@ -529,6 +585,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
529
585
|
"api_endpoint": "",
|
|
530
586
|
"total_records": False,
|
|
531
587
|
"addSnapshotId": False,
|
|
588
|
+
"supports_upsert": False,
|
|
532
589
|
},
|
|
533
590
|
"Items": {
|
|
534
591
|
"object_name": "items",
|
|
@@ -540,6 +597,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
540
597
|
"is_batch": True,
|
|
541
598
|
"total_records": False,
|
|
542
599
|
"addSnapshotId": False,
|
|
600
|
+
"supports_upsert": True,
|
|
543
601
|
},
|
|
544
602
|
"Holdings": {
|
|
545
603
|
"object_name": "holdingsRecords",
|
|
@@ -551,6 +609,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
551
609
|
"is_batch": True,
|
|
552
610
|
"total_records": False,
|
|
553
611
|
"addSnapshotId": False,
|
|
612
|
+
"supports_upsert": True,
|
|
554
613
|
},
|
|
555
614
|
"Instances": {
|
|
556
615
|
"object_name": "instances",
|
|
@@ -562,6 +621,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
562
621
|
"is_batch": True,
|
|
563
622
|
"total_records": False,
|
|
564
623
|
"addSnapshotId": False,
|
|
624
|
+
"supports_upsert": True,
|
|
565
625
|
},
|
|
566
626
|
"Authorities": {
|
|
567
627
|
"object_name": "",
|
|
@@ -569,6 +629,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
569
629
|
"is_batch": False,
|
|
570
630
|
"total_records": False,
|
|
571
631
|
"addSnapshotId": False,
|
|
632
|
+
"supports_upsert": False,
|
|
572
633
|
},
|
|
573
634
|
"SRS": {
|
|
574
635
|
"object_name": "records",
|
|
@@ -576,6 +637,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
576
637
|
"is_batch": True,
|
|
577
638
|
"total_records": True,
|
|
578
639
|
"addSnapshotId": True,
|
|
640
|
+
"supports_upsert": False,
|
|
579
641
|
},
|
|
580
642
|
"Users": {
|
|
581
643
|
"object_name": "users",
|
|
@@ -583,6 +645,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
583
645
|
"is_batch": True,
|
|
584
646
|
"total_records": True,
|
|
585
647
|
"addSnapshotId": False,
|
|
648
|
+
"supports_upsert": False,
|
|
586
649
|
},
|
|
587
650
|
"Organizations": {
|
|
588
651
|
"object_name": "",
|
|
@@ -590,6 +653,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
590
653
|
"is_batch": False,
|
|
591
654
|
"total_records": False,
|
|
592
655
|
"addSnapshotId": False,
|
|
656
|
+
"supports_upsert": False,
|
|
593
657
|
},
|
|
594
658
|
"Orders": {
|
|
595
659
|
"object_name": "",
|
|
@@ -597,6 +661,7 @@ def get_api_info(object_type: str, use_safe: bool = True):
|
|
|
597
661
|
"is_batch": False,
|
|
598
662
|
"total_records": False,
|
|
599
663
|
"addSnapshotId": False,
|
|
664
|
+
"supports_upsert": False,
|
|
600
665
|
},
|
|
601
666
|
}
|
|
602
667
|
|
|
@@ -623,31 +688,6 @@ def chunks(records, number_of_chunks):
|
|
|
623
688
|
yield records[i : i + number_of_chunks]
|
|
624
689
|
|
|
625
690
|
|
|
626
|
-
def get_extradata_endpoint(object_name: str, string_object: str):
|
|
627
|
-
object_types = {
|
|
628
|
-
"precedingSucceedingTitles": "preceding-succeeding-titles",
|
|
629
|
-
"precedingTitles": "preceding-succeeding-titles",
|
|
630
|
-
"succeedingTitles": "preceding-succeeding-titles",
|
|
631
|
-
"boundwithPart": "inventory-storage/bound-with-parts",
|
|
632
|
-
"notes": "notes",
|
|
633
|
-
"course": "coursereserves/courses",
|
|
634
|
-
"courselisting": "coursereserves/courselistings",
|
|
635
|
-
"contacts": "organizations-storage/contacts",
|
|
636
|
-
"interfaces": "organizations-storage/interfaces",
|
|
637
|
-
"account": "accounts",
|
|
638
|
-
"feefineaction": "feefineactions",
|
|
639
|
-
}
|
|
640
|
-
if object_name == "instructor":
|
|
641
|
-
instructor = json.loads(string_object)
|
|
642
|
-
return f'coursereserves/courselistings/{instructor["courseListingId"]}/instructors'
|
|
643
|
-
|
|
644
|
-
if object_name == "interfaceCredential":
|
|
645
|
-
credential = json.loads(string_object)
|
|
646
|
-
return f'organizations-storage/interfaces/{credential["interfaceId"]}/credentials'
|
|
647
|
-
|
|
648
|
-
return object_types[object_name]
|
|
649
|
-
|
|
650
|
-
|
|
651
691
|
def get_human_readable(size, precision=2):
|
|
652
692
|
suffixes = ["B", "KB", "MB", "GB", "TB"]
|
|
653
693
|
suffix_index = 0
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import Annotated
|
|
3
|
-
from typing import List
|
|
4
|
-
import i18n
|
|
2
|
+
from typing import Annotated, List
|
|
5
3
|
|
|
4
|
+
import i18n
|
|
6
5
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
7
6
|
from pydantic import Field
|
|
8
7
|
|
|
9
8
|
from folio_migration_tools.helper import Helper
|
|
10
|
-
from folio_migration_tools.library_configuration import
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
from folio_migration_tools.library_configuration import (
|
|
10
|
+
FileDefinition,
|
|
11
|
+
HridHandling,
|
|
12
|
+
IlsFlavour,
|
|
13
|
+
LibraryConfiguration,
|
|
14
|
+
)
|
|
14
15
|
from folio_migration_tools.marc_rules_transformation.marc_file_processor import (
|
|
15
16
|
MarcFileProcessor,
|
|
16
17
|
)
|
|
@@ -96,6 +97,17 @@ class BibsTransformer(MigrationTaskBase):
|
|
|
96
97
|
),
|
|
97
98
|
),
|
|
98
99
|
] = True
|
|
100
|
+
data_import_marc: Annotated[
|
|
101
|
+
bool,
|
|
102
|
+
Field(
|
|
103
|
+
title="Generate a MARC file for data import overlay of instances",
|
|
104
|
+
description=(
|
|
105
|
+
"If set to true, the process will generate a file of binary MARC records that can"
|
|
106
|
+
"be imported into FOLIO using the Data Import APIs. If set to false, only a file"
|
|
107
|
+
"of FOLIO instance records (and optional SRS records) will be generated."
|
|
108
|
+
),
|
|
109
|
+
)
|
|
110
|
+
] = False
|
|
99
111
|
parse_cataloged_date: Annotated[
|
|
100
112
|
bool,
|
|
101
113
|
Field(
|
|
@@ -153,9 +165,10 @@ class BibsTransformer(MigrationTaskBase):
|
|
|
153
165
|
self,
|
|
154
166
|
task_config: TaskConfiguration,
|
|
155
167
|
library_config: LibraryConfiguration,
|
|
168
|
+
folio_client,
|
|
156
169
|
use_logging: bool = True,
|
|
157
170
|
):
|
|
158
|
-
super().__init__(library_config, task_config, use_logging)
|
|
171
|
+
super().__init__(library_config, task_config, folio_client, use_logging)
|
|
159
172
|
self.processor: MarcFileProcessor
|
|
160
173
|
self.check_source_files(
|
|
161
174
|
self.folder_structure.legacy_records_folder, self.task_configuration.files
|
|
@@ -4,15 +4,19 @@ import logging
|
|
|
4
4
|
import sys
|
|
5
5
|
import time
|
|
6
6
|
import traceback
|
|
7
|
-
import i18n
|
|
8
7
|
from typing import Optional
|
|
9
8
|
|
|
9
|
+
import i18n
|
|
10
10
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
11
11
|
|
|
12
|
-
from folio_migration_tools.custom_exceptions import
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
12
|
+
from folio_migration_tools.custom_exceptions import (
|
|
13
|
+
TransformationProcessError,
|
|
14
|
+
TransformationRecordFailedError,
|
|
15
|
+
)
|
|
16
|
+
from folio_migration_tools.library_configuration import (
|
|
17
|
+
FileDefinition,
|
|
18
|
+
LibraryConfiguration,
|
|
19
|
+
)
|
|
16
20
|
from folio_migration_tools.mapping_file_transformation.courses_mapper import (
|
|
17
21
|
CoursesMapper,
|
|
18
22
|
)
|
|
@@ -41,10 +45,11 @@ class CoursesMigrator(MigrationTaskBase):
|
|
|
41
45
|
self,
|
|
42
46
|
task_configuration: TaskConfiguration,
|
|
43
47
|
library_config: LibraryConfiguration,
|
|
48
|
+
folio_client
|
|
44
49
|
):
|
|
45
50
|
csv.register_dialect("tsv", delimiter="\t")
|
|
46
51
|
self.task_configuration = task_configuration
|
|
47
|
-
super().__init__(library_config, task_configuration)
|
|
52
|
+
super().__init__(library_config, task_configuration, folio_client)
|
|
48
53
|
self.t0 = time.time()
|
|
49
54
|
self.courses_map = self.setup_records_map(
|
|
50
55
|
self.folder_structure.mapping_files_folder
|
|
@@ -5,23 +5,24 @@ import logging
|
|
|
5
5
|
import sys
|
|
6
6
|
import time
|
|
7
7
|
import traceback
|
|
8
|
-
from typing import Annotated
|
|
9
|
-
from typing import List
|
|
10
|
-
from typing import Optional
|
|
8
|
+
from typing import Annotated, List, Optional
|
|
11
9
|
|
|
12
10
|
import i18n
|
|
13
11
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
14
|
-
from folio_uuid.folio_uuid import FolioUUID
|
|
15
12
|
from httpx import HTTPError
|
|
16
13
|
from pydantic import Field
|
|
17
14
|
|
|
18
|
-
from folio_migration_tools.custom_exceptions import
|
|
19
|
-
|
|
15
|
+
from folio_migration_tools.custom_exceptions import (
|
|
16
|
+
TransformationProcessError,
|
|
17
|
+
TransformationRecordFailedError,
|
|
18
|
+
)
|
|
20
19
|
from folio_migration_tools.helper import Helper
|
|
21
20
|
from folio_migration_tools.holdings_helper import HoldingsHelper
|
|
22
|
-
from folio_migration_tools.library_configuration import
|
|
23
|
-
|
|
24
|
-
|
|
21
|
+
from folio_migration_tools.library_configuration import (
|
|
22
|
+
FileDefinition,
|
|
23
|
+
HridHandling,
|
|
24
|
+
LibraryConfiguration,
|
|
25
|
+
)
|
|
25
26
|
from folio_migration_tools.mapping_file_transformation.holdings_mapper import (
|
|
26
27
|
HoldingsMapper,
|
|
27
28
|
)
|
|
@@ -80,9 +81,10 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
80
81
|
self,
|
|
81
82
|
task_config: TaskConfiguration,
|
|
82
83
|
library_config: LibraryConfiguration,
|
|
84
|
+
folio_client,
|
|
83
85
|
use_logging: bool = True,
|
|
84
86
|
):
|
|
85
|
-
super().__init__(library_config, task_config, use_logging)
|
|
87
|
+
super().__init__(library_config, task_config, folio_client, use_logging)
|
|
86
88
|
self.fallback_holdings_type = None
|
|
87
89
|
try:
|
|
88
90
|
self.task_config = task_config
|
|
@@ -250,9 +252,9 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
250
252
|
# Prevent the first item in a boundwith to be overwritten
|
|
251
253
|
# TODO: Find out why not
|
|
252
254
|
# if legacy_id not in self.holdings_id_map:
|
|
253
|
-
self.holdings_id_map[
|
|
254
|
-
|
|
255
|
-
|
|
255
|
+
self.holdings_id_map[legacy_id] = self.mapper.get_id_map_tuple(
|
|
256
|
+
legacy_id, holding, self.object_type
|
|
257
|
+
)
|
|
256
258
|
Helper.write_to_file(holdings_file, holding)
|
|
257
259
|
self.mapper.migration_report.add_general_statistics(
|
|
258
260
|
i18n.t("Holdings Records Written to disk")
|
|
@@ -306,7 +308,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
306
308
|
folio_rec, legacy_id = self.mapper.do_map(
|
|
307
309
|
legacy_record, f"row # {idx}", FOLIONamespaces.holdings
|
|
308
310
|
)
|
|
309
|
-
self.post_process_holding(folio_rec, legacy_id)
|
|
311
|
+
self.post_process_holding(folio_rec, legacy_id, file_def)
|
|
310
312
|
except TransformationProcessError as process_error:
|
|
311
313
|
self.mapper.handle_transformation_process_error(idx, process_error)
|
|
312
314
|
except TransformationRecordFailedError as error:
|
|
@@ -326,7 +328,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
326
328
|
f"Total records processed: {self.total_records:,}"
|
|
327
329
|
)
|
|
328
330
|
|
|
329
|
-
def post_process_holding(self, folio_rec: dict, legacy_id: str):
|
|
331
|
+
def post_process_holding(self, folio_rec: dict, legacy_id: str, file_def: FileDefinition):
|
|
330
332
|
HoldingsHelper.handle_notes(folio_rec)
|
|
331
333
|
HoldingsHelper.remove_empty_holdings_statements(folio_rec)
|
|
332
334
|
|
|
@@ -348,6 +350,7 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
348
350
|
raise TransformationRecordFailedError(legacy_id, "No instance id in parsed record", "")
|
|
349
351
|
|
|
350
352
|
for folio_holding in holdings_from_row:
|
|
353
|
+
self.mapper.perform_additional_mappings(folio_holding, file_def)
|
|
351
354
|
self.merge_holding_in(folio_holding, all_instance_ids, legacy_id)
|
|
352
355
|
self.mapper.report_folio_mapping(folio_holding, self.mapper.schema)
|
|
353
356
|
|
|
@@ -375,7 +378,10 @@ class HoldingsCsvTransformer(MigrationTaskBase):
|
|
|
375
378
|
"""
|
|
376
379
|
if len(instance_ids) > 1:
|
|
377
380
|
# Is boundwith
|
|
378
|
-
bw_key =
|
|
381
|
+
bw_key = (
|
|
382
|
+
f"bw_{incoming_holding['instanceId']}_{incoming_holding['permanentLocationId']}_"
|
|
383
|
+
f"{incoming_holding.get('callNumber', '')}_{'_'.join(sorted(instance_ids))}"
|
|
384
|
+
)
|
|
379
385
|
if bw_key not in self.bound_with_keys:
|
|
380
386
|
self.bound_with_keys.add(bw_key)
|
|
381
387
|
self.holdings[bw_key] = incoming_holding
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
'''Main "script."'''
|
|
2
2
|
import csv
|
|
3
3
|
import logging
|
|
4
|
-
import
|
|
5
|
-
from typing import Annotated
|
|
6
|
-
from typing import List
|
|
4
|
+
from typing import Annotated, List
|
|
7
5
|
|
|
6
|
+
import i18n
|
|
8
7
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
9
8
|
from pydantic import Field
|
|
10
9
|
|
|
11
10
|
from folio_migration_tools.custom_exceptions import TransformationProcessError
|
|
12
11
|
from folio_migration_tools.helper import Helper
|
|
13
|
-
from folio_migration_tools.library_configuration import
|
|
14
|
-
|
|
15
|
-
|
|
12
|
+
from folio_migration_tools.library_configuration import (
|
|
13
|
+
FileDefinition,
|
|
14
|
+
HridHandling,
|
|
15
|
+
LibraryConfiguration,
|
|
16
|
+
)
|
|
16
17
|
from folio_migration_tools.marc_rules_transformation.rules_mapper_holdings import (
|
|
17
18
|
RulesMapperHoldings,
|
|
18
19
|
)
|
|
@@ -162,10 +163,11 @@ class HoldingsMarcTransformer(MigrationTaskBase):
|
|
|
162
163
|
self,
|
|
163
164
|
task_config: TaskConfiguration,
|
|
164
165
|
library_config: LibraryConfiguration,
|
|
166
|
+
folio_client,
|
|
165
167
|
use_logging: bool = True,
|
|
166
168
|
):
|
|
167
169
|
csv.register_dialect("tsv", delimiter="\t")
|
|
168
|
-
super().__init__(library_config, task_config, use_logging)
|
|
170
|
+
super().__init__(library_config, task_config, folio_client, use_logging)
|
|
169
171
|
self.task_config = task_config
|
|
170
172
|
self.holdings_types = list(
|
|
171
173
|
self.folio_client.folio_get_all("/holdings-types", "holdingsTypes")
|
|
@@ -7,20 +7,22 @@ import sys
|
|
|
7
7
|
import time
|
|
8
8
|
import traceback
|
|
9
9
|
import uuid
|
|
10
|
-
import
|
|
11
|
-
from typing import Annotated
|
|
12
|
-
from typing import List
|
|
13
|
-
from typing import Optional
|
|
10
|
+
from typing import Annotated, List, Optional
|
|
14
11
|
|
|
12
|
+
import i18n
|
|
15
13
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
16
14
|
from pydantic import Field
|
|
17
15
|
|
|
18
|
-
from folio_migration_tools.custom_exceptions import
|
|
19
|
-
|
|
16
|
+
from folio_migration_tools.custom_exceptions import (
|
|
17
|
+
TransformationProcessError,
|
|
18
|
+
TransformationRecordFailedError,
|
|
19
|
+
)
|
|
20
20
|
from folio_migration_tools.helper import Helper
|
|
21
|
-
from folio_migration_tools.library_configuration import
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
from folio_migration_tools.library_configuration import (
|
|
22
|
+
FileDefinition,
|
|
23
|
+
HridHandling,
|
|
24
|
+
LibraryConfiguration,
|
|
25
|
+
)
|
|
24
26
|
from folio_migration_tools.mapping_file_transformation.item_mapper import ItemMapper
|
|
25
27
|
from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
|
|
26
28
|
MappingFileMapperBase,
|
|
@@ -75,10 +77,11 @@ class ItemsTransformer(MigrationTaskBase):
|
|
|
75
77
|
self,
|
|
76
78
|
task_config: TaskConfiguration,
|
|
77
79
|
library_config: LibraryConfiguration,
|
|
80
|
+
folio_client,
|
|
78
81
|
use_logging: bool = True,
|
|
79
82
|
):
|
|
80
83
|
csv.register_dialect("tsv", delimiter="\t")
|
|
81
|
-
super().__init__(library_config, task_config, use_logging)
|
|
84
|
+
super().__init__(library_config, task_config, folio_client, use_logging)
|
|
82
85
|
self.task_config = task_config
|
|
83
86
|
self.check_source_files(
|
|
84
87
|
self.folder_structure.legacy_records_folder, self.task_config.files
|
|
@@ -5,22 +5,22 @@ import logging
|
|
|
5
5
|
import sys
|
|
6
6
|
import time
|
|
7
7
|
import traceback
|
|
8
|
-
from datetime import datetime
|
|
9
|
-
from datetime import timedelta
|
|
8
|
+
from datetime import datetime, timedelta
|
|
10
9
|
from typing import Optional
|
|
11
10
|
from urllib.error import HTTPError
|
|
12
|
-
from zoneinfo import ZoneInfo
|
|
13
11
|
|
|
14
|
-
import httpx
|
|
15
12
|
import i18n
|
|
16
13
|
from dateutil import parser as du_parser
|
|
17
14
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
15
|
+
from zoneinfo import ZoneInfo
|
|
18
16
|
|
|
19
17
|
from folio_migration_tools.circulation_helper import CirculationHelper
|
|
20
18
|
from folio_migration_tools.helper import Helper
|
|
21
|
-
from folio_migration_tools.library_configuration import
|
|
22
|
-
|
|
23
|
-
|
|
19
|
+
from folio_migration_tools.library_configuration import (
|
|
20
|
+
FileDefinition,
|
|
21
|
+
FolioRelease,
|
|
22
|
+
LibraryConfiguration,
|
|
23
|
+
)
|
|
24
24
|
from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base import (
|
|
25
25
|
MappingFileMapperBase,
|
|
26
26
|
)
|
|
@@ -51,6 +51,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
51
51
|
self,
|
|
52
52
|
task_configuration: TaskConfiguration,
|
|
53
53
|
library_config: LibraryConfiguration,
|
|
54
|
+
folio_client
|
|
54
55
|
):
|
|
55
56
|
csv.register_dialect("tsv", delimiter="\t")
|
|
56
57
|
self.patron_item_combos: set = set()
|
|
@@ -62,7 +63,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
62
63
|
self.failed_and_not_dupe: dict = {}
|
|
63
64
|
self.migration_report = MigrationReport()
|
|
64
65
|
self.valid_legacy_loans = []
|
|
65
|
-
super().__init__(library_config, task_configuration)
|
|
66
|
+
super().__init__(library_config, task_configuration, folio_client)
|
|
66
67
|
self.circulation_helper = CirculationHelper(
|
|
67
68
|
self.folio_client,
|
|
68
69
|
task_configuration.fallback_service_point_id,
|
|
@@ -168,7 +169,7 @@ class LoansMigrator(MigrationTaskBase):
|
|
|
168
169
|
logging.info("SMTP connection is disabled...")
|
|
169
170
|
|
|
170
171
|
def do_work(self):
|
|
171
|
-
with
|
|
172
|
+
with self.folio_client.get_folio_http_client() as self.http_client:
|
|
172
173
|
logging.info("Starting")
|
|
173
174
|
starting_index = (
|
|
174
175
|
self.task_configuration.starting_row - 1
|
|
@@ -4,18 +4,21 @@ import logging
|
|
|
4
4
|
import sys
|
|
5
5
|
import time
|
|
6
6
|
import traceback
|
|
7
|
-
import
|
|
8
|
-
from typing import List
|
|
9
|
-
from typing import Optional
|
|
7
|
+
from typing import List, Optional
|
|
10
8
|
|
|
9
|
+
import i18n
|
|
11
10
|
from folio_uuid.folio_namespaces import FOLIONamespaces
|
|
12
11
|
|
|
13
|
-
from folio_migration_tools.custom_exceptions import
|
|
14
|
-
|
|
15
|
-
|
|
12
|
+
from folio_migration_tools.custom_exceptions import (
|
|
13
|
+
TransformationFieldMappingError,
|
|
14
|
+
TransformationProcessError,
|
|
15
|
+
TransformationRecordFailedError,
|
|
16
|
+
)
|
|
16
17
|
from folio_migration_tools.helper import Helper
|
|
17
|
-
from folio_migration_tools.library_configuration import
|
|
18
|
-
|
|
18
|
+
from folio_migration_tools.library_configuration import (
|
|
19
|
+
FileDefinition,
|
|
20
|
+
LibraryConfiguration,
|
|
21
|
+
)
|
|
19
22
|
from folio_migration_tools.mapping_file_transformation.manual_fee_fines_mapper import (
|
|
20
23
|
ManualFeeFinesMapper,
|
|
21
24
|
)
|
|
@@ -44,11 +47,12 @@ class ManualFeeFinesTransformer(MigrationTaskBase):
|
|
|
44
47
|
self,
|
|
45
48
|
task_configuration: TaskConfiguration,
|
|
46
49
|
library_config: LibraryConfiguration,
|
|
50
|
+
folio_client,
|
|
47
51
|
use_logging: bool = True,
|
|
48
52
|
):
|
|
49
53
|
csv.register_dialect("tsv", delimiter="\t")
|
|
50
54
|
|
|
51
|
-
super().__init__(library_config, task_configuration, use_logging)
|
|
55
|
+
super().__init__(library_config, task_configuration, folio_client, use_logging)
|
|
52
56
|
self.object_type_name = self.get_object_type().name
|
|
53
57
|
self.task_configuration = task_configuration
|
|
54
58
|
self.check_source_files(
|