folio-data-import 0.3.2__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of folio-data-import might be problematic. Click here for more details.
- folio_data_import/MARCDataImport.py +295 -159
- folio_data_import/UserImport.py +378 -255
- folio_data_import/__main__.py +7 -110
- folio_data_import/_progress.py +27 -0
- folio_data_import/marc_preprocessors/_preprocessors.py +12 -8
- {folio_data_import-0.3.2.dist-info → folio_data_import-0.4.0.dist-info}/METADATA +57 -6
- folio_data_import-0.4.0.dist-info/RECORD +13 -0
- {folio_data_import-0.3.2.dist-info → folio_data_import-0.4.0.dist-info}/WHEEL +1 -1
- folio_data_import-0.4.0.dist-info/entry_points.txt +5 -0
- folio_data_import-0.3.2.dist-info/RECORD +0 -12
- folio_data_import-0.3.2.dist-info/entry_points.txt +0 -5
- {folio_data_import-0.3.2.dist-info → folio_data_import-0.4.0.dist-info/licenses}/LICENSE +0 -0
folio_data_import/UserImport.py
CHANGED
|
@@ -1,19 +1,32 @@
|
|
|
1
|
-
import argparse
|
|
2
1
|
import asyncio
|
|
3
2
|
import datetime
|
|
4
|
-
import getpass
|
|
5
3
|
import json
|
|
6
|
-
import
|
|
4
|
+
import logging
|
|
5
|
+
import sys
|
|
7
6
|
import time
|
|
8
7
|
import uuid
|
|
9
8
|
from datetime import datetime as dt
|
|
9
|
+
from enum import Enum
|
|
10
10
|
from pathlib import Path
|
|
11
|
-
from typing import
|
|
11
|
+
from typing import List, Tuple
|
|
12
12
|
|
|
13
13
|
import aiofiles
|
|
14
14
|
import folioclient
|
|
15
15
|
import httpx
|
|
16
|
+
import typer
|
|
16
17
|
from aiofiles.threadpool.text import AsyncTextIOWrapper
|
|
18
|
+
from rich.logging import RichHandler
|
|
19
|
+
from rich.progress import (
|
|
20
|
+
BarColumn,
|
|
21
|
+
MofNCompleteColumn,
|
|
22
|
+
Progress,
|
|
23
|
+
SpinnerColumn,
|
|
24
|
+
TimeElapsedColumn,
|
|
25
|
+
TimeRemainingColumn,
|
|
26
|
+
)
|
|
27
|
+
from typing_extensions import Annotated
|
|
28
|
+
|
|
29
|
+
from folio_data_import._progress import ItemsPerSecondColumn, UserStatsColumn
|
|
17
30
|
|
|
18
31
|
try:
|
|
19
32
|
utc = datetime.UTC
|
|
@@ -22,6 +35,8 @@ except AttributeError:
|
|
|
22
35
|
|
|
23
36
|
utc = zoneinfo.ZoneInfo("UTC")
|
|
24
37
|
|
|
38
|
+
logger = logging.getLogger(__name__)
|
|
39
|
+
|
|
25
40
|
# Mapping of preferred contact type IDs to their corresponding values
|
|
26
41
|
PREFERRED_CONTACT_TYPES_MAP = {
|
|
27
42
|
"001": "mail",
|
|
@@ -31,6 +46,26 @@ PREFERRED_CONTACT_TYPES_MAP = {
|
|
|
31
46
|
"005": "mobile",
|
|
32
47
|
}
|
|
33
48
|
|
|
49
|
+
|
|
50
|
+
class PreferredContactType(Enum):
|
|
51
|
+
MAIL = "001"
|
|
52
|
+
EMAIL = "002"
|
|
53
|
+
TEXT = "003"
|
|
54
|
+
PHONE = "004"
|
|
55
|
+
MOBILE = "005"
|
|
56
|
+
_001 = "mail"
|
|
57
|
+
_002 = "email"
|
|
58
|
+
_003 = "text"
|
|
59
|
+
_004 = "phone"
|
|
60
|
+
_005 = "mobile"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class UserMatchKeys(Enum):
|
|
64
|
+
USERNAME = "username"
|
|
65
|
+
EMAIL = "email"
|
|
66
|
+
EXTERNAL_SYSTEM_ID = "externalSystemId"
|
|
67
|
+
|
|
68
|
+
|
|
34
69
|
class UserImporter: # noqa: R0902
|
|
35
70
|
"""
|
|
36
71
|
Class to import mod-user-import compatible user objects
|
|
@@ -38,20 +73,21 @@ class UserImporter: # noqa: R0902
|
|
|
38
73
|
from a JSON-lines file into FOLIO
|
|
39
74
|
"""
|
|
40
75
|
|
|
76
|
+
logfile: AsyncTextIOWrapper
|
|
77
|
+
errorfile: AsyncTextIOWrapper
|
|
78
|
+
http_client: httpx.AsyncClient
|
|
79
|
+
|
|
41
80
|
def __init__(
|
|
42
81
|
self,
|
|
43
82
|
folio_client: folioclient.FolioClient,
|
|
44
83
|
library_name: str,
|
|
45
84
|
batch_size: int,
|
|
46
85
|
limit_simultaneous_requests: asyncio.Semaphore,
|
|
47
|
-
logfile: AsyncTextIOWrapper,
|
|
48
|
-
errorfile: AsyncTextIOWrapper,
|
|
49
|
-
http_client: httpx.AsyncClient,
|
|
50
86
|
user_file_path: Path = None,
|
|
51
87
|
user_match_key: str = "externalSystemId",
|
|
52
88
|
only_update_present_fields: bool = False,
|
|
53
89
|
default_preferred_contact_type: str = "002",
|
|
54
|
-
fields_to_protect: List[str] =[],
|
|
90
|
+
fields_to_protect: List[str] = [],
|
|
55
91
|
) -> None:
|
|
56
92
|
self.limit_simultaneous_requests = limit_simultaneous_requests
|
|
57
93
|
self.batch_size = batch_size
|
|
@@ -70,9 +106,6 @@ class UserImporter: # noqa: R0902
|
|
|
70
106
|
self.service_point_map: dict = self.build_ref_data_id_map(
|
|
71
107
|
self.folio_client, "/service-points", "servicepoints", "code"
|
|
72
108
|
)
|
|
73
|
-
self.logfile: AsyncTextIOWrapper = logfile
|
|
74
|
-
self.errorfile: AsyncTextIOWrapper = errorfile
|
|
75
|
-
self.http_client: httpx.AsyncClient = http_client
|
|
76
109
|
self.only_update_present_fields: bool = only_update_present_fields
|
|
77
110
|
self.default_preferred_contact_type: str = default_preferred_contact_type
|
|
78
111
|
self.match_key = user_match_key
|
|
@@ -114,17 +147,36 @@ class UserImporter: # noqa: R0902
|
|
|
114
147
|
except ValueError:
|
|
115
148
|
return False
|
|
116
149
|
|
|
150
|
+
async def setup(self, error_file_path: Path) -> None:
|
|
151
|
+
"""
|
|
152
|
+
Sets up the importer by initializing necessary resources.
|
|
153
|
+
|
|
154
|
+
Args:
|
|
155
|
+
log_file_path (Path): The path to the log file.
|
|
156
|
+
error_file_path (Path): The path to the error file.
|
|
157
|
+
"""
|
|
158
|
+
self.errorfile = await aiofiles.open(error_file_path, "w", encoding="utf-8")
|
|
159
|
+
|
|
160
|
+
async def close(self) -> None:
|
|
161
|
+
"""
|
|
162
|
+
Closes the importer by releasing any resources.
|
|
163
|
+
|
|
164
|
+
"""
|
|
165
|
+
await self.errorfile.close()
|
|
166
|
+
|
|
117
167
|
async def do_import(self) -> None:
|
|
118
168
|
"""
|
|
119
169
|
Main method to import users.
|
|
120
170
|
|
|
121
171
|
This method triggers the process of importing users by calling the `process_file` method.
|
|
122
172
|
"""
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
173
|
+
async with httpx.AsyncClient() as client:
|
|
174
|
+
self.http_client = client
|
|
175
|
+
if self.user_file_path:
|
|
176
|
+
with open(self.user_file_path, "r", encoding="utf-8") as openfile:
|
|
177
|
+
await self.process_file(openfile)
|
|
178
|
+
else:
|
|
179
|
+
raise FileNotFoundError("No user objects file provided")
|
|
128
180
|
|
|
129
181
|
async def get_existing_user(self, user_obj) -> dict:
|
|
130
182
|
"""
|
|
@@ -227,7 +279,7 @@ class UserImporter: # noqa: R0902
|
|
|
227
279
|
self.validate_uuid(address["addressTypeId"])
|
|
228
280
|
and address["addressTypeId"] in self.address_type_map.values()
|
|
229
281
|
):
|
|
230
|
-
|
|
282
|
+
logger.debug(
|
|
231
283
|
f"Row {line_number}: Address type {address['addressTypeId']} is a UUID, "
|
|
232
284
|
f"skipping mapping\n"
|
|
233
285
|
)
|
|
@@ -239,11 +291,7 @@ class UserImporter: # noqa: R0902
|
|
|
239
291
|
mapped_addresses.append(address)
|
|
240
292
|
except KeyError:
|
|
241
293
|
if address["addressTypeId"] not in self.address_type_map.values():
|
|
242
|
-
|
|
243
|
-
f"Row {line_number}: Address type {address['addressTypeId']} not found"
|
|
244
|
-
f", removing address"
|
|
245
|
-
)
|
|
246
|
-
await self.logfile.write(
|
|
294
|
+
logger.error(
|
|
247
295
|
f"Row {line_number}: Address type {address['addressTypeId']} not found"
|
|
248
296
|
f", removing address\n"
|
|
249
297
|
)
|
|
@@ -266,7 +314,7 @@ class UserImporter: # noqa: R0902
|
|
|
266
314
|
self.validate_uuid(user_obj["patronGroup"])
|
|
267
315
|
and user_obj["patronGroup"] in self.patron_group_map.values()
|
|
268
316
|
):
|
|
269
|
-
|
|
317
|
+
logger.debug(
|
|
270
318
|
f"Row {line_number}: Patron group {user_obj['patronGroup']} is a UUID, "
|
|
271
319
|
f"skipping mapping\n"
|
|
272
320
|
)
|
|
@@ -274,11 +322,7 @@ class UserImporter: # noqa: R0902
|
|
|
274
322
|
user_obj["patronGroup"] = self.patron_group_map[user_obj["patronGroup"]]
|
|
275
323
|
except KeyError:
|
|
276
324
|
if user_obj["patronGroup"] not in self.patron_group_map.values():
|
|
277
|
-
|
|
278
|
-
f"Row {line_number}: Patron group {user_obj['patronGroup']} not found, "
|
|
279
|
-
f"removing patron group"
|
|
280
|
-
)
|
|
281
|
-
await self.logfile.write(
|
|
325
|
+
logger.error(
|
|
282
326
|
f"Row {line_number}: Patron group {user_obj['patronGroup']} not found in, "
|
|
283
327
|
f"removing patron group\n"
|
|
284
328
|
)
|
|
@@ -302,25 +346,23 @@ class UserImporter: # noqa: R0902
|
|
|
302
346
|
self.validate_uuid(department)
|
|
303
347
|
and department in self.department_map.values()
|
|
304
348
|
):
|
|
305
|
-
|
|
349
|
+
logger.debug(
|
|
306
350
|
f"Row {line_number}: Department {department} is a UUID, skipping mapping\n"
|
|
307
351
|
)
|
|
308
352
|
mapped_departments.append(department)
|
|
309
353
|
else:
|
|
310
354
|
mapped_departments.append(self.department_map[department])
|
|
311
355
|
except KeyError:
|
|
312
|
-
|
|
313
|
-
f'Row {line_number}: Department "{department}" not found, ' # noqa: B907
|
|
314
|
-
f"excluding department from user"
|
|
315
|
-
)
|
|
316
|
-
await self.logfile.write(
|
|
356
|
+
logger.error(
|
|
317
357
|
f'Row {line_number}: Department "{department}" not found, ' # noqa: B907
|
|
318
358
|
f"excluding department from user\n"
|
|
319
359
|
)
|
|
320
360
|
if mapped_departments:
|
|
321
361
|
user_obj["departments"] = mapped_departments
|
|
322
362
|
|
|
323
|
-
async def update_existing_user(
|
|
363
|
+
async def update_existing_user(
|
|
364
|
+
self, user_obj, existing_user, protected_fields
|
|
365
|
+
) -> Tuple[dict, dict]:
|
|
324
366
|
"""
|
|
325
367
|
Updates an existing user with the provided user object.
|
|
326
368
|
|
|
@@ -338,7 +380,11 @@ class UserImporter: # noqa: R0902
|
|
|
338
380
|
"""
|
|
339
381
|
|
|
340
382
|
await self.set_preferred_contact_type(user_obj, existing_user)
|
|
341
|
-
preferred_contact_type = {
|
|
383
|
+
preferred_contact_type = {
|
|
384
|
+
"preferredContactTypeId": existing_user.get("personal", {}).pop(
|
|
385
|
+
"preferredContactTypeId"
|
|
386
|
+
)
|
|
387
|
+
}
|
|
342
388
|
if self.only_update_present_fields:
|
|
343
389
|
new_personal = user_obj.pop("personal", {})
|
|
344
390
|
existing_personal = existing_user.pop("personal", {})
|
|
@@ -412,30 +458,39 @@ class UserImporter: # noqa: R0902
|
|
|
412
458
|
current_pref_contact = user_obj["personal"].get(
|
|
413
459
|
"preferredContactTypeId", ""
|
|
414
460
|
)
|
|
415
|
-
if mapped_contact_type := dict(
|
|
461
|
+
if mapped_contact_type := dict(
|
|
462
|
+
[(v, k) for k, v in PREFERRED_CONTACT_TYPES_MAP.items()]
|
|
463
|
+
).get(
|
|
416
464
|
current_pref_contact,
|
|
417
465
|
"",
|
|
418
466
|
):
|
|
419
|
-
existing_user["personal"]["preferredContactTypeId"] =
|
|
467
|
+
existing_user["personal"]["preferredContactTypeId"] = (
|
|
468
|
+
mapped_contact_type
|
|
469
|
+
)
|
|
420
470
|
else:
|
|
421
|
-
existing_user["personal"]["preferredContactTypeId"] =
|
|
471
|
+
existing_user["personal"]["preferredContactTypeId"] = (
|
|
472
|
+
current_pref_contact
|
|
473
|
+
if current_pref_contact in PREFERRED_CONTACT_TYPES_MAP
|
|
474
|
+
else self.default_preferred_contact_type
|
|
475
|
+
)
|
|
422
476
|
else:
|
|
423
|
-
|
|
424
|
-
f"Preferred contact type not provided or is not a valid option: {PREFERRED_CONTACT_TYPES_MAP}
|
|
477
|
+
logger.warning(
|
|
478
|
+
f"Preferred contact type not provided or is not a valid option: {PREFERRED_CONTACT_TYPES_MAP} "
|
|
425
479
|
f"Setting preferred contact type to {self.default_preferred_contact_type} or using existing value"
|
|
426
480
|
)
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
481
|
+
mapped_contact_type = (
|
|
482
|
+
existing_user.get("personal", {}).get("preferredContactTypeId", "")
|
|
483
|
+
or self.default_preferred_contact_type
|
|
430
484
|
)
|
|
431
|
-
mapped_contact_type = existing_user.get("personal", {}).get(
|
|
432
|
-
"preferredContactTypeId", ""
|
|
433
|
-
) or self.default_preferred_contact_type
|
|
434
485
|
if "personal" not in existing_user:
|
|
435
486
|
existing_user["personal"] = {}
|
|
436
|
-
existing_user["personal"]["preferredContactTypeId"] =
|
|
487
|
+
existing_user["personal"]["preferredContactTypeId"] = (
|
|
488
|
+
mapped_contact_type or self.default_preferred_contact_type
|
|
489
|
+
)
|
|
437
490
|
|
|
438
|
-
async def create_or_update_user(
|
|
491
|
+
async def create_or_update_user(
|
|
492
|
+
self, user_obj, existing_user, protected_fields, line_number
|
|
493
|
+
) -> dict:
|
|
439
494
|
"""
|
|
440
495
|
Creates or updates a user based on the given user object and existing user.
|
|
441
496
|
|
|
@@ -456,11 +511,7 @@ class UserImporter: # noqa: R0902
|
|
|
456
511
|
self.logs["updated"] += 1
|
|
457
512
|
return existing_user
|
|
458
513
|
except Exception as ee:
|
|
459
|
-
|
|
460
|
-
f"Row {line_number}: User update failed: "
|
|
461
|
-
f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}"
|
|
462
|
-
)
|
|
463
|
-
await self.logfile.write(
|
|
514
|
+
logger.error(
|
|
464
515
|
f"Row {line_number}: User update failed: "
|
|
465
516
|
f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
|
|
466
517
|
)
|
|
@@ -474,11 +525,7 @@ class UserImporter: # noqa: R0902
|
|
|
474
525
|
new_user = await self.create_new_user(user_obj)
|
|
475
526
|
return new_user
|
|
476
527
|
except Exception as ee:
|
|
477
|
-
|
|
478
|
-
f"Row {line_number}: User creation failed: "
|
|
479
|
-
f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}"
|
|
480
|
-
)
|
|
481
|
-
await self.logfile.write(
|
|
528
|
+
logger.error(
|
|
482
529
|
f"Row {line_number}: User creation failed: "
|
|
483
530
|
f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
|
|
484
531
|
)
|
|
@@ -516,7 +563,9 @@ class UserImporter: # noqa: R0902
|
|
|
516
563
|
dict: A dictionary containing the protected fields and their values.
|
|
517
564
|
"""
|
|
518
565
|
protected_fields = {}
|
|
519
|
-
protected_fields_list =
|
|
566
|
+
protected_fields_list = (
|
|
567
|
+
existing_user.get("customFields", {}).get("protectedFields", "").split(",")
|
|
568
|
+
)
|
|
520
569
|
cli_fields = list(self.fields_to_protect)
|
|
521
570
|
# combine and dedupe:
|
|
522
571
|
all_fields = list(dict.fromkeys(protected_fields_list + cli_fields))
|
|
@@ -557,7 +606,15 @@ class UserImporter: # noqa: R0902
|
|
|
557
606
|
existing_pu = {}
|
|
558
607
|
existing_spu = {}
|
|
559
608
|
protected_fields = {}
|
|
560
|
-
return
|
|
609
|
+
return (
|
|
610
|
+
rp_obj,
|
|
611
|
+
spu_obj,
|
|
612
|
+
existing_user,
|
|
613
|
+
protected_fields,
|
|
614
|
+
existing_rp,
|
|
615
|
+
existing_pu,
|
|
616
|
+
existing_spu,
|
|
617
|
+
)
|
|
561
618
|
|
|
562
619
|
async def create_or_update_rp(self, rp_obj, existing_rp, new_user_obj):
|
|
563
620
|
"""
|
|
@@ -572,10 +629,8 @@ class UserImporter: # noqa: R0902
|
|
|
572
629
|
None
|
|
573
630
|
"""
|
|
574
631
|
if existing_rp:
|
|
575
|
-
# print(existing_rp)
|
|
576
632
|
await self.update_existing_rp(rp_obj, existing_rp)
|
|
577
633
|
else:
|
|
578
|
-
# print(new_user_obj)
|
|
579
634
|
await self.create_new_rp(new_user_obj)
|
|
580
635
|
|
|
581
636
|
async def create_new_rp(self, new_user_obj):
|
|
@@ -593,7 +648,6 @@ class UserImporter: # noqa: R0902
|
|
|
593
648
|
"""
|
|
594
649
|
rp_obj = {"holdShelf": True, "delivery": False}
|
|
595
650
|
rp_obj["userId"] = new_user_obj["id"]
|
|
596
|
-
# print(rp_obj)
|
|
597
651
|
response = await self.http_client.post(
|
|
598
652
|
self.folio_client.gateway_url
|
|
599
653
|
+ "/request-preference-storage/request-preference",
|
|
@@ -617,7 +671,6 @@ class UserImporter: # noqa: R0902
|
|
|
617
671
|
None
|
|
618
672
|
"""
|
|
619
673
|
existing_rp.update(rp_obj)
|
|
620
|
-
# print(existing_rp)
|
|
621
674
|
response = await self.http_client.put(
|
|
622
675
|
self.folio_client.gateway_url
|
|
623
676
|
+ f"/request-preference-storage/request-preference/{existing_rp['id']}",
|
|
@@ -668,9 +721,15 @@ class UserImporter: # noqa: R0902
|
|
|
668
721
|
"""
|
|
669
722
|
async with self.limit_simultaneous_requests:
|
|
670
723
|
user_obj = await self.process_user_obj(user)
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
724
|
+
(
|
|
725
|
+
rp_obj,
|
|
726
|
+
spu_obj,
|
|
727
|
+
existing_user,
|
|
728
|
+
protected_fields,
|
|
729
|
+
existing_rp,
|
|
730
|
+
existing_pu,
|
|
731
|
+
existing_spu,
|
|
732
|
+
) = await self.process_existing_user(user_obj)
|
|
674
733
|
await self.map_address_types(user_obj, line_number)
|
|
675
734
|
await self.map_patron_groups(user_obj, line_number)
|
|
676
735
|
await self.map_departments(user_obj, line_number)
|
|
@@ -684,11 +743,7 @@ class UserImporter: # noqa: R0902
|
|
|
684
743
|
rp_obj, existing_rp, new_user_obj
|
|
685
744
|
)
|
|
686
745
|
else:
|
|
687
|
-
|
|
688
|
-
f"Row {line_number}: Creating default request preference object"
|
|
689
|
-
f" for {new_user_obj['id']}"
|
|
690
|
-
)
|
|
691
|
-
await self.logfile.write(
|
|
746
|
+
logger.debug(
|
|
692
747
|
f"Row {line_number}: Creating default request preference object"
|
|
693
748
|
f" for {new_user_obj['id']}\n"
|
|
694
749
|
)
|
|
@@ -699,8 +754,7 @@ class UserImporter: # noqa: R0902
|
|
|
699
754
|
f"{new_user_obj['id']}: "
|
|
700
755
|
f"{str(getattr(getattr(ee, 'response', ee), 'text', str(ee)))}"
|
|
701
756
|
)
|
|
702
|
-
|
|
703
|
-
await self.logfile.write(rp_error_message + "\n")
|
|
757
|
+
logger.error(rp_error_message)
|
|
704
758
|
if not existing_pu:
|
|
705
759
|
try:
|
|
706
760
|
await self.create_perms_user(new_user_obj)
|
|
@@ -710,9 +764,10 @@ class UserImporter: # noqa: R0902
|
|
|
710
764
|
f"{new_user_obj['id']}: "
|
|
711
765
|
f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}"
|
|
712
766
|
)
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
767
|
+
logger.error(pu_error_message)
|
|
768
|
+
await self.handle_service_points_user(
|
|
769
|
+
spu_obj, existing_spu, new_user_obj
|
|
770
|
+
)
|
|
716
771
|
|
|
717
772
|
async def map_service_points(self, spu_obj, existing_user):
|
|
718
773
|
"""
|
|
@@ -730,40 +785,43 @@ class UserImporter: # noqa: R0902
|
|
|
730
785
|
for sp in spu_obj.pop("servicePointsIds", []):
|
|
731
786
|
try:
|
|
732
787
|
if self.validate_uuid(sp) and sp in self.service_point_map.values():
|
|
733
|
-
|
|
788
|
+
logger.debug(
|
|
734
789
|
f"Service point {sp} is a UUID, skipping mapping\n"
|
|
735
790
|
)
|
|
736
791
|
mapped_service_points.append(sp)
|
|
737
792
|
else:
|
|
738
793
|
mapped_service_points.append(self.service_point_map[sp])
|
|
739
794
|
except KeyError:
|
|
740
|
-
|
|
795
|
+
logger.error(
|
|
741
796
|
f'Service point "{sp}" not found, excluding service point from user: '
|
|
742
|
-
f
|
|
797
|
+
f"{self.service_point_map}"
|
|
743
798
|
)
|
|
744
799
|
if mapped_service_points:
|
|
745
800
|
spu_obj["servicePointsIds"] = mapped_service_points
|
|
746
801
|
if "defaultServicePointId" in spu_obj:
|
|
747
|
-
sp_code = spu_obj.pop(
|
|
802
|
+
sp_code = spu_obj.pop("defaultServicePointId", "")
|
|
748
803
|
try:
|
|
749
|
-
if
|
|
750
|
-
|
|
804
|
+
if (
|
|
805
|
+
self.validate_uuid(sp_code)
|
|
806
|
+
and sp_code in self.service_point_map.values()
|
|
807
|
+
):
|
|
808
|
+
logger.debug(
|
|
751
809
|
f"Default service point {sp_code} is a UUID, skipping mapping\n"
|
|
752
810
|
)
|
|
753
811
|
mapped_sp_id = sp_code
|
|
754
812
|
else:
|
|
755
813
|
mapped_sp_id = self.service_point_map[sp_code]
|
|
756
|
-
if mapped_sp_id not in spu_obj.get(
|
|
757
|
-
|
|
814
|
+
if mapped_sp_id not in spu_obj.get("servicePointsIds", []):
|
|
815
|
+
logger.warning(
|
|
758
816
|
f'Default service point "{sp_code}" not found in assigned service points, '
|
|
759
|
-
|
|
817
|
+
"excluding default service point from user"
|
|
760
818
|
)
|
|
761
819
|
else:
|
|
762
|
-
spu_obj[
|
|
820
|
+
spu_obj["defaultServicePointId"] = mapped_sp_id
|
|
763
821
|
except KeyError:
|
|
764
|
-
|
|
822
|
+
logger.error(
|
|
765
823
|
f'Default service point "{sp_code}" not found, excluding default service '
|
|
766
|
-
f
|
|
824
|
+
f"point from user: {existing_user['id']}"
|
|
767
825
|
)
|
|
768
826
|
|
|
769
827
|
async def handle_service_points_user(self, spu_obj, existing_spu, existing_user):
|
|
@@ -837,7 +895,8 @@ class UserImporter: # noqa: R0902
|
|
|
837
895
|
"""
|
|
838
896
|
existing_spu.update(spu_obj)
|
|
839
897
|
response = await self.http_client.put(
|
|
840
|
-
self.folio_client.gateway_url
|
|
898
|
+
self.folio_client.gateway_url
|
|
899
|
+
+ f"/service-points-users/{existing_spu['id']}",
|
|
841
900
|
headers=self.folio_client.okapi_headers,
|
|
842
901
|
json=existing_spu,
|
|
843
902
|
)
|
|
@@ -850,199 +909,263 @@ class UserImporter: # noqa: R0902
|
|
|
850
909
|
Args:
|
|
851
910
|
openfile: The file or file-like object to process.
|
|
852
911
|
"""
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
912
|
+
with Progress( # Set up the progress bar
|
|
913
|
+
"{task.description}",
|
|
914
|
+
SpinnerColumn(),
|
|
915
|
+
BarColumn(),
|
|
916
|
+
MofNCompleteColumn(),
|
|
917
|
+
UserStatsColumn(),
|
|
918
|
+
"[",
|
|
919
|
+
TimeElapsedColumn(),
|
|
920
|
+
"<",
|
|
921
|
+
TimeRemainingColumn(),
|
|
922
|
+
"/",
|
|
923
|
+
ItemsPerSecondColumn(),
|
|
924
|
+
"]",
|
|
925
|
+
) as progress:
|
|
926
|
+
with open(openfile.name, "rb") as f:
|
|
927
|
+
total_lines = sum(
|
|
928
|
+
buf.count(b"\n") for buf in iter(lambda: f.read(1024 * 1024), b"")
|
|
929
|
+
)
|
|
930
|
+
self.progress = progress
|
|
931
|
+
self.task_progress = progress.add_task(
|
|
932
|
+
"Importing users: ", total=total_lines, created=0, updated=0, failed=0
|
|
933
|
+
) # Add a task to the progress bar
|
|
934
|
+
openfile.seek(0)
|
|
935
|
+
tasks = []
|
|
936
|
+
for line_number, user in enumerate(openfile):
|
|
937
|
+
tasks.append(self.process_line(user, line_number))
|
|
938
|
+
if len(tasks) == self.batch_size:
|
|
939
|
+
start = time.time()
|
|
940
|
+
await asyncio.gather(*tasks)
|
|
941
|
+
duration = time.time() - start
|
|
942
|
+
async with self.lock:
|
|
943
|
+
progress.update(
|
|
944
|
+
self.task_progress,
|
|
945
|
+
advance=len(tasks),
|
|
946
|
+
created=self.logs["created"],
|
|
947
|
+
updated=self.logs["updated"],
|
|
948
|
+
failed=self.logs["failed"],
|
|
949
|
+
)
|
|
950
|
+
message = (
|
|
951
|
+
f"{dt.now().isoformat(sep=' ', timespec='milliseconds')}: "
|
|
952
|
+
f"Batch of {self.batch_size} users processed in {duration:.2f} "
|
|
953
|
+
f"seconds. - Users created: {self.logs['created']} - Users updated: "
|
|
954
|
+
f"{self.logs['updated']} - Users failed: {self.logs['failed']}"
|
|
955
|
+
)
|
|
956
|
+
logger.info(message)
|
|
957
|
+
tasks = []
|
|
958
|
+
if tasks:
|
|
857
959
|
start = time.time()
|
|
858
960
|
await asyncio.gather(*tasks)
|
|
859
961
|
duration = time.time() - start
|
|
860
962
|
async with self.lock:
|
|
963
|
+
progress.update(
|
|
964
|
+
self.task_progress,
|
|
965
|
+
advance=len(tasks),
|
|
966
|
+
created=self.logs["created"],
|
|
967
|
+
updated=self.logs["updated"],
|
|
968
|
+
failed=self.logs["failed"],
|
|
969
|
+
)
|
|
861
970
|
message = (
|
|
862
971
|
f"{dt.now().isoformat(sep=' ', timespec='milliseconds')}: "
|
|
863
|
-
f"Batch of {
|
|
864
|
-
f"
|
|
972
|
+
f"Batch of {len(tasks)} users processed in {duration:.2f} seconds. - "
|
|
973
|
+
f"Users created: {self.logs['created']} - Users updated: "
|
|
865
974
|
f"{self.logs['updated']} - Users failed: {self.logs['failed']}"
|
|
866
975
|
)
|
|
867
|
-
|
|
868
|
-
await self.logfile.write(message + "\n")
|
|
869
|
-
tasks = []
|
|
870
|
-
if tasks:
|
|
871
|
-
start = time.time()
|
|
872
|
-
await asyncio.gather(*tasks)
|
|
873
|
-
duration = time.time() - start
|
|
874
|
-
async with self.lock:
|
|
875
|
-
message = (
|
|
876
|
-
f"{dt.now().isoformat(sep=' ', timespec='milliseconds')}: "
|
|
877
|
-
f"Batch of {len(tasks)} users processed in {duration:.2f} seconds. - "
|
|
878
|
-
f"Users created: {self.logs['created']} - Users updated: "
|
|
879
|
-
f"{self.logs['updated']} - Users failed: {self.logs['failed']}"
|
|
880
|
-
)
|
|
881
|
-
print(message)
|
|
882
|
-
await self.logfile.write(message + "\n")
|
|
976
|
+
logger.info(message)
|
|
883
977
|
|
|
884
978
|
|
|
885
|
-
|
|
979
|
+
def set_up_cli_logging():
|
|
886
980
|
"""
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
Parses command line arguments, initializes necessary objects, and starts the import process.
|
|
890
|
-
|
|
891
|
-
Args:
|
|
892
|
-
--tenant_id (str): The tenant id.
|
|
893
|
-
--library_name (str): The name of the library.
|
|
894
|
-
--username (str): The FOLIO username.
|
|
895
|
-
--okapi_url (str): The Okapi URL.
|
|
896
|
-
--user_file_path (str): The path to the user file.
|
|
897
|
-
--limit_async_requests (int): Limit how many http requests can be made at once. Default 10.
|
|
898
|
-
--batch_size (int): How many records to process before logging statistics. Default 250.
|
|
899
|
-
--folio_password (str): The FOLIO password.
|
|
900
|
-
--user_match_key (str): The key to use to match users. Default "externalSystemId".
|
|
901
|
-
--report_file_base_path (str): The base path for the log and error files. Default "./".
|
|
902
|
-
--update_only_present_fields (bool): Only update fields that are present in the new user object.
|
|
903
|
-
--default_preferred_contact_type (str): The default preferred contact type to use if the provided \
|
|
904
|
-
value is not valid or not present. Default "002".
|
|
905
|
-
--fields_to_protect (str): Comma-separated list of top-level or nested (dot-notation) fields to protect.
|
|
906
|
-
|
|
907
|
-
Raises:
|
|
908
|
-
Exception: If an unknown error occurs during the import process.
|
|
909
|
-
|
|
910
|
-
Returns:
|
|
911
|
-
None
|
|
981
|
+
This function sets up logging for the CLI.
|
|
912
982
|
"""
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
)
|
|
920
|
-
parser.add_argument("--library_name", help="The name of the library")
|
|
921
|
-
parser.add_argument("--username", help="The FOLIO username")
|
|
922
|
-
parser.add_argument("--okapi_url", help="The Okapi URL")
|
|
923
|
-
parser.add_argument("--user_file_path", help="The path to the user file")
|
|
924
|
-
parser.add_argument(
|
|
925
|
-
"--limit_async_requests",
|
|
926
|
-
help="Limit how many http requests can be made at once",
|
|
927
|
-
type=int,
|
|
928
|
-
default=10,
|
|
929
|
-
)
|
|
930
|
-
parser.add_argument(
|
|
931
|
-
"--batch_size",
|
|
932
|
-
help="How many user records to process before logging statistics",
|
|
933
|
-
type=int,
|
|
934
|
-
default=250,
|
|
935
|
-
)
|
|
936
|
-
parser.add_argument("--folio_password", help="The FOLIO password")
|
|
937
|
-
parser.add_argument(
|
|
938
|
-
"--user_match_key",
|
|
939
|
-
help="The key to use to match users",
|
|
940
|
-
choices=["externalSystemId", "barcode", "username"],
|
|
941
|
-
default="externalSystemId",
|
|
942
|
-
)
|
|
943
|
-
parser.add_argument(
|
|
944
|
-
"--report_file_base_path",
|
|
945
|
-
help="The base path for the log and error files",
|
|
946
|
-
default="./",
|
|
947
|
-
)
|
|
948
|
-
parser.add_argument(
|
|
949
|
-
"--update_only_present_fields",
|
|
950
|
-
help="Only update fields that are present in the user object",
|
|
951
|
-
action="store_true",
|
|
983
|
+
logger.setLevel(logging.INFO)
|
|
984
|
+
logger.propagate = False
|
|
985
|
+
|
|
986
|
+
# Set up file and stream handlers
|
|
987
|
+
file_handler = logging.FileHandler(
|
|
988
|
+
"folio_user_import_{}.log".format(dt.now().strftime("%Y%m%d%H%M%S"))
|
|
952
989
|
)
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
990
|
+
file_handler.setLevel(logging.INFO)
|
|
991
|
+
file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
|
|
992
|
+
file_handler.setFormatter(file_formatter)
|
|
993
|
+
logger.addHandler(file_handler)
|
|
994
|
+
|
|
995
|
+
if not any(
|
|
996
|
+
isinstance(h, logging.StreamHandler) and h.stream == sys.stderr
|
|
997
|
+
for h in logger.handlers
|
|
998
|
+
):
|
|
999
|
+
stream_handler = RichHandler(
|
|
1000
|
+
show_level=False,
|
|
1001
|
+
show_time=False,
|
|
1002
|
+
omit_repeated_times=False,
|
|
1003
|
+
show_path=False,
|
|
1004
|
+
)
|
|
1005
|
+
stream_handler.setLevel(logging.WARNING)
|
|
1006
|
+
stream_formatter = logging.Formatter("%(message)s")
|
|
1007
|
+
stream_handler.setFormatter(stream_formatter)
|
|
1008
|
+
logger.addHandler(stream_handler)
|
|
1009
|
+
|
|
1010
|
+
# Stop httpx from logging info messages to the console
|
|
1011
|
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
app = typer.Typer()
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
@app.command()
|
|
1018
|
+
def main(
|
|
1019
|
+
gateway_url: Annotated[
|
|
1020
|
+
str,
|
|
1021
|
+
typer.Option(
|
|
1022
|
+
...,
|
|
1023
|
+
prompt="Please enter the FOLIO API Gateway URL",
|
|
1024
|
+
help="The FOLIO API Gateway URL",
|
|
1025
|
+
envvar="FOLIO_GATEWAY_URL",
|
|
959
1026
|
),
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
"
|
|
968
|
-
"(e.g. type,expirationDate)"
|
|
1027
|
+
],
|
|
1028
|
+
tenant_id: Annotated[
|
|
1029
|
+
str,
|
|
1030
|
+
typer.Option(
|
|
1031
|
+
...,
|
|
1032
|
+
prompt="Please enter the FOLIO tenant id",
|
|
1033
|
+
help="The tenant id",
|
|
1034
|
+
envvar="FOLIO_TENANT_ID",
|
|
969
1035
|
),
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
1036
|
+
],
|
|
1037
|
+
username: Annotated[
|
|
1038
|
+
str,
|
|
1039
|
+
typer.Option(
|
|
1040
|
+
...,
|
|
1041
|
+
prompt="Please enter your FOLIO username",
|
|
1042
|
+
help="The FOLIO username",
|
|
1043
|
+
envvar="FOLIO_USERNAME",
|
|
1044
|
+
),
|
|
1045
|
+
],
|
|
1046
|
+
password: Annotated[
|
|
1047
|
+
str,
|
|
1048
|
+
typer.Option(
|
|
1049
|
+
...,
|
|
1050
|
+
prompt="Please enter your FOLIO Password",
|
|
1051
|
+
hide_input=True,
|
|
1052
|
+
help="The FOLIO password",
|
|
1053
|
+
envvar="FOLIO_PASSWORD",
|
|
1054
|
+
),
|
|
1055
|
+
],
|
|
1056
|
+
library_name: Annotated[
|
|
1057
|
+
str,
|
|
1058
|
+
typer.Option(
|
|
1059
|
+
...,
|
|
1060
|
+
prompt="Please enter the library name",
|
|
1061
|
+
help="The name of the library",
|
|
1062
|
+
envvar="FOLIO_LIBRARY_NAME",
|
|
1063
|
+
),
|
|
1064
|
+
],
|
|
1065
|
+
user_file_path: Annotated[
|
|
1066
|
+
Path, typer.Option(..., help="The path to the user file")
|
|
1067
|
+
],
|
|
1068
|
+
member_tenant_id: Annotated[
|
|
1069
|
+
str,
|
|
1070
|
+
typer.Option(
|
|
1071
|
+
help="The FOLIO ECS member tenant id (if applicable)",
|
|
1072
|
+
envvar="FOLIO_MEMBER_TENANT_ID",
|
|
1073
|
+
),
|
|
1074
|
+
] = "",
|
|
1075
|
+
fields_to_protect: Annotated[
|
|
1076
|
+
str,
|
|
1077
|
+
typer.Option(
|
|
1078
|
+
help="Comma-separated list of top-level or nested (dot-notation) fields to protect"
|
|
1079
|
+
),
|
|
1080
|
+
] = "",
|
|
1081
|
+
update_only_present_fields: bool = typer.Option(
|
|
1082
|
+
False,
|
|
1083
|
+
"--update-only-present-fields",
|
|
1084
|
+
help="Only update fields that are present in the new user object",
|
|
1085
|
+
),
|
|
1086
|
+
limit_async_requests: Annotated[
|
|
1087
|
+
int,
|
|
1088
|
+
typer.Option(
|
|
1089
|
+
help="Limit how many http requests can be made at once",
|
|
1090
|
+
envvar="FOLIO_LIMIT_ASYNC_REQUESTS",
|
|
1091
|
+
),
|
|
1092
|
+
] = 10,
|
|
1093
|
+
batch_size: Annotated[
|
|
1094
|
+
int,
|
|
1095
|
+
typer.Option(
|
|
1096
|
+
help="How many user records to process before logging statistics",
|
|
1097
|
+
envvar="FOLIO_USER_IMPORT_BATCH_SIZE",
|
|
1098
|
+
),
|
|
1099
|
+
] = 250,
|
|
1100
|
+
report_file_base_path: Annotated[
|
|
1101
|
+
Path, typer.Option(help="The base path for the log and error files")
|
|
1102
|
+
] = Path.cwd(),
|
|
1103
|
+
user_match_key: UserMatchKeys = typer.Option(
|
|
1104
|
+
UserMatchKeys.EXTERNAL_SYSTEM_ID.value, help="The key to use to match users"
|
|
1105
|
+
),
|
|
1106
|
+
default_preferred_contact_type: PreferredContactType = typer.Option(
|
|
1107
|
+
PreferredContactType.EMAIL.value,
|
|
1108
|
+
case_sensitive=False,
|
|
1109
|
+
help="The default preferred contact type to use if the provided value is not valid or not present",
|
|
1110
|
+
),
|
|
1111
|
+
) -> None:
|
|
1112
|
+
"""
|
|
1113
|
+
Command-line interface to batch import users into FOLIO
|
|
1114
|
+
"""
|
|
1115
|
+
set_up_cli_logging()
|
|
1116
|
+
protect_fields = [f.strip() for f in fields_to_protect.split(",") if f.strip()]
|
|
977
1117
|
|
|
978
|
-
library_name =
|
|
1118
|
+
library_name = library_name
|
|
979
1119
|
|
|
980
1120
|
# Semaphore to limit the number of async HTTP requests active at any given time
|
|
981
|
-
limit_async_requests = asyncio.Semaphore(
|
|
982
|
-
batch_size =
|
|
983
|
-
|
|
984
|
-
folio_client = folioclient.FolioClient(
|
|
985
|
-
args.okapi_url,
|
|
986
|
-
args.tenant_id,
|
|
987
|
-
args.username,
|
|
988
|
-
args.folio_password
|
|
989
|
-
or os.environ.get("FOLIO_PASS", "")
|
|
990
|
-
or getpass.getpass(
|
|
991
|
-
"Enter your FOLIO password: ",
|
|
992
|
-
),
|
|
993
|
-
)
|
|
1121
|
+
limit_async_requests = asyncio.Semaphore(limit_async_requests)
|
|
1122
|
+
batch_size = batch_size
|
|
1123
|
+
|
|
1124
|
+
folio_client = folioclient.FolioClient(gateway_url, tenant_id, username, password)
|
|
994
1125
|
|
|
995
1126
|
# Set the member tenant id if provided to support FOLIO ECS multi-tenant environments
|
|
996
|
-
if
|
|
997
|
-
folio_client.okapi_headers["x-okapi-tenant"] =
|
|
1127
|
+
if member_tenant_id:
|
|
1128
|
+
folio_client.okapi_headers["x-okapi-tenant"] = member_tenant_id
|
|
998
1129
|
|
|
999
|
-
user_file_path =
|
|
1000
|
-
report_file_base_path =
|
|
1001
|
-
log_file_path = (
|
|
1002
|
-
report_file_base_path
|
|
1003
|
-
/ f"log_user_import_{dt.now(utc).strftime('%Y%m%d_%H%M%S')}.log"
|
|
1004
|
-
)
|
|
1130
|
+
user_file_path = user_file_path
|
|
1131
|
+
report_file_base_path = report_file_base_path
|
|
1005
1132
|
error_file_path = (
|
|
1006
1133
|
report_file_base_path
|
|
1007
1134
|
/ f"failed_user_import_{dt.now(utc).strftime('%Y%m%d_%H%M%S')}.txt"
|
|
1008
1135
|
)
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
args.user_match_key,
|
|
1026
|
-
args.update_only_present_fields,
|
|
1027
|
-
args.default_preferred_contact_type,
|
|
1028
|
-
fields_to_protect=protect_fields,
|
|
1029
|
-
)
|
|
1030
|
-
await importer.do_import()
|
|
1031
|
-
except Exception as ee:
|
|
1032
|
-
print(f"An unknown error occurred: {ee}")
|
|
1033
|
-
await logfile.write(f"An error occurred {ee}\n")
|
|
1034
|
-
raise ee
|
|
1136
|
+
try:
|
|
1137
|
+
importer = UserImporter(
|
|
1138
|
+
folio_client,
|
|
1139
|
+
library_name,
|
|
1140
|
+
batch_size,
|
|
1141
|
+
limit_async_requests,
|
|
1142
|
+
user_file_path,
|
|
1143
|
+
user_match_key.value,
|
|
1144
|
+
update_only_present_fields,
|
|
1145
|
+
default_preferred_contact_type.value,
|
|
1146
|
+
fields_to_protect=protect_fields,
|
|
1147
|
+
)
|
|
1148
|
+
asyncio.run(run_user_importer(importer, error_file_path))
|
|
1149
|
+
except Exception as ee:
|
|
1150
|
+
logger.critical(f"An unknown error occurred: {ee}")
|
|
1151
|
+
raise typer.Exit(1)
|
|
1035
1152
|
|
|
1036
1153
|
|
|
1037
|
-
def
|
|
1038
|
-
|
|
1039
|
-
|
|
1154
|
+
async def run_user_importer(importer: UserImporter, error_file_path: Path):
|
|
1155
|
+
try:
|
|
1156
|
+
await importer.setup(error_file_path)
|
|
1157
|
+
await importer.do_import()
|
|
1158
|
+
except Exception as ee:
|
|
1159
|
+
logger.critical(f"An unknown error occurred: {ee}")
|
|
1160
|
+
typer.Exit(1)
|
|
1161
|
+
finally:
|
|
1162
|
+
await importer.close()
|
|
1040
1163
|
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1164
|
+
|
|
1165
|
+
def _main():
|
|
1166
|
+
typer.run(main)
|
|
1044
1167
|
|
|
1045
1168
|
|
|
1046
1169
|
# Run the main function
|
|
1047
1170
|
if __name__ == "__main__":
|
|
1048
|
-
|
|
1171
|
+
app()
|