folio-data-import 0.3.2__py3-none-any.whl → 0.4.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of folio-data-import might be problematic. Click here for more details.

@@ -1,19 +1,32 @@
1
- import argparse
2
1
  import asyncio
3
2
  import datetime
4
- import getpass
5
3
  import json
6
- import os
4
+ import logging
5
+ import sys
7
6
  import time
8
7
  import uuid
9
8
  from datetime import datetime as dt
9
+ from enum import Enum
10
10
  from pathlib import Path
11
- from typing import Tuple, List
11
+ from typing import List, Tuple
12
12
 
13
13
  import aiofiles
14
14
  import folioclient
15
15
  import httpx
16
+ import typer
16
17
  from aiofiles.threadpool.text import AsyncTextIOWrapper
18
+ from rich.logging import RichHandler
19
+ from rich.progress import (
20
+ BarColumn,
21
+ MofNCompleteColumn,
22
+ Progress,
23
+ SpinnerColumn,
24
+ TimeElapsedColumn,
25
+ TimeRemainingColumn,
26
+ )
27
+ from typing_extensions import Annotated
28
+
29
+ from folio_data_import._progress import ItemsPerSecondColumn, UserStatsColumn
17
30
 
18
31
  try:
19
32
  utc = datetime.UTC
@@ -22,6 +35,8 @@ except AttributeError:
22
35
 
23
36
  utc = zoneinfo.ZoneInfo("UTC")
24
37
 
38
+ logger = logging.getLogger(__name__)
39
+
25
40
  # Mapping of preferred contact type IDs to their corresponding values
26
41
  PREFERRED_CONTACT_TYPES_MAP = {
27
42
  "001": "mail",
@@ -31,6 +46,26 @@ PREFERRED_CONTACT_TYPES_MAP = {
31
46
  "005": "mobile",
32
47
  }
33
48
 
49
+
50
+ class PreferredContactType(Enum):
51
+ MAIL = "001"
52
+ EMAIL = "002"
53
+ TEXT = "003"
54
+ PHONE = "004"
55
+ MOBILE = "005"
56
+ _001 = "mail"
57
+ _002 = "email"
58
+ _003 = "text"
59
+ _004 = "phone"
60
+ _005 = "mobile"
61
+
62
+
63
+ class UserMatchKeys(Enum):
64
+ USERNAME = "username"
65
+ EMAIL = "email"
66
+ EXTERNAL_SYSTEM_ID = "externalSystemId"
67
+
68
+
34
69
  class UserImporter: # noqa: R0902
35
70
  """
36
71
  Class to import mod-user-import compatible user objects
@@ -38,20 +73,22 @@ class UserImporter: # noqa: R0902
38
73
  from a JSON-lines file into FOLIO
39
74
  """
40
75
 
76
+ logfile: AsyncTextIOWrapper
77
+ errorfile: AsyncTextIOWrapper
78
+ http_client: httpx.AsyncClient
79
+
41
80
  def __init__(
42
81
  self,
43
82
  folio_client: folioclient.FolioClient,
44
83
  library_name: str,
45
84
  batch_size: int,
46
85
  limit_simultaneous_requests: asyncio.Semaphore,
47
- logfile: AsyncTextIOWrapper,
48
- errorfile: AsyncTextIOWrapper,
49
- http_client: httpx.AsyncClient,
50
86
  user_file_path: Path = None,
51
87
  user_match_key: str = "externalSystemId",
52
88
  only_update_present_fields: bool = False,
53
89
  default_preferred_contact_type: str = "002",
54
- fields_to_protect: List[str] =[],
90
+ fields_to_protect: List[str] = [],
91
+ no_progress: bool = False,
55
92
  ) -> None:
56
93
  self.limit_simultaneous_requests = limit_simultaneous_requests
57
94
  self.batch_size = batch_size
@@ -70,15 +107,13 @@ class UserImporter: # noqa: R0902
70
107
  self.service_point_map: dict = self.build_ref_data_id_map(
71
108
  self.folio_client, "/service-points", "servicepoints", "code"
72
109
  )
73
- self.logfile: AsyncTextIOWrapper = logfile
74
- self.errorfile: AsyncTextIOWrapper = errorfile
75
- self.http_client: httpx.AsyncClient = http_client
76
110
  self.only_update_present_fields: bool = only_update_present_fields
77
111
  self.default_preferred_contact_type: str = default_preferred_contact_type
78
112
  self.match_key = user_match_key
79
113
  self.lock: asyncio.Lock = asyncio.Lock()
80
114
  self.logs: dict = {"created": 0, "updated": 0, "failed": 0}
81
115
  self.fields_to_protect = set(fields_to_protect)
116
+ self.no_progress = no_progress
82
117
 
83
118
  @staticmethod
84
119
  def build_ref_data_id_map(
@@ -114,17 +149,36 @@ class UserImporter: # noqa: R0902
114
149
  except ValueError:
115
150
  return False
116
151
 
152
+ async def setup(self, error_file_path: Path) -> None:
153
+ """
154
+ Sets up the importer by initializing necessary resources.
155
+
156
+ Args:
157
+ log_file_path (Path): The path to the log file.
158
+ error_file_path (Path): The path to the error file.
159
+ """
160
+ self.errorfile = await aiofiles.open(error_file_path, "w", encoding="utf-8")
161
+
162
+ async def close(self) -> None:
163
+ """
164
+ Closes the importer by releasing any resources.
165
+
166
+ """
167
+ await self.errorfile.close()
168
+
117
169
  async def do_import(self) -> None:
118
170
  """
119
171
  Main method to import users.
120
172
 
121
173
  This method triggers the process of importing users by calling the `process_file` method.
122
174
  """
123
- if self.user_file_path:
124
- with open(self.user_file_path, "r", encoding="utf-8") as openfile:
125
- await self.process_file(openfile)
126
- else:
127
- raise FileNotFoundError("No user objects file provided")
175
+ async with httpx.AsyncClient() as client:
176
+ self.http_client = client
177
+ if self.user_file_path:
178
+ with open(self.user_file_path, "r", encoding="utf-8") as openfile:
179
+ await self.process_file(openfile)
180
+ else:
181
+ raise FileNotFoundError("No user objects file provided")
128
182
 
129
183
  async def get_existing_user(self, user_obj) -> dict:
130
184
  """
@@ -227,7 +281,7 @@ class UserImporter: # noqa: R0902
227
281
  self.validate_uuid(address["addressTypeId"])
228
282
  and address["addressTypeId"] in self.address_type_map.values()
229
283
  ):
230
- await self.logfile.write(
284
+ logger.debug(
231
285
  f"Row {line_number}: Address type {address['addressTypeId']} is a UUID, "
232
286
  f"skipping mapping\n"
233
287
  )
@@ -239,11 +293,7 @@ class UserImporter: # noqa: R0902
239
293
  mapped_addresses.append(address)
240
294
  except KeyError:
241
295
  if address["addressTypeId"] not in self.address_type_map.values():
242
- print(
243
- f"Row {line_number}: Address type {address['addressTypeId']} not found"
244
- f", removing address"
245
- )
246
- await self.logfile.write(
296
+ logger.error(
247
297
  f"Row {line_number}: Address type {address['addressTypeId']} not found"
248
298
  f", removing address\n"
249
299
  )
@@ -266,7 +316,7 @@ class UserImporter: # noqa: R0902
266
316
  self.validate_uuid(user_obj["patronGroup"])
267
317
  and user_obj["patronGroup"] in self.patron_group_map.values()
268
318
  ):
269
- await self.logfile.write(
319
+ logger.debug(
270
320
  f"Row {line_number}: Patron group {user_obj['patronGroup']} is a UUID, "
271
321
  f"skipping mapping\n"
272
322
  )
@@ -274,11 +324,7 @@ class UserImporter: # noqa: R0902
274
324
  user_obj["patronGroup"] = self.patron_group_map[user_obj["patronGroup"]]
275
325
  except KeyError:
276
326
  if user_obj["patronGroup"] not in self.patron_group_map.values():
277
- print(
278
- f"Row {line_number}: Patron group {user_obj['patronGroup']} not found, "
279
- f"removing patron group"
280
- )
281
- await self.logfile.write(
327
+ logger.error(
282
328
  f"Row {line_number}: Patron group {user_obj['patronGroup']} not found in, "
283
329
  f"removing patron group\n"
284
330
  )
@@ -302,25 +348,23 @@ class UserImporter: # noqa: R0902
302
348
  self.validate_uuid(department)
303
349
  and department in self.department_map.values()
304
350
  ):
305
- await self.logfile.write(
351
+ logger.debug(
306
352
  f"Row {line_number}: Department {department} is a UUID, skipping mapping\n"
307
353
  )
308
354
  mapped_departments.append(department)
309
355
  else:
310
356
  mapped_departments.append(self.department_map[department])
311
357
  except KeyError:
312
- print(
313
- f'Row {line_number}: Department "{department}" not found, ' # noqa: B907
314
- f"excluding department from user"
315
- )
316
- await self.logfile.write(
358
+ logger.error(
317
359
  f'Row {line_number}: Department "{department}" not found, ' # noqa: B907
318
360
  f"excluding department from user\n"
319
361
  )
320
362
  if mapped_departments:
321
363
  user_obj["departments"] = mapped_departments
322
364
 
323
- async def update_existing_user(self, user_obj, existing_user, protected_fields) -> Tuple[dict, dict]:
365
+ async def update_existing_user(
366
+ self, user_obj, existing_user, protected_fields
367
+ ) -> Tuple[dict, dict]:
324
368
  """
325
369
  Updates an existing user with the provided user object.
326
370
 
@@ -338,7 +382,11 @@ class UserImporter: # noqa: R0902
338
382
  """
339
383
 
340
384
  await self.set_preferred_contact_type(user_obj, existing_user)
341
- preferred_contact_type = {"preferredContactTypeId": existing_user.get("personal", {}).pop("preferredContactTypeId")}
385
+ preferred_contact_type = {
386
+ "preferredContactTypeId": existing_user.get("personal", {}).pop(
387
+ "preferredContactTypeId"
388
+ )
389
+ }
342
390
  if self.only_update_present_fields:
343
391
  new_personal = user_obj.pop("personal", {})
344
392
  existing_personal = existing_user.pop("personal", {})
@@ -412,30 +460,39 @@ class UserImporter: # noqa: R0902
412
460
  current_pref_contact = user_obj["personal"].get(
413
461
  "preferredContactTypeId", ""
414
462
  )
415
- if mapped_contact_type := dict([(v, k) for k, v in PREFERRED_CONTACT_TYPES_MAP.items()]).get(
463
+ if mapped_contact_type := dict(
464
+ [(v, k) for k, v in PREFERRED_CONTACT_TYPES_MAP.items()]
465
+ ).get(
416
466
  current_pref_contact,
417
467
  "",
418
468
  ):
419
- existing_user["personal"]["preferredContactTypeId"] = mapped_contact_type
469
+ existing_user["personal"]["preferredContactTypeId"] = (
470
+ mapped_contact_type
471
+ )
420
472
  else:
421
- existing_user["personal"]["preferredContactTypeId"] = current_pref_contact if current_pref_contact in PREFERRED_CONTACT_TYPES_MAP else self.default_preferred_contact_type
473
+ existing_user["personal"]["preferredContactTypeId"] = (
474
+ current_pref_contact
475
+ if current_pref_contact in PREFERRED_CONTACT_TYPES_MAP
476
+ else self.default_preferred_contact_type
477
+ )
422
478
  else:
423
- print(
424
- f"Preferred contact type not provided or is not a valid option: {PREFERRED_CONTACT_TYPES_MAP}\n"
479
+ logger.warning(
480
+ f"Preferred contact type not provided or is not a valid option: {PREFERRED_CONTACT_TYPES_MAP} "
425
481
  f"Setting preferred contact type to {self.default_preferred_contact_type} or using existing value"
426
482
  )
427
- await self.logfile.write(
428
- f"Preferred contact type not provided or is not a valid option: {PREFERRED_CONTACT_TYPES_MAP}\n"
429
- f"Setting preferred contact type to {self.default_preferred_contact_type} or using existing value\n"
483
+ mapped_contact_type = (
484
+ existing_user.get("personal", {}).get("preferredContactTypeId", "")
485
+ or self.default_preferred_contact_type
430
486
  )
431
- mapped_contact_type = existing_user.get("personal", {}).get(
432
- "preferredContactTypeId", ""
433
- ) or self.default_preferred_contact_type
434
487
  if "personal" not in existing_user:
435
488
  existing_user["personal"] = {}
436
- existing_user["personal"]["preferredContactTypeId"] = mapped_contact_type or self.default_preferred_contact_type
489
+ existing_user["personal"]["preferredContactTypeId"] = (
490
+ mapped_contact_type or self.default_preferred_contact_type
491
+ )
437
492
 
438
- async def create_or_update_user(self, user_obj, existing_user, protected_fields, line_number) -> dict:
493
+ async def create_or_update_user(
494
+ self, user_obj, existing_user, protected_fields, line_number
495
+ ) -> dict:
439
496
  """
440
497
  Creates or updates a user based on the given user object and existing user.
441
498
 
@@ -456,11 +513,7 @@ class UserImporter: # noqa: R0902
456
513
  self.logs["updated"] += 1
457
514
  return existing_user
458
515
  except Exception as ee:
459
- print(
460
- f"Row {line_number}: User update failed: "
461
- f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}"
462
- )
463
- await self.logfile.write(
516
+ logger.error(
464
517
  f"Row {line_number}: User update failed: "
465
518
  f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
466
519
  )
@@ -474,11 +527,7 @@ class UserImporter: # noqa: R0902
474
527
  new_user = await self.create_new_user(user_obj)
475
528
  return new_user
476
529
  except Exception as ee:
477
- print(
478
- f"Row {line_number}: User creation failed: "
479
- f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}"
480
- )
481
- await self.logfile.write(
530
+ logger.error(
482
531
  f"Row {line_number}: User creation failed: "
483
532
  f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
484
533
  )
@@ -516,7 +565,9 @@ class UserImporter: # noqa: R0902
516
565
  dict: A dictionary containing the protected fields and their values.
517
566
  """
518
567
  protected_fields = {}
519
- protected_fields_list = existing_user.get("customFields", {}).get("protectedFields", "").split(",")
568
+ protected_fields_list = (
569
+ existing_user.get("customFields", {}).get("protectedFields", "").split(",")
570
+ )
520
571
  cli_fields = list(self.fields_to_protect)
521
572
  # combine and dedupe:
522
573
  all_fields = list(dict.fromkeys(protected_fields_list + cli_fields))
@@ -557,7 +608,15 @@ class UserImporter: # noqa: R0902
557
608
  existing_pu = {}
558
609
  existing_spu = {}
559
610
  protected_fields = {}
560
- return rp_obj, spu_obj, existing_user, protected_fields, existing_rp, existing_pu, existing_spu
611
+ return (
612
+ rp_obj,
613
+ spu_obj,
614
+ existing_user,
615
+ protected_fields,
616
+ existing_rp,
617
+ existing_pu,
618
+ existing_spu,
619
+ )
561
620
 
562
621
  async def create_or_update_rp(self, rp_obj, existing_rp, new_user_obj):
563
622
  """
@@ -572,10 +631,8 @@ class UserImporter: # noqa: R0902
572
631
  None
573
632
  """
574
633
  if existing_rp:
575
- # print(existing_rp)
576
634
  await self.update_existing_rp(rp_obj, existing_rp)
577
635
  else:
578
- # print(new_user_obj)
579
636
  await self.create_new_rp(new_user_obj)
580
637
 
581
638
  async def create_new_rp(self, new_user_obj):
@@ -593,7 +650,6 @@ class UserImporter: # noqa: R0902
593
650
  """
594
651
  rp_obj = {"holdShelf": True, "delivery": False}
595
652
  rp_obj["userId"] = new_user_obj["id"]
596
- # print(rp_obj)
597
653
  response = await self.http_client.post(
598
654
  self.folio_client.gateway_url
599
655
  + "/request-preference-storage/request-preference",
@@ -617,7 +673,6 @@ class UserImporter: # noqa: R0902
617
673
  None
618
674
  """
619
675
  existing_rp.update(rp_obj)
620
- # print(existing_rp)
621
676
  response = await self.http_client.put(
622
677
  self.folio_client.gateway_url
623
678
  + f"/request-preference-storage/request-preference/{existing_rp['id']}",
@@ -668,9 +723,15 @@ class UserImporter: # noqa: R0902
668
723
  """
669
724
  async with self.limit_simultaneous_requests:
670
725
  user_obj = await self.process_user_obj(user)
671
- rp_obj, spu_obj, existing_user, protected_fields, existing_rp, existing_pu, existing_spu = (
672
- await self.process_existing_user(user_obj)
673
- )
726
+ (
727
+ rp_obj,
728
+ spu_obj,
729
+ existing_user,
730
+ protected_fields,
731
+ existing_rp,
732
+ existing_pu,
733
+ existing_spu,
734
+ ) = await self.process_existing_user(user_obj)
674
735
  await self.map_address_types(user_obj, line_number)
675
736
  await self.map_patron_groups(user_obj, line_number)
676
737
  await self.map_departments(user_obj, line_number)
@@ -684,11 +745,7 @@ class UserImporter: # noqa: R0902
684
745
  rp_obj, existing_rp, new_user_obj
685
746
  )
686
747
  else:
687
- print(
688
- f"Row {line_number}: Creating default request preference object"
689
- f" for {new_user_obj['id']}"
690
- )
691
- await self.logfile.write(
748
+ logger.debug(
692
749
  f"Row {line_number}: Creating default request preference object"
693
750
  f" for {new_user_obj['id']}\n"
694
751
  )
@@ -699,8 +756,7 @@ class UserImporter: # noqa: R0902
699
756
  f"{new_user_obj['id']}: "
700
757
  f"{str(getattr(getattr(ee, 'response', ee), 'text', str(ee)))}"
701
758
  )
702
- print(rp_error_message)
703
- await self.logfile.write(rp_error_message + "\n")
759
+ logger.error(rp_error_message)
704
760
  if not existing_pu:
705
761
  try:
706
762
  await self.create_perms_user(new_user_obj)
@@ -710,9 +766,10 @@ class UserImporter: # noqa: R0902
710
766
  f"{new_user_obj['id']}: "
711
767
  f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}"
712
768
  )
713
- print(pu_error_message)
714
- await self.logfile.write(pu_error_message + "\n")
715
- await self.handle_service_points_user(spu_obj, existing_spu, new_user_obj)
769
+ logger.error(pu_error_message)
770
+ await self.handle_service_points_user(
771
+ spu_obj, existing_spu, new_user_obj
772
+ )
716
773
 
717
774
  async def map_service_points(self, spu_obj, existing_user):
718
775
  """
@@ -730,40 +787,43 @@ class UserImporter: # noqa: R0902
730
787
  for sp in spu_obj.pop("servicePointsIds", []):
731
788
  try:
732
789
  if self.validate_uuid(sp) and sp in self.service_point_map.values():
733
- await self.logfile.write(
790
+ logger.debug(
734
791
  f"Service point {sp} is a UUID, skipping mapping\n"
735
792
  )
736
793
  mapped_service_points.append(sp)
737
794
  else:
738
795
  mapped_service_points.append(self.service_point_map[sp])
739
796
  except KeyError:
740
- print(
797
+ logger.error(
741
798
  f'Service point "{sp}" not found, excluding service point from user: '
742
- f'{self.service_point_map}'
799
+ f"{self.service_point_map}"
743
800
  )
744
801
  if mapped_service_points:
745
802
  spu_obj["servicePointsIds"] = mapped_service_points
746
803
  if "defaultServicePointId" in spu_obj:
747
- sp_code = spu_obj.pop('defaultServicePointId', '')
804
+ sp_code = spu_obj.pop("defaultServicePointId", "")
748
805
  try:
749
- if self.validate_uuid(sp_code) and sp_code in self.service_point_map.values():
750
- await self.logfile.write(
806
+ if (
807
+ self.validate_uuid(sp_code)
808
+ and sp_code in self.service_point_map.values()
809
+ ):
810
+ logger.debug(
751
811
  f"Default service point {sp_code} is a UUID, skipping mapping\n"
752
812
  )
753
813
  mapped_sp_id = sp_code
754
814
  else:
755
815
  mapped_sp_id = self.service_point_map[sp_code]
756
- if mapped_sp_id not in spu_obj.get('servicePointsIds', []):
757
- print(
816
+ if mapped_sp_id not in spu_obj.get("servicePointsIds", []):
817
+ logger.warning(
758
818
  f'Default service point "{sp_code}" not found in assigned service points, '
759
- 'excluding default service point from user'
819
+ "excluding default service point from user"
760
820
  )
761
821
  else:
762
- spu_obj['defaultServicePointId'] = mapped_sp_id
822
+ spu_obj["defaultServicePointId"] = mapped_sp_id
763
823
  except KeyError:
764
- print(
824
+ logger.error(
765
825
  f'Default service point "{sp_code}" not found, excluding default service '
766
- f'point from user: {existing_user["id"]}'
826
+ f"point from user: {existing_user['id']}"
767
827
  )
768
828
 
769
829
  async def handle_service_points_user(self, spu_obj, existing_spu, existing_user):
@@ -837,7 +897,8 @@ class UserImporter: # noqa: R0902
837
897
  """
838
898
  existing_spu.update(spu_obj)
839
899
  response = await self.http_client.put(
840
- self.folio_client.gateway_url + f"/service-points-users/{existing_spu['id']}",
900
+ self.folio_client.gateway_url
901
+ + f"/service-points-users/{existing_spu['id']}",
841
902
  headers=self.folio_client.okapi_headers,
842
903
  json=existing_spu,
843
904
  )
@@ -850,199 +911,269 @@ class UserImporter: # noqa: R0902
850
911
  Args:
851
912
  openfile: The file or file-like object to process.
852
913
  """
853
- tasks = []
854
- for line_number, user in enumerate(openfile):
855
- tasks.append(self.process_line(user, line_number))
856
- if len(tasks) == self.batch_size:
914
+ with Progress( # Set up the progress bar
915
+ "{task.description}",
916
+ SpinnerColumn(),
917
+ BarColumn(),
918
+ MofNCompleteColumn(),
919
+ UserStatsColumn(),
920
+ "[",
921
+ TimeElapsedColumn(),
922
+ "<",
923
+ TimeRemainingColumn(),
924
+ "/",
925
+ ItemsPerSecondColumn(),
926
+ "]",
927
+ ) as progress:
928
+ with open(openfile.name, "rb") as f:
929
+ total_lines = sum(
930
+ buf.count(b"\n") for buf in iter(lambda: f.read(1024 * 1024), b"")
931
+ )
932
+ self.progress = progress
933
+ self.task_progress = progress.add_task(
934
+ "Importing users: ", total=total_lines, created=0, updated=0, failed=0, visible=not self.no_progress
935
+ ) # Add a task to the progress bar
936
+ openfile.seek(0)
937
+ tasks = []
938
+ for line_number, user in enumerate(openfile):
939
+ tasks.append(self.process_line(user, line_number))
940
+ if len(tasks) == self.batch_size:
941
+ start = time.time()
942
+ await asyncio.gather(*tasks)
943
+ duration = time.time() - start
944
+ async with self.lock:
945
+ progress.update(
946
+ self.task_progress,
947
+ advance=len(tasks),
948
+ created=self.logs["created"],
949
+ updated=self.logs["updated"],
950
+ failed=self.logs["failed"],
951
+ )
952
+ message = (
953
+ f"{dt.now().isoformat(sep=' ', timespec='milliseconds')}: "
954
+ f"Batch of {self.batch_size} users processed in {duration:.2f} "
955
+ f"seconds. - Users created: {self.logs['created']} - Users updated: "
956
+ f"{self.logs['updated']} - Users failed: {self.logs['failed']}"
957
+ )
958
+ logger.info(message)
959
+ tasks = []
960
+ if tasks:
857
961
  start = time.time()
858
962
  await asyncio.gather(*tasks)
859
963
  duration = time.time() - start
860
964
  async with self.lock:
965
+ progress.update(
966
+ self.task_progress,
967
+ advance=len(tasks),
968
+ created=self.logs["created"],
969
+ updated=self.logs["updated"],
970
+ failed=self.logs["failed"],
971
+ )
861
972
  message = (
862
973
  f"{dt.now().isoformat(sep=' ', timespec='milliseconds')}: "
863
- f"Batch of {self.batch_size} users processed in {duration:.2f} "
864
- f"seconds. - Users created: {self.logs['created']} - Users updated: "
974
+ f"Batch of {len(tasks)} users processed in {duration:.2f} seconds. - "
975
+ f"Users created: {self.logs['created']} - Users updated: "
865
976
  f"{self.logs['updated']} - Users failed: {self.logs['failed']}"
866
977
  )
867
- print(message)
868
- await self.logfile.write(message + "\n")
869
- tasks = []
870
- if tasks:
871
- start = time.time()
872
- await asyncio.gather(*tasks)
873
- duration = time.time() - start
874
- async with self.lock:
875
- message = (
876
- f"{dt.now().isoformat(sep=' ', timespec='milliseconds')}: "
877
- f"Batch of {len(tasks)} users processed in {duration:.2f} seconds. - "
878
- f"Users created: {self.logs['created']} - Users updated: "
879
- f"{self.logs['updated']} - Users failed: {self.logs['failed']}"
880
- )
881
- print(message)
882
- await self.logfile.write(message + "\n")
978
+ logger.info(message)
883
979
 
884
980
 
885
- async def main() -> None:
981
+ def set_up_cli_logging():
886
982
  """
887
- Entry point of the user import script.
888
-
889
- Parses command line arguments, initializes necessary objects, and starts the import process.
890
-
891
- Args:
892
- --tenant_id (str): The tenant id.
893
- --library_name (str): The name of the library.
894
- --username (str): The FOLIO username.
895
- --okapi_url (str): The Okapi URL.
896
- --user_file_path (str): The path to the user file.
897
- --limit_async_requests (int): Limit how many http requests can be made at once. Default 10.
898
- --batch_size (int): How many records to process before logging statistics. Default 250.
899
- --folio_password (str): The FOLIO password.
900
- --user_match_key (str): The key to use to match users. Default "externalSystemId".
901
- --report_file_base_path (str): The base path for the log and error files. Default "./".
902
- --update_only_present_fields (bool): Only update fields that are present in the new user object.
903
- --default_preferred_contact_type (str): The default preferred contact type to use if the provided \
904
- value is not valid or not present. Default "002".
905
- --fields_to_protect (str): Comma-separated list of top-level or nested (dot-notation) fields to protect.
906
-
907
- Raises:
908
- Exception: If an unknown error occurs during the import process.
909
-
910
- Returns:
911
- None
983
+ This function sets up logging for the CLI.
912
984
  """
913
- parser = argparse.ArgumentParser()
914
- parser.add_argument("--tenant_id", help="The tenant id")
915
- parser.add_argument(
916
- "--member_tenant_id",
917
- help="The FOLIO ECS member tenant id (if applicable)",
918
- default="",
919
- )
920
- parser.add_argument("--library_name", help="The name of the library")
921
- parser.add_argument("--username", help="The FOLIO username")
922
- parser.add_argument("--okapi_url", help="The Okapi URL")
923
- parser.add_argument("--user_file_path", help="The path to the user file")
924
- parser.add_argument(
925
- "--limit_async_requests",
926
- help="Limit how many http requests can be made at once",
927
- type=int,
928
- default=10,
929
- )
930
- parser.add_argument(
931
- "--batch_size",
932
- help="How many user records to process before logging statistics",
933
- type=int,
934
- default=250,
935
- )
936
- parser.add_argument("--folio_password", help="The FOLIO password")
937
- parser.add_argument(
938
- "--user_match_key",
939
- help="The key to use to match users",
940
- choices=["externalSystemId", "barcode", "username"],
941
- default="externalSystemId",
942
- )
943
- parser.add_argument(
944
- "--report_file_base_path",
945
- help="The base path for the log and error files",
946
- default="./",
947
- )
948
- parser.add_argument(
949
- "--update_only_present_fields",
950
- help="Only update fields that are present in the user object",
951
- action="store_true",
985
+ logger.setLevel(logging.INFO)
986
+ logger.propagate = False
987
+
988
+ # Set up file and stream handlers
989
+ file_handler = logging.FileHandler(
990
+ "folio_user_import_{}.log".format(dt.now().strftime("%Y%m%d%H%M%S"))
952
991
  )
953
- parser.add_argument(
954
- "--default_preferred_contact_type",
955
- help=(
956
- "The default preferred contact type to use if the provided value is not present or not valid. "
957
- "Note: '002' is the default, and will be used if the provided value is not valid or not present, "
958
- "unless the existing user object being updated has a valid preferred contact type set."
992
+ file_handler.setLevel(logging.INFO)
993
+ file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
994
+ file_handler.setFormatter(file_formatter)
995
+ logger.addHandler(file_handler)
996
+
997
+ if not any(
998
+ isinstance(h, logging.StreamHandler) and h.stream == sys.stderr
999
+ for h in logger.handlers
1000
+ ):
1001
+ stream_handler = RichHandler(
1002
+ show_level=False,
1003
+ show_time=False,
1004
+ omit_repeated_times=False,
1005
+ show_path=False,
1006
+ )
1007
+ stream_handler.setLevel(logging.WARNING)
1008
+ stream_formatter = logging.Formatter("%(message)s")
1009
+ stream_handler.setFormatter(stream_formatter)
1010
+ logger.addHandler(stream_handler)
1011
+
1012
+ # Stop httpx from logging info messages to the console
1013
+ logging.getLogger("httpx").setLevel(logging.WARNING)
1014
+
1015
+
1016
+ app = typer.Typer()
1017
+
1018
+
1019
+ @app.command()
1020
+ def main(
1021
+ gateway_url: Annotated[
1022
+ str,
1023
+ typer.Option(
1024
+ ...,
1025
+ prompt="Please enter the FOLIO API Gateway URL",
1026
+ help="The FOLIO API Gateway URL",
1027
+ envvar="FOLIO_GATEWAY_URL",
959
1028
  ),
960
- choices=list(PREFERRED_CONTACT_TYPES_MAP.keys()) + list(PREFERRED_CONTACT_TYPES_MAP.values()),
961
- default="002",
962
- )
963
- parser.add_argument(
964
- "--fields-to-protect", # new flag name
965
- dest="fields_to_protect", # sets args.fields_to_protect
966
- help=(
967
- "Comma-separated list of top-level user fields to protect "
968
- "(e.g. type,expirationDate)"
1029
+ ],
1030
+ tenant_id: Annotated[
1031
+ str,
1032
+ typer.Option(
1033
+ ...,
1034
+ prompt="Please enter the FOLIO tenant id",
1035
+ help="The tenant id",
1036
+ envvar="FOLIO_TENANT_ID",
969
1037
  ),
970
- default="",
971
- )
972
- args = parser.parse_args()
973
- protect_fields = [
974
- f.strip() for f in args.fields_to_protect.split(",")
975
- if f.strip()
976
- ]
1038
+ ],
1039
+ username: Annotated[
1040
+ str,
1041
+ typer.Option(
1042
+ ...,
1043
+ prompt="Please enter your FOLIO username",
1044
+ help="The FOLIO username",
1045
+ envvar="FOLIO_USERNAME",
1046
+ ),
1047
+ ],
1048
+ password: Annotated[
1049
+ str,
1050
+ typer.Option(
1051
+ ...,
1052
+ prompt="Please enter your FOLIO Password",
1053
+ hide_input=True,
1054
+ help="The FOLIO password",
1055
+ envvar="FOLIO_PASSWORD",
1056
+ ),
1057
+ ],
1058
+ library_name: Annotated[
1059
+ str,
1060
+ typer.Option(
1061
+ ...,
1062
+ prompt="Please enter the library name",
1063
+ help="The name of the library",
1064
+ envvar="FOLIO_LIBRARY_NAME",
1065
+ ),
1066
+ ],
1067
+ user_file_path: Annotated[
1068
+ Path, typer.Option(..., help="The path to the user file")
1069
+ ],
1070
+ member_tenant_id: Annotated[
1071
+ str,
1072
+ typer.Option(
1073
+ help="The FOLIO ECS member tenant id (if applicable)",
1074
+ envvar="FOLIO_MEMBER_TENANT_ID",
1075
+ ),
1076
+ ] = "",
1077
+ fields_to_protect: Annotated[
1078
+ str,
1079
+ typer.Option(
1080
+ help="Comma-separated list of top-level or nested (dot-notation) fields to protect"
1081
+ ),
1082
+ ] = "",
1083
+ update_only_present_fields: bool = typer.Option(
1084
+ False,
1085
+ "--update-only-present-fields",
1086
+ help="Only update fields that are present in the new user object",
1087
+ ),
1088
+ limit_async_requests: Annotated[
1089
+ int,
1090
+ typer.Option(
1091
+ help="Limit how many http requests can be made at once",
1092
+ envvar="FOLIO_LIMIT_ASYNC_REQUESTS",
1093
+ ),
1094
+ ] = 10,
1095
+ batch_size: Annotated[
1096
+ int,
1097
+ typer.Option(
1098
+ help="How many user records to process before logging statistics",
1099
+ envvar="FOLIO_USER_IMPORT_BATCH_SIZE",
1100
+ ),
1101
+ ] = 250,
1102
+ report_file_base_path: Annotated[
1103
+ Path, typer.Option(help="The base path for the log and error files")
1104
+ ] = Path.cwd(),
1105
+ user_match_key: UserMatchKeys = typer.Option(
1106
+ UserMatchKeys.EXTERNAL_SYSTEM_ID.value, help="The key to use to match users"
1107
+ ),
1108
+ default_preferred_contact_type: PreferredContactType = typer.Option(
1109
+ PreferredContactType.EMAIL.value,
1110
+ case_sensitive=False,
1111
+ help="The default preferred contact type to use if the provided value is not valid or not present",
1112
+ ),
1113
+ no_progress: bool = typer.Option(
1114
+ False,
1115
+ "--no-progress",
1116
+ help="Disable progress bar display during user import",
1117
+ ),
1118
+ ) -> None:
1119
+ """
1120
+ Command-line interface to batch import users into FOLIO
1121
+ """
1122
+ set_up_cli_logging()
1123
+ protect_fields = [f.strip() for f in fields_to_protect.split(",") if f.strip()]
977
1124
 
978
- library_name = args.library_name
1125
+ library_name = library_name
979
1126
 
980
1127
  # Semaphore to limit the number of async HTTP requests active at any given time
981
- limit_async_requests = asyncio.Semaphore(args.limit_async_requests)
982
- batch_size = args.batch_size
983
-
984
- folio_client = folioclient.FolioClient(
985
- args.okapi_url,
986
- args.tenant_id,
987
- args.username,
988
- args.folio_password
989
- or os.environ.get("FOLIO_PASS", "")
990
- or getpass.getpass(
991
- "Enter your FOLIO password: ",
992
- ),
993
- )
1128
+ limit_async_requests = asyncio.Semaphore(limit_async_requests)
1129
+ batch_size = batch_size
1130
+
1131
+ folio_client = folioclient.FolioClient(gateway_url, tenant_id, username, password)
994
1132
 
995
1133
  # Set the member tenant id if provided to support FOLIO ECS multi-tenant environments
996
- if args.member_tenant_id:
997
- folio_client.okapi_headers["x-okapi-tenant"] = args.member_tenant_id
1134
+ if member_tenant_id:
1135
+ folio_client.okapi_headers["x-okapi-tenant"] = member_tenant_id
998
1136
 
999
- user_file_path = Path(args.user_file_path)
1000
- report_file_base_path = Path(args.report_file_base_path)
1001
- log_file_path = (
1002
- report_file_base_path
1003
- / f"log_user_import_{dt.now(utc).strftime('%Y%m%d_%H%M%S')}.log"
1004
- )
1137
+ user_file_path = user_file_path
1138
+ report_file_base_path = report_file_base_path
1005
1139
  error_file_path = (
1006
1140
  report_file_base_path
1007
1141
  / f"failed_user_import_{dt.now(utc).strftime('%Y%m%d_%H%M%S')}.txt"
1008
1142
  )
1009
- async with aiofiles.open(
1010
- log_file_path,
1011
- "w",
1012
- ) as logfile, aiofiles.open(
1013
- error_file_path, "w"
1014
- ) as errorfile, httpx.AsyncClient(timeout=None) as http_client:
1015
- try:
1016
- importer = UserImporter(
1017
- folio_client,
1018
- library_name,
1019
- batch_size,
1020
- limit_async_requests,
1021
- logfile,
1022
- errorfile,
1023
- http_client,
1024
- user_file_path,
1025
- args.user_match_key,
1026
- args.update_only_present_fields,
1027
- args.default_preferred_contact_type,
1028
- fields_to_protect=protect_fields,
1029
- )
1030
- await importer.do_import()
1031
- except Exception as ee:
1032
- print(f"An unknown error occurred: {ee}")
1033
- await logfile.write(f"An error occurred {ee}\n")
1034
- raise ee
1143
+ try:
1144
+ importer = UserImporter(
1145
+ folio_client,
1146
+ library_name,
1147
+ batch_size,
1148
+ limit_async_requests,
1149
+ user_file_path,
1150
+ user_match_key.value,
1151
+ update_only_present_fields,
1152
+ default_preferred_contact_type.value,
1153
+ fields_to_protect=protect_fields,
1154
+ no_progress=no_progress
1155
+ )
1156
+ asyncio.run(run_user_importer(importer, error_file_path))
1157
+ except Exception as ee:
1158
+ logger.critical(f"An unknown error occurred: {ee}")
1159
+ raise typer.Exit(1)
1035
1160
 
1036
1161
 
1037
- def sync_main() -> None:
1038
- """
1039
- Synchronous version of the main function.
1162
+ async def run_user_importer(importer: UserImporter, error_file_path: Path):
1163
+ try:
1164
+ await importer.setup(error_file_path)
1165
+ await importer.do_import()
1166
+ except Exception as ee:
1167
+ logger.critical(f"An unknown error occurred: {ee}")
1168
+ typer.Exit(1)
1169
+ finally:
1170
+ await importer.close()
1040
1171
 
1041
- This function is used to run the main function in a synchronous context.
1042
- """
1043
- asyncio.run(main())
1172
+
1173
+ def _main():
1174
+ typer.run(main)
1044
1175
 
1045
1176
 
1046
1177
  # Run the main function
1047
1178
  if __name__ == "__main__":
1048
- asyncio.run(main())
1179
+ app()