folio-data-import 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of folio-data-import might be problematic. Click here for more details.

@@ -1,19 +1,32 @@
1
- import argparse
2
1
  import asyncio
3
2
  import datetime
4
- import getpass
5
3
  import json
6
- import os
4
+ import logging
5
+ import sys
7
6
  import time
8
7
  import uuid
9
8
  from datetime import datetime as dt
9
+ from enum import Enum
10
10
  from pathlib import Path
11
- from typing import Tuple, List
11
+ from typing import List, Tuple
12
12
 
13
13
  import aiofiles
14
14
  import folioclient
15
15
  import httpx
16
+ import typer
16
17
  from aiofiles.threadpool.text import AsyncTextIOWrapper
18
+ from rich.logging import RichHandler
19
+ from rich.progress import (
20
+ BarColumn,
21
+ MofNCompleteColumn,
22
+ Progress,
23
+ SpinnerColumn,
24
+ TimeElapsedColumn,
25
+ TimeRemainingColumn,
26
+ )
27
+ from typing_extensions import Annotated
28
+
29
+ from folio_data_import._progress import ItemsPerSecondColumn, UserStatsColumn
17
30
 
18
31
  try:
19
32
  utc = datetime.UTC
@@ -22,6 +35,8 @@ except AttributeError:
22
35
 
23
36
  utc = zoneinfo.ZoneInfo("UTC")
24
37
 
38
+ logger = logging.getLogger(__name__)
39
+
25
40
  # Mapping of preferred contact type IDs to their corresponding values
26
41
  PREFERRED_CONTACT_TYPES_MAP = {
27
42
  "001": "mail",
@@ -31,6 +46,26 @@ PREFERRED_CONTACT_TYPES_MAP = {
31
46
  "005": "mobile",
32
47
  }
33
48
 
49
+
50
+ class PreferredContactType(Enum):
51
+ MAIL = "001"
52
+ EMAIL = "002"
53
+ TEXT = "003"
54
+ PHONE = "004"
55
+ MOBILE = "005"
56
+ _001 = "mail"
57
+ _002 = "email"
58
+ _003 = "text"
59
+ _004 = "phone"
60
+ _005 = "mobile"
61
+
62
+
63
+ class UserMatchKeys(Enum):
64
+ USERNAME = "username"
65
+ EMAIL = "email"
66
+ EXTERNAL_SYSTEM_ID = "externalSystemId"
67
+
68
+
34
69
  class UserImporter: # noqa: R0902
35
70
  """
36
71
  Class to import mod-user-import compatible user objects
@@ -38,20 +73,21 @@ class UserImporter: # noqa: R0902
38
73
  from a JSON-lines file into FOLIO
39
74
  """
40
75
 
76
+ logfile: AsyncTextIOWrapper
77
+ errorfile: AsyncTextIOWrapper
78
+ http_client: httpx.AsyncClient
79
+
41
80
  def __init__(
42
81
  self,
43
82
  folio_client: folioclient.FolioClient,
44
83
  library_name: str,
45
84
  batch_size: int,
46
85
  limit_simultaneous_requests: asyncio.Semaphore,
47
- logfile: AsyncTextIOWrapper,
48
- errorfile: AsyncTextIOWrapper,
49
- http_client: httpx.AsyncClient,
50
86
  user_file_path: Path = None,
51
87
  user_match_key: str = "externalSystemId",
52
88
  only_update_present_fields: bool = False,
53
89
  default_preferred_contact_type: str = "002",
54
- fields_to_protect: List[str] =[],
90
+ fields_to_protect: List[str] = [],
55
91
  ) -> None:
56
92
  self.limit_simultaneous_requests = limit_simultaneous_requests
57
93
  self.batch_size = batch_size
@@ -70,9 +106,6 @@ class UserImporter: # noqa: R0902
70
106
  self.service_point_map: dict = self.build_ref_data_id_map(
71
107
  self.folio_client, "/service-points", "servicepoints", "code"
72
108
  )
73
- self.logfile: AsyncTextIOWrapper = logfile
74
- self.errorfile: AsyncTextIOWrapper = errorfile
75
- self.http_client: httpx.AsyncClient = http_client
76
109
  self.only_update_present_fields: bool = only_update_present_fields
77
110
  self.default_preferred_contact_type: str = default_preferred_contact_type
78
111
  self.match_key = user_match_key
@@ -114,17 +147,36 @@ class UserImporter: # noqa: R0902
114
147
  except ValueError:
115
148
  return False
116
149
 
150
+ async def setup(self, error_file_path: Path) -> None:
151
+ """
152
+ Sets up the importer by initializing necessary resources.
153
+
154
+ Args:
155
+ log_file_path (Path): The path to the log file.
156
+ error_file_path (Path): The path to the error file.
157
+ """
158
+ self.errorfile = await aiofiles.open(error_file_path, "w", encoding="utf-8")
159
+
160
+ async def close(self) -> None:
161
+ """
162
+ Closes the importer by releasing any resources.
163
+
164
+ """
165
+ await self.errorfile.close()
166
+
117
167
  async def do_import(self) -> None:
118
168
  """
119
169
  Main method to import users.
120
170
 
121
171
  This method triggers the process of importing users by calling the `process_file` method.
122
172
  """
123
- if self.user_file_path:
124
- with open(self.user_file_path, "r", encoding="utf-8") as openfile:
125
- await self.process_file(openfile)
126
- else:
127
- raise FileNotFoundError("No user objects file provided")
173
+ async with httpx.AsyncClient() as client:
174
+ self.http_client = client
175
+ if self.user_file_path:
176
+ with open(self.user_file_path, "r", encoding="utf-8") as openfile:
177
+ await self.process_file(openfile)
178
+ else:
179
+ raise FileNotFoundError("No user objects file provided")
128
180
 
129
181
  async def get_existing_user(self, user_obj) -> dict:
130
182
  """
@@ -227,7 +279,7 @@ class UserImporter: # noqa: R0902
227
279
  self.validate_uuid(address["addressTypeId"])
228
280
  and address["addressTypeId"] in self.address_type_map.values()
229
281
  ):
230
- await self.logfile.write(
282
+ logger.debug(
231
283
  f"Row {line_number}: Address type {address['addressTypeId']} is a UUID, "
232
284
  f"skipping mapping\n"
233
285
  )
@@ -239,11 +291,7 @@ class UserImporter: # noqa: R0902
239
291
  mapped_addresses.append(address)
240
292
  except KeyError:
241
293
  if address["addressTypeId"] not in self.address_type_map.values():
242
- print(
243
- f"Row {line_number}: Address type {address['addressTypeId']} not found"
244
- f", removing address"
245
- )
246
- await self.logfile.write(
294
+ logger.error(
247
295
  f"Row {line_number}: Address type {address['addressTypeId']} not found"
248
296
  f", removing address\n"
249
297
  )
@@ -266,7 +314,7 @@ class UserImporter: # noqa: R0902
266
314
  self.validate_uuid(user_obj["patronGroup"])
267
315
  and user_obj["patronGroup"] in self.patron_group_map.values()
268
316
  ):
269
- await self.logfile.write(
317
+ logger.debug(
270
318
  f"Row {line_number}: Patron group {user_obj['patronGroup']} is a UUID, "
271
319
  f"skipping mapping\n"
272
320
  )
@@ -274,11 +322,7 @@ class UserImporter: # noqa: R0902
274
322
  user_obj["patronGroup"] = self.patron_group_map[user_obj["patronGroup"]]
275
323
  except KeyError:
276
324
  if user_obj["patronGroup"] not in self.patron_group_map.values():
277
- print(
278
- f"Row {line_number}: Patron group {user_obj['patronGroup']} not found, "
279
- f"removing patron group"
280
- )
281
- await self.logfile.write(
325
+ logger.error(
282
326
  f"Row {line_number}: Patron group {user_obj['patronGroup']} not found in, "
283
327
  f"removing patron group\n"
284
328
  )
@@ -302,25 +346,23 @@ class UserImporter: # noqa: R0902
302
346
  self.validate_uuid(department)
303
347
  and department in self.department_map.values()
304
348
  ):
305
- await self.logfile.write(
349
+ logger.debug(
306
350
  f"Row {line_number}: Department {department} is a UUID, skipping mapping\n"
307
351
  )
308
352
  mapped_departments.append(department)
309
353
  else:
310
354
  mapped_departments.append(self.department_map[department])
311
355
  except KeyError:
312
- print(
313
- f'Row {line_number}: Department "{department}" not found, ' # noqa: B907
314
- f"excluding department from user"
315
- )
316
- await self.logfile.write(
356
+ logger.error(
317
357
  f'Row {line_number}: Department "{department}" not found, ' # noqa: B907
318
358
  f"excluding department from user\n"
319
359
  )
320
360
  if mapped_departments:
321
361
  user_obj["departments"] = mapped_departments
322
362
 
323
- async def update_existing_user(self, user_obj, existing_user, protected_fields) -> Tuple[dict, dict]:
363
+ async def update_existing_user(
364
+ self, user_obj, existing_user, protected_fields
365
+ ) -> Tuple[dict, dict]:
324
366
  """
325
367
  Updates an existing user with the provided user object.
326
368
 
@@ -338,7 +380,11 @@ class UserImporter: # noqa: R0902
338
380
  """
339
381
 
340
382
  await self.set_preferred_contact_type(user_obj, existing_user)
341
- preferred_contact_type = {"preferredContactTypeId": existing_user.get("personal", {}).pop("preferredContactTypeId")}
383
+ preferred_contact_type = {
384
+ "preferredContactTypeId": existing_user.get("personal", {}).pop(
385
+ "preferredContactTypeId"
386
+ )
387
+ }
342
388
  if self.only_update_present_fields:
343
389
  new_personal = user_obj.pop("personal", {})
344
390
  existing_personal = existing_user.pop("personal", {})
@@ -412,30 +458,39 @@ class UserImporter: # noqa: R0902
412
458
  current_pref_contact = user_obj["personal"].get(
413
459
  "preferredContactTypeId", ""
414
460
  )
415
- if mapped_contact_type := dict([(v, k) for k, v in PREFERRED_CONTACT_TYPES_MAP.items()]).get(
461
+ if mapped_contact_type := dict(
462
+ [(v, k) for k, v in PREFERRED_CONTACT_TYPES_MAP.items()]
463
+ ).get(
416
464
  current_pref_contact,
417
465
  "",
418
466
  ):
419
- existing_user["personal"]["preferredContactTypeId"] = mapped_contact_type
467
+ existing_user["personal"]["preferredContactTypeId"] = (
468
+ mapped_contact_type
469
+ )
420
470
  else:
421
- existing_user["personal"]["preferredContactTypeId"] = current_pref_contact if current_pref_contact in PREFERRED_CONTACT_TYPES_MAP else self.default_preferred_contact_type
471
+ existing_user["personal"]["preferredContactTypeId"] = (
472
+ current_pref_contact
473
+ if current_pref_contact in PREFERRED_CONTACT_TYPES_MAP
474
+ else self.default_preferred_contact_type
475
+ )
422
476
  else:
423
- print(
424
- f"Preferred contact type not provided or is not a valid option: {PREFERRED_CONTACT_TYPES_MAP}\n"
477
+ logger.warning(
478
+ f"Preferred contact type not provided or is not a valid option: {PREFERRED_CONTACT_TYPES_MAP} "
425
479
  f"Setting preferred contact type to {self.default_preferred_contact_type} or using existing value"
426
480
  )
427
- await self.logfile.write(
428
- f"Preferred contact type not provided or is not a valid option: {PREFERRED_CONTACT_TYPES_MAP}\n"
429
- f"Setting preferred contact type to {self.default_preferred_contact_type} or using existing value\n"
481
+ mapped_contact_type = (
482
+ existing_user.get("personal", {}).get("preferredContactTypeId", "")
483
+ or self.default_preferred_contact_type
430
484
  )
431
- mapped_contact_type = existing_user.get("personal", {}).get(
432
- "preferredContactTypeId", ""
433
- ) or self.default_preferred_contact_type
434
485
  if "personal" not in existing_user:
435
486
  existing_user["personal"] = {}
436
- existing_user["personal"]["preferredContactTypeId"] = mapped_contact_type or self.default_preferred_contact_type
487
+ existing_user["personal"]["preferredContactTypeId"] = (
488
+ mapped_contact_type or self.default_preferred_contact_type
489
+ )
437
490
 
438
- async def create_or_update_user(self, user_obj, existing_user, protected_fields, line_number) -> dict:
491
+ async def create_or_update_user(
492
+ self, user_obj, existing_user, protected_fields, line_number
493
+ ) -> dict:
439
494
  """
440
495
  Creates or updates a user based on the given user object and existing user.
441
496
 
@@ -456,11 +511,7 @@ class UserImporter: # noqa: R0902
456
511
  self.logs["updated"] += 1
457
512
  return existing_user
458
513
  except Exception as ee:
459
- print(
460
- f"Row {line_number}: User update failed: "
461
- f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}"
462
- )
463
- await self.logfile.write(
514
+ logger.error(
464
515
  f"Row {line_number}: User update failed: "
465
516
  f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
466
517
  )
@@ -474,11 +525,7 @@ class UserImporter: # noqa: R0902
474
525
  new_user = await self.create_new_user(user_obj)
475
526
  return new_user
476
527
  except Exception as ee:
477
- print(
478
- f"Row {line_number}: User creation failed: "
479
- f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}"
480
- )
481
- await self.logfile.write(
528
+ logger.error(
482
529
  f"Row {line_number}: User creation failed: "
483
530
  f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}\n"
484
531
  )
@@ -516,7 +563,9 @@ class UserImporter: # noqa: R0902
516
563
  dict: A dictionary containing the protected fields and their values.
517
564
  """
518
565
  protected_fields = {}
519
- protected_fields_list = existing_user.get("customFields", {}).get("protectedFields", "").split(",")
566
+ protected_fields_list = (
567
+ existing_user.get("customFields", {}).get("protectedFields", "").split(",")
568
+ )
520
569
  cli_fields = list(self.fields_to_protect)
521
570
  # combine and dedupe:
522
571
  all_fields = list(dict.fromkeys(protected_fields_list + cli_fields))
@@ -557,7 +606,15 @@ class UserImporter: # noqa: R0902
557
606
  existing_pu = {}
558
607
  existing_spu = {}
559
608
  protected_fields = {}
560
- return rp_obj, spu_obj, existing_user, protected_fields, existing_rp, existing_pu, existing_spu
609
+ return (
610
+ rp_obj,
611
+ spu_obj,
612
+ existing_user,
613
+ protected_fields,
614
+ existing_rp,
615
+ existing_pu,
616
+ existing_spu,
617
+ )
561
618
 
562
619
  async def create_or_update_rp(self, rp_obj, existing_rp, new_user_obj):
563
620
  """
@@ -572,10 +629,8 @@ class UserImporter: # noqa: R0902
572
629
  None
573
630
  """
574
631
  if existing_rp:
575
- # print(existing_rp)
576
632
  await self.update_existing_rp(rp_obj, existing_rp)
577
633
  else:
578
- # print(new_user_obj)
579
634
  await self.create_new_rp(new_user_obj)
580
635
 
581
636
  async def create_new_rp(self, new_user_obj):
@@ -593,7 +648,6 @@ class UserImporter: # noqa: R0902
593
648
  """
594
649
  rp_obj = {"holdShelf": True, "delivery": False}
595
650
  rp_obj["userId"] = new_user_obj["id"]
596
- # print(rp_obj)
597
651
  response = await self.http_client.post(
598
652
  self.folio_client.gateway_url
599
653
  + "/request-preference-storage/request-preference",
@@ -617,7 +671,6 @@ class UserImporter: # noqa: R0902
617
671
  None
618
672
  """
619
673
  existing_rp.update(rp_obj)
620
- # print(existing_rp)
621
674
  response = await self.http_client.put(
622
675
  self.folio_client.gateway_url
623
676
  + f"/request-preference-storage/request-preference/{existing_rp['id']}",
@@ -668,9 +721,15 @@ class UserImporter: # noqa: R0902
668
721
  """
669
722
  async with self.limit_simultaneous_requests:
670
723
  user_obj = await self.process_user_obj(user)
671
- rp_obj, spu_obj, existing_user, protected_fields, existing_rp, existing_pu, existing_spu = (
672
- await self.process_existing_user(user_obj)
673
- )
724
+ (
725
+ rp_obj,
726
+ spu_obj,
727
+ existing_user,
728
+ protected_fields,
729
+ existing_rp,
730
+ existing_pu,
731
+ existing_spu,
732
+ ) = await self.process_existing_user(user_obj)
674
733
  await self.map_address_types(user_obj, line_number)
675
734
  await self.map_patron_groups(user_obj, line_number)
676
735
  await self.map_departments(user_obj, line_number)
@@ -684,11 +743,7 @@ class UserImporter: # noqa: R0902
684
743
  rp_obj, existing_rp, new_user_obj
685
744
  )
686
745
  else:
687
- print(
688
- f"Row {line_number}: Creating default request preference object"
689
- f" for {new_user_obj['id']}"
690
- )
691
- await self.logfile.write(
746
+ logger.debug(
692
747
  f"Row {line_number}: Creating default request preference object"
693
748
  f" for {new_user_obj['id']}\n"
694
749
  )
@@ -699,8 +754,7 @@ class UserImporter: # noqa: R0902
699
754
  f"{new_user_obj['id']}: "
700
755
  f"{str(getattr(getattr(ee, 'response', ee), 'text', str(ee)))}"
701
756
  )
702
- print(rp_error_message)
703
- await self.logfile.write(rp_error_message + "\n")
757
+ logger.error(rp_error_message)
704
758
  if not existing_pu:
705
759
  try:
706
760
  await self.create_perms_user(new_user_obj)
@@ -710,9 +764,10 @@ class UserImporter: # noqa: R0902
710
764
  f"{new_user_obj['id']}: "
711
765
  f"{str(getattr(getattr(ee, 'response', str(ee)), 'text', str(ee)))}"
712
766
  )
713
- print(pu_error_message)
714
- await self.logfile.write(pu_error_message + "\n")
715
- await self.handle_service_points_user(spu_obj, existing_spu, new_user_obj)
767
+ logger.error(pu_error_message)
768
+ await self.handle_service_points_user(
769
+ spu_obj, existing_spu, new_user_obj
770
+ )
716
771
 
717
772
  async def map_service_points(self, spu_obj, existing_user):
718
773
  """
@@ -730,40 +785,43 @@ class UserImporter: # noqa: R0902
730
785
  for sp in spu_obj.pop("servicePointsIds", []):
731
786
  try:
732
787
  if self.validate_uuid(sp) and sp in self.service_point_map.values():
733
- await self.logfile.write(
788
+ logger.debug(
734
789
  f"Service point {sp} is a UUID, skipping mapping\n"
735
790
  )
736
791
  mapped_service_points.append(sp)
737
792
  else:
738
793
  mapped_service_points.append(self.service_point_map[sp])
739
794
  except KeyError:
740
- print(
795
+ logger.error(
741
796
  f'Service point "{sp}" not found, excluding service point from user: '
742
- f'{self.service_point_map}'
797
+ f"{self.service_point_map}"
743
798
  )
744
799
  if mapped_service_points:
745
800
  spu_obj["servicePointsIds"] = mapped_service_points
746
801
  if "defaultServicePointId" in spu_obj:
747
- sp_code = spu_obj.pop('defaultServicePointId', '')
802
+ sp_code = spu_obj.pop("defaultServicePointId", "")
748
803
  try:
749
- if self.validate_uuid(sp_code) and sp_code in self.service_point_map.values():
750
- await self.logfile.write(
804
+ if (
805
+ self.validate_uuid(sp_code)
806
+ and sp_code in self.service_point_map.values()
807
+ ):
808
+ logger.debug(
751
809
  f"Default service point {sp_code} is a UUID, skipping mapping\n"
752
810
  )
753
811
  mapped_sp_id = sp_code
754
812
  else:
755
813
  mapped_sp_id = self.service_point_map[sp_code]
756
- if mapped_sp_id not in spu_obj.get('servicePointsIds', []):
757
- print(
814
+ if mapped_sp_id not in spu_obj.get("servicePointsIds", []):
815
+ logger.warning(
758
816
  f'Default service point "{sp_code}" not found in assigned service points, '
759
- 'excluding default service point from user'
817
+ "excluding default service point from user"
760
818
  )
761
819
  else:
762
- spu_obj['defaultServicePointId'] = mapped_sp_id
820
+ spu_obj["defaultServicePointId"] = mapped_sp_id
763
821
  except KeyError:
764
- print(
822
+ logger.error(
765
823
  f'Default service point "{sp_code}" not found, excluding default service '
766
- f'point from user: {existing_user["id"]}'
824
+ f"point from user: {existing_user['id']}"
767
825
  )
768
826
 
769
827
  async def handle_service_points_user(self, spu_obj, existing_spu, existing_user):
@@ -837,7 +895,8 @@ class UserImporter: # noqa: R0902
837
895
  """
838
896
  existing_spu.update(spu_obj)
839
897
  response = await self.http_client.put(
840
- self.folio_client.gateway_url + f"/service-points-users/{existing_spu['id']}",
898
+ self.folio_client.gateway_url
899
+ + f"/service-points-users/{existing_spu['id']}",
841
900
  headers=self.folio_client.okapi_headers,
842
901
  json=existing_spu,
843
902
  )
@@ -850,199 +909,263 @@ class UserImporter: # noqa: R0902
850
909
  Args:
851
910
  openfile: The file or file-like object to process.
852
911
  """
853
- tasks = []
854
- for line_number, user in enumerate(openfile):
855
- tasks.append(self.process_line(user, line_number))
856
- if len(tasks) == self.batch_size:
912
+ with Progress( # Set up the progress bar
913
+ "{task.description}",
914
+ SpinnerColumn(),
915
+ BarColumn(),
916
+ MofNCompleteColumn(),
917
+ UserStatsColumn(),
918
+ "[",
919
+ TimeElapsedColumn(),
920
+ "<",
921
+ TimeRemainingColumn(),
922
+ "/",
923
+ ItemsPerSecondColumn(),
924
+ "]",
925
+ ) as progress:
926
+ with open(openfile.name, "rb") as f:
927
+ total_lines = sum(
928
+ buf.count(b"\n") for buf in iter(lambda: f.read(1024 * 1024), b"")
929
+ )
930
+ self.progress = progress
931
+ self.task_progress = progress.add_task(
932
+ "Importing users: ", total=total_lines, created=0, updated=0, failed=0
933
+ ) # Add a task to the progress bar
934
+ openfile.seek(0)
935
+ tasks = []
936
+ for line_number, user in enumerate(openfile):
937
+ tasks.append(self.process_line(user, line_number))
938
+ if len(tasks) == self.batch_size:
939
+ start = time.time()
940
+ await asyncio.gather(*tasks)
941
+ duration = time.time() - start
942
+ async with self.lock:
943
+ progress.update(
944
+ self.task_progress,
945
+ advance=len(tasks),
946
+ created=self.logs["created"],
947
+ updated=self.logs["updated"],
948
+ failed=self.logs["failed"],
949
+ )
950
+ message = (
951
+ f"{dt.now().isoformat(sep=' ', timespec='milliseconds')}: "
952
+ f"Batch of {self.batch_size} users processed in {duration:.2f} "
953
+ f"seconds. - Users created: {self.logs['created']} - Users updated: "
954
+ f"{self.logs['updated']} - Users failed: {self.logs['failed']}"
955
+ )
956
+ logger.info(message)
957
+ tasks = []
958
+ if tasks:
857
959
  start = time.time()
858
960
  await asyncio.gather(*tasks)
859
961
  duration = time.time() - start
860
962
  async with self.lock:
963
+ progress.update(
964
+ self.task_progress,
965
+ advance=len(tasks),
966
+ created=self.logs["created"],
967
+ updated=self.logs["updated"],
968
+ failed=self.logs["failed"],
969
+ )
861
970
  message = (
862
971
  f"{dt.now().isoformat(sep=' ', timespec='milliseconds')}: "
863
- f"Batch of {self.batch_size} users processed in {duration:.2f} "
864
- f"seconds. - Users created: {self.logs['created']} - Users updated: "
972
+ f"Batch of {len(tasks)} users processed in {duration:.2f} seconds. - "
973
+ f"Users created: {self.logs['created']} - Users updated: "
865
974
  f"{self.logs['updated']} - Users failed: {self.logs['failed']}"
866
975
  )
867
- print(message)
868
- await self.logfile.write(message + "\n")
869
- tasks = []
870
- if tasks:
871
- start = time.time()
872
- await asyncio.gather(*tasks)
873
- duration = time.time() - start
874
- async with self.lock:
875
- message = (
876
- f"{dt.now().isoformat(sep=' ', timespec='milliseconds')}: "
877
- f"Batch of {len(tasks)} users processed in {duration:.2f} seconds. - "
878
- f"Users created: {self.logs['created']} - Users updated: "
879
- f"{self.logs['updated']} - Users failed: {self.logs['failed']}"
880
- )
881
- print(message)
882
- await self.logfile.write(message + "\n")
976
+ logger.info(message)
883
977
 
884
978
 
885
- async def main() -> None:
979
+ def set_up_cli_logging():
886
980
  """
887
- Entry point of the user import script.
888
-
889
- Parses command line arguments, initializes necessary objects, and starts the import process.
890
-
891
- Args:
892
- --tenant_id (str): The tenant id.
893
- --library_name (str): The name of the library.
894
- --username (str): The FOLIO username.
895
- --okapi_url (str): The Okapi URL.
896
- --user_file_path (str): The path to the user file.
897
- --limit_async_requests (int): Limit how many http requests can be made at once. Default 10.
898
- --batch_size (int): How many records to process before logging statistics. Default 250.
899
- --folio_password (str): The FOLIO password.
900
- --user_match_key (str): The key to use to match users. Default "externalSystemId".
901
- --report_file_base_path (str): The base path for the log and error files. Default "./".
902
- --update_only_present_fields (bool): Only update fields that are present in the new user object.
903
- --default_preferred_contact_type (str): The default preferred contact type to use if the provided \
904
- value is not valid or not present. Default "002".
905
- --fields_to_protect (str): Comma-separated list of top-level or nested (dot-notation) fields to protect.
906
-
907
- Raises:
908
- Exception: If an unknown error occurs during the import process.
909
-
910
- Returns:
911
- None
981
+ This function sets up logging for the CLI.
912
982
  """
913
- parser = argparse.ArgumentParser()
914
- parser.add_argument("--tenant_id", help="The tenant id")
915
- parser.add_argument(
916
- "--member_tenant_id",
917
- help="The FOLIO ECS member tenant id (if applicable)",
918
- default="",
919
- )
920
- parser.add_argument("--library_name", help="The name of the library")
921
- parser.add_argument("--username", help="The FOLIO username")
922
- parser.add_argument("--okapi_url", help="The Okapi URL")
923
- parser.add_argument("--user_file_path", help="The path to the user file")
924
- parser.add_argument(
925
- "--limit_async_requests",
926
- help="Limit how many http requests can be made at once",
927
- type=int,
928
- default=10,
929
- )
930
- parser.add_argument(
931
- "--batch_size",
932
- help="How many user records to process before logging statistics",
933
- type=int,
934
- default=250,
935
- )
936
- parser.add_argument("--folio_password", help="The FOLIO password")
937
- parser.add_argument(
938
- "--user_match_key",
939
- help="The key to use to match users",
940
- choices=["externalSystemId", "barcode", "username"],
941
- default="externalSystemId",
942
- )
943
- parser.add_argument(
944
- "--report_file_base_path",
945
- help="The base path for the log and error files",
946
- default="./",
947
- )
948
- parser.add_argument(
949
- "--update_only_present_fields",
950
- help="Only update fields that are present in the user object",
951
- action="store_true",
983
+ logger.setLevel(logging.INFO)
984
+ logger.propagate = False
985
+
986
+ # Set up file and stream handlers
987
+ file_handler = logging.FileHandler(
988
+ "folio_user_import_{}.log".format(dt.now().strftime("%Y%m%d%H%M%S"))
952
989
  )
953
- parser.add_argument(
954
- "--default_preferred_contact_type",
955
- help=(
956
- "The default preferred contact type to use if the provided value is not present or not valid. "
957
- "Note: '002' is the default, and will be used if the provided value is not valid or not present, "
958
- "unless the existing user object being updated has a valid preferred contact type set."
990
+ file_handler.setLevel(logging.INFO)
991
+ file_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
992
+ file_handler.setFormatter(file_formatter)
993
+ logger.addHandler(file_handler)
994
+
995
+ if not any(
996
+ isinstance(h, logging.StreamHandler) and h.stream == sys.stderr
997
+ for h in logger.handlers
998
+ ):
999
+ stream_handler = RichHandler(
1000
+ show_level=False,
1001
+ show_time=False,
1002
+ omit_repeated_times=False,
1003
+ show_path=False,
1004
+ )
1005
+ stream_handler.setLevel(logging.WARNING)
1006
+ stream_formatter = logging.Formatter("%(message)s")
1007
+ stream_handler.setFormatter(stream_formatter)
1008
+ logger.addHandler(stream_handler)
1009
+
1010
+ # Stop httpx from logging info messages to the console
1011
+ logging.getLogger("httpx").setLevel(logging.WARNING)
1012
+
1013
+
1014
+ app = typer.Typer()
1015
+
1016
+
1017
+ @app.command()
1018
+ def main(
1019
+ gateway_url: Annotated[
1020
+ str,
1021
+ typer.Option(
1022
+ ...,
1023
+ prompt="Please enter the FOLIO API Gateway URL",
1024
+ help="The FOLIO API Gateway URL",
1025
+ envvar="FOLIO_GATEWAY_URL",
959
1026
  ),
960
- choices=list(PREFERRED_CONTACT_TYPES_MAP.keys()) + list(PREFERRED_CONTACT_TYPES_MAP.values()),
961
- default="002",
962
- )
963
- parser.add_argument(
964
- "--fields-to-protect", # new flag name
965
- dest="fields_to_protect", # sets args.fields_to_protect
966
- help=(
967
- "Comma-separated list of top-level user fields to protect "
968
- "(e.g. type,expirationDate)"
1027
+ ],
1028
+ tenant_id: Annotated[
1029
+ str,
1030
+ typer.Option(
1031
+ ...,
1032
+ prompt="Please enter the FOLIO tenant id",
1033
+ help="The tenant id",
1034
+ envvar="FOLIO_TENANT_ID",
969
1035
  ),
970
- default="",
971
- )
972
- args = parser.parse_args()
973
- protect_fields = [
974
- f.strip() for f in args.fields_to_protect.split(",")
975
- if f.strip()
976
- ]
1036
+ ],
1037
+ username: Annotated[
1038
+ str,
1039
+ typer.Option(
1040
+ ...,
1041
+ prompt="Please enter your FOLIO username",
1042
+ help="The FOLIO username",
1043
+ envvar="FOLIO_USERNAME",
1044
+ ),
1045
+ ],
1046
+ password: Annotated[
1047
+ str,
1048
+ typer.Option(
1049
+ ...,
1050
+ prompt="Please enter your FOLIO Password",
1051
+ hide_input=True,
1052
+ help="The FOLIO password",
1053
+ envvar="FOLIO_PASSWORD",
1054
+ ),
1055
+ ],
1056
+ library_name: Annotated[
1057
+ str,
1058
+ typer.Option(
1059
+ ...,
1060
+ prompt="Please enter the library name",
1061
+ help="The name of the library",
1062
+ envvar="FOLIO_LIBRARY_NAME",
1063
+ ),
1064
+ ],
1065
+ user_file_path: Annotated[
1066
+ Path, typer.Option(..., help="The path to the user file")
1067
+ ],
1068
+ member_tenant_id: Annotated[
1069
+ str,
1070
+ typer.Option(
1071
+ help="The FOLIO ECS member tenant id (if applicable)",
1072
+ envvar="FOLIO_MEMBER_TENANT_ID",
1073
+ ),
1074
+ ] = "",
1075
+ fields_to_protect: Annotated[
1076
+ str,
1077
+ typer.Option(
1078
+ help="Comma-separated list of top-level or nested (dot-notation) fields to protect"
1079
+ ),
1080
+ ] = "",
1081
+ update_only_present_fields: bool = typer.Option(
1082
+ False,
1083
+ "--update-only-present-fields",
1084
+ help="Only update fields that are present in the new user object",
1085
+ ),
1086
+ limit_async_requests: Annotated[
1087
+ int,
1088
+ typer.Option(
1089
+ help="Limit how many http requests can be made at once",
1090
+ envvar="FOLIO_LIMIT_ASYNC_REQUESTS",
1091
+ ),
1092
+ ] = 10,
1093
+ batch_size: Annotated[
1094
+ int,
1095
+ typer.Option(
1096
+ help="How many user records to process before logging statistics",
1097
+ envvar="FOLIO_USER_IMPORT_BATCH_SIZE",
1098
+ ),
1099
+ ] = 250,
1100
+ report_file_base_path: Annotated[
1101
+ Path, typer.Option(help="The base path for the log and error files")
1102
+ ] = Path.cwd(),
1103
+ user_match_key: UserMatchKeys = typer.Option(
1104
+ UserMatchKeys.EXTERNAL_SYSTEM_ID.value, help="The key to use to match users"
1105
+ ),
1106
+ default_preferred_contact_type: PreferredContactType = typer.Option(
1107
+ PreferredContactType.EMAIL.value,
1108
+ case_sensitive=False,
1109
+ help="The default preferred contact type to use if the provided value is not valid or not present",
1110
+ ),
1111
+ ) -> None:
1112
+ """
1113
+ Command-line interface to batch import users into FOLIO
1114
+ """
1115
+ set_up_cli_logging()
1116
+ protect_fields = [f.strip() for f in fields_to_protect.split(",") if f.strip()]
977
1117
 
978
- library_name = args.library_name
1118
+ library_name = library_name
979
1119
 
980
1120
  # Semaphore to limit the number of async HTTP requests active at any given time
981
- limit_async_requests = asyncio.Semaphore(args.limit_async_requests)
982
- batch_size = args.batch_size
983
-
984
- folio_client = folioclient.FolioClient(
985
- args.okapi_url,
986
- args.tenant_id,
987
- args.username,
988
- args.folio_password
989
- or os.environ.get("FOLIO_PASS", "")
990
- or getpass.getpass(
991
- "Enter your FOLIO password: ",
992
- ),
993
- )
1121
+ limit_async_requests = asyncio.Semaphore(limit_async_requests)
1122
+ batch_size = batch_size
1123
+
1124
+ folio_client = folioclient.FolioClient(gateway_url, tenant_id, username, password)
994
1125
 
995
1126
  # Set the member tenant id if provided to support FOLIO ECS multi-tenant environments
996
- if args.member_tenant_id:
997
- folio_client.okapi_headers["x-okapi-tenant"] = args.member_tenant_id
1127
+ if member_tenant_id:
1128
+ folio_client.okapi_headers["x-okapi-tenant"] = member_tenant_id
998
1129
 
999
- user_file_path = Path(args.user_file_path)
1000
- report_file_base_path = Path(args.report_file_base_path)
1001
- log_file_path = (
1002
- report_file_base_path
1003
- / f"log_user_import_{dt.now(utc).strftime('%Y%m%d_%H%M%S')}.log"
1004
- )
1130
+ user_file_path = user_file_path
1131
+ report_file_base_path = report_file_base_path
1005
1132
  error_file_path = (
1006
1133
  report_file_base_path
1007
1134
  / f"failed_user_import_{dt.now(utc).strftime('%Y%m%d_%H%M%S')}.txt"
1008
1135
  )
1009
- async with aiofiles.open(
1010
- log_file_path,
1011
- "w",
1012
- ) as logfile, aiofiles.open(
1013
- error_file_path, "w"
1014
- ) as errorfile, httpx.AsyncClient(timeout=None) as http_client:
1015
- try:
1016
- importer = UserImporter(
1017
- folio_client,
1018
- library_name,
1019
- batch_size,
1020
- limit_async_requests,
1021
- logfile,
1022
- errorfile,
1023
- http_client,
1024
- user_file_path,
1025
- args.user_match_key,
1026
- args.update_only_present_fields,
1027
- args.default_preferred_contact_type,
1028
- fields_to_protect=protect_fields,
1029
- )
1030
- await importer.do_import()
1031
- except Exception as ee:
1032
- print(f"An unknown error occurred: {ee}")
1033
- await logfile.write(f"An error occurred {ee}\n")
1034
- raise ee
1136
+ try:
1137
+ importer = UserImporter(
1138
+ folio_client,
1139
+ library_name,
1140
+ batch_size,
1141
+ limit_async_requests,
1142
+ user_file_path,
1143
+ user_match_key.value,
1144
+ update_only_present_fields,
1145
+ default_preferred_contact_type.value,
1146
+ fields_to_protect=protect_fields,
1147
+ )
1148
+ asyncio.run(run_user_importer(importer, error_file_path))
1149
+ except Exception as ee:
1150
+ logger.critical(f"An unknown error occurred: {ee}")
1151
+ raise typer.Exit(1)
1035
1152
 
1036
1153
 
1037
- def sync_main() -> None:
1038
- """
1039
- Synchronous version of the main function.
1154
+ async def run_user_importer(importer: UserImporter, error_file_path: Path):
1155
+ try:
1156
+ await importer.setup(error_file_path)
1157
+ await importer.do_import()
1158
+ except Exception as ee:
1159
+ logger.critical(f"An unknown error occurred: {ee}")
1160
+ typer.Exit(1)
1161
+ finally:
1162
+ await importer.close()
1040
1163
 
1041
- This function is used to run the main function in a synchronous context.
1042
- """
1043
- asyncio.run(main())
1164
+
1165
+ def _main():
1166
+ typer.run(main)
1044
1167
 
1045
1168
 
1046
1169
  # Run the main function
1047
1170
  if __name__ == "__main__":
1048
- asyncio.run(main())
1171
+ app()