folio-migration-tools 1.9.0rc11__py3-none-any.whl → 1.9.0rc13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. folio_migration_tools/__main__.py +1 -2
  2. folio_migration_tools/library_configuration.py +21 -1
  3. folio_migration_tools/mapper_base.py +78 -4
  4. folio_migration_tools/mapping_file_transformation/courses_mapper.py +2 -1
  5. folio_migration_tools/mapping_file_transformation/holdings_mapper.py +8 -4
  6. folio_migration_tools/mapping_file_transformation/item_mapper.py +4 -11
  7. folio_migration_tools/mapping_file_transformation/manual_fee_fines_mapper.py +1 -0
  8. folio_migration_tools/mapping_file_transformation/mapping_file_mapper_base.py +3 -19
  9. folio_migration_tools/mapping_file_transformation/notes_mapper.py +2 -0
  10. folio_migration_tools/mapping_file_transformation/order_mapper.py +4 -1
  11. folio_migration_tools/mapping_file_transformation/organization_mapper.py +7 -4
  12. folio_migration_tools/mapping_file_transformation/user_mapper.py +3 -1
  13. folio_migration_tools/marc_rules_transformation/holdings_statementsparser.py +7 -14
  14. folio_migration_tools/marc_rules_transformation/rules_mapper_authorities.py +1 -0
  15. folio_migration_tools/marc_rules_transformation/rules_mapper_base.py +83 -4
  16. folio_migration_tools/marc_rules_transformation/rules_mapper_bibs.py +10 -5
  17. folio_migration_tools/marc_rules_transformation/rules_mapper_holdings.py +39 -33
  18. folio_migration_tools/migration_tasks/bibs_transformer.py +13 -3
  19. folio_migration_tools/migration_tasks/holdings_csv_transformer.py +42 -21
  20. folio_migration_tools/migration_tasks/holdings_marc_transformer.py +39 -22
  21. folio_migration_tools/migration_tasks/items_transformer.py +4 -3
  22. folio_migration_tools/migration_tasks/migration_task_base.py +22 -1
  23. folio_migration_tools/migration_tasks/orders_transformer.py +2 -0
  24. folio_migration_tools/migration_tasks/user_transformer.py +1 -0
  25. folio_migration_tools/transaction_migration/legacy_loan.py +2 -1
  26. folio_migration_tools/translations/en.json +12 -1
  27. {folio_migration_tools-1.9.0rc11.dist-info → folio_migration_tools-1.9.0rc13.dist-info}/METADATA +2 -2
  28. {folio_migration_tools-1.9.0rc11.dist-info → folio_migration_tools-1.9.0rc13.dist-info}/RECORD +31 -31
  29. {folio_migration_tools-1.9.0rc11.dist-info → folio_migration_tools-1.9.0rc13.dist-info}/WHEEL +1 -1
  30. {folio_migration_tools-1.9.0rc11.dist-info → folio_migration_tools-1.9.0rc13.dist-info}/LICENSE +0 -0
  31. {folio_migration_tools-1.9.0rc11.dist-info → folio_migration_tools-1.9.0rc13.dist-info}/entry_points.txt +0 -0
@@ -157,7 +157,6 @@ def main():
157
157
  print("Task failure. Halting.")
158
158
  sys.exit(1)
159
159
  logging.info("Work done. Shutting down")
160
- sys.exit(0)
161
160
  except json.decoder.JSONDecodeError as json_error:
162
161
  logging.critical(json_error)
163
162
  print(json_error.doc)
@@ -198,7 +197,7 @@ def main():
198
197
  logging.exception("Unhandled exception")
199
198
  print(f"\n{ee}")
200
199
  sys.exit(ee.__class__.__name__)
201
-
200
+ sys.exit(0)
202
201
 
203
202
  def inheritors(base_class):
204
203
  subclasses = set()
@@ -32,7 +32,27 @@ class FileDefinition(BaseModel):
32
32
  ] = ""
33
33
  discovery_suppressed: Annotated[bool, Field(title="Discovery suppressed")] = False
34
34
  staff_suppressed: Annotated[bool, Field(title="Staff suppressed")] = False
35
- service_point_id: Annotated[str, Field(title="Service point ID")] = ""
35
+ service_point_id: Annotated[
36
+ str,
37
+ Field(
38
+ title="Service point ID",
39
+ description=(
40
+ "Service point to be used for "
41
+ "transactions created from this file (Loans-only)."
42
+ ),
43
+ )
44
+ ] = ""
45
+ statistical_code: Annotated[
46
+ str,
47
+ Field(
48
+ title="Statistical code",
49
+ description=(
50
+ "Statistical code (code) to be used inventory records created from "
51
+ "this file (Instances, Holdings, Items). Specify multiple codes using "
52
+ "multi_field_delimiter."
53
+ ),
54
+ )
55
+ ] = ""
36
56
  create_source_records: Annotated[
37
57
  bool,
38
58
  Field(
@@ -6,12 +6,13 @@ import sys
6
6
  import uuid
7
7
  from datetime import datetime, timezone
8
8
  from pathlib import Path
9
- from typing import Dict, List
9
+ from typing import Dict, List, Optional, Tuple, Union
10
10
 
11
11
  import i18n
12
12
  from folio_uuid.folio_namespaces import FOLIONamespaces
13
13
  from folio_uuid.folio_uuid import FolioUUID
14
14
  from folioclient import FolioClient
15
+ from pymarc import Record
15
16
 
16
17
  from folio_migration_tools.custom_exceptions import (
17
18
  TransformationFieldMappingError,
@@ -19,7 +20,8 @@ from folio_migration_tools.custom_exceptions import (
19
20
  TransformationRecordFailedError,
20
21
  )
21
22
  from folio_migration_tools.extradata_writer import ExtradataWriter
22
- from folio_migration_tools.library_configuration import LibraryConfiguration
23
+ from folio_migration_tools.helper import Helper
24
+ from folio_migration_tools.library_configuration import FileDefinition, LibraryConfiguration
23
25
  from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
24
26
  RefDataMapping,
25
27
  )
@@ -34,8 +36,9 @@ class MapperBase:
34
36
  def __init__(
35
37
  self,
36
38
  library_configuration: LibraryConfiguration,
39
+ task_configuration: AbstractTaskConfiguration,
37
40
  folio_client: FolioClient,
38
- parent_id_map: dict[str, tuple] = {},
41
+ parent_id_map: Dict[str, Tuple] = {},
39
42
  ):
40
43
  logging.info("MapperBase initiating")
41
44
  self.parent_id_map: dict[str, tuple] = parent_id_map
@@ -43,7 +46,7 @@ class MapperBase:
43
46
  self.start_datetime = datetime.now(timezone.utc)
44
47
  self.folio_client: FolioClient = folio_client
45
48
  self.library_configuration: LibraryConfiguration = library_configuration
46
- self.task_configuration: AbstractTaskConfiguration
49
+ self.task_configuration: AbstractTaskConfiguration = task_configuration
47
50
  self.mapped_folio_fields: dict = {}
48
51
  self.migration_report: MigrationReport = MigrationReport()
49
52
  self.num_criticalerrors = 0
@@ -440,6 +443,74 @@ class MapperBase:
440
443
  )
441
444
  )
442
445
 
446
+ def map_statistical_codes(self, folio_record: dict, file_def: FileDefinition, legacy_record: Optional[Union[dict, Record]] = None):
447
+ """Map statistical codes to the folio record.
448
+
449
+ This method checks if the file definition contains statistical codes and
450
+ if so, it splits the codes by the multi-field delimiter and adds them to
451
+ the folio record's 'statisticalCodeIds' field. If the field does not exist,
452
+ it initializes it as an empty list before appending the codes.
453
+
454
+ Args:
455
+ folio_record (dict): The FOLIO record to which the statistical codes will be added.
456
+ file_def (FileDefinition): The file definition containing the statistical codes.
457
+ legacy_record (Optional[Union[dict, Record]]): The legacy record from which the statistical codes are derived.
458
+ """
459
+ if file_def.statistical_code:
460
+ code_strings = file_def.statistical_code.split(
461
+ self.library_configuration.multi_field_delimiter
462
+ )
463
+ folio_record["statisticalCodeIds"] = folio_record.get("statisticalCodeIds", []) + code_strings
464
+
465
+ def setup_statistical_codes_map(self, statistical_codes_map):
466
+ if statistical_codes_map:
467
+ self.statistical_codes_mapping = RefDataMapping(
468
+ self.folio_client,
469
+ "/statistical-codes",
470
+ "statisticalCodes",
471
+ statistical_codes_map,
472
+ "code",
473
+ "StatisticalCodeMapping",
474
+ )
475
+ logging.info(f"Statistical codes mapping set up {self.statistical_codes_mapping.mapped_legacy_keys}")
476
+ else:
477
+ self.statistical_codes_mapping = None
478
+ logging.info("Statistical codes map is not set up")
479
+
480
+ def get_statistical_code(self, legacy_item: dict, folio_prop_name: str, index_or_id):
481
+ if self.statistical_codes_mapping:
482
+ return self.get_mapped_ref_data_value(
483
+ self.statistical_codes_mapping,
484
+ legacy_item,
485
+ index_or_id,
486
+ folio_prop_name,
487
+ True,
488
+ )
489
+ self.migration_report.add(
490
+ "StatisticalCodeMapping",
491
+ i18n.t("Mapping not set up"),
492
+ )
493
+ return ""
494
+
495
+ def map_statistical_code_ids(
496
+ self, legacy_ids, folio_record: dict
497
+ ):
498
+ if stat_codes := {x: None for x in folio_record.pop("statisticalCodeIds", [])}:
499
+ folio_code_ids = set()
500
+ for stat_code in stat_codes:
501
+ if stat_code_id := self.get_statistical_code({"legacy_stat_code": stat_code}, "statisticalCodeId", legacy_ids):
502
+ folio_code_ids.add(stat_code_id)
503
+ else:
504
+ Helper.log_data_issue(
505
+ legacy_ids,
506
+ i18n.t(
507
+ "Statistical code '%{code}' not found in FOLIO",
508
+ code=stat_code,
509
+ ),
510
+ stat_code,
511
+ )
512
+ folio_record["statisticalCodeIds"] = list(folio_code_ids)
513
+
443
514
  @property
444
515
  def base_string_for_folio_uuid(self):
445
516
  if self.library_configuration.use_gateway_url_for_uuids and not self.library_configuration.is_ecs:
@@ -462,6 +533,9 @@ class MapperBase:
462
533
  )
463
534
  return location_map
464
535
 
536
+ @staticmethod
537
+ def get_object_type() -> FOLIONamespaces:
538
+ raise NotImplementedError("This method should be overridden in subclasses")
465
539
 
466
540
  def flatten(my_dict: dict, path=""):
467
541
  for k, v in iter(my_dict.items()):
@@ -31,13 +31,13 @@ class CoursesMapper(MappingFileMapperBase):
31
31
  self.user_cache: dict = {}
32
32
  self.notes_mapper: NotesMapper = NotesMapper(
33
33
  library_configuration,
34
+ None,
34
35
  self.folio_client,
35
36
  course_map,
36
37
  FOLIONamespaces.note,
37
38
  True,
38
39
  )
39
40
  self.composite_course_schema = self.get_composite_course_schema()
40
- self.task_configuration = task_configuration
41
41
  super().__init__(
42
42
  folio_client,
43
43
  self.composite_course_schema,
@@ -45,6 +45,7 @@ class CoursesMapper(MappingFileMapperBase):
45
45
  None,
46
46
  FOLIONamespaces.course,
47
47
  library_configuration,
48
+ task_configuration
48
49
  )
49
50
  self.course_map = course_map
50
51
  if terms_map:
@@ -15,7 +15,7 @@ from folio_migration_tools.mapping_file_transformation.mapping_file_mapper_base
15
15
  from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
16
16
  RefDataMapping,
17
17
  )
18
-
18
+ from folio_migration_tools.task_configuration import AbstractTaskConfiguration
19
19
 
20
20
  class HoldingsMapper(MappingFileMapperBase):
21
21
  def __init__(
@@ -26,6 +26,7 @@ class HoldingsMapper(MappingFileMapperBase):
26
26
  call_number_type_map,
27
27
  instance_id_map,
28
28
  library_configuration: LibraryConfiguration,
29
+ task_config: AbstractTaskConfiguration,
29
30
  statistical_codes_map=None,
30
31
  ):
31
32
  holdings_schema = folio_client.get_holdings_schema()
@@ -37,6 +38,7 @@ class HoldingsMapper(MappingFileMapperBase):
37
38
  statistical_codes_map,
38
39
  FOLIONamespaces.holdings,
39
40
  library_configuration,
41
+ task_config
40
42
  )
41
43
  self.holdings_map = holdings_map
42
44
 
@@ -58,8 +60,10 @@ class HoldingsMapper(MappingFileMapperBase):
58
60
  "CallNumberTypeMapping",
59
61
  )
60
62
 
61
- def perform_additional_mappings(self, folio_rec, file_def):
63
+ def perform_additional_mappings(self, legacy_ids, folio_rec, file_def):
62
64
  self.handle_suppression(folio_rec, file_def)
65
+ self.map_statistical_codes(folio_rec, file_def)
66
+ self.map_statistical_code_ids(legacy_ids, folio_rec)
63
67
 
64
68
  def handle_suppression(self, folio_record, file_def: FileDefinition):
65
69
  folio_record["discoverySuppress"] = file_def.discovery_suppressed
@@ -73,8 +77,8 @@ class HoldingsMapper(MappingFileMapperBase):
73
77
  return self.get_location_id(legacy_item, index_or_id, folio_prop_name)
74
78
  elif folio_prop_name == "callNumberTypeId":
75
79
  return self.get_call_number_type_id(legacy_item, folio_prop_name, index_or_id)
76
- elif folio_prop_name.startswith("statisticalCodeIds"):
77
- return self.get_statistical_code(legacy_item, folio_prop_name, index_or_id)
80
+ # elif folio_prop_name.startswith("statisticalCodeIds"):
81
+ # return self.get_statistical_code(legacy_item, folio_prop_name, index_or_id)
78
82
 
79
83
  mapped_value = super().get_prop(
80
84
  legacy_item, folio_prop_name, index_or_id, schema_default_value
@@ -52,8 +52,8 @@ class ItemMapper(MappingFileMapperBase):
52
52
  statistical_codes_map,
53
53
  FOLIONamespaces.items,
54
54
  library_configuration,
55
+ task_configuration,
55
56
  )
56
- self.task_configuration = task_configuration
57
57
  self.item_schema = self.folio_client.get_item_schema()
58
58
  self.items_map = items_map
59
59
  self.holdings_id_map = holdings_id_map
@@ -117,8 +117,10 @@ class ItemMapper(MappingFileMapperBase):
117
117
  "LocationMapping",
118
118
  )
119
119
 
120
- def perform_additional_mappings(self, folio_rec, file_def):
120
+ def perform_additional_mappings(self, legacy_ids, folio_rec, file_def):
121
121
  self.handle_suppression(folio_rec, file_def)
122
+ self.map_statistical_codes(folio_rec, file_def)
123
+ self.map_statistical_code_ids(legacy_ids, folio_rec)
122
124
 
123
125
  def handle_suppression(self, folio_record, file_def: FileDefinition):
124
126
  folio_record["discoverySuppress"] = file_def.discovery_suppressed
@@ -219,15 +221,6 @@ class ItemMapper(MappingFileMapperBase):
219
221
  return self.get_mapped_ref_data_value(
220
222
  self.loan_type_mapping, legacy_item, folio_prop_name, index_or_id
221
223
  )
222
- elif folio_prop_name.startswith("statisticalCodeIds"):
223
- statistical_code_id = self.get_statistical_code(
224
- legacy_item, folio_prop_name, index_or_id
225
- )
226
- self.migration_report.add(
227
- "StatisticalCodeMapping",
228
- f"{folio_prop_name} -> {statistical_code_id}",
229
- )
230
- return statistical_code_id
231
224
 
232
225
  mapped_value = super().get_prop(
233
226
  legacy_item, folio_prop_name, index_or_id, schema_default_value
@@ -46,6 +46,7 @@ class ManualFeeFinesMapper(MappingFileMapperBase):
46
46
  None,
47
47
  FOLIONamespaces.fees_fines,
48
48
  library_configuration,
49
+ task_configuration,
49
50
  ignore_legacy_identifier,
50
51
  )
51
52
 
@@ -20,10 +20,8 @@ from folio_migration_tools.custom_exceptions import (
20
20
  )
21
21
  from folio_migration_tools.library_configuration import LibraryConfiguration
22
22
  from folio_migration_tools.mapper_base import MapperBase
23
- from folio_migration_tools.mapping_file_transformation.ref_data_mapping import (
24
- RefDataMapping,
25
- )
26
23
  from folio_migration_tools.migration_report import MigrationReport
24
+ from folio_migration_tools.task_configuration import AbstractTaskConfiguration
27
25
 
28
26
  empty_vals = ["Not mapped", None, ""]
29
27
 
@@ -37,9 +35,10 @@ class MappingFileMapperBase(MapperBase):
37
35
  statistical_codes_map,
38
36
  uuid_namespace: UUID,
39
37
  library_configuration: LibraryConfiguration,
38
+ task_config: AbstractTaskConfiguration,
40
39
  ignore_legacy_identifier=False,
41
40
  ):
42
- super().__init__(library_configuration, folio_client)
41
+ super().__init__(library_configuration, task_config, folio_client)
43
42
  self.uuid_namespace = uuid_namespace
44
43
  self.ignore_legacy_identifier = ignore_legacy_identifier
45
44
  self.schema = schema
@@ -101,21 +100,6 @@ class MappingFileMapperBase(MapperBase):
101
100
  )
102
101
  csv.register_dialect("tsv", delimiter="\t")
103
102
 
104
- def setup_statistical_codes_map(self, statistical_codes_map):
105
- if statistical_codes_map:
106
- self.statistical_codes_mapping = RefDataMapping(
107
- self.folio_client,
108
- "/statistical-codes",
109
- "statisticalCodes",
110
- statistical_codes_map,
111
- "code",
112
- "StatisticalCodeMapping",
113
- )
114
- logging.info("Statistical codes mapping set up")
115
- else:
116
- self.statistical_codes_mapping = None
117
- logging.info("Statistical codes map is not set up")
118
-
119
103
  def setup_field_map(self, ignore_legacy_identifier):
120
104
  field_map = {} # Map of folio_fields and source fields as an array
121
105
  for k in self.record_map["data"]:
@@ -14,6 +14,7 @@ class NotesMapper(MappingFileMapperBase):
14
14
  def __init__(
15
15
  self,
16
16
  library_configuration: LibraryConfiguration,
17
+ task_configuration,
17
18
  folio_client: FolioClient,
18
19
  record_map: dict,
19
20
  object_type: FOLIONamespaces,
@@ -28,6 +29,7 @@ class NotesMapper(MappingFileMapperBase):
28
29
  None,
29
30
  object_type,
30
31
  library_configuration,
32
+ task_configuration,
31
33
  ignore_legacy_identifier,
32
34
  )
33
35
 
@@ -30,6 +30,7 @@ class CompositeOrderMapper(MappingFileMapperBase):
30
30
  self,
31
31
  folio_client: FolioClient,
32
32
  library_configuration: LibraryConfiguration,
33
+ task_configuration,
33
34
  composite_order_map: dict,
34
35
  organizations_id_map: dict,
35
36
  instance_id_map: dict,
@@ -53,6 +54,7 @@ class CompositeOrderMapper(MappingFileMapperBase):
53
54
  None,
54
55
  FOLIONamespaces.orders,
55
56
  library_configuration,
57
+ task_configuration,
56
58
  )
57
59
  logging.info("Loading Instance ID map...")
58
60
  self.instance_id_map = instance_id_map
@@ -80,6 +82,7 @@ class CompositeOrderMapper(MappingFileMapperBase):
80
82
  self.folio_client: FolioClient = folio_client
81
83
  self.notes_mapper: NotesMapper = NotesMapper(
82
84
  library_configuration,
85
+ None,
83
86
  self.folio_client,
84
87
  composite_order_map,
85
88
  FOLIONamespaces.note,
@@ -366,7 +369,7 @@ class CompositeOrderMapper(MappingFileMapperBase):
366
369
 
367
370
  def perform_additional_mapping(self, index_or_id, composite_order):
368
371
  self.validate_po_number(index_or_id, composite_order.get("poNumber"))
369
-
372
+
370
373
  # Get organization UUID from FOLIO
371
374
  composite_order["vendor"] = self.get_folio_organization_uuid(
372
375
  index_or_id, composite_order.get("vendor")
@@ -23,6 +23,7 @@ class OrganizationMapper(MappingFileMapperBase):
23
23
  self,
24
24
  folio_client: FolioClient,
25
25
  library_configuration: LibraryConfiguration,
26
+ task_config,
26
27
  organization_map: dict,
27
28
  organization_types_map,
28
29
  address_categories_map,
@@ -43,6 +44,7 @@ class OrganizationMapper(MappingFileMapperBase):
43
44
  None,
44
45
  FOLIONamespaces.organizations,
45
46
  library_configuration,
47
+ task_config,
46
48
  )
47
49
  self.organization_schema = organization_schema
48
50
  # Set up reference data maps
@@ -56,6 +58,7 @@ class OrganizationMapper(MappingFileMapperBase):
56
58
  self.folio_client: FolioClient = folio_client
57
59
  self.notes_mapper: NotesMapper = NotesMapper(
58
60
  library_configuration,
61
+ None,
59
62
  self.folio_client,
60
63
  organization_map,
61
64
  FOLIONamespaces.note,
@@ -79,28 +82,28 @@ class OrganizationMapper(MappingFileMapperBase):
79
82
  False,
80
83
  )
81
84
 
82
- elif re.compile("addresses\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
85
+ elif re.compile(r"addresses\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
83
86
  return self.get_mapped_ref_data_value(
84
87
  self.address_categories_map,
85
88
  *value_tuple,
86
89
  False,
87
90
  )
88
91
 
89
- elif re.compile("emails\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
92
+ elif re.compile(r"emails\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
90
93
  return self.get_mapped_ref_data_value(
91
94
  self.email_categories_map,
92
95
  *value_tuple,
93
96
  False,
94
97
  )
95
98
 
96
- elif re.compile("phoneNumbers\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
99
+ elif re.compile(r"phoneNumbers\[(\d+)\]\.categories\[(\d+)\]").fullmatch(folio_prop_name):
97
100
  return self.get_mapped_ref_data_value(
98
101
  self.phone_categories_map,
99
102
  *value_tuple,
100
103
  False,
101
104
  )
102
105
 
103
- elif re.compile("interfaces\[(\d+)\]\.interfaceCredential.interfaceId").fullmatch(
106
+ elif re.compile(r"interfaces\[(\d+)\]\.interfaceCredential.interfaceId").fullmatch(
104
107
  folio_prop_name
105
108
  ):
106
109
  return "replace_with_interface_id"
@@ -47,10 +47,12 @@ class UserMapper(MappingFileMapperBase):
47
47
  None,
48
48
  FOLIONamespaces.users,
49
49
  library_config,
50
+ task_config
50
51
  )
51
- self.task_config = task_config
52
+ self.task_config = self.task_configuration
52
53
  self.notes_mapper: NotesMapper = NotesMapper(
53
54
  self.library_configuration,
55
+ None,
54
56
  self.folio_client,
55
57
  self.record_map,
56
58
  FOLIONamespaces.users,
@@ -165,23 +165,16 @@ class HoldingsStatementsParser:
165
165
  TransformationFieldMappingError: _description_
166
166
  """
167
167
  for f in marc_record.get_fields(field_textual):
168
- codes = [sf.code for sf in f.subfields]
169
- if "a" not in codes and "z" not in codes and "x" not in codes:
170
- raise TransformationFieldMappingError(
171
- legacy_ids,
172
- i18n.t(
173
- "%{field} subfields a, x, and z missing from field", field=field_textual
174
- ),
175
- f,
176
- )
177
- if not (
178
- len(f.get_subfields("a")) == 0
179
- or len(f.get_subfields("z")) == 0
180
- or len(f.get_subfields("x")) == 0
168
+ if all(
169
+ [
170
+ len("".join(f.get_subfields("a")).strip()) == 0,
171
+ len("".join(f.get_subfields("z")).strip()) == 0,
172
+ len("".join(f.get_subfields("x")).strip()) == 0,
173
+ ]
181
174
  ):
182
175
  raise TransformationFieldMappingError(
183
176
  legacy_ids,
184
- i18n.t("%{field} a,x and z are all empty", field=field_textual),
177
+ i18n.t("%{field} a, x and z are missing or empty", field=field_textual),
185
178
  f,
186
179
  )
187
180
  return_dict["statements"].append(
@@ -62,6 +62,7 @@ class AuthorityMapper(RulesMapperBase):
62
62
  folio_client,
63
63
  library_configuration,
64
64
  task_configuration,
65
+ None,
65
66
  self.get_authority_json_schema(folio_client, library_configuration),
66
67
  Conditions(folio_client, self, "auth", library_configuration.folio_release),
67
68
  )
@@ -6,14 +6,14 @@ import urllib.parse
6
6
  import uuid
7
7
  from abc import abstractmethod
8
8
  from textwrap import wrap
9
- from typing import List, Tuple
9
+ from typing import Dict, List, Tuple
10
10
 
11
11
  import i18n
12
12
  import pymarc
13
13
  from dateutil.parser import parse
14
14
  from folio_uuid.folio_uuid import FOLIONamespaces, FolioUUID
15
15
  from folioclient import FolioClient
16
- from pymarc import Field, Record, Subfield
16
+ from pymarc import Field, Optional, Record, Subfield
17
17
 
18
18
  from folio_migration_tools.custom_exceptions import (
19
19
  TransformationFieldMappingError,
@@ -35,18 +35,18 @@ class RulesMapperBase(MapperBase):
35
35
  folio_client: FolioClient,
36
36
  library_configuration: LibraryConfiguration,
37
37
  task_configuration,
38
+ statistical_codes_map: Optional[Dict],
38
39
  schema: dict,
39
40
  conditions=None,
40
41
  parent_id_map: dict[str, tuple] = None,
41
42
  ):
42
- super().__init__(library_configuration, folio_client, parent_id_map)
43
+ super().__init__(library_configuration, task_configuration, folio_client, parent_id_map)
43
44
  self.parsed_records = 0
44
45
  self.id_map: dict[str, tuple] = {}
45
46
  self.start = time.time()
46
47
  self.last_batch_time = time.time()
47
48
  self.folio_client: FolioClient = folio_client
48
49
  self.schema: dict = schema
49
- self.task_configuration = task_configuration
50
50
  self.conditions = conditions
51
51
  self.item_json_schema = ""
52
52
  self.mappings: dict = {}
@@ -61,6 +61,8 @@ class RulesMapperBase(MapperBase):
61
61
  self.migration_report,
62
62
  self.task_configuration.deactivate035_from001,
63
63
  )
64
+
65
+ self.setup_statistical_codes_map(statistical_codes_map)
64
66
  logging.info("Current user id is %s", self.folio_client.current_user)
65
67
 
66
68
  def print_progress(self):
@@ -813,6 +815,83 @@ class RulesMapperBase(MapperBase):
813
815
  )
814
816
  data_import_marc_file.write(marc_record.as_marc())
815
817
 
818
+
819
+ def map_statistical_codes(
820
+ self,
821
+ folio_record: dict,
822
+ file_def: FileDefinition,
823
+ marc_record: Record,
824
+ ):
825
+ """Map statistical codes to FOLIO instance
826
+
827
+ This method first calls the base class method to map statistical codes
828
+ from the file_def. Then, it checks to see if there are any MARC field
829
+ mappings defined in the task configuration. If so, it creates a list
830
+ of lists where the first element is the MARC field tag, and the remaining
831
+ elements are the subfields to be used for mapping. It then iterates
832
+ through the MARC fields, retrieves the values based on the subfields.
833
+ Finally, it adds the mapped codes to the folio_record's statisticalCodeIds.
834
+
835
+ Args:
836
+ legacy_ids (List[str]): The legacy IDs of the folio record
837
+ folio_record (dict): The Dictionary representation of the FOLIO record
838
+ marc_record (Record): The pymarc Record object
839
+ file_def (FileDefinition): The file definition object from which marc_record was read
840
+ """
841
+ super().map_statistical_codes(folio_record, file_def)
842
+ if self.task_configuration.statistical_code_mapping_fields:
843
+ stat_code_marc_fields = []
844
+ for mapping in self.task_configuration.statistical_code_mapping_fields:
845
+ stat_code_marc_fields.append(mapping.split("$"))
846
+ for field_map in stat_code_marc_fields:
847
+ mapped_codes = self.map_stat_codes_from_marc_field(field_map, marc_record, self.library_configuration.multi_field_delimiter)
848
+ folio_record['statisticalCodeIds'] = folio_record.get("statisticalCodeIds", []) + mapped_codes
849
+
850
+ @staticmethod
851
+ def map_stat_codes_from_marc_field(field_map: List[str], marc_record: Record, multi_field_delimiter: str="<delimiter>") -> List[str]:
852
+ """Map statistical codes from MARC field to FOLIO instance.
853
+
854
+ This function extracts statistical codes from a MARC field based on the provided field map.
855
+ It supports multiple subfields and uses a delimiter to handle concatenated values.
856
+
857
+ Args:
858
+ field_map (List[str]): A list where the first element is the MARC field tag, and the remaining elements are subfields to extract values from.
859
+ marc_record (Record): The MARC record to process.
860
+ multi_field_delimiter (str): A delimiter used to concatenate multiple subfield values that should be individual mapped values.
861
+
862
+ Returns:
863
+ str: A string of statistical codes extracted from the MARC field, formatted as "<field>_<subfield>:<value>".
864
+ """
865
+ field_values = []
866
+ if len(field_map) == 2:
867
+ subfields = []
868
+ for mf in marc_record.get_fields(field_map[0]):
869
+ subfields.extend(
870
+ multi_field_delimiter.join(
871
+ mf.get_subfields(field_map[1])
872
+ ).split(multi_field_delimiter)
873
+ )
874
+ field_values.extend(
875
+ [
876
+ f"{field_map[0]}_{field_map[1]}:{x}" for
877
+ x in subfields
878
+ ]
879
+ )
880
+ elif len(field_map) > 2:
881
+ for mf in marc_record.get_fields(field_map[0]):
882
+ for sf in field_map[1:]:
883
+ field_values.extend(
884
+ [
885
+ f"{field_map[0]}_{sf}:{x}" for x in multi_field_delimiter.join(
886
+ mf.get_subfields(sf)
887
+ ).split(multi_field_delimiter)
888
+ ]
889
+ )
890
+ elif field_map:
891
+ for mf in marc_record.get_fields(field_map[0]):
892
+ field_values.append(f"{field_map[0]}:{mf.value()}")
893
+ return field_values
894
+
816
895
  def save_source_record(
817
896
  self,
818
897
  srs_records_file,