folio-data-import 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

folio_data_import/__main__.py

@@ -1,115 +1,12 @@
-import argparse
-import asyncio
-import glob
-from getpass import getpass
-from pathlib import Path
+import typer
 
-import folioclient
-import inquirer
+from folio_data_import.MARCDataImport import main as marc_app
+from folio_data_import.UserImport import main as users_main
 
-from folio_data_import.MARCDataImport import MARCImportJob
-
-
-async def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--record-type", type=str, help="The record type to import", default="MARC21"
-    )
-    parser.add_argument("--gateway_url", type=str, help="The FOLIO API Gateway URL")
-    parser.add_argument("--tenant_id", type=str, help="The FOLIO tenant ID")
-    parser.add_argument("--username", type=str, help="The FOLIO username")
-    parser.add_argument("--password", type=str, help="The FOLIO password", default="")
-    parser.add_argument(
-        "--marc_file_path",
-        type=str,
-        help="The MARC file (or file glob, using shell globbing syntax) to import",
-    )
-    parser.add_argument(
-        "--import_profile_name",
-        type=str,
-        help="The name of the data import job profile to use",
-        default="",
-    )
-    parser.add_argument(
-        "--batch_size",
-        type=int,
-        help="The number of source records to include in a record batch sent to FOLIO.",
-        default=10,
-    )
-    parser.add_argument(
-        "--batch_delay",
-        type=float,
-        help="The number of seconds to wait between record batches.",
-        default=0.0,
-    )
-    parser.add_argument(
-        "--consolidate",
-        action="store_true",
-        help=(
-            "Consolidate records into a single job. "
-            "Default is to create a new job for each MARC file."
-        ),
-    )
-    parser.add_argument(
-        "--no-progress",
-        action="store_true",
-        help="Disable progress bars (eg. for running in a CI environment)",
-    )
-    args = parser.parse_args()
-    if not args.password:
-        args.password = getpass("Enter FOLIO password: ")
-    folio_client = folioclient.FolioClient(
-        args.gateway_url, args.tenant_id, args.username, args.password
-    )
-    if not args.import_profile_name:
-        import_profiles = folio_client.folio_get(
-            "/data-import-profiles/jobProfiles",
-            "jobProfiles",
-            query_params={"limit": "1000"},
-        )
-        import_profile_names = [
-            profile["name"]
-            for profile in import_profiles
-            if args.record_type.lower() in profile["dataType"].lower()
-        ]
-        questions = [
-            inquirer.List(
-                "import_profile_name",
-                message="Select an import profile",
-                choices=import_profile_names,
-            )
-        ]
-        answers = inquirer.prompt(questions)
-        args.import_profile_name = answers["import_profile_name"]
-
-    if args.record_type.lower() == "marc21":
-        marc_files = [Path(x) for x in glob.glob(args.marc_file_path, root_dir="./")]
-        print(marc_files)
-        try:
-            await MARCImportJob(
-                folio_client,
-                marc_files,
-                args.import_profile_name,
-                batch_size=args.batch_size,
-                batch_delay=args.batch_delay,
-                consolidate=bool(args.consolidate),
-                no_progress=bool(args.no_progress),
-            ).do_work()
-        except Exception as e:
-            print("Error importing files: " + str(e))
-            raise
-    elif args.record_type.lower() == "users":
-        print(
-            "User import not yet implemented. Run UserImport.py directly "
-            "or use folio-user-import CLI."
-        )
-    else:
-        print("Record type not supported. Supported types are: MARC21")
-
-
-def sync_main():
-    asyncio.run(main())
+app = typer.Typer()
 
+app.command(name="marc")(marc_app)
+app.command(name="users")(users_main)
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    app()

folio_data_import/_progress.py

@@ -0,0 +1,27 @@
+from rich.progress import ProgressColumn, Task
+from rich.text import Text
+
+class ItemsPerSecondColumn(ProgressColumn):
+    """Renders the speed in items per second."""
+
+    def render(self, task: Task) -> Text:
+        if task.speed is None:
+            return Text("?", style="progress.data.speed")
+        return Text(f"{task.speed:.0f}rec/s", style="progress.data.speed")
+
+class UserStatsColumn(ProgressColumn):
+    def render(self, task: Task) -> Text:
+        created = task.fields.get("created", 0)
+        updated = task.fields.get("updated", 0)
+        failed = task.fields.get("failed", 0)
+        created_string = f"Created: {created}"
+        updated_string = f"Updated: {updated}"
+        failed_string = f"Failed: {failed}"
+        text = Text("(")
+        text.append(created_string, style="green")
+        text.append(" | ")
+        text.append(updated_string, style="cyan")
+        text.append(" | ")
+        text.append(failed_string, style="red")
+        text.append(")")
+        return text

folio_data_import/marc_preprocessors/_preprocessors.py

@@ -3,17 +3,19 @@ import sys
 from typing import Callable, Dict, List, Tuple, Union
 import pymarc
 import logging
+import re
 
 from pymarc.record import Record
 
 logger = logging.getLogger("folio_data_import.MARCDataImport")
 
+
 class MARCPreprocessor:
     """
     A class to preprocess MARC records for data import into FOLIO.
     """
 
-    def __init__(self, preprocessors: Union[str,List[Callable]], **kwargs):
+    def __init__(self, preprocessors: Union[str, List[Callable]], **kwargs):
         """
         Initialize the MARCPreprocessor with a list of preprocessors.
 
@@ -361,7 +363,7 @@ def clean_empty_fields(record: Record, **kwargs) -> Record:
 
     for field in record.get_fields(*MAPPED_FIELDS.keys()):
         len_subs = len(field.subfields)
-        subfield_value = bool(field.subfields[0].value) if len_subs else False
+        subfield_value = bool(re.sub(r"[.,-]", "", field.subfields[0].value).strip()) if len_subs else False
         if int(field.tag) > 9 and len_subs == 0:
             logger.log(
                 26,
@@ -415,7 +417,7 @@ def clean_empty_fields(record: Record, **kwargs) -> Record:
     return record
 
 
-def fix_leader(record: Record, **kwargs) -> Record:
+def fix_bib_leader(record: Record, **kwargs) -> Record:
     """
     Fixes the leader of the record by setting the record status to 'c' (modified
     record) and the type of record to 'a' (language material).
@@ -448,6 +450,7 @@ def fix_leader(record: Record, **kwargs) -> Record:
         record.leader = pymarc.Leader(record.leader[:6] + "a" + record.leader[7:])
     return record
 
+
 def move_authority_subfield_9_to_0_all_controllable_fields(record: Record, **kwargs) -> Record:
     """
     Move subfield 9 from authority fields to subfield 0. This is useful when
@@ -460,15 +463,16 @@ def move_authority_subfield_9_to_0_all_controllable_fields(record: Record, **kwa
         Record: The preprocessed MARC record.
     """
     controlled_fields = [
-            "100", "110", "111", "130",
-            "600", "610", "611", "630", "650", "651", "655",
-            "700", "710", "711", "730",
-            "800", "810", "811", "830"
-        ]
+        "100", "110", "111", "130",
+        "600", "610", "611", "630", "650", "651", "655",
+        "700", "710", "711", "730",
+        "800", "810", "811", "830", "880"
+    ]
     for field in record.get_fields(*controlled_fields):
-        for subfield in list(field.get_subfields("9")):
+        _subfields = field.get_subfields("9")
+        for subfield in _subfields:
             field.add_subfield("0", subfield)
-            field.delete_subfield("9", subfield)
+            field.delete_subfield("9")
             logger.log(
                 26,
                 "DATA ISSUE\t%s\t%s\t%s",
@@ -478,6 +482,7 @@ def move_authority_subfield_9_to_0_all_controllable_fields(record: Record, **kwa
             )
     return record
 
+
 def ordinal(n):
     s = ("th", "st", "nd", "rd") + ("th",) * 10
     v = n % 100

{folio_data_import-0.3.1.dist-info → folio_data_import-0.4.0.dist-info}/METADATA

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: folio_data_import
-Version: 0.3.1
+Version: 0.4.0
 Summary: A python module to interact with the data importing capabilities of the open-source FOLIO ILS
 License: MIT
+License-File: LICENSE
 Author: Brooks Travis
 Author-email: brooks.travis@gmail.com
 Requires-Python: >=3.9,<4.0
@@ -13,6 +14,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
 Requires-Dist: flake8-bandit (>=4.1.1,<5.0.0)
 Requires-Dist: flake8-black (>=0.3.6,<0.4.0)
@@ -24,7 +26,7 @@ Requires-Dist: inquirer (>=3.4.0,<4.0.0)
 Requires-Dist: pyhumps (>=3.8.0,<4.0.0)
 Requires-Dist: pymarc (>=5.2.2,<6.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: tqdm (>=4.66.5,<5.0.0)
+Requires-Dist: typer (>=0.17.4,<0.18.0)
 Description-Content-Type: text/markdown
 
 # folio_data_import
@@ -42,7 +44,16 @@ This project is designed to import data into the FOLIO LSP. It provides a simple
 
 ## Installation
 
-To install the project using Poetry, follow these steps:
+Using `pip`
+```shell
+pip install folio_data_import
+```
+or `uv pip`
+```shell
+uv pip install folio_data_import
+```
+
+To install the project from the git repo using Poetry, follow these steps:
 
 1. Clone the repository.
 2. Navigate to the project directory: `$ cd /path/to/folio_data_import`.
@@ -58,8 +69,27 @@ Make sure to activate the virtual environment created by Poetry before running t
 2. Run the application and follow the prompts to import the data.
 3. Monitor the import progress and handle any errors or conflicts that may arise.
 
+### folio-data-import
+This command provides access to subcommands for importing user and marc data. To import users:
+```shell
+folio-data-import users --help
+```
+(for more information, see [folio-user-import](#folio-user-import), below)
+
+For MARC data:
+```shell
+folio-data-import marc --help
+```
+(for more information, see [folio-marc-import](#folio-marc-import), below)
+
+As an added convenience, this script can also install tab-completions for itself in your shell:
+```shell
+folio-data-import --install-completion
+```
+Once installed, you can `[tab][tab]` after typing `--` and be presented with a list of availabl command options.
+
 ### folio-user-import
-When this package is installed via PyPI or using `poetry install` from this repository, it installs a convenience script in your `$PATH` called `folio-user-import`. To view all command line options for this script, run `folio-user-import -h`. In addition to supporting `mod-user-import`-style JSON objects, this script also allows you to manage service point assignments for users by specifying a `servicePointsUser` object in the JSON object, using service point codes in place of UUIDs in the `defaultServicePointId` and `servicePointIds` fields:
+When this package is installed via PyPI or using `poetry install` from this repository, it installs a convenience script in your `$PATH` called `folio-user-import`. To view all command line options for this script, run `folio-user-import --help`. In addition to supporting `mod-user-import`-style JSON objects, this script also allows you to manage service point assignments for users by specifying a `servicePointsUser` object in the JSON object, using service point codes in place of UUIDs in the `defaultServicePointId` and `servicePointIds` fields:
 ```
 {
     "username": "checkin-all",
@@ -109,7 +139,7 @@ Unlike mod-user-import, this importer does not require `externalSystemId` as the
 
 Another point of departure from the behavior of `mod-user-import` is the handling of `preferredContactTypeId`. This importer will accept either the `"001", "002", "003"...` values stored by FOLIO, or the human-friendly strings used by `mod-user-import` (`"mail", "email", "text", "phone", "mobile"`). It will also __*set a customizable default for all users that do not otherwise have a valid value specified*__ (using `--default_preferred_contact_type`), unless a (valid) value is already present in the user record being updated.
 
-#### Field Protection (*experimental*)
+#### Per-record Field Protection (*experimental*)
 
 This script offers a rudimentary field protection implementation using custom fields. To enable this functionality, create a text custom field that has the field name `protectedFields`. In this field, you can specify a comma-separated list of User schema field names, using dot-notation for nested fields. This protection should support all standard fields except addresses within `personal.addresses`. If you include `personal.addresses` in a user record, any existing addresses will be replaced by the new values.
 
@@ -123,11 +153,32 @@ This script offers a rudimentary field protection implementation using custom fi
 
 Would result in `preferredFirstName`, `barcode`, and `telephone` remaining unchanged, regardless of the contents of the incoming records.
 
+#### Job-level field protection
+
+To protect fields for all records in a particular import job, you can pass a list of field paths with the `--fields-to-protect` flag. These protections will be applied in combination with any record-level protections specified.
+
+##### Example
+```Shell
+folio-user-import ... --fields-to-protect "personal.preferredFirstName,customFields.exampleCustomField"
+```
 
 #### How to use:
 1. Generate a JSON lines (one JSON object per line) file of FOLIO user objects in the style of [mod-user-import](https://github.com/folio-org/mod-user-import)
 2. Run the script and specify the required arguments (and any desired optional arguments), including the path to your file of user objects
+3. Watch the pretty progress bars...
+
+### folio-marc-import
+`folio-marc-import` provides direct access to the MARC import functionality of FOLIO. It can be used to import any file (or files) of binary MARC records via FOLIO's Data Import system using the [change-manager](https://github.com/folio-org/mod-source-record-manager?tab=readme-ov-file#data-import-workflow) APIs directly.
 
+#### How to use:
+1. Have a binary MARC21 file (or directory of files)
+2. Have a [Data Import Job Profile](https://docs.folio.org/docs/metadata/additional-topics/jobprofiles/) that you want to use to import your records already set up in FOLIO
+3. Run the script and specify the required arguments. (`folio-marc-import --help` for more details)
+4. Select the job profile you want to use
+5. Watch the pretty progress bars...
+
+#### A note on logging
+The import logs and job summaries provided by FOLIO can be unreliable in certain circumstances. The scripts have been written to balance the need to retrieve job summary information at the end of each job with the the need to move on to the next import job. If you don't see a job summary when your job completes, check Data Import in FOLIO (Data Import > Actions > View all logs...)
 
 ## Contributing
 

folio_data_import-0.4.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+folio_data_import/MARCDataImport.py,sha256=ubTd7ROtqLavSmPNma2pI8ixjFZs7FEp5_oIuD0ONqo,42945
+folio_data_import/UserImport.py,sha256=ic2msCmHsU3cFWPd1edBbC7ltTCYxV8OFCvGd0cBcDQ,43048
+folio_data_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+folio_data_import/__main__.py,sha256=asGlYSuLpvNW5cwxcAH03iXtMcP_DpCdwhpYZy01org,269
+folio_data_import/_progress.py,sha256=QsZNHLjjNW3Q_NHIAuLgsigGXnV-COAvxIM6vO2ggnM,1010
+folio_data_import/custom_exceptions.py,sha256=1xw1BI5fW7WDd37zUIOcw0DAvrFKtklnqmbRhZXSAiE,1093
+folio_data_import/marc_preprocessors/__init__.py,sha256=urExfNTQoZsDCtDPcUY9EEC5OFcUihxhYEQkQFVzbMY,30
+folio_data_import/marc_preprocessors/_preprocessors.py,sha256=PuXWWzDLr0AJVS0rFcK5MiP_apSm0ewwTNRd_n8hMgc,16584
+folio_data_import-0.4.0.dist-info/METADATA,sha256=Ecy3voKtb7f647G1MHwvgWOH5LBJYaAHjB7yXGoFlMI,8530
+folio_data_import-0.4.0.dist-info/WHEEL,sha256=M5asmiAlL6HEcOq52Yi5mmk9KmTVjY2RDPtO4p9DMrc,88
+folio_data_import-0.4.0.dist-info/entry_points.txt,sha256=jRCg5w4I8s4qYRexIJJeVLSNZ142WVTn25tM84MetxM,174
+folio_data_import-0.4.0.dist-info/licenses/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
+folio_data_import-0.4.0.dist-info/RECORD,,

{folio_data_import-0.3.1.dist-info → folio_data_import-0.4.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.0
 Root-Is-Purelib: true
 Tag: py3-none-any

folio_data_import-0.4.0.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+folio-data-import=folio_data_import.__main__:app
+folio-marc-import=folio_data_import.MARCDataImport:app
+folio-user-import=folio_data_import.UserImport:app
+

folio_data_import-0.3.1.dist-info/RECORD

@@ -1,12 +0,0 @@
-folio_data_import/MARCDataImport.py,sha256=Qfpc3NtlQe4VUobXsFhmSnScBhiNRGEzBN7FZrdsBYc,37633
-folio_data_import/UserImport.py,sha256=4Bn_Z1xX5DvnodscW9NgGeBZgvFeShVoIbxDGrt6BMo,41748
-folio_data_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-folio_data_import/__main__.py,sha256=kav_uUsnrIjGjVxQkk3exLKrc1mah9t2x3G6bGS-5I0,3710
-folio_data_import/custom_exceptions.py,sha256=1xw1BI5fW7WDd37zUIOcw0DAvrFKtklnqmbRhZXSAiE,1093
-folio_data_import/marc_preprocessors/__init__.py,sha256=urExfNTQoZsDCtDPcUY9EEC5OFcUihxhYEQkQFVzbMY,30
-folio_data_import/marc_preprocessors/_preprocessors.py,sha256=5JnxCoLDsZKEwrAazTXrI1b0CNthvhcTEKgQZILIjVk,16533
-folio_data_import-0.3.1.dist-info/LICENSE,sha256=qJX7wxMC7ky9Kq4v3zij8MjGEiC5wsB7pYeOhLj5TDk,1083
-folio_data_import-0.3.1.dist-info/METADATA,sha256=VsbdMezeVPB5jyYAB6o-9k6fhfgnEa5i_T2GphBOtzg,6069
-folio_data_import-0.3.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-folio_data_import-0.3.1.dist-info/entry_points.txt,sha256=498SxWVXeEMRNw3PUf-eoReZvKewmYwPBtZhIUPr_Jg,192
-folio_data_import-0.3.1.dist-info/RECORD,,

folio_data_import-0.3.1.dist-info/entry_points.txt

@@ -1,5 +0,0 @@
-[console_scripts]
-folio-data-import=folio_data_import.__main__:sync_main
-folio-marc-import=folio_data_import.MARCDataImport:sync_main
-folio-user-import=folio_data_import.UserImport:sync_main
-