aspace-wb-scripts 0.1.0__py3-none-any.whl

aspace_wb/__init__.py

@@ -0,0 +1,44 @@
+"""
+ASpace Workbench Scripts Package
+
+This package provides tools for American Philosophical Society (APS) staff to prepare
+CSV files for ingesting files into Islandora 8 using Workbench. It automates metadata
+extraction from media files and ArchivesSpace Bulk Update Spreadsheets.
+"""
+
+__version__ = "0.1.0"
+__author__ = "American Philosophical Society"
+__email__ = "cds@amphilsoc.org"
+
+from .core.specs import *
+
+__all__ = [
+    "FILESTOUPLOAD_DIR",
+    "METADATA_DIR",
+    "CV_DIR",
+    "FIELDS_DIR",
+    "AS_AGENTS_FILENAME",
+    "ISO639_FILENAME",
+    "CNAIR_SUBJECTS_FILENAME",
+    "RELATOR_CODES_FILENAME",
+    "LANGUAGE_NAMES",
+    "LANGUAGE_CODES",
+    "CNAIR_SUBJECTS",
+    "RELATOR_CODES",
+    "VALIDATE_ERROR_PREFIX",
+    "AS_DIGITAL_OBJECT_NODE_PREFIX",
+    "AS_DIGITAL_OBJECT_FILE_URI_PREFIX",
+    "BOOK_TITLE_URL_ALIAS_LENGTH",
+    "field_digital_origin",
+    "field_reformatting_quality",
+    "field_model_BOOK",
+    "field_resource_type_BOOK",
+    "file_SINGLE_PREFIX",
+    "url_alias_PREFIX",
+    "WB_FIELDS_REQUIRED_AT_INPUT_SINGLE",
+    "WB_FIELDS_REQUIRED_AT_INPUT_BOOK",
+    "WB_FIELDS_REQUIRED_AT_WORKBENCH_SINGLE",
+    "WB_FIELDS_REQUIRED_AT_WORKBENCH_BOOK",
+    "WB_FIELDS_ORDERED_WITH_DESCRIPTION",
+    "extension_to_WB_field",
+]

aspace_wb/cli/__init__.py

@@ -0,0 +1,3 @@
+"""
+Command-line interface modules for ASpace Workbench Scripts.
+"""

aspace_wb/cli/create_aspace_dos.py

@@ -0,0 +1,14 @@
+"""
+CLI entry point for create_aspace_dos command
+"""
+
+
+def main():
+    """Entry point that calls the original Create_ASpace_DOs.py script"""
+    # Import and run the original script
+    from aspace_wb.scripts import create_aspace_dos  # NOQA
+    # The original script runs on import, so we don't need to call anything
+
+
+if __name__ == "__main__":
+    main()

aspace_wb/cli/create_blank_fillable.py

@@ -0,0 +1,14 @@
+"""
+CLI entry point for create_blank_fillable command
+"""
+
+
+def main():
+    """Entry point that calls the original Create_Blank_Fillable.py script"""
+    # Import and run the original script
+    from aspace_wb.scripts import create_blank_fillable  # NOQA
+    # The original script runs on import, so we don't need to call anything
+
+
+if __name__ == "__main__":
+    main()

aspace_wb/cli/create_fillable.py

@@ -0,0 +1,14 @@
+"""
+CLI entry point for create_fillable command
+"""
+
+
+def main():
+    """Entry point that calls the original Create_Fillable.py script"""
+    # Import and run the original script
+    from aspace_wb.scripts import create_fillable  # NOQA
+    # The original script runs on import, so we don't need to call anything
+
+
+if __name__ == "__main__":
+    main()

aspace_wb/cli/filled_to_wb.py

@@ -0,0 +1,14 @@
+"""
+CLI entry point for filled_to_wb command
+"""
+
+
+def main():
+    """Entry point that calls the original Filled_to_WB.py script"""
+    # Import and run the original script
+    from aspace_wb.scripts import filled_to_wb  # NOQA
+    # The original script runs on import, so we don't need to call anything
+
+
+if __name__ == "__main__":
+    main()

aspace_wb/cli/validate_filled.py

@@ -0,0 +1,14 @@
+"""
+CLI entry point for validate_filled command
+"""
+
+
+def main():
+    """Entry point that calls the original Validate_Filled.py script"""
+    # Import and run the original script
+    from aspace_wb.scripts import validate_filled  # NOQA
+    # The original script runs on import, so we don't need to call anything
+
+
+if __name__ == "__main__":
+    main()

aspace_wb/core/__init__.py

@@ -0,0 +1,3 @@
+"""
+Core functionality and constants for ASpace Workbench Scripts.
+"""

aspace_wb/core/specs.py

@@ -0,0 +1,348 @@
+"""
+File defining constants, Workbench fields, and mappings between different fields
+"""
+
+import os
+from importlib import resources
+from aspace_wb.utils import use_CSVs
+
+"""
+Locations of files & directories; controlled vocabularies; misc
+"""
+
+# directories and files
+FILESTOUPLOAD_DIR = "files_to_upload"
+METADATA_DIR = "metadata"
+CV_DIR = "vocabularies"  # controlled vocabularies
+FIELDS_DIR = "fields"
+AS_AGENTS_FILENAME = "agents_in_AS.csv"
+ISO639_FILENAME = "iso639.csv"
+CNAIR_SUBJECTS_FILENAME = "cnair_subject.csv"
+RELATOR_CODES_FILENAME = "relator.csv"
+
+
+# Package data paths
+def get_package_data_path(data_type: str, filename: str) -> str:
+    """Get path to package data file"""
+    import aspace_wb
+
+    return str(
+        resources.files(aspace_wb.__name__).joinpath("data", data_type, filename)
+    )
+
+
+# controlled vocabularies, from the package data
+LANGUAGE_NAMES = use_CSVs.CSV_col_to_list(
+    get_package_data_path("vocabularies", ISO639_FILENAME), 0
+)
+LANGUAGE_CODES = use_CSVs.CSV_col_to_list(
+    get_package_data_path("vocabularies", ISO639_FILENAME), 1
+)
+CNAIR_SUBJECTS = use_CSVs.CSV_col_to_list(
+    get_package_data_path("vocabularies", CNAIR_SUBJECTS_FILENAME), 0
+)
+RELATOR_CODES = use_CSVs.CSV_col_to_list(
+    get_package_data_path("vocabularies", RELATOR_CODES_FILENAME), 0
+)
+
+# other
+VALIDATE_ERROR_PREFIX = "!! ERROR - "
+AS_DIGITAL_OBJECT_NODE_PREFIX = "islandora8_"
+AS_DIGITAL_OBJECT_FILE_URI_PREFIX = "https://diglib.amphilsoc.org/node/"
+BOOK_TITLE_URL_ALIAS_LENGTH = 92
+
+"""
+Workbench fields
+
+"""
+
+# field defaults - ones with _BOOK are for book only
+field_digital_origin = "Reformatted digital"
+field_reformatting_quality = "Preservation"
+field_model_BOOK = "Paged Content"
+field_resource_type_BOOK = "Collection"
+
+# field text prefixes (referencing locations within the upload server)
+file_SINGLE_PREFIX = "/mnt/ingest/data/"
+url_alias_PREFIX = "/islandora/object/"
+
+# required fields that must exist in a "fields" csv BEFORE running create_fillable.py
+WB_FIELDS_REQUIRED_AT_INPUT_SINGLE = (
+    "file",
+    "field_resource_type",
+    "field_model",
+    "field_member_of",
+    "title",
+    "field_digital_origin",
+    "field_parent_collection_call_num",
+    "field_collection2",
+    "field_collection_url",
+    "field_reformatting_quality",
+)
+
+WB_FIELDS_REQUIRED_AT_INPUT_BOOK = (
+    "file",
+    "field_resource_type",
+    "field_model",
+    "field_member_of",
+    "title",
+    "field_digital_origin",
+    "field_parent_collection_call_num",
+    "field_collection2",
+    "field_collection_url",
+    "field_reformatting_quality",
+    "total_scans",
+)
+
+# fields that will fill downwards through gaps
+WB_FIELDS_DOWNFILLING = (
+    "field_member_of",
+    "field_collection2",
+    "field_parent_collection_call_num",
+    "field_collection_url",
+)
+
+# fields required after filled_to_wb.py to exist, even if/necessarily empty
+WB_FIELDS_EXPORT_KEEP_EMPTY_SINGLE = ("field_member_of", "parent_id")
+
+WB_FIELDS_EXPORT_KEEP_EMPTY_BOOK = (
+    "field_member_of",
+    "parent_id",
+    "field_weight",
+    "field_display_hints",
+)
+
+# fields that field_linked_agent gets split into and reformed from
+# this is only called once out of several places it could be called (in which individual field names are typed out)
+WB_field_linked_agent_SPLIT_FIELDS = (
+    "field_linked_agent_NAME",
+    "field_linked_agent_ROLE",
+    "field_linked_agent_TYPE",
+)
+
+# field names and their descriptions, as tuples
+# 1. these display to the user in the Fillable sheet (not every field needs one or will display in that sheet)
+# 2. the order here is the order that the Workbench csv will output
+
+WB_FIELDS_ORDERED_WITH_DESCRIPTION = (
+    ("file", "Name of file/folder. Autogenerated"),
+    ("total_scans", "Number of scans in a book. Autogenerated"),
+    ("field_resource_type", "Autogenerated"),
+    ("field_model", "Autogenerated"),
+    (
+        "field_member_of",
+        "Existing collection node to place object under. LEAVE BLANK if creating new nodes. Fills down.",
+    ),
+    ("title", "Title of object"),
+    ("field_metadata_title", ""),
+    ("id", ""),
+    ("parent_id", ""),
+    ("field_weight", ""),
+    ("field_display_hints", ""),
+    ("field_alternative_title", ""),
+    ("field_subtitle", ""),
+    ("field_linked_agent", ""),
+    (
+        "field_linked_agent_NAME",
+        "Name part of linked agent. Consult Library of Congress. Pipe-separate.",
+    ),
+    (
+        "field_linked_agent_ROLE",
+        "Role (relator code) part of linked agent. Pipe-separate, one for each of NAME",
+    ),
+    (
+        "field_linked_agent_TYPE",
+        "Type part of linked agent. Leave blank if all are type 'person'. Otherwise pipe-separate abbreviations (p|c|f) or full (person|corporate_body|family), one for each NAME",
+    ),
+    ("field_speaker_affiliation", ""),
+    ("field_family", ""),
+    ("field_description_long", "Scope and Contents-style description"),
+    ("field_conference", ""),
+    ("field_table_of_contents", ""),
+    ("field_note", "Additional note, e.g. caveats about scanning or metadata quality"),
+    ("field_biographical_note", ""),
+    ("field_genre", ""),
+    ("field_physical_form", ""),
+    ("field_edtf_date_created", "EDTF form of date (either this or text)"),
+    ("field_date_created_text", "Text form of date (either this or EDTF)"),
+    ("field_subject", ""),
+    ("field_cnair_subject", "Consult csv here, Airtable, or Indigenous Subject Guide"),
+    ("field_local_subject", ""),
+    ("field_subject_olac", "Legacy CNAIR only. Not for new materials."),
+    ("field_subjects_name", ""),
+    ("field_geographic_subject", "Consult Library of Congress"),
+    ("field_coordinates", ""),
+    ("field_temporal_subject", ""),
+    ("field_language", "ISO639 language names and codes. Enter either."),
+    ("field_olac_discourse_type", ""),
+    ("field_olac_linguistic_type", ""),
+    ("field_place2", ""),
+    ("field_publisher", ""),
+    ("field_place_published", ""),
+    ("field_extent", "Autogenerated if possible"),
+    ("field_legacy_identifier", ""),
+    (
+        "field_original_format",
+        "See https://diglib.amphilsoc.org/taxonomy/vocab/original_format/all (first letter capitalized only)",
+    ),
+    ("field_digital_origin", "Value assumed. Change if incorrect."),
+    ("field_reformatting_quality", "Value assumed. Change if incorrect."),
+    ("field_legacy_model", ""),
+    (
+        "field_date_digitized",
+        "Date digitized, estimated from file metadata. Change if you know this to be wrong.",
+    ),
+    ("field_date_modified", ""),
+    (
+        "field_recording_number",
+        "Legacy audio only. Use field_local_identifier for new audio.",
+    ),
+    (
+        "field_program_number",
+        "Legacy audio only. Use field_local_identifier for new audio.",
+    ),
+    ("field_legacy_type", ""),
+    ("field_classification", ""),
+    ("field_edition", ""),
+    ("field_collection2", "Collection title, e.g. John Tukey Papers. Fills down."),
+    (
+        "field_parent_collection_call_num",
+        "Collection call number e.g. Mss.Ms.Coll.42. Fills down.",
+    ),
+    (
+        "field_collection_url",
+        "Collection URL, e.g. https://as.amphilsoc.org/repositories/2/resources/3085. Fills down.",
+    ),
+    ("field_restrictions_on_access", ""),
+    ("field_mods_relateditem_titleinfo", "Collection title to relate to this object"),
+    ("field_related_materials_note", "Note on related collection"),
+    ("field_related_resource_url", "Collection URL to relate to this object"),
+    ("field_related_object", "DigLib node to relate to this object"),
+    ("field_access_restricted", ""),
+    ("field_mods_subject_displaylabel", ""),
+    ("field_administrative_notes", "Internal (non-public) note"),
+    (
+        "field_access_terms",
+        "Uploading restricted items is not yet worked out. Autogenerated for audio.",
+    ),
+    ("TEI", ""),
+    ("url_alias", "An alternate URL, for long/repeated titles. Prefix will be filled."),
+    (
+        "field_local_identifier",
+        "Equivalent to Component Unique Identifier. Use for alternate (non-title/chrono) arrangement in a collection.",
+    ),
+    (
+        "field_internet_media_type",
+        "Media Type, a file format identification standard. Autogenerated",
+    ),
+)
+
+# all the above fields as a list, for simpler access
+WB_FIELDS_ALL = [x[0] for x in WB_FIELDS_ORDERED_WITH_DESCRIPTION]
+
+# File extensions to Workbench values
+# This is fairly repetitive as field_display_hints and field_internet_media_type are based on specific extensions
+# for field_internet_media_type standard consult https://www.iana.org/assignments/media-types/media-types.xhtml and/or https://mimetype.io
+#  (for some reason these are distinct. wav is not included in IANA.)
+# field_access_terms, field_model and field_resource_type always come in a group
+extension_to_WB_field = (
+    {
+        "extension": ".wav",
+        "field_access_terms": "Restricted Audio",
+        "field_display_hints": None,
+        "field_model": "Audio",
+        "field_resource_type": "Sound",
+        "field_internet_media_type": "audio/wav",  # https://mimetype.io/audio/vnd.wav official but apparently less supported than audio/wav, see discussion on linked github page
+    },
+    {
+        "extension": ".mp3",
+        "field_access_terms": "Restricted Audio",
+        "field_display_hints": None,
+        "field_model": "Audio",
+        "field_resource_type": "Sound",
+        "field_internet_media_type": "audio/mpeg",
+    },
+    {
+        "extension": ".pdf",
+        "field_access_terms": None,
+        "field_display_hints": "PDFjs",
+        "field_model": "Digital Document",
+        "field_resource_type": "Collection",
+        "field_internet_media_type": "application/pdf",
+    },
+    {
+        "extension": ".tif",
+        "field_model": "Image",
+        "field_access_terms": None,
+        "field_display_hints": "Open Seadragon",
+        "field_resource_type": "Still Image",
+        "field_internet_media_type": "image/tiff",
+    },
+    {
+        "extension": ".jpg",
+        "field_model": "Image",
+        "field_access_terms": None,
+        "field_display_hints": "Open Seadragon",
+        "field_resource_type": "Still Image",
+        "field_internet_media_type": "image/jpeg",
+    },
+    {
+        "extension": ".jp2",
+        "field_model": "Image",
+        "field_access_terms": None,
+        "field_display_hints": "Open Seadragon",
+        "field_resource_type": "Still Image",
+        "field_internet_media_type": "image/jp2",
+    },
+    {
+        "extension": ".mp4",
+        "field_model": "Video",
+        "field_access_terms": None,
+        "field_display_hints": None,
+        "field_resource_type": "Moving Image",
+        "field_internet_media_type": "video/mp4",
+    },
+    {
+        "extension": ".mov",
+        "field_model": "Video",
+        "field_access_terms": None,
+        "field_display_hints": None,
+        "field_resource_type": "Moving Image",
+        "field_internet_media_type": "video/quicktime",
+    },
+    {
+        "extension": ".mts",
+        "field_model": "Video",
+        "field_access_terms": None,
+        "field_display_hints": None,
+        "field_resource_type": "Moving Image",
+        "field_internet_media_type": "video/MP2T",
+    },
+)
+
+# list of extensions, for simple access
+EXTENSIONS = [x["extension"] for x in extension_to_WB_field]
+
+"""
+Check that runs on any use, to validate that all Workbench fields referenced are in WB_FIELDS_ALL
+this will flag an error if we update the fields but forget something
+"""
+
+
+def _WB_fields_consistency_check():
+    for x in (
+        WB_FIELDS_REQUIRED_AT_INPUT_SINGLE,
+        WB_FIELDS_REQUIRED_AT_INPUT_BOOK,
+        WB_FIELDS_DOWNFILLING,
+        WB_FIELDS_EXPORT_KEEP_EMPTY_SINGLE,
+        WB_FIELDS_EXPORT_KEEP_EMPTY_BOOK,
+    ):
+        for y in x:
+            if y not in WB_FIELDS_ALL:
+                raise ValueError(
+                    "Workbench fields error: this field is referenced in one WB_FIELDS tuple but not WB_FIELDS_ALL: "
+                    + str(y)
+                )
+
+
+_WB_fields_consistency_check()

aspace_wb/data/fields/cnairaudio.csv

@@ -0,0 +1,26 @@
+file
+field_collection2
+field_parent_collection_call_num
+field_collection_url
+field_member_of
+field_local_identifier
+title
+url_alias
+field_edtf_date_created
+field_date_created_text
+field_original_format
+field_extent
+field_description_long
+field_note
+field_cnair_subject
+field_language
+field_geographic_subject
+field_linked_agent
+field_related_materials_note
+field_mods_relateditem_titleinfo
+field_related_resource_url
+field_date_digitized
+field_resource_type
+field_model
+field_digital_origin
+field_reformatting_quality

aspace_wb/data/fields/cnairbook.csv

@@ -0,0 +1,26 @@
+file
+field_collection2
+field_parent_collection_call_num
+field_collection_url
+field_member_of
+field_local_identifier
+title
+url_alias
+field_edtf_date_created
+field_date_created_text
+field_description_long
+field_note
+field_cnair_subject
+field_language
+field_geographic_subject
+field_linked_agent
+field_related_materials_note
+field_mods_relateditem_titleinfo
+field_related_resource_url
+field_extent
+total_scans
+field_date_digitized
+field_resource_type
+field_model
+field_digital_origin
+field_reformatting_quality

aspace_wb/data/fields/cnairimage.csv

@@ -0,0 +1,24 @@
+file
+field_collection2
+field_parent_collection_call_num
+field_collection_url
+field_member_of
+field_local_identifier
+title
+url_alias
+field_edtf_date_created
+field_date_created_text
+field_description_long
+field_note
+field_cnair_subject
+field_language
+field_geographic_subject
+field_linked_agent
+field_related_materials_note
+field_mods_relateditem_titleinfo
+field_related_resource_url
+field_date_digitized
+field_resource_type
+field_model
+field_digital_origin
+field_reformatting_quality

aspace_wb/data/fields/example_minimum_book.csv

@@ -0,0 +1,11 @@
+file
+field_resource_type
+field_model
+field_member_of
+title
+field_digital_origin
+field_parent_collection_call_num
+field_collection2
+field_collection_url
+field_reformatting_quality
+total_scans

aspace_wb/data/fields/example_minimum_single.csv

@@ -0,0 +1,10 @@
+file
+field_resource_type
+field_model
+field_member_of
+title
+field_digital_origin
+field_parent_collection_call_num
+field_collection2
+field_collection_url
+field_reformatting_quality