aipscan 0.9.0.dev10__tar.gz → 0.9.0rc2__tar.gz

{aipscan-0.9.0.dev10 → aipscan-0.9.0rc2}/.gitignore

@@ -10,3 +10,4 @@ node_modules/
 __pycache__/
 *.py[codz]
 .DS_Store
+/output

{aipscan-0.9.0.dev10 → aipscan-0.9.0rc2}/AIPscan/Aggregator/database_helpers.py

@@ -2,13 +2,15 @@
 database.
 """
 
-from datetime import datetime
+import os
 
 from celery.utils.log import get_task_logger
+from dateutil.parser import parse
 from lxml import etree
 
 from AIPscan import db
 from AIPscan.Aggregator import tasks
+from AIPscan.Aggregator.downloads import get_download_root
 from AIPscan.Aggregator.task_helpers import _tz_neutral_date
 from AIPscan.Aggregator.task_helpers import get_storage_service_api_url
 from AIPscan.models import AIP
@@ -247,6 +249,16 @@ def create_or_update_pipeline(origin_pipeline, storage_service):
     return pipeline
 
 
+def _optional_file_date(value):
+    """Keep absent or unusable PREMIS file dates unknown."""
+    if value is None or value == () or value == "":
+        return None
+    try:
+        return parse(value).replace(tzinfo=None, microsecond=0)
+    except ValueError, TypeError, OverflowError:
+        return None
+
+
 def _get_file_properties(fs_entry):
     """Retrieve file properties from FSEntry
 
@@ -270,14 +282,17 @@ def _get_file_properties(fs_entry):
 
     try:
         for premis_object in fs_entry.get_premis_objects():
-            file_info["size"] = premis_object.size
+            size = premis_object.size
+            file_info["size"] = None if size == () else size
             key_alias = premis_object.format_registry_key
-            file_info["date_created"] = _tz_neutral_date(
+            file_info["date_created"] = _optional_file_date(
                 premis_object.date_created_by_application
             )
             if not isinstance(key_alias, tuple):
                 file_info["puid"] = key_alias
-            file_info["file_format"] = premis_object.format_name
+            name_alias = premis_object.format_name
+            if not isinstance(name_alias, tuple):
+                file_info["file_format"] = name_alias
             version_alias = premis_object.format_version
             if not isinstance(version_alias, tuple):
                 file_info["format_version"] = version_alias
@@ -406,13 +421,15 @@ def process_aip_data(aip, mets):
 
 
 def create_fetch_job(datetime_obj_start, timestamp_str, storage_server_id):
+    download_root = get_download_root()
+    download_directory = os.path.join(download_root, timestamp_str, "")
     fetch_job = FetchJob(
         total_packages=None,
         total_deleted_aips=None,
         total_aips=None,
         download_start=datetime_obj_start,
         download_end=None,
-        download_directory=f"AIPscan/Aggregator/downloads/{timestamp_str}/",
+        download_directory=download_directory,
         storage_service_id=storage_server_id,
     )
     db.session.add(fetch_job)
@@ -453,7 +470,6 @@ def update_fetch_job(fetch_job_id, processed_packages, total_packages_count):
     obj.total_sips = total_sips
     obj.total_replicas = total_replicas
     obj.total_deleted_aips = total_deleted_aips
-    obj.download_end = datetime.now().replace(microsecond=0)
     db.session.commit()
 
     return obj

aipscan-0.9.0rc2/AIPscan/Aggregator/downloads.py

@@ -0,0 +1,24 @@
+"""Helpers for working with Aggregator download directories."""
+
+import os
+from importlib import resources
+
+from flask import current_app
+
+DEFAULT_DOWNLOAD_ROOT = os.fspath(resources.files(__package__).joinpath("downloads"))
+
+
+def get_download_root(app=None) -> str:
+    """Return the base directory used to stage download artifacts."""
+    if app is not None:
+        return app.config.get("AGGREGATOR_DOWNLOAD_ROOT", DEFAULT_DOWNLOAD_ROOT)
+
+    try:
+        application = current_app._get_current_object()
+    except RuntimeError:
+        env_override = os.getenv("AGGREGATOR_DOWNLOAD_ROOT")
+        if env_override:
+            return env_override
+        return DEFAULT_DOWNLOAD_ROOT
+
+    return application.config.get("AGGREGATOR_DOWNLOAD_ROOT", DEFAULT_DOWNLOAD_ROOT)

{aipscan-0.9.0.dev10 → aipscan-0.9.0rc2}/AIPscan/Aggregator/forms.py

@@ -1,7 +1,10 @@
 from flask_wtf import FlaskForm
 from wtforms import BooleanField
+from wtforms import IntegerField
 from wtforms import StringField
 from wtforms.validators import DataRequired
+from wtforms.validators import InputRequired
+from wtforms.validators import NumberRange
 from wtforms.widgets import PasswordInput
 
 
@@ -12,6 +15,10 @@ class StorageServiceForm(FlaskForm):
     api_key = StringField(
         "API key", validators=[DataRequired()], widget=PasswordInput(hide_value=False)
     )
-    download_limit = StringField("Download limit", default="20")
-    download_offset = StringField("Download offset", default="0")
+    download_limit = IntegerField(
+        "Download limit", default=20, validators=[InputRequired(), NumberRange(min=1)]
+    )
+    download_offset = IntegerField(
+        "Download offset", default=0, validators=[InputRequired(), NumberRange(min=0)]
+    )
     default = BooleanField("Default")

{aipscan-0.9.0.dev10 → aipscan-0.9.0rc2}/AIPscan/Aggregator/mets_parse_helpers.py

@@ -40,17 +40,14 @@ def parse_mets_with_metsrw(mets_file):
     return mets
 
 
-def get_aip_original_name(mets):
-    """Retrieve PREMIS original name from a METSDocument object.
+def get_aip_original_name(mets, package_uuid):
+    """Retrieve the package name from PREMIS or its physical directory label.
 
     If the original name cannot be reliably retrieved from the METS file
     a METSError exception is returned to be handled by the caller as
     desired.
     """
 
-    # Negated as we're going to want to remove this length of values.
-    NAMESUFFIX = -len("-00000000-0000-0000-0000-000000000000")
-
     # The transfer directory prefix is a directory prefix that can also
     # exist in a dmdSec intellectual entity and we want to identify and
     # ignore those.
@@ -66,16 +63,22 @@ def get_aip_original_name(mets):
             full_name = dmd_element.find(
                 ELEM_ORIGINAL_NAME_PATTERN, namespaces=NAMESPACES
             )
-            if full_name is not None and full_name.text.startswith(TRANSFER_DIR_PREFIX):
+            if full_name is None or not full_name.text:
+                continue
+            if full_name.text.startswith(TRANSFER_DIR_PREFIX):
                 # We don't want this value, it will usually represent an
                 # directory entity.
                 continue
-            try:
-                original_name = full_name.text[:NAMESUFFIX]
-            except AttributeError:
-                continue
+            original_name = full_name.text
 
-    # There should be a transfer name in every METS.
+    # Older Archivematica METS only identify the package in the physical
+    # structure map. METSRW represents its top-level divs as parentless entries.
+    if not original_name:
+        roots = [entry for entry in mets.all_files() if entry.parent is None]
+        if len(roots) == 1 and roots[0].type == "Directory":
+            original_name = roots[0].label or ""
+
+    original_name = original_name.removesuffix(f"-{package_uuid}")
     if original_name == "":
         raise METSError("Cannot locate transfer name in METS")
 
@@ -94,14 +97,15 @@ def download_mets(
     numbered_subdir = create_numbered_subdirs(timestamp, package_list_no)
     download_file = os.path.join(numbered_subdir, f"METS.{package_uuid}.xml")
 
-    mets_response = requests.get(
-        get_mets_url(storage_service, package_uuid, relative_path_to_mets),
-        stream=True,
-    )
-
     try:
-        mets_hash = stream_write_and_hash(mets_response, download_file)
-    finally:
-        mets_response.close()
+        with requests.get(
+            get_mets_url(storage_service, package_uuid, relative_path_to_mets),
+            stream=True,
+            timeout=(10, 120),
+        ) as mets_response:
+            mets_response.raise_for_status()
+            mets_hash = stream_write_and_hash(mets_response, download_file)
+    except requests.RequestException:
+        raise METSError(f"Cannot download METS for AIP {package_uuid}") from None
 
     return download_file, mets_hash

{aipscan-0.9.0.dev10 → aipscan-0.9.0rc2}/AIPscan/Aggregator/task_helpers.py

@@ -9,6 +9,7 @@ from datetime import datetime
 from dateutil.parser import ParserError
 from dateutil.parser import parse
 
+from AIPscan.Aggregator.downloads import get_download_root
 from AIPscan.Aggregator.types import StorageServicePackage
 
 
@@ -27,7 +28,7 @@ def get_packages_directory(timestamp):
     """Create a path which we will use to store packages downloaded from
     the storage service plus other metadata.
     """
-    return os.path.join("AIPscan", "Aggregator", "downloads", timestamp, "packages")
+    return os.path.join(get_download_root(), timestamp, "packages")
 
 
 def parse_package_list_file(filepath, logger=None, remove_after_parsing=False):
@@ -129,12 +130,11 @@ def create_numbered_subdirs(timestamp, package_list_number):
     """Check for the existence and create a container folder for our
     METS files as required.
     """
-    AGGREGATOR_DOWNLOADS = os.path.join("AIPscan", "Aggregator", "downloads")
     METS_FOLDER = "mets"
 
     # Create a package list numbered subdirectory if it doesn't exist.
     numbered_subdir = os.path.join(
-        AGGREGATOR_DOWNLOADS, timestamp, METS_FOLDER, str(package_list_number)
+        get_download_root(), timestamp, METS_FOLDER, str(package_list_number)
     )
     if not os.path.exists(numbered_subdir):
         os.makedirs(numbered_subdir)

{aipscan-0.9.0.dev10 → aipscan-0.9.0rc2}/AIPscan/Aggregator/tasks.py

@@ -1,10 +1,12 @@
 import json
 import os
 import shutil
+import time
 from datetime import datetime
 
 import requests
 from celery.utils.log import get_task_logger
+from flask import current_app
 
 from AIPscan import db
 from AIPscan import typesense_helpers
@@ -127,15 +129,25 @@ def workflow_coordinator(
     # Wait for package lists task to finish downloading all package
     # lists.
     task = package_lists_request.AsyncResult(package_lists_task.id, app=celery)
-    while True:
-        if (task.state == "SUCCESS") or (task.state == "FAILURE"):
-            break
-
-    if isinstance(package_lists_task.info, TaskError):
-        # Re-raise.
-        raise (package_lists_task.info)
+    idle_timeout = current_app.config["PACKAGE_LIST_IDLE_TIMEOUT_SECONDS"]
+    deadline = time.monotonic() + idle_timeout
+    completed_lists = 0
+    while task.state not in {"SUCCESS", "FAILURE"}:
+        now = time.monotonic()
+        info = task.info
+        progress = info if isinstance(info, dict) else {}
+        if progress.get("completedPackageLists", 0) > completed_lists:
+            completed_lists = progress["completedPackageLists"]
+            deadline = now + idle_timeout
+        if now >= deadline:
+            raise TaskError(
+                f"Package listing made no progress for {idle_timeout} seconds"
+            )
+        time.sleep(0.2)
+    if task.state == "FAILURE":
+        raise TaskError("Package listing failed; check the worker logs")
 
-    total_package_lists = package_lists_task.info["totalPackageLists"]
+    total_package_lists = task.info["totalPackageLists"]
 
     all_packages = []
     for package_list_no in range(1, total_package_lists + 1):
@@ -152,7 +164,7 @@ def workflow_coordinator(
 
         all_packages = all_packages + packages
 
-    total_packages_count = package_lists_task.info["totalPackages"]
+    total_packages_count = task.info["totalPackages"]
 
     obj = database_helpers.update_fetch_job(
         fetch_job_id, all_packages, total_packages_count
@@ -165,7 +177,11 @@ def make_request(request_url, request_url_without_api_key):
     """Make our request to the storage service and return a valid
     response to our caller or raise a TaskError for celery.
     """
-    response = requests.get(request_url)
+    try:
+        response = requests.get(request_url, timeout=(10, 120))
+    except requests.RequestException:
+        # Request exceptions may contain the URL, including its API key.
+        raise TaskError(f"Cannot connect to: {request_url_without_api_key}") from None
     if response.status_code != requests.codes.ok:
         err = f"Check the URL and API details, cannot connect to: `{request_url_without_api_key}`"
         logger.error(err)
@@ -225,7 +241,8 @@ def package_lists_request(self, storage_service_id, timestamp, packages_director
         self.update_state(
             state=IN_PROGRESS,
             meta={
-                "message": f"Total packages: {total_packages} Total package lists: {total_package_lists}"
+                "completedPackageLists": packages_count,
+                "message": f"Total packages: {total_packages} Total package lists: {total_package_lists}",
             },
         )
 
@@ -292,8 +309,6 @@ def get_mets(
     This function relies on being able to use mets-reader-writer which
     is the primary object we will be passing about.
 
-    TODO: Log METS errors.
-
     The "customlogger" argument allows an external logger to be specified when
     the task's logic is executed, using the task's "apply" method, by an
     external application like a batch script.
@@ -305,13 +320,16 @@ def get_mets(
 
     # Download METS file
     storage_service = db.session.get(StorageService, storage_service_id)
-    download_file, mets_hash = download_mets(
-        storage_service,
-        package_uuid,
-        relative_path_to_mets,
-        timestamp_str,
-        package_list_no,
-    )
+    try:
+        download_file, mets_hash = download_mets(
+            storage_service,
+            package_uuid,
+            relative_path_to_mets,
+            timestamp_str,
+            package_list_no,
+        )
+    except METSError as err:
+        raise TaskError(str(err)) from err
     mets_name = os.path.basename(download_file)
 
     # If METS file's hash matches an existing value, this is a duplicate of an
@@ -331,12 +349,11 @@ def get_mets(
 
     try:
         mets = parse_mets_with_metsrw(download_file)
-    except METSError:
-        # An error we need to log and report back to the user.
-        return
+    except METSError as err:
+        raise TaskError(f"Cannot parse METS for AIP {package_uuid}: {err}") from err
 
     try:
-        original_name = get_aip_original_name(mets)
+        original_name = get_aip_original_name(mets, package_uuid)
     except METSError:
         # Some other error with the METS file that we might want to
         # log and act upon.
@@ -344,26 +361,29 @@ def get_mets(
 
     # Delete records of any previous versions of this AIP, which will shortly
     # be replaced by new records from the updated METS.
-    previous_aips = AIP.query.filter_by(uuid=package_uuid).all()
-    for previous_aip in previous_aips:
-        tasklogger.info(
-            f"Deleting record for AIP {package_uuid} to replace from newer METS"
-        )
-        database_helpers.delete_aip_object(previous_aip)
-
-    aip = database_helpers.create_aip_object(
-        package_uuid=package_uuid,
-        transfer_name=original_name,
-        create_date=mets.createdate,
-        mets_sha256=mets_hash,
-        size=aip_size,
-        storage_service_id=storage_service_id,
-        storage_location_id=storage_location_id,
-        fetch_job_id=fetch_job_id,
-        origin_pipeline_id=origin_pipeline_id,
-    )
+    try:
+        previous_aips = AIP.query.filter_by(uuid=package_uuid).all()
+        for previous_aip in previous_aips:
+            tasklogger.info(
+                f"Deleting record for AIP {package_uuid} to replace from newer METS"
+            )
+            database_helpers.delete_aip_object(previous_aip)
 
-    database_helpers.process_aip_data(aip, mets)
+        aip = database_helpers.create_aip_object(
+            package_uuid=package_uuid,
+            transfer_name=original_name,
+            create_date=mets.createdate,
+            mets_sha256=mets_hash,
+            size=aip_size,
+            storage_service_id=storage_service_id,
+            storage_location_id=storage_location_id,
+            fetch_job_id=fetch_job_id,
+            origin_pipeline_id=origin_pipeline_id,
+        )
+        database_helpers.process_aip_data(aip, mets)
+    except Exception as err:
+        db.session.rollback()
+        raise TaskError(f"Cannot import METS for AIP {package_uuid}") from err
 
     # Delete downloaded METS file.
     try:

{aipscan-0.9.0.dev10 → aipscan-0.9.0rc2}/AIPscan/Aggregator/templates/edit_storage_service.html

@@ -23,8 +23,14 @@
     {% for error in form.api_key.errors %}
     <span style="color: red;">[{{ error }}]</span>
     {% endfor %}</td></tr>
-  <tr><td><strong>{{ form.download_limit.label }}</strong></td><td>{{ form.download_limit(size=48) }} </td></tr>
-  <tr><td><strong>{{ form.download_offset.label }}</strong></td><td>{{ form.download_offset(size=48) }} </td></tr>
+  <tr><td><strong>{{ form.download_limit.label }}</strong></td><td>{{ form.download_limit(size=48) }}
+    {% for error in form.download_limit.errors %}
+    <span style="color: red;">[{{ error }}]</span>
+    {% endfor %}</td></tr>
+  <tr><td><strong>{{ form.download_offset.label }}</strong></td><td>{{ form.download_offset(size=48) }}
+    {% for error in form.download_offset.errors %}
+    <span style="color: red;">[{{ error }}]</span>
+    {% endfor %}</td></tr>
   <tr><td><strong>{{ form.default.label }}</strong></td><td>{{ form.default() }}</td></tr>
 </table>
 <button type="submit" class="btn btn-success">Save</button>

aipscan-0.9.0rc2/AIPscan/Aggregator/tests/fixtures/legacy_mets/README.md

@@ -0,0 +1,23 @@
+# Legacy METS fixture
+
+`production-aip-mets-file.xml` is copied unchanged from
+[mets-reader-writer] at revision `0affd1133b3fee11c1ee911eb445ba8504c9be1c`.
+Its introducing commit, [1a1fcc93], describes it as generated by an
+Archivematica QA deployment. The document identifies Archivematica 1.6 and
+uses PREMIS 2.2. Its original file, `abc.txt`, has size and format metadata
+but no `dateCreatedByApplication`.
+
+The upstream suite uses this fixture for METS XMLSchema and Archivematica
+Schematron validation. AIPscan's import tests vary only the original file's
+optional creation date in memory; the fixture on disk remains unchanged.
+
+AIP name tests also use this fixture to recover the package name from the
+physical structure map when no PREMIS package name is available. They vary
+the root directory label in memory to exercise UUID suffix handling.
+
+The Dagger scenario packages this METS unchanged and also creates a variant
+with the original file's format identification removed. It verifies both
+packages through Storage Service, Celery, CSV exports, and browser charts.
+
+[mets-reader-writer]: https://github.com/artefactual-labs/mets-reader-writer/blob/0affd1133b3fee11c1ee911eb445ba8504c9be1c/fixtures/production-aip-mets-file.xml
+[1a1fcc93]: https://github.com/artefactual-labs/mets-reader-writer/commit/1a1fcc93e73237e1a7c252b416dc4b6e443e9185