pathling 9.9.0.dev0__tar.gz → 9.10.0.dev0__tar.gz

{pathling-9.9.0.dev0 → pathling-9.10.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pathling
-Version: 9.9.0.dev0
+Version: 9.10.0.dev0
 Summary: Python API for Pathling
 Project-URL: Homepage, https://github.com/aehrc/pathling
 Author-email: "Australian e-Health Research Centre, CSIRO" <pathling@csiro.au>

{pathling-9.9.0.dev0 → pathling-9.10.0.dev0}/pathling/_version.py

@@ -2,8 +2,8 @@
 # Auto generated from POM project version.
 # Please do not modify.
 #
-__version__="9.9.0.dev0"
-__java_version__="9.9.0-SNAPSHOT"
+__version__="9.10.0.dev0"
+__java_version__="9.10.0-SNAPSHOT"
 __scala_version__="2.13"
 __delta_version__="4.0.0"
 __hadoop_version__="3.4.1"

{pathling-9.9.0.dev0 → pathling-9.10.0.dev0}/pathling/cli/config.py

@@ -55,7 +55,15 @@ PROJECT_CONFIG_FILENAME = "pathling.toml"
 
 # Valid top-level keys in the config file.
 VALID_CONFIG_KEYS = frozenset(
-    {"tx-server", "fhir-version", "terminology-auth", "bulk-auth", "spark", "tx-store"}
+    {
+        "tx-server",
+        "fhir-version",
+        "terminology-auth",
+        "bulk-auth",
+        "spark",
+        "tx-store",
+        "package-registry",
+    }
 )
 
 # Valid keys within the [terminology-auth] and [bulk-auth] tables.
@@ -181,6 +189,10 @@ class CliConfig:
     :param tx_server_explicit: whether the terminology server URL was set
            explicitly (via flag or config key) rather than falling back to the
            built-in default. Drives the store-wins conflict warning.
+    :param package_registry: the FHIR package registry that a package import
+           checks a package against, or None to leave the choice to the library
+           default. Concerns package distribution rather than the store, so it
+           is a top-level key rather than part of ``[tx-store]``.
     """
 
     tx_server: str = DEFAULT_TX_SERVER
@@ -192,6 +204,7 @@ class CliConfig:
     bulk_auth_table: Optional[dict] = None
     tx_store: Optional[TxStore] = None
     tx_server_explicit: bool = False
+    package_registry: Optional[str] = None
 
 
 def _load_toml(path: Path) -> dict:
@@ -751,4 +764,5 @@ def resolve_config(
         bulk_auth_table=file_data.get("bulk-auth"),
         tx_store=resolved_tx_store,
         tx_server_explicit=tx_server_explicit,
+        package_registry=file_data.get("package-registry"),
     )

{pathling-9.9.0.dev0 → pathling-9.10.0.dev0}/pathling/cli/import_terminology.py

@@ -35,13 +35,26 @@ from rich.console import Console
 
 from pathling.cli import session
 from pathling.cli.config import CliConfig
-from pathling.cli.errors import EXIT_USAGE, CliError
+from pathling.cli.errors import EXIT_USAGE, CliError, unwrap_java_exception
 from pathling.cli.render import progress_status
 
 if TYPE_CHECKING:
+    from pyspark.sql import Row, SparkSession
+
     from pathling import PathlingContext
     from pathling.cli.main import CliContext
 
+# The name of the manifest table within a terminology store.
+MANIFEST_TABLE = "manifest"
+
+# The fixed phrase the library uses when a package does not match the checksum
+# its registry publishes. The library must not name a CLI flag, so the hint
+# naming --no-verify is added here, keyed off this phrase.
+MISMATCH_PHRASE = "does not match the registry checksum"
+
+# The hint appended to a checksum mismatch failure.
+MISMATCH_HINT = "Re-run with --no-verify to import it anyway."
+
 
 def _import_context(config: CliConfig, console: Console) -> PathlingContext:
     """Creates a context for importing into a store.
@@ -84,6 +97,91 @@ def _resolve_storage_path(config: CliConfig, storage_path: Optional[str]) -> str
     )
 
 
+def read_latest_import_row(
+    spark: SparkSession, store: str, source: str
+) -> Optional[Row]:
+    """Reads the most recent manifest row recorded for a source.
+
+    The store manifest carries the provenance of each import: the source hash,
+    the package identity, and the outcome of the registry check. The latest row
+    for this source describes the import that has just completed.
+
+    A failure to read the manifest yields None rather than an error: the import
+    has already committed, and reporting less detail is better than failing a
+    command whose work succeeded.
+
+    :param spark: the Spark session to read with.
+    :param store: the terminology store path.
+    :param source: the import source, matched against the manifest ``source``
+           column.
+    :return: the latest manifest row for the source, or None when there is none
+             or the manifest cannot be read.
+    """
+    from pyspark.sql import functions as F
+
+    manifest_path = f"{store.rstrip('/')}/{MANIFEST_TABLE}"
+    try:
+        rows = (
+            spark.read.format("delta")
+            .load(manifest_path)
+            .filter(F.col("source") == source)
+            .orderBy(F.col("imported_at").desc())
+            .take(1)
+        )
+    except Exception:  # noqa: BLE001 - detail only; the import already committed.
+        return None
+    return rows[0] if rows else None
+
+
+def format_import_summary(
+    command_noun: str, source: str, store: str, row: Optional[Row]
+) -> str:
+    """Renders the completion line for a finished import.
+
+    The line always names what was imported, from where, and into where. Where
+    the manifest records provenance it is reported in parentheses: the source
+    hash, and for a package the identity and the outcome of the registry check.
+
+    :param command_noun: what was imported, such as ``"SNOMED CT"``.
+    :param source: the import source as given on the command line.
+    :param store: the resolved store path.
+    :param row: the manifest row describing the import, or None when it could
+           not be read.
+    :return: the completion line.
+    """
+    summary = f"Imported {command_noun} from {source} into {store}"
+    if row is None:
+        return summary
+    values = row.asDict() if hasattr(row, "asDict") else dict(row)
+    sha256 = values.get("source_sha256")
+    if sha256 is None:
+        # A directory source has no single set of bytes to fingerprint.
+        return summary
+    status = values.get("package_verification")
+    name = values.get("package_name")
+    version = values.get("package_version")
+    identity = f"{name} {version}" if name and version else None
+    if status == "verified":
+        registry = values.get("package_registry")
+        detail = f"verified {identity} against {registry}"
+    elif status == "unverified":
+        subject = identity if identity else "package"
+        detail = (
+            f"{subject} not verified against a registry; re-run with --verbose "
+            "for the reason"
+        )
+    elif status == "skipped":
+        detail = (
+            f"{identity}, verification skipped"
+            if identity
+            else "package verification skipped"
+        )
+    else:
+        # Not a package: the hash alone is the provenance.
+        return f"{summary} (sha256 {sha256})"
+    return f"{summary} ({detail}; sha256 {sha256})"
+
+
 @click.command(name="import-snomed")
 @click.argument("source")
 @click.argument("storage_path", required=False)
@@ -142,20 +240,46 @@ def import_snomed(
         pc.import_snomed(
             source, resolved_path, edition_uri, dense_id_order, resolved_dialect
         )
-    click.echo(f"Imported SNOMED CT from {source} into {resolved_path}")
+    row = read_latest_import_row(pc.spark, resolved_path, source)
+    click.echo(format_import_summary("SNOMED CT", source, resolved_path, row))
 
 
 @click.command(name="import-fhir-terminology")
 @click.argument("source")
 @click.argument("storage_path", required=False)
+@click.option(
+    "--no-verify",
+    "no_verify",
+    is_flag=True,
+    help=(
+        "Do not check a FHIR NPM package against its registry's published "
+        "checksum. Use for offline imports or packages not published to a "
+        "registry."
+    ),
+)
+@click.option(
+    "--package-registry",
+    "package_registry",
+    metavar="URL",
+    help=(
+        "The FHIR package registry to check a package against. Falls back to "
+        "the 'package-registry' config key, then https://packages.fhir.org."
+    ),
+)
 @click.pass_obj
 def import_fhir_terminology(
-    obj: CliContext, source: str, storage_path: Optional[str]
+    obj: CliContext,
+    source: str,
+    storage_path: Optional[str],
+    no_verify: bool,
+    package_registry: Optional[str],
 ) -> None:
     """Import FHIR CodeSystem, ValueSet, and ConceptMap resources into a store.
 
     The source may be a JSON file, a directory of JSON files, or a FHIR NPM
-    package (.tgz). STORAGE_PATH may be omitted when 'tx-store.path' (or
+    package (.tgz). A package is checked against the checksum its registry
+    publishes; a package that does not match fails the import and leaves the
+    store unchanged. STORAGE_PATH may be omitted when 'tx-store.path' (or
     --tx-store) is set.
 
     Example:
@@ -165,10 +289,27 @@ def import_fhir_terminology(
     config = obj.config
     console = obj.console
     resolved_path = _resolve_storage_path(config, storage_path)
+    # The flag wins over the config key; with neither set no registry is named
+    # and the library default applies, so the CLI never holds that URL.
+    resolved_registry = (
+        package_registry if package_registry is not None else config.package_registry
+    )
     pc = _import_context(config, console)
     with progress_status(console, "Importing FHIR terminology...", config.verbose):
-        pc.import_fhir_terminology(source, resolved_path)
-    click.echo(f"Imported FHIR terminology from {source} into {resolved_path}")
+        try:
+            pc.import_fhir_terminology(
+                source,
+                resolved_path,
+                verify_package=not no_verify,
+                package_registry=resolved_registry,
+            )
+        except Exception as exc:  # noqa: BLE001 - enrich a mismatch failure.
+            root_message = unwrap_java_exception(exc)
+            if MISMATCH_PHRASE not in root_message:
+                raise
+            raise CliError(f"{root_message} {MISMATCH_HINT}") from exc
+    row = read_latest_import_row(pc.spark, resolved_path, source)
+    click.echo(format_import_summary("FHIR terminology", source, resolved_path, row))
 
 
 #: The terminology import commands registered by the root command group.

{pathling-9.9.0.dev0 → pathling-9.10.0.dev0}/pathling/context.py

@@ -254,8 +254,8 @@ class PathlingContext:
                unversioned SNOMED reference when the local store holds multiple editions.
         :param expansion_cache_size: the maximum number of value set expansions cached per executor
                in local mode.
-        :param dialect_aliases: additional dialect tags recognised in local mode when a display or
-               designation is requested in a particular language, mapping a language tag to the
+        :param dialect_aliases: additional dialect tags recognised in local mode when a display
+               is requested in a particular language, mapping a language tag to the
                identifier of the SNOMED CT language reference set that serves it (for example
                ``{"en-NZ": "271000210107"}``). An entry for a tag that is already recognised
                replaces the built-in mapping for it.
@@ -473,18 +473,46 @@ class PathlingContext:
             options = builder.build()
         self._jpc.importSnomed(source, storage_path, options)
 
-    def import_fhir_terminology(self, source: str, storage_path: str) -> None:
+    def import_fhir_terminology(
+        self,
+        source: str,
+        storage_path: str,
+        verify_package: bool = True,
+        package_registry: Optional[str] = None,
+    ) -> None:
         """
         Imports FHIR R4 CodeSystem, ValueSet, and ConceptMap resources into a local terminology
         store.
 
+        A FHIR NPM package (``.tgz``) is checked against the checksum its registry publishes before
+        anything is written: a package that does not match fails the import and leaves the store
+        untouched. A package the registry does not describe, or a registry that cannot be reached,
+        is imported and recorded as unverified. For an offline import, or a package that was never
+        published to a registry, pass ``verify_package=False``; the import is then recorded as
+        having skipped verification.
+
         :param source: the path to a JSON file, a directory of JSON files, or a FHIR NPM package
                (``.tgz``), on any filesystem accessible through the Hadoop FileSystem API
         :param storage_path: the terminology store location, created if absent
+        :param verify_package: whether a package source is checked against its registry's published
+               checksum; ``True`` by default
+        :param package_registry: the FHIR package registry to check a package against; when omitted
+               the library default (``https://packages.fhir.org``) is used
         :raises: the mapped JVM ``TerminologyImportException`` if the source contains no importable
-                 resources or an invalid resource; the store is left unmodified
+                 resources or an invalid resource, or the package does not match the registry
+                 checksum; the store is left unmodified
         """
-        self._jpc.importFhirTerminology(source, storage_path)
+        options = None
+        if not verify_package or package_registry is not None:
+            jvm = self._spark._jvm
+            builder = (
+                jvm.au.csiro.pathling.library.terminology.FhirImportOptions.builder()
+            )
+            builder = builder.verifyPackage(verify_package)
+            if package_registry is not None:
+                builder = builder.packageRegistry(package_registry)
+            options = builder.build()
+        self._jpc.importFhirTerminology(source, storage_path, options)
 
     def encode(
         self,