ecosemantic-databases 0.1.8__py3-none-any.whl

ecosemantic/databases/__init__.py

@@ -0,0 +1 @@
+"""Native EcoSemantic bundles built from a source database's own files."""

ecosemantic/databases/bafu/__init__.py

@@ -0,0 +1 @@
+"""Build a native EcoSemantic bundle from BAFU's own files."""

ecosemantic/databases/bafu/build.py

@@ -0,0 +1,44 @@
+"""Build a BAFU zolca into one openLCA JSON-LD bundle, read back by the core's importer."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from ecosemantic.databases.zolca.export.write import NotCarried, export
+from ecosemantic.errors import Conflict
+from ecosemantic.package.background import read_background
+
+
+@dataclass(frozen=True)
+class Built:
+    """What a build wrote: the bundle, the records it holds by folder, and the rows its package does
+    not carry."""
+
+    bundle: Path
+    records: dict[str, int]
+    not_carried: tuple[NotCarried, ...]
+
+
+def build(zolca: Path, *, version: str, out_dir: Path) -> Built:
+    """Write a zolca as `<out_dir>/bafu-<version>.zip` and import it under `bafu-<version>`. The
+    package is exported to a staging file beside the destination and moved into place only once the
+    core reads it, so a bundle the core refuses leaves nothing under the final name; a destination
+    that exists is `Conflict`."""
+    destination = out_dir / f"bafu-{version}.zip"
+    staging = out_dir / f".bafu-{version}.staging.zip"
+    if destination.exists():
+        raise Conflict(f"{destination.name} exists; name another destination")
+    out_dir.mkdir(parents=True, exist_ok=True)
+    written = export(zolca, staging)
+    try:
+        read_background(written.bundle, f"bafu-{version}", foreground=frozenset())
+        os.replace(staging, destination)
+    finally:
+        staging.unlink(missing_ok=True)
+    return Built(
+        bundle=destination,
+        records=written.records,
+        not_carried=written.not_carried,
+    )

ecosemantic/databases/bafu/cli.py

@@ -0,0 +1,124 @@
+"""The command line: the subcommands a BAFU source answers."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import cast
+
+from lca_data_interface import NexusSession
+
+from ecosemantic.databases import command
+from ecosemantic.databases.bafu.build import Built, build
+from ecosemantic.databases.bafu.download import Downloads, Terms, download, translating
+from ecosemantic.databases.bafu.probe import Probe, probe
+from ecosemantic.databases.bafu.profile import RELEASES, read_profile, recorded
+from ecosemantic.databases.bafu.validate import Counted, record, validate
+from ecosemantic.databases.command import Command
+
+
+def _download(args: argparse.Namespace) -> dict[str, object] | Terms:
+    """Fetch a release's packs through Nexus and answer each file's path and the hash the profile
+    pins; while the terms are not accepted, answer the licence file and the citation instead."""
+    profile = read_profile(args.version)
+    with translating(), NexusSession.from_environment(base=profile.server) as session:
+        fetched = download(profile, session, accept_terms=args.accept_terms, results=args.results)
+    if isinstance(fetched, Terms):
+        return fetched
+    return {
+        "inventory": {"path": str(fetched.inventory), "sha256": profile.inventory.sha256},
+        "results": (
+            None
+            if fetched.results is None
+            else {"path": str(fetched.results), "sha256": profile.results.sha256}
+        ),
+        "licence": str(fetched.licence),
+        "citation": fetched.citation,
+    }
+
+
+def _probe(args: argparse.Namespace) -> Probe:
+    """Read a zolca and answer what it holds and what the core would not take of it."""
+    return probe(Path(args.zolca))
+
+
+def _fetched(version: str) -> Path:
+    """The zolca a release's profile pins, fetched through Nexus with the pack's terms accepted, so
+    the answer is always the downloaded file."""
+    profile = read_profile(version)
+    with translating(), NexusSession.from_environment(base=profile.server) as session:
+        fetched = cast(Downloads, download(profile, session, accept_terms=True, results=False))
+    return fetched.inventory
+
+
+def _build(args: argparse.Namespace) -> Built:
+    """Build the zolca given, or the one the release's profile pins, into one bundle under
+    `--out`."""
+    zolca = Path(args.zolca) if args.zolca is not None else _fetched(args.version)
+    return build(zolca, version=args.version, out_dir=Path(args.out))
+
+
+def _validate(args: argparse.Namespace) -> Counted:
+    """Compare a bundle with its release's workbook and the release's two lists, record the run
+    when asked, and answer the differences counted by column."""
+    profile = read_profile(args.version)
+    validation = validate(
+        Path(args.bundle),
+        results=Path(args.results),
+        profile=profile,
+        unlinked=recorded(profile.unlinked),
+        differences=recorded(profile.differences),
+        compiled=Path(args.compiled) if args.compiled is not None else None,
+    )
+    if args.record:
+        record(profile, validation, directory=RELEASES)
+    return validation.counted()
+
+
+def _code(answer: object) -> int:
+    """2 for the answer that asks for the pack's terms to be accepted, 1 for a validation that
+    found a mismatch, and 0 for an answer that stands."""
+    if isinstance(answer, Terms):
+        return 2
+    return 1 if isinstance(answer, Counted) and bool(answer.mismatches) else 0
+
+
+COMMANDS: dict[str, Command] = {
+    "download": Command("Fetch a release's pinned packs from Nexus.", _download),
+    "probe": Command("Read a zolca and list what it holds.", _probe),
+    "build": Command("Write a release's bundle and import it.", _build),
+    "validate": Command("Compare a bundle with its release's workbook.", _validate),
+}
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="ecosemantic-databases bafu",
+        description="Build a native EcoSemantic bundle from a BAFU release.",
+    )
+    subcommands = parser.add_subparsers(dest="command", required=True)
+    for name, each in COMMANDS.items():
+        sub = subcommands.add_parser(name, help=each.help)
+        if name == "download":
+            sub.add_argument("version")
+            sub.add_argument("--accept-terms", action="store_true")
+            sub.add_argument("--results", action="store_true")
+        elif name == "probe":
+            sub.add_argument("zolca")
+        elif name == "build":
+            sub.add_argument("version")
+            sub.add_argument("--out", required=True)
+            sub.add_argument("--zolca")
+        elif name == "validate":
+            sub.add_argument("version")
+            sub.add_argument("--bundle", required=True)
+            sub.add_argument("--results", required=True)
+            sub.add_argument("--compiled")
+            sub.add_argument("--record", action="store_true")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run one command; 0 on success, 1 on a typed error or a validation that found a mismatch,
+    and 2 when the terms are not accepted."""
+    return command.main(_parser(), COMMANDS, argv, code=_code)

ecosemantic/databases/bafu/differences.py

@@ -0,0 +1,52 @@
+"""The (process, column) pairs a BAFU validation knows differ beyond its tolerance, keyed by the
+process's id and the workbook column's header so neither BAFU's names nor its values are written
+into the package."""
+
+from __future__ import annotations
+
+import csv
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+from ecosemantic.errors import Invalid
+
+COLUMNS = ("process_id", "column")
+
+
+@dataclass(frozen=True)
+class Difference:
+    """One pair whose two scores differ beyond the tolerance."""
+
+    process_id: str
+    column: str
+
+
+def _order(row: Difference) -> tuple[str, str]:
+    """A row's sort position: its process, then its column."""
+    return (row.process_id, row.column)
+
+
+def read_differences(path: Path) -> tuple[Difference, ...]:
+    """The pairs a list holds, in process and column order; a pair the list holds twice is
+    `Invalid`."""
+    with path.open(newline="", encoding="utf-8") as read:
+        rows = [
+            Difference(process_id=record["process_id"], column=record["column"])
+            for record in csv.DictReader(read)
+        ]
+    seen: set[tuple[str, str]] = set()
+    for row in rows:
+        if _order(row) in seen:
+            raise Invalid(f"{path} holds {row.process_id} under {row.column!r} twice")
+        seen.add(_order(row))
+    return tuple(sorted(rows, key=_order))
+
+
+def write_differences(path: Path, rows: Iterable[Difference]) -> None:
+    """Write the pairs as CSV, in process and column order."""
+    with path.open("w", newline="", encoding="utf-8") as written:
+        writer = csv.writer(written, lineterminator="\n")
+        writer.writerow(COLUMNS)
+        for row in sorted(rows, key=_order):
+            writer.writerow((row.process_id, row.column))

ecosemantic/databases/bafu/download.py

@@ -0,0 +1,142 @@
+"""Fetch BAFU's openLCA pack and its LCIA results pack from Nexus, refusing a file whose hash is not
+the one its profile pins."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final, Protocol
+
+from lca_data_interface import Downloaded
+from lca_data_interface import Pack as NexusPack
+from lca_data_interface.errors import (
+    CredentialsMissing,
+    LcaDataInterfaceError,
+    LicenceNotAccepted,
+    LoginRefused,
+    MissingField,
+    NotInCatalog,
+    NotOnServer,
+    ServiceError,
+    ShortDownload,
+    TokenRequired,
+)
+
+from ecosemantic.databases.bafu.profile import Pack, Profile
+from ecosemantic.errors import Conflict, EcoSemanticError, Invalid, NotFound, Unavailable
+
+_TRANSLATIONS: Final[dict[type[LcaDataInterfaceError], type[EcoSemanticError]]] = {
+    CredentialsMissing: Unavailable,
+    LoginRefused: Unavailable,
+    TokenRequired: Unavailable,
+    ShortDownload: Unavailable,
+    ServiceError: Unavailable,
+    NotInCatalog: NotFound,
+    NotOnServer: NotFound,
+    MissingField: Invalid,
+    LicenceNotAccepted: Invalid,
+}
+
+
+class Nexus(Protocol):
+    """The three calls a download makes on a Nexus session; `NexusSession` satisfies it and a test
+    doubles it."""
+
+    def pack(self, uuid: str) -> NexusPack: ...
+
+    def licence(self, pack: NexusPack) -> Path: ...
+
+    def download(
+        self,
+        pack: NexusPack,
+        *,
+        version: int,
+        file_format: str,
+        accept_licence: bool,
+    ) -> Downloaded: ...
+
+
+@dataclass(frozen=True)
+class Terms:
+    """What a download answers while a pack's terms are not accepted: the licence file to read and
+    the citation to carry."""
+
+    licence: Path
+    citation: str
+
+
+@dataclass(frozen=True)
+class Downloads:
+    """The files a download fetched, the pack's licence file, and the release's citation."""
+
+    inventory: Path
+    results: Path | None
+    licence: Path
+    citation: str
+
+
+@contextmanager
+def translating() -> Iterator[None]:
+    """A client call whose errors leave as the core's, of the kind and with the names the spec's
+    *Errors* table gives them."""
+    try:
+        yield
+    except LcaDataInterfaceError as error:
+        raise _translated(error) from error
+
+
+def download(
+    profile: Profile, session: Nexus, *, accept_terms: bool, results: bool
+) -> Downloads | Terms:
+    """Fetch a release's packs through Nexus, each file's SHA-256 compared with the profile's.
+
+    Without `accept_terms` the pack's licence file is fetched and nothing is downloaded: the answer
+    is the file a person reads to accept the terms. `results` also fetches the LCIA results pack,
+    which only a validation reads.
+    """
+    with translating():
+        inventory_pack = session.pack(profile.inventory.pack)
+        licence = session.licence(inventory_pack)
+        if not accept_terms:
+            return Terms(licence=licence, citation=profile.citation)
+        inventory = session.download(
+            inventory_pack,
+            version=profile.inventory.version,
+            file_format=profile.inventory.format,
+            accept_licence=True,
+        )
+        results_file = _fetched(profile.results, session) if results else None
+    return Downloads(
+        inventory=_checked(profile.inventory, inventory),
+        results=None if results_file is None else _checked(profile.results, results_file),
+        licence=licence,
+        citation=profile.citation,
+    )
+
+
+def _fetched(pinned: Pack, session: Nexus) -> Downloaded:
+    """One pack's file at the version and the format the profile pins, its terms accepted."""
+    return session.download(
+        session.pack(pinned.pack),
+        version=pinned.version,
+        file_format=pinned.format,
+        accept_licence=True,
+    )
+
+
+def _checked(pinned: Pack, downloaded: Downloaded) -> Path:
+    """The path of a file whose SHA-256 is the one the profile pins; a different hash is a conflict
+    naming the file and both hashes."""
+    if downloaded.sha256 != pinned.sha256:
+        raise Conflict(
+            f"{downloaded.path.name} hashes to {downloaded.sha256}, the profile pins "
+            f"{pinned.sha256}"
+        )
+    return downloaded.path
+
+
+def _translated(error: LcaDataInterfaceError) -> EcoSemanticError:
+    """The core's error for one the client raised."""
+    return _TRANSLATIONS[type(error)](str(error))

ecosemantic/databases/bafu/probe.py

@@ -0,0 +1,168 @@
+"""What a BAFU zolca holds and what of it the core's reader takes: its schema version, its tables,
+the counts its export becomes, the inputs that stay unlinked, the exchanges the core refuses and
+the rows its package does not carry."""
+
+from __future__ import annotations
+
+import tempfile
+from collections import Counter
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+import olca_schema as o
+from olca_schema.zipio import ZipReader
+
+from ecosemantic.databases.lists import distinct_unlinked
+from ecosemantic.databases.zolca.derby.database import Database
+from ecosemantic.databases.zolca.export.write import NotCarried, export
+from ecosemantic.errors import Invalid, NotSupported
+from ecosemantic.model import units
+from ecosemantic.package.background import Import, exchange_refusal, read_background
+from ecosemantic.package.olca import uncertainty_refusal
+
+BUNDLE = "probe.zip"
+DATABASE = "bafu-probe"
+VERSION_TABLE = "OPENLCA_VERSION"
+VERSION_COLUMN = "VERSION"
+PREFIX = "ecosemantic-bafu-probe-"
+
+
+@dataclass(frozen=True)
+class TableProbe:
+    """One table of the database: its rows, and the SQL type each column is stored as."""
+
+    name: str
+    rows: int
+    columns: dict[str, str]
+
+
+@dataclass(frozen=True)
+class Probe:
+    """What a zolca holds and what the core would take of it: its schema version, every table with
+    its rows and column types, what its export becomes in the core, the unlinked inputs by their
+    flow's top category, the exchanges the core refuses by its own reason, the rows the package
+    does not carry, and the factors method categories set apart."""
+
+    schema_version: str
+    tables: tuple[TableProbe, ...]
+    processes: int
+    flows: int
+    methods: int
+    categories: int
+    unlinked: dict[str, int]
+    refused: dict[str, int]
+    not_carried: tuple[NotCarried, ...]
+    set_apart: int
+
+
+def _schema_version(database: Database) -> str:
+    """The openLCA schema version the database stores."""
+    stored = list(database.rows(VERSION_TABLE))
+    if len(stored) != 1:
+        raise Invalid(f"{database.path} stores {len(stored)} {VERSION_TABLE} rows, not one")
+    return str(stored[0][VERSION_COLUMN])
+
+
+def _tables(database: Database) -> tuple[TableProbe, ...]:
+    """Every table of schema `APP`, its rows counted and each column's SQL type named."""
+    return tuple(
+        TableProbe(
+            name=name,
+            rows=sum(1 for _ in database.rows(table)),
+            columns={column.name: column.type.name for column in table.columns},
+        )
+        for name, table in sorted(database.tables().items())
+    )
+
+
+def _read(bundle: Path) -> tuple[tuple[o.Process, ...], dict[str, o.Flow]]:
+    """A package's processes, and its flows by id."""
+    with ZipReader(str(bundle)) as reader:
+        processes = tuple(reader.read_each(o.Process))
+        flows = {flow.id: flow for flow in reader.read_each(o.Flow)}
+    return processes, flows
+
+
+def _factors(bundle: Path) -> list[o.ImpactFactor]:
+    """Every impact factor the package's categories carry."""
+    with ZipReader(str(bundle)) as reader:
+        return [
+            factor
+            for category in reader.read_each(o.ImpactCategory)
+            for factor in category.impact_factors or []
+        ]
+
+
+def _symbols(records: Iterable[o.Exchange | o.ImpactFactor]) -> set[str]:
+    """The unit symbols a run of exchanges or impact factors names."""
+    return {
+        record.unit.name
+        for record in records
+        if record.unit is not None and record.unit.name is not None
+    }
+
+
+def _known(symbol: str) -> bool:
+    """Whether the core's unit table holds a symbol."""
+    try:
+        units.by_symbol(symbol)
+    except NotSupported:
+        return False
+    return True
+
+
+def _lacking(bundle: Path, processes: Sequence[o.Process]) -> tuple[str, ...]:
+    """Every unit symbol the package's exchanges and impact factors name that the core's table
+    lacks, in one sorted tuple."""
+    named = _symbols(
+        exchange for process in processes for exchange in process.exchanges or []
+    ) | _symbols(_factors(bundle))
+    return tuple(sorted(symbol for symbol in named if not _known(symbol)))
+
+
+def _unlinked(imported: Import, flows: dict[str, o.Flow]) -> dict[str, int]:
+    """The import's distinct unlinked inputs, counted by the top category of their flow."""
+    tops = {flow_id: (flow.category or "").split("/")[0] for flow_id, flow in flows.items()}
+    return dict(Counter(row.flow_category for row in distinct_unlinked(imported, tops)))
+
+
+def _refused(processes: Sequence[o.Process], flows: dict[str, o.Flow]) -> Counter[str]:
+    """What the core's two refusals answer over the package's exchanges, counted by reason."""
+    return Counter(
+        reason
+        for process in processes
+        for exchange in process.exchanges or []
+        if exchange.flow is not None
+        and exchange.flow.id in flows
+        and (
+            reason := uncertainty_refusal(exchange.uncertainty, exchange.amount or 0.0)
+            or exchange_refusal(exchange, flows[exchange.flow.id])
+        )
+    )
+
+
+def probe(zolca: Path) -> Probe:
+    """Read a zolca through the export and the core's importer in a temporary directory, and answer
+    what it holds, what the core refuses of it and what its package does not carry; a unit symbol
+    the core's table lacks is `NotSupported`, every lacking symbol named at once."""
+    database = Database(zolca)
+    with tempfile.TemporaryDirectory(prefix=PREFIX) as directory:
+        written = export(zolca, Path(directory) / BUNDLE)
+        processes, flows = _read(written.bundle)
+        lacking = _lacking(written.bundle, processes)
+        if lacking:
+            raise NotSupported(f"{zolca}: the core's unit table lacks {', '.join(lacking)}")
+        imported = read_background(written.bundle, DATABASE, foreground=frozenset())
+        return Probe(
+            schema_version=_schema_version(database),
+            tables=_tables(database),
+            processes=imported.processes,
+            flows=len(imported.flows),
+            methods=len({method.key[1] for method in imported.methods}),
+            categories=len(imported.methods),
+            unlinked=_unlinked(imported, flows),
+            refused=dict(_refused(processes, flows)),
+            not_carried=written.not_carried,
+            set_apart=sum(method.set_apart for method in imported.methods),
+        )

ecosemantic/databases/bafu/profile.py

@@ -0,0 +1,91 @@
+"""The facts a BAFU release pins: the two packs Nexus serves, the workbook's columns mapped to the
+method and category each scores, its total column, the lists its validation holds it to, and what a
+validated run recorded."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from ecosemantic.databases import profiles
+
+RELEASES = Path(__file__).parent / "releases"
+
+
+class Pack(BaseModel):
+    """One file a release is read from: the datafile Nexus serves, its version, its format, and the
+    SHA-256 of the bytes the profile pins."""
+
+    model_config = ConfigDict(frozen=True)
+
+    pack: str
+    version: int
+    format: str
+    sha256: str
+
+
+class Column(BaseModel):
+    """One workbook column that scores a category, and the method and category it scores."""
+
+    model_config = ConfigDict(frozen=True)
+
+    header: str
+    method: str
+    category: str
+
+
+class Total(BaseModel):
+    """The workbook column that sums one method's categories."""
+
+    model_config = ConfigDict(frozen=True)
+
+    header: str
+    method: str
+
+
+class Validated(BaseModel):
+    """What a validated run measured: the processes and columns it compared, how many known
+    differences it holds, and the largest relative difference inside the tolerance."""
+
+    model_config = ConfigDict(frozen=True)
+
+    processes: int
+    columns: int
+    differences: int
+    largest_rel_diff_within_tolerance: float
+
+
+class Profile(BaseModel):
+    """One BAFU release's pinned facts: its inventory and results packs, the workbook's columns and
+    total, the file names of its two lists, and the tolerance and counts a validated run
+    recorded."""
+
+    model_config = ConfigDict(frozen=True)
+
+    version: str
+    server: str
+    citation: str
+    inventory: Pack
+    results: Pack
+    columns: tuple[Column, ...]
+    total: Total
+    unlinked: str
+    differences: str
+    tolerance: float
+    validated: Validated | None = None
+
+
+def versions() -> tuple[str, ...]:
+    """The release labels the profiles directory carries, newest first."""
+    return profiles.versions(RELEASES)
+
+
+def recorded(name: str) -> Path | None:
+    """The list file a profile names, or nothing when the release has not recorded it yet."""
+    return profiles.recorded(RELEASES, name)
+
+
+def read_profile(version: str) -> Profile:
+    """The profile named for a release label; a label without one is not found."""
+    return profiles.read_profile(Profile, RELEASES, version)