kgm 2.0.0__py3-none-any.whl

kgm/__init__.py

@@ -0,0 +1,13 @@
+import importlib.metadata
+
+__version__ = importlib.metadata.version(__package__)
+
+from .documentor import (
+    catalogue as create_catalogue,
+)
+from .documentor import (
+    table as create_table,
+)
+from .labeller import label as label
+from .loader import load as load
+from .validator import validate as validate

kgm/cli/__init__.py

@@ -0,0 +1,18 @@
+from kgm.cli.app import app
+from kgm.cli.commands.label import app as label_app
+from kgm.cli.commands.validate import validate_command
+
+app.add_typer(label_app, name="label")
+
+from kgm.cli.commands.document import app as document_app
+
+app.add_typer(document_app, name="document")
+
+from kgm.cli.commands.load import app as load_app
+
+app.add_typer(load_app, name="load")
+
+from kgm.cli.commands.event.cli import event_app
+from kgm.cli.commands.sync import sync_command
+
+app.add_typer(event_app)

kgm/cli/app.py

@@ -0,0 +1,60 @@
+import logging
+import os
+import sys
+from typing import Annotated
+
+import typer
+
+from kgm import __version__
+from kgm.cli.console import console
+
+
+# Configure logging
+def setup_logging():
+    """Set up logging configuration for the CLI."""
+    # Get log level from environment or default to WARNING
+    log_level_str = os.getenv("KGM_LOG_LEVEL", "WARNING").upper()
+    log_level = getattr(logging, log_level_str, logging.WARNING)
+
+    # Create logger
+    logger = logging.getLogger("kgm")
+    logger.setLevel(log_level)
+
+    # Remove any existing handlers
+    for handler in logger.handlers[:]:
+        logger.removeHandler(handler)
+
+    # Create console handler
+    console_handler = logging.StreamHandler(sys.stderr)
+    console_handler.setLevel(log_level)
+
+    # Create formatter
+    formatter = logging.Formatter(
+        "%(asctime)s [%(levelname)s] %(name)s:%(lineno)d %(message)s"
+    )
+    console_handler.setFormatter(formatter)
+
+    # Add handler to logger
+    logger.addHandler(console_handler)
+
+
+# Set up logging when the module is imported
+setup_logging()
+
+app = typer.Typer(
+    invoke_without_command=True,
+    context_settings={
+        "help_option_names": ["-h", "--help"],
+    },
+    add_completion=False,
+)
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    version: Annotated[bool, typer.Option("--version", "-v", is_eager=True)] = False,
+):
+    """KGM top-level Command Line Interface. Ask for help (-h) for each Command"""
+    if version:
+        console.print(__version__)
+        raise typer.Exit()

kgm/cli/commands/document.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+import typer
+
+from kgm.documentor import TableFormats, catalogue, table
+
+app = typer.Typer(help="Create documentation from a KGM")
+
+
+@app.command(
+    name="table",
+    help="Create a Markdown or ASCIIDOC table for the resources listed in a KGM",
+)
+def table_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be documented"
+    ),
+    table_format: TableFormats = typer.Option(
+        TableFormats.markdown,
+        "--format",
+        "-f",
+        help="The format of the table to be created",
+    ),
+) -> None:
+    print(table(manifest, table_format))
+
+
+@app.command(
+    name="catalogue",
+    help="Add the resources listed in a KGM to a catalogue RDF file",
+)
+def catalogue_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be documented"
+    ),
+) -> None:
+    print(catalogue(manifest).serialize(format="longturtle"))

kgm/cli/commands/event/asb.py

@@ -0,0 +1,52 @@
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from kgm.event.asb_client import AzureServiceBusEventClient
+from kgm.event.syncer import sync_rdf_delta
+from kgm.utils import make_httpx_client
+
+app = typer.Typer()
+
+
+@app.command(
+    name="azure-service-bus",
+    help="Synchronize a KGM's resources by sending RDF patch logs to Azure Service Bus.",
+)
+def event_sync_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be loaded"
+    ),
+    endpoint: str = typer.Argument(..., help="The URL of the SPARQL Endpoint"),
+    connection: str = typer.Argument(
+        ..., help="The Azure Service Bus connection string"
+    ),
+    topic: str = typer.Argument(..., help="The Azure Service Bus topic name"),
+    subscription: str = typer.Argument(
+        ..., help="The Azure Service Bus subscription name"
+    ),
+    session: str = typer.Argument(..., help="The Azure Service Bus session ID"),
+    websocket: bool = typer.Option(False, "--websocket", help="Use WebSockets"),
+    username: Annotated[
+        str, typer.Option("--username", "-u", help="SPARQL Endpoint username")
+    ] = None,
+    password: Annotated[
+        str, typer.Option("--password", "-p", help="SPARQL Endpoint password")
+    ] = None,
+    timeout: Annotated[
+        int, typer.Option("--timeout", "-t", help="Timeout per request")
+    ] = 60,
+):
+    cwd = Path.cwd()
+    http_client = make_httpx_client(username, password, timeout)
+    event_client = AzureServiceBusEventClient(
+        connection, topic, subscription, session, websocket
+    )
+    try:
+        sync_rdf_delta(cwd, manifest, endpoint, http_client, event_client)
+        print(
+            "The KGM synchronization event has been sent to Azure Service Bus."
+        )
+    finally:
+        http_client.close()

kgm/cli/commands/event/cli.py

@@ -0,0 +1,19 @@
+import typer
+
+from kgm.cli.commands.event.rdf_delta import app as rdf_delta_app
+
+event_app = typer.Typer(name="event", help="Event-based KGMs actions")
+sync_app = typer.Typer()
+event_app.add_typer(
+    sync_app, name="sync", help="Synchronize a KGM with an event system"
+)
+sync_app.add_typer(rdf_delta_app)
+
+try:
+    import azure.servicebus
+
+    from kgm.cli.commands.event.asb import app as asb_app
+
+    sync_app.add_typer(asb_app)
+except ImportError:
+    pass

kgm/cli/commands/event/rdf_delta.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from kgm.event.client import DeltaEventClient
+from kgm.event.syncer import sync_rdf_delta
+from kgm.utils import make_httpx_client
+
+app = typer.Typer()
+
+
+@app.command(
+    name="rdf-delta",
+    help="Synchronize a KGM's resources by sending RDF patch logs to RDF Delta.",
+)
+def event_sync_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be loaded"
+    ),
+    endpoint: str = typer.Argument(..., help="The URL of the SPARQL Endpoint"),
+    delta_url: str = typer.Argument(..., help="The URL of the RDF Delta endpoint"),
+    delta_datasource: str = typer.Argument(
+        ..., help="The name of the RDF Delta datasource"
+    ),
+    username: Annotated[
+        str, typer.Option("--username", "-u", help="SPARQL Endpoint username")
+    ] = None,
+    password: Annotated[
+        str, typer.Option("--password", "-p", help="SPARQL Endpoint password")
+    ] = None,
+    timeout: Annotated[
+        int, typer.Option("--timeout", "-t", help="Timeout per request")
+    ] = 60,
+):
+    cwd = Path.cwd()
+    http_client = make_httpx_client(username, password, timeout)
+    event_client = DeltaEventClient(delta_url, delta_datasource)
+    try:
+        sync_rdf_delta(cwd, manifest, endpoint, http_client, event_client)
+        print("The KGM synchronization event has been sent to RDF Delta.")
+    finally:
+        http_client.close()

kgm/cli/commands/label.py

@@ -0,0 +1,97 @@
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from kgm.labeller import LabellerOutputTypes, label
+from kgm.utils import make_httpx_client
+
+app = typer.Typer(
+    help="Discover labels missing from data in a in a KGM and patch them"
+)
+
+
+@app.command(
+    name="iris",
+    help="Find all the IRIs of objects in the Manifest's resources without labels",
+)
+def iris_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be labelled"
+    ),
+    context: str = typer.Argument(
+        None,
+        help="The path of an RDF file, a directory of RDF files or the URL of a SPARQL endpoint from which to obtain labels",
+    ),
+    username: Annotated[
+        str, typer.Option("--username", "-u", help="SPARQL Endpoint username")
+    ] = None,
+    password: Annotated[
+        str, typer.Option("--password", "-p", help="SPARQL Endpoint password")
+    ] = None,
+) -> None:
+    for iri in label(
+        manifest,
+        LabellerOutputTypes.iris,
+        context,
+        make_httpx_client(username, password),
+    ):
+        print(str(iri))
+
+
+@app.command(
+    name="rdf",
+    help="Create labels for all the objects in the Manifest's resources without labels",
+)
+def rdf_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be labelled"
+    ),
+    context: str = typer.Argument(
+        None,
+        help="The path of an RDF file, a directory of RDF files or the URL of a SPARQL endpoint from which to obtain labels",
+    ),
+    username: Annotated[
+        str, typer.Option("--username", "-u", help="SPARQL Endpoint username")
+    ] = None,
+    password: Annotated[
+        str, typer.Option("--password", "-p", help="SPARQL Endpoint password")
+    ] = None,
+) -> None:
+    print(
+        label(
+            manifest,
+            LabellerOutputTypes.rdf,
+            context,
+            make_httpx_client(username, password),
+        ).serialize(format="longturtle")
+    )
+
+
+@app.command(
+    name="manifest",
+    help="Create labels for all the objects in the Manifest's resources without labels and store them as a new Manifest resource",
+)
+def manifest_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be labelled"
+    ),
+    context: Path = typer.Argument(
+        ...,
+        help="The path of an RDF file, a directory of RDF files or the URL of a SPARQL endpoint from which to obtain labels",
+    ),
+    username: Annotated[
+        str, typer.Option("--username", "-u", help="SPARQL Endpoint username")
+    ] = None,
+    password: Annotated[
+        str, typer.Option("--password", "-p", help="SPARQL Endpoint password")
+    ] = None,
+) -> None:
+    label(
+        manifest,
+        LabellerOutputTypes.manifest,
+        context,
+        make_httpx_client(username, password),
+    )
+
+    print("A new Resource containing labels has been added to the Manifest")

kgm/cli/commands/load.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from kgm.loader import load
+
+app = typer.Typer(help="Load a KGM's content into a file or DB")
+
+
+@app.command(
+    name="sparql", help="Load a KGM's resources into a SPARQL Endpoint"
+)
+def sparql_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be loaded"
+    ),
+    endpoint: str = typer.Argument(..., help="The URL of the SPARQL Endpoint"),
+    username: Annotated[
+        str, typer.Option("--username", "-u", help="SPARQL Endpoint username.")
+    ] = None,
+    password: Annotated[
+        str, typer.Option("--password", "-p", help="SPARQL Endpoint password.")
+    ] = None,
+    timeout: Annotated[
+        int, typer.Option("--timeout", "-t", help="Timeout per request")
+    ] = 60,
+) -> None:
+    load(
+        manifest,
+        sparql_endpoint=endpoint,
+        sparql_username=username,
+        sparql_password=password,
+        timeout=timeout,
+    )
+
+
+@app.command(
+    name="file", help="Load a KGM's resources into a single RDF quads file"
+)
+def file_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be loaded"
+    ),
+    file: Path = typer.Argument(..., help="The path of the quads file"),
+) -> None:
+    load(manifest, destination_file=file)

kgm/cli/commands/sync.py

@@ -0,0 +1,75 @@
+import collections
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.table import Table
+
+from kgm.cli.app import app
+from kgm.cli.console import console
+from kgm.syncer import sync
+from kgm.utils import make_httpx_client
+
+
+@app.command(
+    name="sync",
+    help="Synchronize a KGM's resources with loaded copies of them in a SPARQL Endpoint",
+)
+def sync_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be loaded"
+    ),
+    endpoint: str = typer.Argument(..., help="The URL of the SPARQL Endpoint"),
+    update_remote: bool = typer.Argument(
+        True, help="Copy more recent local artifacts to DB"
+    ),
+    update_local: bool = typer.Argument(
+        True, help="Copy more recent DB artifacts to local"
+    ),
+    add_remote: bool = typer.Argument(True, help="Add new local artifacts to DB"),
+    add_local: bool = typer.Argument(True, help="Add new DB artifacts to local"),
+    username: Annotated[
+        str, typer.Option("--username", "-u", help="SPARQL Endpoint username")
+    ] = None,
+    password: Annotated[
+        str, typer.Option("--password", "-p", help="SPARQL Endpoint password")
+    ] = None,
+    timeout: Annotated[
+        int, typer.Option("--timeout", "-t", help="Timeout per request")
+    ] = 60,
+    response_format: str = typer.Option(
+        "table",
+        "--response-format",
+        "-f",
+        help="The response format of the SPARQL query. Either 'table' (default) or 'json'",
+    ),
+) -> None:
+    r = sync(
+        manifest,
+        endpoint,
+        make_httpx_client(username, password, timeout),
+        update_remote,
+        update_local,
+        add_remote,
+        add_local,
+    )
+
+    if response_format == "json":
+        print(json.dumps(r, indent=4))
+    else:
+        console.print(result_as_rich_table(r))
+
+
+def result_as_rich_table(sync_status: dict):
+    t = Table()
+    t.add_column("Artifact")
+    t.add_column("Main Entity")
+    t.add_column("Direction")
+
+    for k, v in collections.OrderedDict(sorted(sync_status.items())).items():
+        t.add_row(str(k), str(v["main_entity"]), v["direction"])
+
+    # json.dumps(sync_status, indent=4)
+
+    return t

kgm/cli/commands/validate.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+import typer
+
+from kgm.cli.app import app
+from kgm.validator import validate
+
+
+@app.command(
+    name="validate", help="Validate the structure and content of a KGM"
+)
+def validate_command(
+    manifest: Path = typer.Argument(
+        ..., help="The path of the KGM file to be validated"
+    ),
+) -> None:
+    validate(manifest)

kgm/cli/console.py

@@ -0,0 +1,3 @@
+from rich.console import Console
+
+console = Console()

kgm/definednamespaces.py

@@ -0,0 +1,41 @@
+from rdflib.namespace import DefinedNamespace, Namespace
+from rdflib.term import URIRef
+
+
+class MRR(DefinedNamespace):
+    _NS = Namespace("https://prez.dev/ManifestResourceRoles/")
+    _fail = True
+
+    CatalogueData: URIRef
+    CatalogueModel: URIRef
+    ResourceData: URIRef
+    ResourceModel: URIRef
+    CatalogueAndResourceModel: URIRef
+    CompleteCatalogueAndResourceLabels: URIRef
+    IncompleteCatalogueAndResourceLabels: URIRef
+
+
+class OLIS(DefinedNamespace):
+    _NS = Namespace("https://olis.dev/")
+    _fail = True
+
+    NamedGraph: URIRef
+    RealGraph: URIRef
+    SystemGraph: URIRef
+    VirtualGraph: URIRef
+
+    isAliasFor: URIRef
+
+
+class PREZ(DefinedNamespace):
+    _NS = Namespace("https://prez.dev/")
+    _fail = True
+
+    Manifest: URIRef
+
+
+class MVT(DefinedNamespace):
+    _NS = Namespace("https://prez.dev/ManifestVersionTypes/")
+    _fail = True
+
+    GitCommitHash: URIRef