prezmanifest 0.2.2__tar.gz

prezmanifest-0.2.2/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2024-2025, KurrawongAI
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

prezmanifest-0.2.2/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.1
+Name: prezmanifest
+Version: 0.2.2
+Summary: A Python package that provides a series of functions to work with Prez Manifests.
+License: BSD-3-Clause
+Author: Nicholas Car
+Author-email: nick@kurrawong.ai
+Requires-Python: >=3.12,<4.0
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: kurra (>=0.6.1,<0.7.0)
+Requires-Dist: labelify (>=0.3.8,<0.4.0)
+Requires-Dist: pyshacl (>=0.29.0,<0.30.0)
+Description-Content-Type: text/plain
+
+= Prez Manifest
+
+A Prez Manifest is an RDF file that describes and links to a set of resources that can be loaded into an RDF database for the http://prez.dev[Prez graph database publication system] to provide access to.
+
+The Prez Manifest specification is online at:
+
+* https://prez.dev/manifest/
+
+This repository contains the `prez-manifest` Python package that provides a series of functions to work with Prez Manifests. The functions provided are:
+
+* `create_table`: creates an ASCIIDOC or Markdown table of Manifest content from a Manifest file
+* `validate`: validates that a Manifest file conforms to the specification and that all linked-to assets are available
+* `load`: loads a Manifest file, and all the content it specifies, into either an n-quads file or a Fuseki database
+
+
+== Installation & Use
+
+This Python package is intended to be used on the command line on Linux/UNIX-like systems and/or as a Python library, called directly from other Python code.
+
+It can be installed using https://python-poetry.org[Poetry]. It may one day be published on https://pypi.org[PyPI].
+
+Please see the `documentor.py`, `loader.py`, & `validator.py` files in the `prezmanifest` folder and the test files in `test` for documentation text and examples of use.
+
+
+== Testing
+
+Run `python -m pytest` in the top-level folder to test. You must have Docker Desktop running to allow all loader tests to be executed.
+
+
+== License
+
+This code is available for reuse according to the https://opensource.org/license/bsd-3-clause[BSD 3-Clause License].
+
+&copy; 2024-2025 KurrawongAI
+
+
+== Contact
+
+For all matters, please contact:
+
+*KurrawongAI* +
+info@kurrawong.ai +
+https://kurrawong.ai +

prezmanifest-0.2.2/README.adoc

@@ -0,0 +1,43 @@
+= Prez Manifest
+
+A Prez Manifest is an RDF file that describes and links to a set of resources that can be loaded into an RDF database for the http://prez.dev[Prez graph database publication system] to provide access to.
+
+The Prez Manifest specification is online at:
+
+* https://prez.dev/manifest/
+
+This repository contains the `prez-manifest` Python package that provides a series of functions to work with Prez Manifests. The functions provided are:
+
+* `create_table`: creates an ASCIIDOC or Markdown table of Manifest content from a Manifest file
+* `validate`: validates that a Manifest file conforms to the specification and that all linked-to assets are available
+* `load`: loads a Manifest file, and all the content it specifies, into either an n-quads file or a Fuseki database
+
+
+== Installation & Use
+
+This Python package is intended to be used on the command line on Linux/UNIX-like systems and/or as a Python library, called directly from other Python code.
+
+It can be installed using https://python-poetry.org[Poetry]. It may one day be published on https://pypi.org[PyPI].
+
+Please see the `documentor.py`, `loader.py`, & `validator.py` files in the `prezmanifest` folder and the test files in `test` for documentation text and examples of use.
+
+
+== Testing
+
+Run `python -m pytest` in the top-level folder to test. You must have Docker Desktop running to allow all loader tests to be executed.
+
+
+== License
+
+This code is available for reuse according to the https://opensource.org/license/bsd-3-clause[BSD 3-Clause License].
+
+© 2024-2025 KurrawongAI
+
+
+== Contact
+
+For all matters, please contact:
+
+*KurrawongAI* +
+info@kurrawong.ai +
+https://kurrawong.ai +

prezmanifest-0.2.2/prezmanifest/__init__.py

@@ -0,0 +1,9 @@
+import importlib.metadata
+
+__version__ = importlib.metadata.version(__package__)
+
+from .definednamespaces import MRR, OLIS, PREZ
+from .validator import validate
+from .documentor import create_table
+from .loader import load
+from .labeller import label

prezmanifest-0.2.2/prezmanifest/definednamespaces.py

@@ -0,0 +1,37 @@
+from rdflib.namespace import DefinedNamespace, Namespace
+from rdflib.term import URIRef
+
+
+class MRR(DefinedNamespace):
+
+    _NS = Namespace("https://prez.dev/ManifestResourceRoles/")
+    _fail = True
+
+    CatalogueData: URIRef
+    CatalogueModel: URIRef
+    ResourceData: URIRef
+    ResourceModel: URIRef
+    CatalogueAndResourceModel: URIRef
+    CompleteCatalogueAndResourceLabels: URIRef
+    IncompleteCatalogueAndResourceLabels: URIRef
+
+
+class OLIS(DefinedNamespace):
+
+    _NS = Namespace("https://olis.dev/")
+    _fail = True
+
+    NamedGraph: URIRef
+    RealGraph: URIRef
+    SystemGraph: URIRef
+    VirtualGraph: URIRef
+
+    isAliasFor: URIRef
+
+
+class PREZ(DefinedNamespace):
+
+    _NS = Namespace("https://prez.dev/")
+    _fail = True
+
+    Manifest: URIRef

prezmanifest-0.2.2/prezmanifest/documentor.py

@@ -0,0 +1,209 @@
+"""
+This script creates a "Prez Resources" table in either Markdown or ASCIIDOC from a Manifest file which it validates first
+
+Run this script with the -h flag for more help, i.e. ~$ python documentor.py -h
+
+Example:
+
+Input:
+
+PREFIX mrr: <https://prez.dev/ManifestResourceRoles/>
+PREFIX prez: <https://prez.dev/>
+PREFIX prof: <http://www.w3.org/ns/dx/prof/>
+PREFIX schema: <https://schema.org/>
+
+[]
+    a prez:Manifest ;
+    prof:hasResource
+        [
+            prof:hasArtifact "catalogue.ttl" ;
+            prof:hasRole mrr:ContainerData ;
+            schema:description "The definition of, and medata for, the container which here is a dcat:Catalog object" ;
+            schema:name "Catalogue Definition"
+        ] ,
+        [
+            prof:hasArtifact "vocabs/*.ttl" ;
+            prof:hasRole mrr:ContentData ;
+            schema:description "skos:ConceptScheme objects in RDF (Turtle) files in the vocabs/ folder" ;
+            schema:name "Content"
+        ] ,
+        [
+            prof:hasArtifact "https://github.com/RDFLib/prez/blob/main/prez/reference_data/profiles/ogc_records_profile.ttl" ;
+            prof:hasRole mrr:ContainerAndContentModel ;
+            schema:description "The default Prez profile for Records API" ;
+            schema:name "Profile Definition"
+        ] ,
+        [
+            prof:hasArtifact "_background/labels.ttl" ;
+            prof:hasRole mrr:CompleteContainerAndContentLabels ;
+            schema:description "An RDF file containing all the labels for the container content" ;
+        ] ;
+    .
+
+Output:
+
+Resource | Role | Description
+--- | --- | ---
+Catalogue Definition, [`catalogue.ttl`](catalogue.ttl) | [Container Data](https://prez.dev/ManifestResourceRoles/ContainerData) | The definition of, and medata for, the container which here is a dcat:Catalog object
+Content, [`vocabs/*.ttl`](vocabs/*.ttl) | [Content Data](https://prez.dev/ManifestResourceRoles/ContentData) | skos:ConceptScheme objects in RDF (Turtle) files in the vocabs/ folder
+Profile Definition, [`ogc_records_profile.ttl`](https://github.com/RDFLib/prez/blob/main/prez/reference_data/profiles/ogc_records_profile.ttl) | [Container & Content Model](https://prez.dev/ManifestResourceRoles/ContainerAndContentModel) | The default Prez profile for Records API
+Labels file, [`_background/labels.ttl`](_background/labels.ttl) | [Complete Content and Container Labels](https://prez.dev/ManifestResourceRoles/CompleteContainerAndContentLabels) | An RDF file containing all the labels for the container content
+"""
+
+import argparse
+import sys
+from pathlib import Path
+from urllib.parse import ParseResult, urlparse
+
+from rdflib import DCAT, DCTERMS, PROF, SDO, SKOS, Graph
+from kurra.utils import load_graph
+
+try:
+    from prezmanifest import MRR, validate, __version__
+    from prezmanifest.utils import *
+except ImportError:
+    import sys
+
+    sys.path.append(str(Path(__file__).parent.parent.resolve()))
+    from prezmanifest import MRR, validate, __version__
+    from prezmanifest.utils import *
+
+
+def create_table(manifest: Path, t="markdown") -> str:
+    # load and validate manifest
+    validate(manifest)
+    manifest_graph = load_graph(manifest)
+
+    # add in MRR vocab
+    manifest_graph += load_graph(Path(__file__).parent / "mrr.ttl")
+
+    if t == "asciidoc":
+        header = "|===\n| Resource | Role | Description\n\n"
+    else:
+        header = "Resource | Role | Description\n--- | --- | ---\n"
+
+    body = ""
+    for s, o in manifest_graph.subject_objects(PROF.hasResource):
+        a = str(manifest_graph.value(o, PROF.hasArtifact))
+        if t == "asciidoc":
+            artifact = f'link:{a}[`{a.split("/")[-1] if a.startswith("http") else a}`]'
+        else:
+            artifact = f'[`{a.split("/")[-1] if a.startswith("http") else a}`]({a})'
+        role_iri = manifest_graph.value(o, PROF.hasRole)
+        role_label = manifest_graph.value(role_iri, SKOS.prefLabel)
+        if t == "asciidoc":
+            role = f"{role_iri}[{role_label}]"
+        else:
+            role = f"[{role_label}]({role_iri})"
+        name = manifest_graph.value(o, SDO.name)
+        description = manifest_graph.value(o, SDO.description)
+        n = f"{name}, {artifact}" if name is not None else f"{artifact}"
+        d = description if description is not None else ""
+        if t == "asciidoc":
+            body += f"| {n} | {role} | {d}\n"
+        else:
+            body += f"{n} | {role} | {d}\n"
+
+    if t == "asciidoc":
+        footer = "|===\n"
+    else:
+        footer = ""
+
+    return (header + body + footer).strip()
+
+
+def create_catalogue(manifest: Path):
+    MANIFEST_ROOT_DIR = manifest.parent
+    # load and validate manifest
+    validate(manifest)
+    manifest_graph = load_graph(manifest)
+
+    for s, o in manifest_graph.subject_objects(PROF.hasResource):
+        for role in manifest_graph.objects(o, PROF.hasRole):
+            if role == MRR.CatalogueData:
+                for artifact in manifest_graph.objects(o, PROF.hasArtifact):
+                    # the artifact can only be a triples file (not a quads file)
+                    catalogue = load_graph(MANIFEST_ROOT_DIR / artifact)
+
+    # get the IRI of the catalogue
+    catalogue_iri = catalogue.value(predicate=RDF.type, object=DCAT.Catalog)
+
+    # non-catalogue resources
+    for s, o in manifest_graph.subject_objects(PROF.hasResource):
+        for role in manifest_graph.objects(o, PROF.hasRole):
+            if role in [
+                MRR.CompleteCatalogueAndResourceLabels,
+                MRR.IncompleteCatalogueAndResourceLabels,
+                MRR.ResourceData,
+            ]:
+                for artifact in manifest_graph.objects(o, PROF.hasArtifact):
+                    for f in get_files_from_artifact(manifest, artifact):
+                        for iri in sorted(get_identifier_from_file(f)):
+                            if iri != URIRef("urn:x-rdflib:default"):
+                                catalogue.add((catalogue_iri, DCTERMS.hasPart, iri))
+
+    return catalogue
+
+
+def setup_cli_parser(args=None):
+    def url_file_or_folder(input: str) -> ParseResult | Path:
+        parsed = urlparse(input)
+        if all([parsed.scheme, parsed.netloc]):
+            return parsed
+        path = Path(input)
+        if path.is_file():
+            return path
+        if path.is_dir():
+            return path
+        raise argparse.ArgumentTypeError(
+            f"{input} is not a valid input. Must be a file, folder or sparql endpoint"
+        )
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version="{version}".format(version=__version__),
+    )
+
+    parser.add_argument(
+        "function",
+        help="The documentation function you wish to perform",
+        choices=["table", "catalogue"],
+    )
+
+    parser.add_argument(
+        "-t",
+        "--type",
+        help="The type of markup you want to export: Markdown or ASCCIDOC. Only relevant for the 'table' function",
+        choices=["markdown", "asciidoc"],
+        default="markdown",
+    )
+
+    parser.add_argument(
+        "input",
+        help="A Prez Manifest file",
+        type=url_file_or_folder,
+    )
+
+    return parser.parse_args(args)
+
+
+def cli(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    args = setup_cli_parser(args)
+
+    if args.function == "table":
+        print(create_table(args.input, t=args.type))
+    else:
+        print(create_catalogue(args.input).serialize(format="longturtle"))
+
+
+if __name__ == "__main__":
+    retval = cli(sys.argv[1:])
+    if retval is not None:
+        sys.exit(retval)

prezmanifest-0.2.2/prezmanifest/get_iris_from_files.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+from rdflib import Graph
+from rdflib.namespace import RDF, SKOS
+
+FILES_DIR = Path("something")
+
+for f in FILES_DIR.glob("*.ttl"):
+    g = Graph()
+    g.parse(f)
+    iri = g.value(predicate=RDF.type, object=SKOS.ConceptScheme)
+
+    # print(f"{f.name}, {iri}")
+
+    print(f"<{iri}> ,")

prezmanifest-0.2.2/prezmanifest/labeller.py

@@ -0,0 +1,188 @@
+"""
+Assesses a given Manifest, finds any IRIs in any of the given resources missing labels and tries to patch them from
+a given source of labels, such as KurrawongAI's Semantic Background (https://github.com/kurrawong/semantic-background)
+repository.
+"""
+import argparse
+import sys
+from pathlib import Path
+from typing import Literal as TLiteral
+from urllib.parse import ParseResult, urlparse
+
+from kurra.utils import load_graph
+from labelify import find_missing_labels, extract_labels
+from rdflib import Graph, BNode, Literal
+from rdflib.namespace import PROF, RDF
+
+from prezmanifest.utils import get_files_from_artifact
+
+try:
+    from prezmanifest import MRR, OLIS, validate, load, __version__
+except ImportError:
+    import sys
+
+    sys.path.append(str(Path(__file__).parent.parent.resolve()))
+    from prezmanifest import MRR, OLIS, validate, __version__
+
+
+def label(
+    manifest: Path,
+    output: TLiteral["iris", "rdf", "manifest"] = "manifest",
+    additional_context: Path | str | Graph = None,
+) -> set | Graph | None:
+    """"Main function for labeller module"""
+    # create the target from the Manifest
+    manifest_content_graph = load(manifest, return_data_type="Graph")
+
+    output_types = ["iris", "rdf", "manifest"]
+    if output not in output_types:
+        raise ValueError(f"Parameter output is {output} but must be one of {', '.join(output_types)}")
+
+    # determine if any labelling context is given in Manifest
+    context_graph = Graph()
+    for s, o in manifest_content_graph.subject_objects(PROF.hasResource):
+        for role in manifest_content_graph.objects(o, PROF.hasRole):
+            if role in [
+                MRR.IncompleteCatalogueAndResourceLabels,
+                MRR.CompleteCatalogueAndResourceLabels,
+            ]:
+                for artifact in manifest_content_graph.objects(o, PROF.hasArtifact):
+                    artifact: Literal
+                    for f in get_files_from_artifact(manifest, artifact):
+                        context_graph += load_graph(f)
+
+    if output == "iris":
+        return find_missing_labels(manifest_content_graph + context_graph, additional_context)
+
+    elif output == "rdf":
+        iris = find_missing_labels(manifest_content_graph, context_graph)
+
+        if additional_context is not None:
+            return extract_labels(iris, additional_context)
+        else:
+            return None
+
+    else:  # output == manifest
+        # If this is selected, generate the "rdf" output and create a resource for it in the Manifest
+        # If there are no more missing labels then we have an mrr:CompleteCatalogueAndResourceLabels
+        # else add mrr:IncompleteCatalogueAndResourceLabels
+
+        # Generate labels for any IRIs missing them, using context given in the Manifest and any
+        # Additional Context supplied
+        manifest_only_graph = load_graph(manifest)
+        rdf_addition = label(manifest, "rdf", additional_context)
+
+        if len(rdf_addition) > 0:
+            new_artifact = manifest.parent / "labels-additional.ttl"
+            rdf_addition.serialize(destination=new_artifact, format="longturtle")
+            new_resource = BNode()
+
+            # Find the role of any context in the Manifest
+            manifest_iri = None
+            context_roles = []
+            for s, o in manifest_only_graph.subject_objects(PROF.hasResource):
+                manifest_iri = s
+                for role in manifest_only_graph.objects(o, PROF.hasRole):
+                    if role in [
+                        MRR.IncompleteCatalogueAndResourceLabels,
+                        MRR.CompleteCatalogueAndResourceLabels,
+                    ]:
+                        context_roles.append(role)
+
+            if MRR.CompleteCatalogueAndResourceLabels in context_roles and len(context_roles) == 1:
+                # If a CompleteCatalogueAndResourceLabels is present in Manifest and yet more labels were discovered,
+                # change CompleteCatalogueAndResourceLabels to IncompleteCatalogueAndResourceLabels and add another
+                for s, o in manifest_content_graph.subject_objects(PROF.hasRole):
+                    if o == MRR.CompleteCatalogueAndResourceLabels:
+                        manifest_only_graph.remove((s, PROF.hasRole, o))
+                        manifest_only_graph.add((manifest_iri, PROF.hasResource, new_resource))
+                        manifest_only_graph.add((new_resource, PROF.hasRole, MRR.IncompleteCatalogueAndResourceLabels))
+                        manifest_only_graph.add((new_resource, PROF.hasArtifact, Literal(new_artifact.name)))
+            else:
+                # If an IncompleteCatalogueAndResourceLabels was present, add another IncompleteCatalogueAndResourceLabels
+                # which together make a CompleteCatalogueAndResourceLabels
+
+                # If none was present, add an IncompleteCatalogueAndResourceLabels or a CompleteCatalogueAndResourceLabels
+                # TODO: test for completeness of labelling and add in CompleteCatalogueAndResourceLabels if complete
+                manifest_only_graph.add((manifest_iri, PROF.hasResource, new_resource))
+                manifest_only_graph.add((new_resource, PROF.hasRole, MRR.IncompleteCatalogueAndResourceLabels))
+                manifest_only_graph.add((new_resource, PROF.hasArtifact, Literal(new_artifact.name)))
+
+            manifest_only_graph.serialize(destination=manifest, format="longturtle")
+
+        else:
+            raise Warning("No new labels have been generated for content in this Manifest. "
+                          "This could be because none were missing or because no new labels can be found in any "
+                          "supplied additional context.")
+
+
+def setup_cli_parser(args=None):
+    def url_file_or_folder(input: str) -> ParseResult | Path:
+        parsed = urlparse(input)
+        if all([parsed.scheme, parsed.netloc]):
+            return parsed
+        path = Path(input)
+        if path.is_file():
+            return path
+        if path.is_dir():
+            return path
+        raise argparse.ArgumentTypeError(
+            f"{input} is not a valid input. Must be a file, folder or sparql endpoint"
+        )
+
+    parser = argparse.ArgumentParser()
+    group = parser.add_mutually_exclusive_group(required=True)
+
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version="{version}".format(version=__version__),
+    )
+
+    group.add_argument(
+        "-o",
+        "--output",
+        help="The form of output you want",
+        choices=["iris", "rdf", "manifest"],
+        default="manifest",
+    )
+
+    parser.add_argument(
+        "-a",
+        "--additional-context",
+        help="File, Folder or Sparql Endpoint to read additional context RDF from",
+        type=url_file_or_folder,
+    )
+
+    parser.add_argument(
+        "manifest",
+        help="A Manifest file to process",
+        type=Path,
+    )
+
+    return parser.parse_args(args)
+
+
+def cli(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    args = setup_cli_parser(args)
+
+    x = label(args.manifest, args.output, args.additional_context)
+
+    if args.output == "iris":
+        print("\n".join([str(iri) for iri in x]))
+    elif args.output == "rdf":
+        if x is not None:
+            print(x.serialize(format="longturtle"))
+
+    else:  # manifest
+        pass
+
+
+if __name__ == "__main__":
+    retval = cli(sys.argv[1:])
+    if retval is not None:
+        sys.exit(retval)

prezmanifest-0.2.2/prezmanifest/loader.py

@@ -0,0 +1,244 @@
+"""
+Either creates an n-quads files containing the content of a Manifest file or uploads the content to Fuseki.
+
+It creates:
+
+ 1. A Named Graph for each resource using the item's IRI as the graph IRI
+ 2. A Named Graph for the catalogue, either using the catalogue's IRI as the graph IRI + "-catalogue" if given, or by making one up - a Blank Node
+ 3. All the triples in resources with roles mrr:CompleteCatalogueAndResourceLabels & mrr:IncompleteCatalogueAndResourceLabels within a Named Graph with IRI <https://background>
+ 4. An Olis Virtual Graph, <https://olis.dev/VirtualGraph> object using the catalogue IRI, if give, which is as an alias for all the Named Graphs from 1., 2. & 3.
+ 5. Multiple entries in the System Graph - Named Graph with IRI <https://olis.dev/SystemGraph> - for each Named and the Virtual Graph from 1., 2. & 3.
+
+Run this script with the -h flag for more help, i.e. ~$ python loader.py -h
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+from kurra.format import make_dataset, export_quads
+from kurra.fuseki import upload
+from kurra.utils import load_graph
+from rdflib import DCAT, DCTERMS, OWL, PROF, RDF, SDO, SKOS
+from rdflib import Graph, URIRef, Dataset
+from typing import Literal as TLiteral
+import logging
+
+try:
+    from prezmanifest import MRR, OLIS, validate, __version__
+    from prezmanifest.utils import get_files_from_artifact
+except ImportError:
+    import sys
+
+    sys.path.append(str(Path(__file__).parent.parent.resolve()))
+    from prezmanifest import MRR, OLIS, validate, __version__
+    from prezmanifest.utils import get_files_from_artifact
+
+
+def load(
+    manifest: Path,
+    sparql_endpoint: str = None,
+    destination_file: Path = None,
+    return_data_type: TLiteral["Graph", "Dataset", None] = None
+) -> None | Graph | Dataset:
+    """Loads a catalogue of data from a prezmanifest file, whose content are valid according to the Prez Manifest Model
+    (https://kurrawong.github.io/prez.dev/manifest/) either into a specified quads file in the Trig format, or into a
+    given SPARQL Endpoint."""
+
+    if return_data_type == "Dataset":
+        dataset_holder = Dataset()
+
+    if return_data_type == "Graph":
+        graph_holder = Graph()
+
+    return_data_value_error_message = "return_data_type was set to an invalid value. Must be one of Dataset or Graph or None"
+
+    def _export(data: Graph | Dataset, iri, sparql_endpoint, destination_file, return_data_type, append=False):
+        if type(data) is Dataset:
+            if iri is not None:
+                raise ValueError("If the data is a Dataset, the parameter iri must be None")
+
+            if destination_file is not None:
+                export_quads(data, destination_file)
+            elif sparql_endpoint is not None:
+                for g in data.graphs():
+                    if g.identifier != URIRef("urn:x-rdflib:default"):
+                        _export(g, g.identifier, sparql_endpoint, None, None)
+            else:
+                if return_data_type == "Dataset":
+                    return data
+                elif return_data_type == "Graph":
+                    gx = Graph()
+                    for g in data.graphs():
+                        if g.identifier != URIRef("urn:x-rdflib:default"):
+                            for s, p, o in g.triples((None, None, None)):
+                                gx.add((s, p, o))
+                    return gx
+
+        elif type(data) is Graph:
+            if iri is None:
+                raise ValueError("If the data is a GRaph, the parameter iri must not be None")
+
+            msg = f"exporting {iri} "
+            if destination_file is not None:
+                msg += f"to file {destination_file} "
+                export_quads(make_dataset(data, iri), destination_file)
+            elif sparql_endpoint is not None:
+                msg += f"to SPARQL Endpoint {sparql_endpoint}"
+                upload(sparql_endpoint, data, iri, append)
+            else:  # returning data
+                if return_data_type == "Dataset":
+                    msg += "to Dataset"
+                    for s, p, o in data:
+                        dataset_holder.add((s, p, o, iri))
+                elif return_data_type == "Graph":
+                    msg += "to Graph"
+                    for s, p, o in data:
+                        graph_holder.add((s, p, o))
+                else:
+                    raise ValueError(return_data_value_error_message)
+
+            logging.info(msg)
+
+    if sum(x is not None for x in [sparql_endpoint, destination_file, return_data_type]) != 1:
+        raise ValueError(
+            "You must specify exactly 1 of sparql_endpoint, destination_file or return_data_type",
+        )
+
+    MANIFEST_ROOT_DIR = manifest.parent
+    # load and validate manifest
+    validate(manifest)
+    manifest_graph = load_graph(manifest)
+
+    vg = Graph()
+    vg_iri = None
+
+    for s, o in manifest_graph.subject_objects(PROF.hasResource):
+        for role in manifest_graph.objects(o, PROF.hasRole):
+            # The catalogue - must be processed first
+            if role == MRR.CatalogueData:
+                for artifact in manifest_graph.objects(o, PROF.hasArtifact):
+                    # load the Catalogue, determine the Virtual Graph & Catalogue IRIs
+                    # and fail if we can't see a Catalogue object
+                    c = load_graph(MANIFEST_ROOT_DIR / str(artifact))
+                    vg_iri = c.value(predicate=RDF.type, object=DCAT.Catalog)
+                    if vg_iri is None:
+                        raise ValueError(
+                            f"ERROR: Could not create a Virtual Graph as no Catalog found in the Catalogue data"
+                        )
+                    catalogue_iri = URIRef(str(vg_iri) + "-catalogue")
+
+                    # add to the System Graph
+                    vg.add((vg_iri, RDF.type, OLIS.VirtualGraph))
+                    vg.add((vg_iri, OLIS.isAliasFor, catalogue_iri))
+                    vg_name = c.value(
+                        subject=vg_iri,
+                        predicate=SDO.name | DCTERMS.title | SKOS.prefLabel,
+                    ) or str(vg_iri)
+                    vg.add((vg_iri, SDO.name, vg_name))
+
+                    # export the Catalogue data
+                    _export(c, catalogue_iri, sparql_endpoint, destination_file, return_data_type)
+
+        # non-catalogue resources
+        for s, o in manifest_graph.subject_objects(PROF.hasResource):
+            for role in manifest_graph.objects(o, PROF.hasRole):
+                # The data files & background - must be processed after Catalogue
+                if role in [
+                    MRR.CompleteCatalogueAndResourceLabels,
+                    MRR.IncompleteCatalogueAndResourceLabels,
+                    MRR.ResourceData,
+                ]:
+                    for artifact in manifest_graph.objects(o, PROF.hasArtifact):
+                        for f in get_files_from_artifact(manifest, artifact):
+                            if str(f.name).endswith(".ttl"):
+                                fg = Graph().parse(f)
+                                # fg.bind("rdf", RDF)
+
+                                if role == MRR.ResourceData:
+                                    resource_iri = fg.value(predicate=RDF.type, object=SKOS.ConceptScheme) or fg.value(predicate=RDF.type, object=OWL.Ontology)
+
+                                if role in [
+                                    MRR.CompleteCatalogueAndResourceLabels,
+                                    MRR.IncompleteCatalogueAndResourceLabels,
+                                ]:
+                                    resource_iri = URIRef("http://background")
+
+                                if resource_iri is None:
+                                    raise ValueError(f"Could not determine Resource IRI for file {f}")
+
+                                vg.add((vg_iri, OLIS.isAliasFor, resource_iri))
+
+                                # export one Resource
+                                _export(fg, resource_iri, sparql_endpoint, destination_file, return_data_type)
+                            elif str(f.name).endswith(".trig"):
+                                d = Dataset()
+                                d.parse(f)
+                                for g in d.graphs():
+                                    if g.identifier != URIRef("urn:x-rdflib:default"):
+                                        vg.add((vg_iri, OLIS.isAliasFor, g.identifier))
+                                _export(d, None, sparql_endpoint, destination_file, return_data_type)
+
+
+        # export the System Graph
+        _export(vg, OLIS.SystemGraph, sparql_endpoint, destination_file, return_data_type, append=True)
+
+    if return_data_type == "Dataset":
+        return dataset_holder
+    elif return_data_type == "Graph":
+        return graph_holder
+    elif return_data_type is None:
+        pass  # return nothing
+    else:
+        raise ValueError(return_data_value_error_message)
+
+
+def setup_cli_parser(args=None):
+    parser = argparse.ArgumentParser()
+    group = parser.add_mutually_exclusive_group(required=True)
+
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version="{version}".format(version=__version__),
+    )
+
+    group.add_argument(
+        "-e",
+        "--endpoint",
+        help="The SPARQL endpoint you want to load the data into. Cannot be specified when destination is.",
+    )
+
+    group.add_argument(
+        "-d",
+        "--destination",
+        help="The n-quads file you want to export the data into. Cannot be specified when endpoint is.",
+    )
+
+    parser.add_argument(
+        "manifest",
+        help="A Manifest file to process",
+        type=Path,
+    )
+
+    return parser.parse_args(args)
+
+
+def cli(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    args = setup_cli_parser(args)
+
+    load(
+        Path(args.manifest),
+        args.endpoint if args.endpoint is not None else None,
+        Path(args.destination) if args.destination is not None else None,
+    )
+
+
+if __name__ == "__main__":
+    retval = cli(sys.argv[1:])
+    if retval is not None:
+        sys.exit(retval)

prezmanifest-0.2.2/prezmanifest/mrr.ttl

@@ -0,0 +1,87 @@
+PREFIX cs: <https://prez.dev/ManifestResourceRoles>
+PREFIX : <https://prez.dev/ManifestResourceRoles/>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX schema: <https://schema.org/>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+
+
+:CompleteCatalogueAndResourceLabels
+    a skos:Concept ;
+    skos:definition "All the labels - possibly including names, descriptions & seeAlso links - for the Catalogue and Resource objects" ;
+    skos:inScheme cs: ;
+    skos:prefLabel "Complete Catalogue and Resource Labels" ;
+.
+
+:CatalogueAndResourceModel
+    a skos:Concept ;
+    skos:definition "The default model for the container and the content. Must be a set of SHACL Shapes" ;
+    skos:inScheme cs: ;
+    skos:prefLabel "Catalogue & Resource Model" ;
+.
+
+:CatalogueData
+    a skos:Concept ;
+    skos:definition "Data for the container, usually a Catalogue, including the identity of it and each item fo content" ;
+    skos:inScheme cs: ;
+    skos:prefLabel "Catalogue Data" ;
+.
+
+:CatalogueModel
+    a skos:Concept ;
+    skos:definition "The default model for the container. Must be a set of SHACL Shapes" ;
+    skos:inScheme cs: ;
+    skos:prefLabel "Catalogue Model" ;
+.
+
+:ResourceData
+    a skos:Concept ;
+    skos:definition "Data for the content of the container" ;
+    skos:inScheme cs: ;
+    skos:prefLabel "Resource Data" ;
+.
+
+:ResourceModel
+    a skos:Concept ;
+    skos:definition "The default model for the content. Must be a set of SHACL Shapes" ;
+    skos:inScheme cs: ;
+    skos:prefLabel "Resource Model" ;
+.
+
+:IncompleteCatalogueAndResourceLabels
+    a skos:Concept ;
+    skos:definition "Some of the labels - possibly including names, descriptions & seeAlso links - for the Catalogue and Resource objects" ;
+    skos:inScheme cs: ;
+    skos:prefLabel "Incomplete Catalogue and Resource Labels" ;
+.
+
+<https://kurrawong.ai>
+    a schema:Organization ;
+    schema:name "KurrawongAI" ;
+    schema:url "https://kurrawong.ai"^^xsd:anyURI ;
+.
+
+:1.0.0
+    rdfs:label "1.0.0" ;
+.
+
+cs:
+    a skos:ConceptScheme ;
+    owl:versionIRI :1.0.0 ;
+    owl:versionInfo "1.0.0: First public version" ;
+    skos:definition "This vocabulary is a test vocabulary showing the use of images within Concept definitions"@en ;
+    skos:hasTopConcept
+        :CompleteCatalogueAndResourceLabels ,
+        :CatalogueAndResourceModel ,
+        :CatalogueData ,
+        :CatalogueModel ,
+        :ResourceData ,
+        :IncompleteCatalogueAndResourceLabels ;
+    skos:historyNote "Create in 2024 for Prez data loading" ;
+    skos:prefLabel "Manifest Resource Roles Vocabulary" ;
+    schema:creator <https://kurrawong.ai> ;
+    schema:dateCreated "2024-11-29"^^xsd:date ;
+    schema:dateModified "2024-12-17"^^xsd:date ;
+    schema:publisher <https://kurrawong.ai> ;
+.

prezmanifest-0.2.2/prezmanifest/utils.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+from typing import List
+from collections.abc import Generator
+from rdflib import Literal, URIRef, Graph, Dataset
+from rdflib.namespace import OWL, RDF, RDFS, SKOS
+
+
+def get_files_from_artifact(manifest: Path, artifact: Literal) -> List[Path] | Generator[Path]:
+    """Returns an iterable (list or generator) of Path objects for files within an artifact literal.
+
+    This function will correctly interpret artifacts such as 'file.ttl', '*.ttl', '**/*.trig' etc."""
+    if not "*" in str(artifact):
+        return [manifest.parent / Path(str(artifact))]
+    else:
+        artifact_str = str(artifact)
+        glob_marker_location = artifact_str.find("*")
+        glob_parts = [artifact_str[:glob_marker_location], artifact_str[glob_marker_location:]]
+
+        return Path(manifest.parent / Path(glob_parts[0])).glob(glob_parts[1])
+
+
+def get_identifier_from_file(file: Path) -> List[URIRef]:
+    """Returns a list if RDFLib graph identifier (URIRefs) from a triples or quads file
+    for all owl:Ontology and skos:ConceptScheme objects"""
+    if file.name.endswith(".ttl"):
+        g = Graph().parse(file)
+        return [
+            g.value(predicate=RDF.type, object=SKOS.ConceptScheme) or
+            g.value(predicate=RDF.type, object=OWL.Ontology)
+        ]
+    elif file.name.endswith(".trig"):
+        gs = []
+        d = Dataset()
+        d.parse(file, format="trig")
+        for g in d.graphs():
+            gs.append(g.identifier)
+        return gs
+    else:
+        return []

prezmanifest-0.2.2/prezmanifest/validator.py

@@ -0,0 +1,104 @@
+"""Validate a Prez Manifest file.
+
+This script performs both SHACL validation to ensure the Manifest is valid according to the Prez Manifest
+specification (see https://prez.dev/manifest/) and checks that all the resources indicated by the Manifest
+- whether local files/folders or remote resources on the Internet - are reachable.
+
+~$ python validate.py {MANIFEST_FILE_PATH}"""
+
+import argparse
+import sys
+from pathlib import Path
+
+import httpx
+from kurra.utils import load_graph
+from pyshacl import validate as shacl_validate
+from rdflib import Graph
+from rdflib.namespace import PROF
+
+try:
+    from prezmanifest import __version__
+except ImportError:
+    import sys
+
+    sys.path.append(str(Path(__file__).parent.parent.resolve()))
+    from prezmanifest import __version__
+
+
+def validate(manifest: Path) -> bool:
+    ME = Path(__file__)
+    MANIFEST_ROOT_DIR = manifest.parent
+
+    # SHACL validation
+    manifest_graph = load_graph(manifest)
+    mrr_vocab_graph = load_graph(ME.parent / "mrr.ttl")
+    data_graph = manifest_graph + mrr_vocab_graph
+    shacl_graph = load_graph(ME.parent / "validator.ttl")
+    valid, v_graph, v_text = shacl_validate(data_graph, shacl_graph=shacl_graph)
+
+    if not valid:
+        raise ValueError(f"SHACL invalid:\n\n{v_text}")
+
+    # Content link validation
+    for s, o in manifest_graph.subject_objects(PROF.hasResource):
+        for artifact in manifest_graph.objects(o, PROF.hasArtifact):
+            artifact_str = str(artifact)
+            if "http" in artifact_str:
+                r = httpx.get(artifact_str)
+                if 200 <= r.status_code < 400:
+                    pass
+                else:
+                    raise ValueError(
+                        f"Remote content link non-resolving: {artifact_str}"
+                    )
+            elif "*" in artifact_str:
+                glob_parts = artifact_str.split("*")
+                dir = Path(manifest.parent / Path(glob_parts[0]))
+                if not Path(dir).is_dir():
+                    raise ValueError(
+                        f"The content link {artifact_str} is not a directory"
+                    )
+            else:
+                # It must be a local
+                if not (MANIFEST_ROOT_DIR / artifact_str).is_file():
+                    print(
+                        f"Content link {MANIFEST_ROOT_DIR / artifact_str} is invalid - not a file"
+                    )
+
+    return manifest_graph
+
+
+def setup_cli_parser(args=None):
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version="{version}".format(version=__version__),
+    )
+
+    parser.add_argument(
+        "manifest",
+        help="A Manifest file to process",
+        type=Path,
+    )
+
+    return parser.parse_args(args)
+
+
+def cli(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    args = setup_cli_parser(args)
+
+    validate(args.manifest)
+
+
+if __name__ == "__main__":
+    retval = cli(sys.argv[1:])
+    if retval is not None:
+        sys.exit(retval)
+

prezmanifest-0.2.2/prezmanifest/validator.ttl

@@ -0,0 +1,126 @@
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX prez: <https://prez.dev/>
+PREFIX prof: <http://www.w3.org/ns/dx/prof/>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX schema: <https://schema.org/>
+PREFIX sh: <http://www.w3.org/ns/shacl#>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX val: <https://prez.dev/manifest-validator/>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+
+<https://prez.dev/manifest-validator>
+    a owl:Ontology ;
+    owl:versionIRI val:1.0.0 ;
+    owl:versionInfo "1.0.0: First public version" ;
+    schema:creator <https://orcid.org/0000-0002-8742-7730> ;
+    schema:dateCreated "2024-11-29"^^xsd:date ;
+    schema:dateModified "2024-11-29"^^xsd:date ;
+    schema:definition "The SHACL Shapes Validator for the Prez Ontology Manifest Model"@en ;
+    schema:name "Prez Manifest Validator" ;
+    schema:publisher <https://kurrawong.ai> ;
+.
+
+val:ShapeN01
+    a sh:NodeShape ;
+    sh:property val:ShapeP01 ;
+    sh:targetClass prez:Manifest ;
+.
+
+val:ShapeN02
+    a sh:NodeShape ;
+    sh:property
+        val:ShapeP02 ,
+        val:ShapeP03 ,
+        val:ShapeP04 ,
+        val:ShapeP05 ;
+    sh:targetObjectsOf prof:hasResource ;
+.
+
+val:ShapeP01
+    a sh:PropertyShape ;
+    sh:nodeKind sh:BlankNodeOrIRI ;
+    sh:message "ShapeP01: You must supply at least one value for prof:hasResource for a prez:Manifest and it must be either a Blank Node or an IRI" ;
+    sh:minCount 1 ;
+    sh:path prof:hasResource ;
+.
+
+val:ShapeP02
+    a sh:PropertyShape ;
+    sh:maxCount 1 ;
+    sh:message "ShapeP02: You must supply exactly one value for prof:hasArtifact for a prez:Manifest's Resource Descriptors and it must be a string that can be interpreted as a file path, either relative to the manifest file, or an absolute system path or a URL" ;
+    sh:minCount 1 ;
+    sh:nodeKind sh:Literal ;
+    sh:path prof:hasArtifact ;
+.
+
+val:ShapeP03
+    a sh:PropertyShape ;
+    sh:maxCount 1 ;
+    sh:message "ShapeP03: You must supply exactly one value for prof:hasRole for a prez:Manifest's Resource Descriptors with the value selected from the Prez Manifest Resource Roles Vocabulary" ;
+    sh:minCount 1 ;
+    sh:nodeKind sh:IRI ;
+    sh:path prof:hasRole ;
+.
+
+val:ShapeP04
+    a sh:PropertyShape ;
+    sh:maxCount 1 ;
+    sh:message "ShapeP04: If you supply a schema:description for a prez:Manifest's Resource Descriptors, it must be either a plain string or a langString literal" ;
+    sh:or (
+
+        [
+            sh:datatype xsd:string ;
+        ]
+
+        [
+            sh:datatype rdf:langString ;
+        ]
+    ) ;
+    sh:path schema:description ;
+.
+
+val:ShapeP05
+    a sh:PropertyShape ;
+    sh:maxCount 1 ;
+    sh:message "ShapeP05: If you supply a schema:name for a prez:Manifest's Resource Descriptors, it must be either a plain string or a langString literal" ;
+    sh:or (
+
+        [
+            sh:datatype xsd:string ;
+        ]
+
+        [
+            sh:datatype rdf:langString ;
+        ]
+    ) ;
+    sh:path schema:name ;
+.
+
+val:ShapeN03
+    a sh:NodeShape ;
+    sh:targetObjectsOf prof:hasRole ;
+    sh:property [
+        a sh:PropertyShape ;
+        sh:minCount 1 ;
+        sh:maxCount 1 ;
+        sh:path skos:inScheme ;
+    ] ;
+.
+
+val:1.0.0
+    schema:name "1.0.0" ;
+.
+
+<https://orcid.org/0000-0002-8742-7730>
+    a schema:Person ;
+    schema:email "nick@kurrawong.ai"^^xsd:anyURI ;
+    schema:identifier "https://orcid.org/0000-0002-8742-7730"^^xsd:anyURI ;
+    schema:memberOf <https://kurrawong.ai> ;
+    schema:name "Nicholas J. Car" ;
+.
+
+<https://kurrawong.ai>
+    a schema:Organization ;
+    schema:name "KurrawongAI" ;
+    schema:url "https://kurrawong.ai"^^xsd:anyURI ;
+.

prezmanifest-0.2.2/pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "prezmanifest"
+version = "0.2.2"
+description = "A Python package that provides a series of functions to work with Prez Manifests."
+authors = ["Nicholas Car <nick@kurrawong.ai>"]
+license = "BSD-3-Clause"
+readme = "README.adoc"
+
+[tool.poetry.dependencies]
+python = ">=3.12,<4.0"
+pyshacl = "^0.29.0"
+kurra = "^0.6.1"
+labelify = "^0.3.8"
+
+[tool.poetry.dev-dependencies]
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.4"
+black = "^24.10.0"
+testcontainers = "^4.9.0"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"