pyjelly 0.2.3__tar.gz → 0.3.0__tar.gz

{pyjelly-0.2.3 → pyjelly-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyjelly
-Version: 0.2.3
+Version: 0.3.0
 Summary: Jelly-RDF implementation for Python
 Project-URL: Homepage, https://w3id.org/jelly/pyjelly
 Project-URL: Documentation, https://w3id.org/jelly/pyjelly

pyjelly-0.3.0/docs/examples/rdflib/04_parse_grouped.py

@@ -0,0 +1,23 @@
+from rdflib import Dataset, Graph
+
+# libraries to load example jelly stream data
+import gzip
+import urllib.request
+from typing import cast, IO
+
+from pyjelly.integrations.rdflib.parse import parse_jelly_grouped
+
+url = "https://w3id.org/riverbench/datasets/dbpedia-live/dev/files/jelly_10K.jelly.gz"
+
+# load, uncompress .gz file, and pass to jelly parser
+with (
+    urllib.request.urlopen(url) as resp,
+    cast(IO[bytes], gzip.GzipFile(fileobj=resp)) as jelly_stream,
+):
+    graphs = parse_jelly_grouped(
+        jelly_stream,
+        graph_factory=lambda: Graph(),
+        dataset_factory=lambda: Dataset(),
+    )
+    for i, graph in enumerate(graphs):
+        print(f"Graph {i} in the stream has {len(graph)} triples")

{pyjelly-0.2.3 → pyjelly-0.3.0}/docs/getting-started.md

@@ -33,6 +33,14 @@ To load RDF data from a `.jelly` file:
 
 RDFLib will reconstruct the graph from the serialized Jelly stream.
 
+### Streaming graph parser
+
+To process a Jelly stream frame-by-frame, loading each as a separate RDFLib graph:
+
+{{ code_example('rdflib/04_parse_grouped.py') }}
+
+Because `parse_jelly_grouped` returns a generator, each iteration receives **one** graph, keeping memory usage bounded to the current frame. Thus, large datasets and live streams can be processed efficiently.
+
 ### File extension support
 
 You can generally omit the `format="jelly"` parameter if the file ends in `.jelly` – RDFLib will auto-detect the format:

{pyjelly-0.2.3 → pyjelly-0.3.0}/docs/overview.md

@@ -26,11 +26,6 @@ pyjelly is suitable for:
 
 pyjelly supports all [*physical* stream types]({{ proto_link("specification/reference/#physicalstreamtype") }}) including `TRIPLES`, `QUADS` and `GRAPHS`.
 
-However, only the following [*logical* stream types]({{ proto_link("specification/serialization/#logical-stream-types") }}) are currently supported: `UNSPECIFIED`, `FLAT_TRIPLES`, `FLAT_QUADS`.
-
-!!! warning
-    Grouped logical stream types are **not yet supported**. Frames with grouped logical types will raise an error at parse time.
-
 See the full [stream type matrix]({{ proto_link("serialization/#consistency-with-physical-stream-types") }}) for an overview of valid combinations.
 
 ### Conformance to the Jelly protocol

{pyjelly-0.2.3 → pyjelly-0.3.0}/mkdocs.yml

@@ -125,4 +125,5 @@ nav:
       - "contributing/index.md"
       - Code of conduct: "contributing/code_of_conduct.md"
   - Changelog: "https://github.com/Jelly-RDF/pyjelly/releases"
+  - Licensing and citation: "https://w3id.org/jelly/dev/licensing"
   - Back to main Jelly docs ↩: "https://w3id.org/jelly/"

pyjelly-0.3.0/pyjelly/integrations/rdflib/parse.py

@@ -0,0 +1,411 @@
+from __future__ import annotations
+
+from collections.abc import Generator, Iterable
+from typing import IO, Any, Callable
+from typing_extensions import Never, override
+
+import rdflib
+from rdflib.graph import DATASET_DEFAULT_GRAPH_ID, Dataset, Graph
+from rdflib.parser import InputSource
+from rdflib.parser import Parser as RDFLibParser
+
+from pyjelly import jelly
+from pyjelly.errors import JellyConformanceError
+from pyjelly.options import StreamTypes
+from pyjelly.parse.decode import Adapter, Decoder, ParserOptions, ParsingMode
+from pyjelly.parse.ioutils import get_options_and_frames
+
+
+class RDFLibAdapter(Adapter):
+    """
+    RDFLib adapter class, is extended by triples and quads implementations.
+
+    Args:
+        Adapter (_type_): abstract adapter class
+
+    """
+
+    @override
+    def iri(self, iri: str) -> rdflib.URIRef:
+        return rdflib.URIRef(iri)
+
+    @override
+    def bnode(self, bnode: str) -> rdflib.BNode:
+        return rdflib.BNode(bnode)
+
+    @override
+    def default_graph(self) -> rdflib.URIRef:
+        return DATASET_DEFAULT_GRAPH_ID
+
+    @override
+    def literal(
+        self,
+        lex: str,
+        language: str | None = None,
+        datatype: str | None = None,
+    ) -> rdflib.Literal:
+        return rdflib.Literal(lex, lang=language, datatype=datatype)
+
+
+def _adapter_missing(feature: str, *, stream_types: StreamTypes) -> Never:
+    """
+    Raise error if functionality is missing in adapter.
+
+    TODO: currently not used anywhere due to logical types being removed
+
+    Args:
+        feature (str): function which is not implemented
+        stream_types (StreamTypes): what combination of physical/logical types
+            triggered the error
+
+    Raises:
+        NotImplementedError: raises error with message with missing functionality
+            and types encountered
+
+    Returns:
+        Never: only raises errors
+
+    """
+    physical_type_name = jelly.PhysicalStreamType.Name(stream_types.physical_type)
+    logical_type_name = jelly.LogicalStreamType.Name(stream_types.logical_type)
+    msg = (
+        f"adapter with {physical_type_name} and {logical_type_name} "
+        f"does not implement {feature}"
+    )
+    raise NotImplementedError(msg)
+
+
+class RDFLibTriplesAdapter(RDFLibAdapter):
+    """
+    Triples adapter RDFLib implementation.
+
+    Notes: has internal graph object which tracks
+        triples and namespaces and can get flushed between frames.
+    """
+
+    def __init__(
+        self,
+        options: ParserOptions,
+        graph_factory: Callable[[], Graph],
+        parsing_mode: ParsingMode = ParsingMode.FLAT,
+    ) -> None:
+        super().__init__(options=options, parsing_mode=parsing_mode)
+        self.graph = graph_factory()
+        self.graph_factory = graph_factory
+        self.parsing_mode = parsing_mode
+
+    @override
+    def triple(self, terms: Iterable[Any]) -> Any:
+        self.graph.add(tuple(terms))
+
+    @override
+    def namespace_declaration(self, name: str, iri: str) -> None:
+        self.graph.bind(name, self.iri(iri))
+
+    def frame(self) -> Graph:
+        """
+        Finalize one frame in triples stream.
+
+        Returns:
+           Graph: frame content as a separate Graph
+                and starts a new Graph
+
+        """
+        this_graph = self.graph
+        self.graph = self.graph_factory()
+        return this_graph
+
+
+class RDFLibQuadsBaseAdapter(RDFLibAdapter):
+    def __init__(
+        self,
+        options: ParserOptions,
+        dataset_factory: Callable[[], Dataset],
+        parsing_mode: ParsingMode = ParsingMode.FLAT,
+    ) -> None:
+        super().__init__(options=options, parsing_mode=parsing_mode)
+        self.dataset = dataset_factory()
+        self.dataset_factory = dataset_factory
+
+    @override
+    def frame(self) -> Dataset:
+        current_dataset = self.dataset
+        self.dataset = self.dataset_factory()
+        return current_dataset
+
+
+class RDFLibQuadsAdapter(RDFLibQuadsBaseAdapter):
+    """
+    Extended RDFLib adapter for the QUADS physical type.
+
+    Notes:
+        Adds triples and namespaces directly to
+        dataset, so RDFLib handles the rest.
+
+    Args:
+        RDFLibQuadsBaseAdapter (_type_): base quads adapter
+            (shared with graphs physical type)
+
+    """
+
+    @override
+    def namespace_declaration(self, name: str, iri: str) -> None:
+        self.dataset.bind(name, self.iri(iri))
+
+    @override
+    def quad(self, terms: Iterable[Any]) -> Any:
+        self.dataset.add(tuple(terms))
+
+
+class RDFLibGraphsAdapter(RDFLibQuadsBaseAdapter):
+    """
+    Extension of RDFLibQuadsBaseAdapter for the GRAPHS physical type.
+
+    Notes: introduces graph start/end, checks if graph exists,
+        dataset store management.
+
+    Args:
+        RDFLibQuadsBaseAdapter (_type_): base adapter for quads management.
+
+    Raises:
+        JellyConformanceError: if no graph_start was encountered
+
+    """
+
+    _graph_id: str | None
+
+    def __init__(
+        self,
+        options: ParserOptions,
+        dataset_factory: Callable[[], Dataset],
+        parsing_mode: ParsingMode = ParsingMode.FLAT,
+    ) -> None:
+        super().__init__(
+            options=options,
+            dataset_factory=dataset_factory,
+            parsing_mode=parsing_mode,
+        )
+        self._graph_id = None
+
+    @property
+    def graph(self) -> None:
+        if self._graph_id is None:
+            msg = "new graph was not started"
+            raise JellyConformanceError(msg)
+
+    @override
+    def graph_start(self, graph_id: str) -> None:
+        self._graph_id = graph_id
+
+    @override
+    def namespace_declaration(self, name: str, iri: str) -> None:
+        self.dataset.bind(name, self.iri(iri))
+
+    @override
+    def triple(self, terms: Iterable[Any]) -> None:
+        self.dataset.add((*terms, self._graph_id))
+
+    @override
+    def graph_end(self) -> None:
+        self._graph_id = None
+
+
+def parse_triples_stream(
+    frames: Iterable[jelly.RdfStreamFrame],
+    options: ParserOptions,
+    graph_factory: Callable[[], Graph],
+    parsing_mode: ParsingMode = ParsingMode.FLAT,
+) -> Generator[Graph]:
+    """
+    Parse flat triple stream.
+
+    Args:
+        frames (Iterable[jelly.RdfStreamFrame]): iterator over stream frames
+        options (ParserOptions): stream options
+        graph_factory (Callable): Lambda to construct a graph
+        parsing_mode (ParsingMode): specifies whether this is
+            a flat or grouped parsing.
+
+    Yields:
+        Generator[Graph]: RDFLib Graph(s)
+
+    """
+    adapter = RDFLibTriplesAdapter(
+        options, graph_factory=graph_factory, parsing_mode=parsing_mode
+    )
+    decoder = Decoder(adapter=adapter)
+    for frame in frames:
+        g = decoder.decode_frame(frame)
+        if g is not None:
+            yield g
+
+    if parsing_mode is ParsingMode.FLAT:
+        yield adapter.graph
+
+
+def parse_quads_stream(
+    frames: Iterable[jelly.RdfStreamFrame],
+    options: ParserOptions,
+    dataset_factory: Callable[[], Dataset],
+    parsing_mode: ParsingMode = ParsingMode.FLAT,
+) -> Generator[Dataset]:
+    """
+    Parse flat quads stream.
+
+    Args:
+        frames (Iterable[jelly.RdfStreamFrame]): iterator over stream frames
+        options (ParserOptions): stream options
+        dataset_factory (Callable): Lambda to construct a dataset
+        parsing_mode (ParsingMode): specifies whether this is
+            a flat or grouped parsing.
+
+    Yields:
+        Generator[Dataset]: RDFLib dataset(s)
+
+    """
+    adapter_class: type[RDFLibQuadsBaseAdapter]
+    if options.stream_types.physical_type == jelly.PHYSICAL_STREAM_TYPE_QUADS:
+        adapter_class = RDFLibQuadsAdapter
+    else:
+        adapter_class = RDFLibGraphsAdapter
+    adapter = adapter_class(
+        options=options,
+        dataset_factory=dataset_factory,
+        parsing_mode=parsing_mode,
+    )
+    decoder = Decoder(adapter=adapter)
+    for frame in frames:
+        ds = decoder.decode_frame(frame)
+        if ds is not None:
+            yield ds
+
+    if parsing_mode is ParsingMode.FLAT:
+        yield adapter.dataset
+
+
+def parse_jelly_grouped(
+    inp: IO[bytes],
+    graph_factory: Callable[[], Graph],
+    dataset_factory: Callable[[], Dataset],
+) -> Generator[Any] | Generator[Graph] | Generator[Dataset]:
+    """
+    Take jelly file and return generators based on the detected logical type.
+
+    Yields one graph/dataset per frame.
+
+    Args:
+        inp (IO[bytes]): input jelly buffered binary stream
+        graph_factory (Callable): lambda to construct a Graph
+        dataset_factory (Callable): lambda to construct a Dataset
+
+    Raises:
+        NotImplementedError: is raised if a logical type is not implemented
+
+    Yields:
+        Generator[Any] | Generator[Dataset] | Generator[Graph]:
+            returns generators for graphs/datasets based on the type of input
+
+    """
+    options, frames = get_options_and_frames(inp)
+
+    if options.stream_types.physical_type == jelly.PHYSICAL_STREAM_TYPE_TRIPLES:
+        yield from parse_triples_stream(
+            frames=frames,
+            options=options,
+            graph_factory=graph_factory,
+            parsing_mode=ParsingMode.GROUPED,
+        )
+        return
+
+    if options.stream_types.physical_type in (
+        jelly.PHYSICAL_STREAM_TYPE_QUADS,
+        jelly.PHYSICAL_STREAM_TYPE_GRAPHS,
+    ):
+        yield from parse_quads_stream(
+            frames=frames,
+            options=options,
+            dataset_factory=dataset_factory,
+            parsing_mode=ParsingMode.GROUPED,
+        )
+        return
+
+    physical_type_name = jelly.PhysicalStreamType.Name(
+        options.stream_types.physical_type
+    )
+    msg = f"the stream type {physical_type_name} is not supported "
+    raise NotImplementedError(msg)
+
+
+def parse_jelly_flat(
+    inp: IO[bytes],
+    graph_factory: Callable[[], Graph],
+    dataset_factory: Callable[[], Dataset],
+) -> Any | Dataset | Graph:
+    """
+    Parse jelly file with FLAT physical type into one Graph/Dataset.
+
+    Args:
+        inp (IO[bytes]): input jelly buffered binary stream
+        graph_factory (Callable): lambda to construct a Graph
+        dataset_factory (Callable): lambda to construct a Dataset
+
+    Raises:
+        NotImplementedError: if physical type is not supported
+
+    Returns:
+        RDFLib Graph or Dataset
+
+    """
+    options, frames = get_options_and_frames(inp)
+
+    if options.stream_types.physical_type == jelly.PHYSICAL_STREAM_TYPE_TRIPLES:
+        return next(
+            parse_triples_stream(
+                frames=frames,
+                options=options,
+                graph_factory=graph_factory,
+                parsing_mode=ParsingMode.FLAT,
+            )
+        )
+
+    if options.stream_types.physical_type in (
+        jelly.PHYSICAL_STREAM_TYPE_QUADS,
+        jelly.PHYSICAL_STREAM_TYPE_GRAPHS,
+    ):
+        return next(
+            parse_quads_stream(
+                frames=frames,
+                options=options,
+                dataset_factory=dataset_factory,
+                parsing_mode=ParsingMode.FLAT,
+            )
+        )
+    physical_type_name = jelly.PhysicalStreamType.Name(
+        options.stream_types.physical_type
+    )
+    msg = f"the stream type {physical_type_name} is not supported "
+    raise NotImplementedError(msg)
+
+
+class RDFLibJellyParser(RDFLibParser):
+    def parse(self, source: InputSource, sink: Graph) -> None:
+        """
+        Parse jelly file into provided RDFLib Graph.
+
+        Args:
+            source (InputSource): jelly file as buffered binary stream InputSource obj
+            sink (Graph): RDFLib Graph
+
+        Raises:
+            TypeError: raises error if invalid input
+
+        """
+        inp = source.getByteStream()  # type: ignore[no-untyped-call]
+        if inp is None:
+            msg = "expected source to be a stream of bytes"
+            raise TypeError(msg)
+        parse_jelly_flat(
+            inp,
+            graph_factory=lambda: Graph(store=sink.store, identifier=sink.identifier),
+            dataset_factory=lambda: Dataset(store=sink.store),
+        )

{pyjelly-0.2.3 → pyjelly-0.3.0}/pyjelly/integrations/rdflib/serialize.py

@@ -23,6 +23,17 @@ from pyjelly.serialize.streams import (
 
 class RDFLibTermEncoder(TermEncoder):
     def encode_any(self, term: object, slot: Slot) -> RowsAndTerm:
+        """
+        Encode term based on its RDFLib object.
+
+        Args:
+            term (object): term to encode
+            slot (Slot): its place in statement.
+
+        Returns:
+            RowsAndTerm: encoded extra rows and a jelly term to encode
+
+        """
         if slot is Slot.graph and term == DATASET_DEFAULT_GRAPH_ID:
             return self.encode_default_graph()
 
@@ -60,6 +71,21 @@ def triples_stream_frames(
     stream: TripleStream,
     data: Graph | Dataset,
 ) -> Generator[jelly.RdfStreamFrame]:
+    """
+    Serialize a Graph/Dataset into jelly frames.
+
+    Args:
+        stream (TripleStream): stream that specifies triples processing
+        data (Graph | Dataset): Graph/Dataset to serialize.
+
+    Notes:
+        if Dataset is given, its graphs are unpacked and iterated over
+        if flow is GraphsFrameFlow, emits a frame per graph.
+
+    Yields:
+        Generator[jelly.RdfStreamFrame]: jelly frames.
+
+    """
     stream.enroll()
     if stream.options.params.namespace_declarations:
         namespace_declarations(data, stream)
@@ -68,6 +94,7 @@ def triples_stream_frames(
         for terms in graph:
             if frame := stream.triple(terms):
                 yield frame
+        # this part turns each graph to a frame for graphs logical type
         if frame := stream.flow.frame_from_graph():
             yield frame
     if stream.stream_types.flat and (frame := stream.flow.to_stream_frame()):
@@ -79,6 +106,20 @@ def quads_stream_frames(
     stream: QuadStream,
     data: Dataset,
 ) -> Generator[jelly.RdfStreamFrame]:
+    """
+    Serialize a Dataset into jelly frames.
+
+    Notes:
+        Emits one frame per dataset if flow is of DatasetsFrameFlow.
+
+    Args:
+        stream (QuadStream): stream that specifies quads processing
+        data (Dataset): Dataset to serialize.
+
+    Yields:
+        Generator[jelly.RdfStreamFrame]: jelly frames
+
+    """
     assert isinstance(data, Dataset)
     stream.enroll()
     if stream.options.params.namespace_declarations:
@@ -97,6 +138,21 @@ def graphs_stream_frames(
     stream: GraphStream,
     data: Dataset,
 ) -> Generator[jelly.RdfStreamFrame]:
+    """
+    Serialize a Dataset into jelly frames as a stream of graphs.
+
+    Notes:
+        If flow of DatasetsFrameFlow type, the whole dataset
+        will be encoded into one frame.
+
+    Args:
+        stream (GraphStream): stream that specifies graphs processing
+        data (Dataset): Dataset to serialize.
+
+    Yields:
+        Generator[jelly.RdfStreamFrame]: jelly frames
+
+    """
     assert isinstance(data, Dataset)
     stream.enroll()
     if stream.options.params.namespace_declarations:
@@ -171,6 +227,18 @@ class RDFLibJellySerializer(RDFLibSerializer):
         options: SerializerOptions | None = None,
         **unused: Any,
     ) -> None:
+        """
+        Serialize self.store content to Jelly format.
+
+        Args:
+            out (IO[bytes]): output buffered writer
+            stream (Stream | None, optional): Jelly stream object. Defaults to None.
+            options (SerializerOptions | None, optional): Serializer options
+                if defined beforehand, e.g., read from a separate file.
+                Defaults to None.
+            **unused(Any): unused args for RDFLib serialize
+
+        """
         if options is None:
             options = self.guess_options()
         if stream is None: