pyjelly 0.2.3__py3-none-any.whl → 0.4.0__py3-none-any.whl

pyjelly/integrations/rdflib/serialize.py

@@ -1,12 +1,14 @@
+# ruff: noqa: I001
 from __future__ import annotations
 
 from collections.abc import Generator
 from functools import singledispatch
-from typing import IO, Any
+from typing import Any, IO
 from typing_extensions import override
 
 import rdflib
-from rdflib.graph import DATASET_DEFAULT_GRAPH_ID, Dataset, Graph, QuotedGraph
+from rdflib import Graph
+from rdflib.graph import DATASET_DEFAULT_GRAPH_ID, Dataset, QuotedGraph
 from rdflib.serializer import Serializer as RDFLibSerializer
 
 from pyjelly import jelly
@@ -18,11 +20,22 @@ from pyjelly.serialize.streams import (
     SerializerOptions,
     Stream,
     TripleStream,
-)
+)  # ruff: enable
 
 
 class RDFLibTermEncoder(TermEncoder):
     def encode_any(self, term: object, slot: Slot) -> RowsAndTerm:
+        """
+        Encode term based on its RDFLib object.
+
+        Args:
+            term (object): term to encode
+            slot (Slot): its place in statement.
+
+        Returns:
+            RowsAndTerm: encoded extra rows and a jelly term to encode
+
+        """
         if slot is Slot.graph and term == DATASET_DEFAULT_GRAPH_ID:
             return self.encode_default_graph()
 
@@ -50,7 +63,10 @@ def namespace_declarations(store: Graph, stream: Stream) -> None:
 
 
 @singledispatch
-def stream_frames(stream: Stream, data: Graph) -> Generator[jelly.RdfStreamFrame]:  # noqa: ARG001
+def stream_frames(
+    stream: Stream,
+    data: Graph,  # noqa: ARG001
+) -> Generator[jelly.RdfStreamFrame]:
     msg = f"invalid stream implementation {stream}"
     raise TypeError(msg)
 
@@ -60,6 +76,21 @@ def triples_stream_frames(
     stream: TripleStream,
     data: Graph | Dataset,
 ) -> Generator[jelly.RdfStreamFrame]:
+    """
+    Serialize a Graph/Dataset into jelly frames.
+
+    Args:
+        stream (TripleStream): stream that specifies triples processing
+        data (Graph | Dataset): Graph/Dataset to serialize.
+
+    Notes:
+        if Dataset is given, its graphs are unpacked and iterated over
+        if flow is GraphsFrameFlow, emits a frame per graph.
+
+    Yields:
+        Generator[jelly.RdfStreamFrame]: jelly frames.
+
+    """
     stream.enroll()
     if stream.options.params.namespace_declarations:
         namespace_declarations(data, stream)
@@ -68,6 +99,7 @@ def triples_stream_frames(
         for terms in graph:
             if frame := stream.triple(terms):
                 yield frame
+        # this part turns each graph to a frame for graphs logical type
         if frame := stream.flow.frame_from_graph():
             yield frame
     if stream.stream_types.flat and (frame := stream.flow.to_stream_frame()):
@@ -79,6 +111,20 @@ def quads_stream_frames(
     stream: QuadStream,
     data: Dataset,
 ) -> Generator[jelly.RdfStreamFrame]:
+    """
+    Serialize a Dataset into jelly frames.
+
+    Notes:
+        Emits one frame per dataset if flow is of DatasetsFrameFlow.
+
+    Args:
+        stream (QuadStream): stream that specifies quads processing
+        data (Dataset): Dataset to serialize.
+
+    Yields:
+        Generator[jelly.RdfStreamFrame]: jelly frames
+
+    """
     assert isinstance(data, Dataset)
     stream.enroll()
     if stream.options.params.namespace_declarations:
@@ -97,6 +143,21 @@ def graphs_stream_frames(
     stream: GraphStream,
     data: Dataset,
 ) -> Generator[jelly.RdfStreamFrame]:
+    """
+    Serialize a Dataset into jelly frames as a stream of graphs.
+
+    Notes:
+        If flow of DatasetsFrameFlow type, the whole dataset
+        will be encoded into one frame.
+
+    Args:
+        stream (GraphStream): stream that specifies graphs processing
+        data (Dataset): Dataset to serialize.
+
+    Yields:
+        Generator[jelly.RdfStreamFrame]: jelly frames
+
+    """
     assert isinstance(data, Dataset)
     stream.enroll()
     if stream.options.params.namespace_declarations:
@@ -109,6 +170,48 @@ def graphs_stream_frames(
         yield frame
 
 
+def guess_options(sink: Graph | Dataset) -> SerializerOptions:
+    """
+    Guess the serializer options based on the store type.
+
+    >>> guess_options(Graph()).logical_type
+    1
+    >>> guess_options(Dataset()).logical_type
+    2
+    """
+    logical_type = (
+        jelly.LOGICAL_STREAM_TYPE_FLAT_QUADS
+        if isinstance(sink, Dataset)
+        else jelly.LOGICAL_STREAM_TYPE_FLAT_TRIPLES
+    )
+    return SerializerOptions(logical_type=logical_type)
+
+
+def guess_stream(options: SerializerOptions, sink: Graph | Dataset) -> Stream:
+    """
+    Return an appropriate stream implementation for the given options.
+
+    Notes: if base(!) logical type is GRAPHS and Dataset is given,
+        initializes TripleStream
+
+    >>> graph_ser = RDFLibJellySerializer(Graph())
+    >>> ds_ser = RDFLibJellySerializer(Dataset())
+
+    >>> type(guess_stream(guess_options(graph_ser.store), graph_ser.store))
+    <class 'pyjelly.serialize.streams.TripleStream'>
+    >>> type(guess_stream(guess_options(ds_ser.store), ds_ser.store))
+    <class 'pyjelly.serialize.streams.QuadStream'>
+    """
+    stream_cls: type[Stream]
+    if (options.logical_type % 10) != jelly.LOGICAL_STREAM_TYPE_GRAPHS and isinstance(
+        sink, Dataset
+    ):
+        stream_cls = QuadStream
+    else:
+        stream_cls = TripleStream
+    return stream_cls.for_rdflib(options=options)
+
+
 class RDFLibJellySerializer(RDFLibSerializer):
     """
     RDFLib serializer for writing graphs in Jelly RDF stream format.
@@ -124,43 +227,6 @@ class RDFLibJellySerializer(RDFLibSerializer):
             raise NotImplementedError(msg)
         super().__init__(store)
 
-    def guess_options(self) -> SerializerOptions:
-        """
-        Guess the serializer options based on the store type.
-
-        >>> RDFLibJellySerializer(Graph()).guess_options().logical_type
-        1
-        >>> RDFLibJellySerializer(Dataset()).guess_options().logical_type
-        2
-        """
-        logical_type = (
-            jelly.LOGICAL_STREAM_TYPE_FLAT_QUADS
-            if isinstance(self.store, Dataset)
-            else jelly.LOGICAL_STREAM_TYPE_FLAT_TRIPLES
-        )
-        return SerializerOptions(logical_type=logical_type)
-
-    def guess_stream(self, options: SerializerOptions) -> Stream:
-        """
-        Return an appropriate stream implementation for the given options.
-
-        >>> graph_ser = RDFLibJellySerializer(Graph())
-        >>> ds_ser = RDFLibJellySerializer(Dataset())
-
-        >>> type(graph_ser.guess_stream(graph_ser.guess_options()))
-        <class 'pyjelly.serialize.streams.TripleStream'>
-        >>> type(ds_ser.guess_stream(ds_ser.guess_options()))
-        <class 'pyjelly.serialize.streams.QuadStream'>
-        """
-        stream_cls: type[Stream]
-        if options.logical_type != jelly.LOGICAL_STREAM_TYPE_GRAPHS and isinstance(
-            self.store, Dataset
-        ):
-            stream_cls = QuadStream
-        else:
-            stream_cls = TripleStream
-        return stream_cls.for_rdflib(options=options)
-
     @override
     def serialize(  # type: ignore[override]
         self,
@@ -171,10 +237,69 @@ class RDFLibJellySerializer(RDFLibSerializer):
         options: SerializerOptions | None = None,
         **unused: Any,
     ) -> None:
+        """
+        Serialize self.store content to Jelly format.
+
+        Args:
+            out (IO[bytes]): output buffered writer
+            stream (Stream | None, optional): Jelly stream object. Defaults to None.
+            options (SerializerOptions | None, optional): Serializer options
+                if defined beforehand, e.g., read from a separate file.
+                Defaults to None.
+            **unused(Any): unused args for RDFLib serialize
+
+        """
         if options is None:
-            options = self.guess_options()
+            options = guess_options(self.store)
         if stream is None:
-            stream = self.guess_stream(options)
+            stream = guess_stream(options, self.store)
         write = write_delimited if stream.options.params.delimited else write_single
         for stream_frame in stream_frames(stream, self.store):
             write(stream_frame, out)
+
+
+def grouped_stream_to_frames(
+    sink_generator: Generator[Graph] | Generator[Dataset],
+    options: SerializerOptions | None = None,
+) -> Generator[jelly.RdfStreamFrame]:
+    """
+    Transform Graphs/Datasets into Jelly frames, one frame per Graph/Dataset.
+
+    Note: options are guessed if not provided.
+
+    Args:
+        sink_generator (Generator[Graph] | Generator[Dataset]): Generator of
+            Graphs/Dataset to transform.
+        options (SerializerOptions | None, optional): stream options to use.
+            Options are guessed based on the sink store type. Defaults to None.
+
+    Yields:
+        Generator[jelly.RdfStreamFrame]: produced Jelly frames
+
+    """
+    stream = None
+    for sink in sink_generator:
+        if not stream:
+            if options is None:
+                options = guess_options(sink)
+            stream = guess_stream(options, sink)
+        yield from stream_frames(stream, sink)
+
+
+def grouped_stream_to_file(
+    stream: Generator[Graph] | Generator[Dataset],
+    output_file: IO[bytes],
+    **kwargs: Any,
+) -> None:
+    """
+    Write stream of Graphs/Datasets to a binary file.
+
+    Args:
+        stream (Generator[Graph] | Generator[Dataset]): Generator of
+            Graphs/Dataset to transform.
+        output_file (IO[bytes]): opened output file.
+        **kwargs (Any): options to pass to stream.
+
+    """
+    for frame in grouped_stream_to_frames(stream, **kwargs):
+        write_delimited(frame, output_file)

pyjelly/jelly/rdf_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: rdf.proto
-# Protobuf Python Version: 5.29.0
+# Protobuf Python Version: 6.31.0
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -11,8 +11,8 @@ from google.protobuf import symbol_database as _symbol_database
 from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
-    5,
-    29,
+    6,
+    31,
     0,
     '',
     'rdf.proto'

pyjelly/jelly/rdf_pb2.pyi

@@ -2,7 +2,8 @@ from google.protobuf.internal import containers as _containers
 from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
-from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union
+from collections.abc import Iterable as _Iterable, Mapping as _Mapping
+from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
 
 DESCRIPTOR: _descriptor.FileDescriptor
 

pyjelly/parse/decode.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 from abc import ABCMeta, abstractmethod
-from collections.abc import Iterable, Sequence
+from collections.abc import Iterable, Iterator, Sequence
+from enum import Enum, auto
 from typing import Any, ClassVar, NamedTuple
 from typing_extensions import Never
 
@@ -10,6 +11,21 @@ from pyjelly.options import LookupPreset, StreamParameters, StreamTypes
 from pyjelly.parse.lookup import LookupDecoder
 
 
+class ParsingMode(Enum):
+    """
+    Specifies how jelly frames should be treated.
+
+    Modes:
+    FLAT
+        Yield all frames as one Graph or Dataset.
+    GROUPED
+        Yield one Graph/Dataset per frame (grouped parsing).
+    """
+
+    FLAT = auto()
+    GROUPED = auto()
+
+
 class ParserOptions(NamedTuple):
     stream_types: StreamTypes
     lookup_preset: LookupPreset
@@ -21,6 +37,21 @@ def options_from_frame(
     *,
     delimited: bool,
 ) -> ParserOptions:
+    """
+    Fill stream options based on the options row.
+
+    Notes:
+        generalized_statements, rdf_star, and namespace declarations
+        are set to false by default
+
+    Args:
+        frame (jelly.RdfStreamFrame): first non-empty frame from the stream
+        delimited (bool): derived delimited flag
+
+    Returns:
+        ParserOptions: filled options with types/lookups/stream parameters information
+
+    """
     row = frame.rows[0]
     options = row.options
     return ParserOptions(
@@ -52,8 +83,11 @@ def _adapter_missing(feature: str, *, stream_types: StreamTypes) -> Never:
 
 
 class Adapter(metaclass=ABCMeta):
-    def __init__(self, options: ParserOptions) -> None:
+    def __init__(
+        self, options: ParserOptions, parsing_mode: ParsingMode = ParsingMode.FLAT
+    ) -> None:
         self.options = options
+        self.parsing_mode = parsing_mode
 
     # Obligatory abstract methods--all adapters must implement these
     @abstractmethod
@@ -106,6 +140,18 @@ class Adapter(metaclass=ABCMeta):
 
 class Decoder:
     def __init__(self, adapter: Adapter) -> None:
+        """
+        Initialize decoder.
+
+        Initializes decoder with a lookup tables with preset sizes,
+        integration-dependent adapter and empty repeated terms dictionary.
+
+        Args:
+            adapter (Adapter): integration-dependent adapter that specifies terms
+            conversion to specific objects, framing,
+            namespace declarations, and graphs/datasets forming.
+
+        """
         self.adapter = adapter
         self.names = LookupDecoder(lookup_size=self.options.lookup_preset.max_names)
         self.prefixes = LookupDecoder(
@@ -120,13 +166,43 @@ class Decoder:
     def options(self) -> ParserOptions:
         return self.adapter.options
 
-    def decode_frame(self, frame: jelly.RdfStreamFrame) -> Any:
+    def iter_rows(self, frame: jelly.RdfStreamFrame) -> Iterator[Any]:
+        """
+        Iterate through rows in the frame.
+
+        Args:
+            frame (jelly.RdfStreamFrame): jelly frame
+        Yields:
+            Iterator[Any]: decoded rows
+
+        """
         for row_owner in frame.rows:
             row = getattr(row_owner, row_owner.WhichOneof("row"))
-            self.decode_row(row)
-        return self.adapter.frame()
+            decoded_row = self.decode_row(row)
+            if isinstance(
+                row, (jelly.RdfTriple, jelly.RdfQuad, jelly.RdfNamespaceDeclaration)
+            ):
+                yield decoded_row
 
     def decode_row(self, row: Any) -> Any | None:
+        """
+        Decode a row based on its type.
+
+        Notes: uses custom adapters to decode triples/quads, namespace declarations,
+               graph start/end.
+
+        Args:
+            row (Any): protobuf row message
+
+        Raises:
+            TypeError: raises error if this type of protobuf message does not have
+                       a respective handler
+
+        Returns:
+            Any | None: decoded row -
+                        result from calling decode_row (row type appropriate handler)
+
+        """
         try:
             decode_row = self.row_handlers[type(row)]
         except KeyError:
@@ -145,15 +221,51 @@ class Decoder:
         assert lookup_preset.max_names == options.max_name_table_size
 
     def ingest_prefix_entry(self, entry: jelly.RdfPrefixEntry) -> None:
+        """
+        Update prefix lookup table based on the table entry.
+
+        Args:
+            entry (jelly.RdfPrefixEntry): prefix message, containing id and value
+
+        """
         self.prefixes.assign_entry(index=entry.id, value=entry.value)
 
     def ingest_name_entry(self, entry: jelly.RdfNameEntry) -> None:
+        """
+        Update name lookup table based on the table entry.
+
+        Args:
+            entry (jelly.RdfNameEntry): name message, containing id and value
+
+        """
         self.names.assign_entry(index=entry.id, value=entry.value)
 
     def ingest_datatype_entry(self, entry: jelly.RdfDatatypeEntry) -> None:
+        """
+        Update datatype lookup table based on the table entry.
+
+        Args:
+            entry (jelly.RdfDatatypeEntry): name message, containing id and value
+
+        """
         self.datatypes.assign_entry(index=entry.id, value=entry.value)
 
     def decode_term(self, term: Any) -> Any:
+        """
+        Decode a term based on its type: IRI/literal/BN/default graph.
+
+        Notes: requires a custom adapter with implemented methods for terms decoding.
+
+        Args:
+            term (Any): IRI/literal/BN(string)/Default graph message
+
+        Raises:
+            TypeError: raises error if no handler for the term is found
+
+        Returns:
+            Any: decoded term (currently, rdflib objects, e.g., rdflib.term.URIRef)
+
+        """
         try:
             decode_term = self.term_handlers[type(term)]
         except KeyError:
@@ -162,6 +274,16 @@ class Decoder:
         return decode_term(self, term)
 
     def decode_iri(self, iri: jelly.RdfIri) -> Any:
+        """
+        Decode RdfIri message to IRI using a custom adapter.
+
+        Args:
+            iri (jelly.RdfIri): RdfIri message
+
+        Returns:
+            Any: IRI, based on adapter implementation, e.g., rdflib.term.URIRef
+
+        """
         name = self.names.decode_name_term_index(iri.name_id)
         prefix = self.prefixes.decode_prefix_term_index(iri.prefix_id)
         return self.adapter.iri(iri=prefix + name)
@@ -170,9 +292,32 @@ class Decoder:
         return self.adapter.default_graph()
 
     def decode_bnode(self, bnode: str) -> Any:
+        """
+        Decode string message to blank node (BN) using a custom adapter.
+
+        Args:
+            bnode (str): blank node id
+
+        Returns:
+            Any: blank node object from the custom adapter
+
+        """
         return self.adapter.bnode(bnode)
 
     def decode_literal(self, literal: jelly.RdfLiteral) -> Any:
+        """
+        Decode RdfLiteral to literal based on custom adapter implementation.
+
+        Notes: checks for langtag existence;
+               for datatype checks for non-zero table size and datatype field presence
+
+        Args:
+            literal (jelly.RdfLiteral): RdfLiteral message
+
+        Returns:
+            Any: literal returned by the custom adapter
+
+        """
         language = datatype = None
         if literal.langtag:
             language = literal.langtag
@@ -203,6 +348,22 @@ class Decoder:
         statement: jelly.RdfTriple | jelly.RdfQuad,
         oneofs: Sequence[str],
     ) -> Any:
+        """
+        Decode a triple/quad message.
+
+        Notes: also updates repeated terms dictionary
+
+        Args:
+            statement (jelly.RdfTriple | jelly.RdfQuad): triple/quad message
+            oneofs (Sequence[str]): terms s/p/o/g(if quads)
+
+        Raises:
+            ValueError: if a missing repeated term is encountered
+
+        Returns:
+            Any: a list of decoded terms
+
+        """
         terms = []
         for oneof in oneofs:
             field = statement.WhichOneof(oneof)

pyjelly/parse/ioutils.py

@@ -55,25 +55,47 @@ def delimited_jelly_hint(header: bytes) -> bool:
 
 def frame_iterator(inp: IO[bytes]) -> Generator[jelly.RdfStreamFrame]:
     while frame := parse_length_prefixed(jelly.RdfStreamFrame, inp):
-        if frame.rows:
-            yield frame
+        yield frame
 
 
 def get_options_and_frames(
     inp: IO[bytes],
 ) -> tuple[ParserOptions, Iterator[jelly.RdfStreamFrame]]:
+    """
+    Return stream options and frames from the buffered binary stream.
+
+    Args:
+        inp (IO[bytes]): jelly buffered binary stream
+
+    Raises:
+        JellyConformanceError: if no non-empty frames detected in the delimited stream
+        JellyConformanceError: if non-delimited,
+            error is raised if no rows are detected (empty frame)
+
+    Returns:
+        tuple[ParserOptions, Iterator[jelly.RdfStreamFrame]]: ParserOptions holds:
+            stream types, lookup presets and other stream options
+
+    """
     is_delimited = delimited_jelly_hint(bytes_read := inp.read(3))
     inp.seek(-len(bytes_read), os.SEEK_CUR)
 
     if is_delimited:
+        first_frame = None
+        skipped_frames = []
         frames = frame_iterator(inp)
-        first_frame = next(frames, None)
+        for frame in frames:
+            if not frame.rows:
+                skipped_frames.append(frame)
+            else:
+                first_frame = frame
+                break
         if first_frame is None:
             msg = "No non-empty frames found in the stream"
             raise JellyConformanceError(msg)
 
         options = options_from_frame(first_frame, delimited=True)
-        return options, chain((first_frame,), frames)
+        return options, chain(skipped_frames, (first_frame,), frames)
 
     frame = parse(jelly.RdfStreamFrame, inp.read())