pyjelly 0.7.1__cp311-cp311-macosx_11_0_x86_64.whl

pyjelly/parse/decode.py

@@ -0,0 +1,447 @@
+from __future__ import annotations
+
+from abc import ABCMeta, abstractmethod
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from enum import Enum, auto
+from typing import Any, ClassVar, NamedTuple
+from typing_extensions import Never
+
+from mypy_extensions import mypyc_attr
+
+from pyjelly import jelly
+from pyjelly.options import MAX_VERSION, LookupPreset, StreamParameters, StreamTypes
+from pyjelly.parse.lookup import LookupDecoder
+
+RowHandler = Callable[[Any], Any | None]
+TermHandler = Callable[[Any], Any | None]
+RdfStreamOptions = jelly.RdfStreamOptions
+
+
+class ParsingMode(Enum):
+    """
+    Specifies how jelly frames should be treated.
+
+    Modes:
+    FLAT
+        Yield all frames as one Graph or Dataset.
+    GROUPED
+        Yield one Graph/Dataset per frame (grouped parsing).
+    """
+
+    FLAT = auto()
+    GROUPED = auto()
+
+
+@mypyc_attr(allow_interpreted_subclasses=True)
+class ParserOptions(NamedTuple):
+    stream_types: StreamTypes
+    lookup_preset: LookupPreset
+    params: StreamParameters
+
+
+def options_from_frame(
+    frame: jelly.RdfStreamFrame,
+    *,
+    delimited: bool,
+) -> ParserOptions:
+    """
+    Fill stream options based on the options row.
+
+    Notes:
+        generalized_statements, rdf_star, and namespace declarations
+        are set to false by default
+
+    Args:
+        frame (jelly.RdfStreamFrame): first non-empty frame from the stream
+        delimited (bool): derived delimited flag
+
+    Returns:
+        ParserOptions: filled options with types/lookups/stream parameters information
+
+    """
+    row = frame.rows[0]
+    options = row.options
+    nd = getattr(options, "namespace_declarations", False) or (
+        options.version >= MAX_VERSION
+    )
+    return ParserOptions(
+        stream_types=StreamTypes(
+            physical_type=options.physical_type,
+            logical_type=options.logical_type,
+        ),
+        lookup_preset=LookupPreset(
+            max_names=options.max_name_table_size,
+            max_prefixes=options.max_prefix_table_size,
+            max_datatypes=options.max_datatype_table_size,
+        ),
+        params=StreamParameters(
+            stream_name=options.stream_name,
+            generalized_statements=options.generalized_statements,
+            rdf_star=options.rdf_star,
+            version=options.version,
+            delimited=delimited,
+            namespace_declarations=nd,
+        ),
+    )
+
+
+def _adapter_missing(feature: str, *, stream_types: StreamTypes) -> Never:
+    physical_type_name = jelly.PhysicalStreamType.Name(stream_types.physical_type)
+    logical_type_name = jelly.LogicalStreamType.Name(stream_types.logical_type)
+    msg = (
+        f"adapter with {physical_type_name} and {logical_type_name} "
+        f"does not implement {feature}"
+    )
+    raise NotImplementedError(msg)
+
+
+@mypyc_attr(allow_interpreted_subclasses=True)
+class Adapter(metaclass=ABCMeta):
+    def __init__(
+        self, options: ParserOptions, parsing_mode: ParsingMode = ParsingMode.FLAT
+    ) -> None:
+        self.options = options
+        self.parsing_mode = parsing_mode
+
+    # Obligatory abstract methods--all adapters must implement these
+    @abstractmethod
+    def iri(self, iri: str) -> Any:
+        raise NotImplementedError
+
+    @abstractmethod
+    def default_graph(self) -> Any:
+        raise NotImplementedError
+
+    @abstractmethod
+    def bnode(self, bnode: str) -> Any:
+        raise NotImplementedError
+
+    @abstractmethod
+    def literal(
+        self,
+        lex: str,
+        language: str | None = None,
+        datatype: str | None = None,
+    ) -> Any:
+        raise NotImplementedError
+
+    # Optional abstract methods--not required to be implemented by all adapters
+    def triple(self, terms: Iterable[Any]) -> Any:  # noqa: ARG002
+        _adapter_missing("decoding triples", stream_types=self.options.stream_types)
+
+    def quad(self, terms: Iterable[Any]) -> Any:  # noqa: ARG002
+        _adapter_missing("decoding quads", stream_types=self.options.stream_types)
+
+    def graph_start(self, graph_id: Any) -> Any:  # noqa: ARG002
+        _adapter_missing(
+            "decoding graph start markers", stream_types=self.options.stream_types
+        )
+
+    def graph_end(self) -> Any:
+        _adapter_missing(
+            "decoding graph end markers", stream_types=self.options.stream_types
+        )
+
+    def namespace_declaration(self, name: str, iri: str) -> Any:  # noqa: ARG002
+        _adapter_missing(
+            "decoding namespace declarations",
+            stream_types=self.options.stream_types,
+        )
+
+    def quoted_triple(self, terms: Iterable[Any]) -> Any:  # noqa: ARG002
+        _adapter_missing(
+            "decoding quoted triple", stream_types=self.options.stream_types
+        )
+
+    def frame(self) -> Any:
+        return None
+
+
+@mypyc_attr(allow_interpreted_subclasses=True)
+class Decoder:
+    _ROW_HANDLER_NAMES: ClassVar[Mapping[type[Any], str]] = {
+        jelly.RdfStreamOptions: "validate_stream_options",
+        jelly.RdfPrefixEntry: "ingest_prefix_entry",
+        jelly.RdfNameEntry: "ingest_name_entry",
+        jelly.RdfDatatypeEntry: "ingest_datatype_entry",
+        jelly.RdfTriple: "decode_triple",
+        jelly.RdfQuad: "decode_quad",
+        jelly.RdfGraphStart: "decode_graph_start",
+        jelly.RdfGraphEnd: "decode_graph_end",
+        jelly.RdfNamespaceDeclaration: "decode_namespace_declaration",
+    }
+
+    _TERM_HANDLER_NAMES: ClassVar[Mapping[type[Any], str]] = {
+        jelly.RdfIri: "decode_iri",
+        str: "decode_bnode",
+        jelly.RdfLiteral: "decode_literal",
+        jelly.RdfDefaultGraph: "decode_default_graph",
+        jelly.RdfTriple: "decode_quoted_triple",
+    }
+
+    def __init__(self, adapter: Adapter) -> None:
+        """
+        Initialize decoder.
+
+        Initializes decoder with a lookup tables with preset sizes,
+        integration-dependent adapter and empty repeated terms dictionary.
+
+        Args:
+            adapter (Adapter): integration-dependent adapter that specifies terms
+            conversion to specific objects, framing,
+            namespace declarations, and graphs/datasets forming.
+
+        """
+        self.adapter = adapter
+        self.names = LookupDecoder(lookup_size=self.options.lookup_preset.max_names)
+        self.prefixes = LookupDecoder(
+            lookup_size=self.options.lookup_preset.max_prefixes
+        )
+        self.datatypes = LookupDecoder(
+            lookup_size=self.options.lookup_preset.max_datatypes
+        )
+        self.repeated_terms: dict[str, jelly.RdfIri | str | jelly.RdfLiteral] = {}
+
+        self.row_handlers: dict[type[Any], RowHandler] = {
+            t: getattr(self, name) for t, name in self._ROW_HANDLER_NAMES.items()
+        }
+        self.term_handlers: dict[type[Any], TermHandler] = {
+            t: getattr(self, name) for t, name in self._TERM_HANDLER_NAMES.items()
+        }
+
+    @property
+    def options(self) -> ParserOptions:
+        return self.adapter.options
+
+    def iter_rows(self, frame: jelly.RdfStreamFrame) -> Iterator[Any]:
+        """
+        Iterate through rows in the frame.
+
+        Args:
+            frame (jelly.RdfStreamFrame): jelly frame
+        Yields:
+            Iterator[Any]: decoded rows
+
+        """
+        for row_owner in frame.rows:
+            row = getattr(row_owner, row_owner.WhichOneof("row"))
+            decoded_row = self.decode_row(row)
+            if isinstance(
+                row, (jelly.RdfTriple, jelly.RdfQuad, jelly.RdfNamespaceDeclaration)
+            ):
+                yield decoded_row
+
+    def decode_row(self, row: Any) -> Any | None:
+        """
+        Decode a row based on its type.
+
+        Notes: uses custom adapters to decode triples/quads, namespace declarations,
+               graph start/end.
+
+        Args:
+            row (Any): protobuf row message
+
+        Raises:
+            TypeError: raises error if this type of protobuf message does not have
+                       a respective handler
+
+        Returns:
+            Any | None: decoded row -
+                        result from calling decode_row (row type appropriate handler)
+
+        """
+        handler = self.row_handlers.get(type(row))
+        if handler is None:
+            msg = f"decoder not implemented for {type(row)}"
+            raise TypeError(msg) from None
+        return handler(row)
+
+    def validate_stream_options(self, options: jelly.RdfStreamOptions) -> None:
+        stream_types, lookup_preset, params = self.options
+        assert stream_types.physical_type == options.physical_type
+        assert stream_types.logical_type == options.logical_type
+        assert params.stream_name == options.stream_name
+        assert params.version >= options.version
+        assert lookup_preset.max_prefixes == options.max_prefix_table_size
+        assert lookup_preset.max_datatypes == options.max_datatype_table_size
+        assert lookup_preset.max_names == options.max_name_table_size
+
+    def ingest_prefix_entry(self, entry: jelly.RdfPrefixEntry) -> None:
+        """
+        Update prefix lookup table based on the table entry.
+
+        Args:
+            entry (jelly.RdfPrefixEntry): prefix message, containing id and value
+
+        """
+        self.prefixes.assign_entry(index=entry.id, value=entry.value)
+
+    def ingest_name_entry(self, entry: jelly.RdfNameEntry) -> None:
+        """
+        Update name lookup table based on the table entry.
+
+        Args:
+            entry (jelly.RdfNameEntry): name message, containing id and value
+
+        """
+        self.names.assign_entry(index=entry.id, value=entry.value)
+
+    def ingest_datatype_entry(self, entry: jelly.RdfDatatypeEntry) -> None:
+        """
+        Update datatype lookup table based on the table entry.
+
+        Args:
+            entry (jelly.RdfDatatypeEntry): name message, containing id and value
+
+        """
+        self.datatypes.assign_entry(index=entry.id, value=entry.value)
+
+    def decode_term(self, term: Any) -> Any:
+        """
+        Decode a term based on its type: IRI/literal/BN/default graph.
+
+        Notes: requires a custom adapter with implemented methods for terms decoding.
+
+        Args:
+            term (Any): IRI/literal/BN(string)/Default graph message
+
+        Raises:
+            TypeError: raises error if no handler for the term is found
+
+        Returns:
+            Any: decoded term (currently, rdflib objects, e.g., rdflib.term.URIRef)
+
+        """
+        decode_term = self.term_handlers.get(type(term))
+        if decode_term is None:
+            msg = f"decoder not implemented for {type(term)}"
+            raise TypeError(msg) from None
+        return decode_term(term)
+
+    def decode_iri(self, iri: jelly.RdfIri) -> Any:
+        """
+        Decode RdfIri message to IRI using a custom adapter.
+
+        Args:
+            iri (jelly.RdfIri): RdfIri message
+
+        Returns:
+            Any: IRI, based on adapter implementation, e.g., rdflib.term.URIRef
+
+        """
+        name = self.names.decode_name_term_index(iri.name_id)
+        prefix = self.prefixes.decode_prefix_term_index(iri.prefix_id)
+        return self.adapter.iri(iri=prefix + name)
+
+    def decode_default_graph(self, _: jelly.RdfDefaultGraph) -> Any:
+        return self.adapter.default_graph()
+
+    def decode_bnode(self, bnode: str) -> Any:
+        """
+        Decode string message to blank node (BN) using a custom adapter.
+
+        Args:
+            bnode (str): blank node id
+
+        Returns:
+            Any: blank node object from the custom adapter
+
+        """
+        return self.adapter.bnode(bnode)
+
+    def decode_literal(self, literal: jelly.RdfLiteral) -> Any:
+        """
+        Decode RdfLiteral to literal based on custom adapter implementation.
+
+        Notes: checks for langtag existence;
+               for datatype checks for non-zero table size and datatype field presence
+
+        Args:
+            literal (jelly.RdfLiteral): RdfLiteral message
+
+        Returns:
+            Any: literal returned by the custom adapter
+
+        """
+        language = datatype = None
+        if literal.langtag:
+            language = literal.langtag
+        elif self.datatypes.lookup_size and literal.HasField("datatype"):
+            datatype = self.datatypes.decode_datatype_term_index(literal.datatype)
+        return self.adapter.literal(
+            lex=literal.lex,
+            language=language,
+            datatype=datatype,
+        )
+
+    def decode_namespace_declaration(
+        self,
+        declaration: jelly.RdfNamespaceDeclaration,
+    ) -> Any:
+        iri = self.decode_iri(declaration.value)
+        return self.adapter.namespace_declaration(declaration.name, iri)
+
+    def decode_graph_start(self, graph_start: jelly.RdfGraphStart) -> Any:
+        term = getattr(graph_start, graph_start.WhichOneof("graph"))
+        return self.adapter.graph_start(self.decode_term(term))
+
+    def decode_graph_end(self, _: jelly.RdfGraphEnd) -> Any:
+        return self.adapter.graph_end()
+
+    def decode_statement(
+        self,
+        statement: jelly.RdfTriple | jelly.RdfQuad,
+        oneofs: Sequence[str],
+    ) -> Any:
+        """
+        Decode a triple/quad message.
+
+        Notes: also updates repeated terms dictionary
+
+        Args:
+            statement (jelly.RdfTriple | jelly.RdfQuad): triple/quad message
+            oneofs (Sequence[str]): terms s/p/o/g(if quads)
+
+        Raises:
+            ValueError: if a missing repeated term is encountered
+
+        Returns:
+            Any: a list of decoded terms
+
+        """
+        terms = []
+        for oneof in oneofs:
+            field = statement.WhichOneof(oneof)
+            if field:
+                jelly_term = getattr(statement, field)
+                decoded_term = self.decode_term(jelly_term)
+                self.repeated_terms[oneof] = decoded_term
+            else:
+                decoded_term = self.repeated_terms[oneof]
+                if decoded_term is None:
+                    msg = f"missing repeated term {oneof}"
+                    raise ValueError(msg)
+            terms.append(decoded_term)
+        return terms
+
+    def decode_triple(self, triple: jelly.RdfTriple) -> Any:
+        terms = self.decode_statement(triple, ("subject", "predicate", "object"))
+        return self.adapter.triple(terms)
+
+    def decode_quoted_triple(self, triple: jelly.RdfTriple) -> Any:
+        oneofs: Sequence[str] = ("subject", "predicate", "object")
+        terms = []
+        for oneof in oneofs:
+            field = triple.WhichOneof(oneof)
+            if field:
+                jelly_term = getattr(triple, field)
+                decoded_term = self.decode_term(jelly_term)
+            else:
+                msg = "repeated terms are not allowed in quoted triples"
+                raise ValueError(msg)
+            terms.append(decoded_term)
+        return self.adapter.quoted_triple(terms)
+
+    def decode_quad(self, quad: jelly.RdfQuad) -> Any:
+        terms = self.decode_statement(quad, ("subject", "predicate", "object", "graph"))
+        return self.adapter.quad(terms)

pyjelly/parse/ioutils.py

@@ -0,0 +1,115 @@
+import io
+import os
+from collections.abc import Generator, Iterator
+from itertools import chain
+from typing import IO
+
+from google.protobuf.proto import parse, parse_length_prefixed
+
+from pyjelly import jelly
+from pyjelly.errors import JellyConformanceError
+from pyjelly.parse.decode import ParserOptions, options_from_frame
+
+
+def delimited_jelly_hint(header: bytes) -> bool:
+    """
+    Detect whether a Jelly file is delimited from its first 3 bytes.
+
+    Truth table (notation: `0A` = `0x0A`, `NN` = `not 0x0A`, `??` = _don't care_):
+
+    | Byte 1 | Byte 2 | Byte 3 | Result                                   |
+    |--------|--------|--------|------------------------------------------|
+    | `NN`   |  `??`  |  `??`  | Delimited                                |
+    | `0A`   |  `NN`  |  `??`  | Non-delimited                            |
+    | `0A`   |  `0A`  |  `NN`  | Delimited (size = 10)                    |
+    | `0A`   |  `0A`  |  `0A`  | Non-delimited (stream options size = 10) |
+
+    >>> delimited_jelly_hint(bytes([0x00, 0x00, 0x00]))
+    True
+
+    >>> delimited_jelly_hint(bytes([0x00, 0x00, 0x0A]))
+    True
+
+    >>> delimited_jelly_hint(bytes([0x00, 0x0A, 0x00]))
+    True
+
+    >>> delimited_jelly_hint(bytes([0x00, 0x0A, 0x0A]))
+    True
+
+    >>> delimited_jelly_hint(bytes([0x0A, 0x00, 0x00]))
+    False
+
+    >>> delimited_jelly_hint(bytes([0x0A, 0x00, 0x0A]))
+    False
+
+    >>> delimited_jelly_hint(bytes([0x0A, 0x0A, 0x00]))
+    True
+
+    >>> delimited_jelly_hint(bytes([0x0A, 0x0A, 0x0A]))
+    False
+    """
+    magic = 0x0A
+    return len(header) >= 3 and (  # noqa: PLR2004
+        header[0] != magic or (header[1] == magic and header[2] != magic)
+    )
+
+
+def frame_iterator(inp: IO[bytes]) -> Generator[jelly.RdfStreamFrame]:
+    while frame := parse_length_prefixed(jelly.RdfStreamFrame, inp):
+        yield frame
+
+
+def get_options_and_frames(
+    inp: IO[bytes],
+) -> tuple[ParserOptions, Iterator[jelly.RdfStreamFrame]]:
+    """
+    Return stream options and frames from the buffered binary stream.
+
+    Args:
+        inp (IO[bytes]): jelly buffered binary stream
+
+    Raises:
+        JellyConformanceError: if no non-empty frames detected in the delimited stream
+        JellyConformanceError: if non-delimited,
+            error is raised if no rows are detected (empty frame)
+
+    Returns:
+        tuple[ParserOptions, Iterator[jelly.RdfStreamFrame]]: ParserOptions holds:
+            stream types, lookup presets and other stream options
+
+    """
+    if not inp.seekable():
+        # Input may not be seekable (e.g. a network stream) -- then we need to buffer
+        # it to determine if it's delimited.
+        # See also: https://github.com/Jelly-RDF/pyjelly/issues/298
+        inp = io.BufferedReader(inp)  # type: ignore[arg-type, type-var, unused-ignore]
+        is_delimited = delimited_jelly_hint(inp.peek(3))
+    else:
+        is_delimited = delimited_jelly_hint(bytes_read := inp.read(3))
+        inp.seek(-len(bytes_read), os.SEEK_CUR)
+
+    if is_delimited:
+        first_frame = None
+        skipped_frames = []
+        frames = frame_iterator(inp)
+        for frame in frames:
+            if not frame.rows:
+                skipped_frames.append(frame)
+            else:
+                first_frame = frame
+                break
+        if first_frame is None:
+            msg = "No non-empty frames found in the stream"
+            raise JellyConformanceError(msg)
+
+        options = options_from_frame(first_frame, delimited=True)
+        return options, chain(skipped_frames, (first_frame,), frames)
+
+    frame = parse(jelly.RdfStreamFrame, inp.read())
+
+    if not frame.rows:
+        msg = "The stream is corrupted (only contains an empty frame)"
+        raise JellyConformanceError(msg)
+
+    options = options_from_frame(frame, delimited=False)
+    return options, iter((frame,))

pyjelly/parse/lookup.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+from pyjelly.errors import JellyAssertionError, JellyConformanceError
+from pyjelly.options import MAX_LOOKUP_SIZE
+
+
+@dataclass
+class LookupDecoder:
+    """
+    Shared base for RDF lookup encoders using Jelly compression.
+
+    Tracks the last assigned and last reused index.
+
+    Parameters
+    ----------
+    lookup_size
+        Maximum lookup size.
+
+    """
+
+    last_assigned_index: int
+    last_reused_index: int
+
+    def __init__(self, *, lookup_size: int) -> None:
+        if lookup_size > MAX_LOOKUP_SIZE:
+            msg = f"lookup size cannot be larger than {MAX_LOOKUP_SIZE}"
+            raise JellyAssertionError(msg)
+        self.lookup_size = lookup_size
+        placeholders = (None,) * lookup_size
+        self.data: deque[str | None] = deque(placeholders, maxlen=lookup_size)
+        self.last_assigned_index = 0
+        self.last_reused_index = 0
+
+    def assign_entry(self, index: int, value: str) -> None:
+        previous_index = self.last_assigned_index
+        if index == 0:
+            index = previous_index + 1
+        assert index > 0
+        self.data[index - 1] = value
+        self.last_assigned_index = index
+
+    def at(self, index: int) -> str:
+        self.last_reused_index = index
+        value = self.data[index - 1]
+        if value is None:
+            msg = f"invalid resolved index {index}"
+            raise IndexError(msg)
+        return value
+
+    def decode_prefix_term_index(self, index: int) -> str:
+        actual_index = index or self.last_reused_index
+        if actual_index == 0:
+            return ""
+        return self.at(actual_index)
+
+    def decode_name_term_index(self, index: int) -> str:
+        actual_index = index or self.last_reused_index + 1
+        if actual_index == 0:
+            msg = "0 is not a valid name term index"
+            raise JellyConformanceError(msg)
+        return self.at(actual_index)
+
+    def decode_datatype_term_index(self, index: int) -> str | None:
+        if index == 0:
+            msg = "0 is not a valid datatype term index"
+            raise JellyConformanceError(msg)
+        return self.at(index)