odatadol 0.0.2__py3-none-any.whl

odatadol/__init__.py

@@ -0,0 +1,64 @@
+"""Read an OData v4 entity set as a Mapping, with the standard library only.
+
+    >>> from odatadol import ODataReader, BasicAuth                     # doctest: +SKIP
+    >>> store = ODataReader('https://example.com/odata', 'Products',    # doctest: +SKIP
+    ...                     key='ProductID', credentials=BasicAuth('me', 'pw'))
+    >>> len(store); store[42]; list(store)                              # doctest: +SKIP
+
+Under the facade are plain functions, usable on their own:
+
+- :func:`odata_get`: the one (``GET``) request, through a ``fetch`` seam.
+- :func:`entity_count`, :func:`entity_records`: ``$count`` and paged records, with
+  ``$select``/``$filter``/``$orderby``/``$top`` passed through.
+- :func:`entity_columns`: a ``$select`` too wide for one URL, read as column groups
+  joined on a key and checked to be one moment (:class:`UnstableRead` otherwise).
+- :func:`entity_metadata`: ``$metadata`` as ``{name: {type, nullable, annotations}}``.
+
+Read-only by construction: no function takes an HTTP verb.
+"""
+
+from odatadol.metadata import entity_metadata, parse_entity_metadata
+from odatadol.read import (
+    DEFAULT_PAGE_SIZE,
+    MAX_URL_CHARS,
+    UnstableRead,
+    entity_columns,
+    entity_count,
+    entity_records,
+    entity_url,
+    quote_name,
+)
+from odatadol.store import ODataReader, odata_literal
+from odatadol.transport import (
+    BasicAuth,
+    BearerToken,
+    Credentials,
+    Fetch,
+    ODataError,
+    odata_get,
+    redact,
+    urllib_fetch,
+)
+
+__all__ = [
+    "ODataReader",
+    "odata_literal",
+    "BasicAuth",
+    "BearerToken",
+    "Credentials",
+    "Fetch",
+    "ODataError",
+    "UnstableRead",
+    "odata_get",
+    "urllib_fetch",
+    "redact",
+    "entity_url",
+    "quote_name",
+    "entity_count",
+    "entity_records",
+    "entity_columns",
+    "entity_metadata",
+    "parse_entity_metadata",
+    "DEFAULT_PAGE_SIZE",
+    "MAX_URL_CHARS",
+]

odatadol/metadata.py

@@ -0,0 +1,167 @@
+"""An entity type's declared properties, read from the service's CSDL ``$metadata``.
+
+:func:`entity_metadata` returns ``{name: {'type': ..., 'nullable': ...,
+'annotations': {...}}}`` in declaration order, with Edm type names intact
+(``Edm.Decimal``, ``Edm.Binary``, ``Collection(Edm.String)``), so a caller can decide how
+to render a value or which properties are not worth fetching (a binary, a stream, a
+server-side global). ``annotations`` is present only when the property has any.
+"""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from typing import Any
+
+from odatadol.transport import Credentials, Fetch, ODataError, odata_get
+
+__all__ = ["EDM_NAMESPACE", "entity_metadata", "parse_entity_metadata"]
+
+#: The CSDL namespace of OData v4.
+EDM_NAMESPACE = "http://docs.oasis-open.org/odata/ns/edm"
+#: The attributes an annotation's constant value may sit in, in the order looked for.
+_ANNOTATION_VALUE_ATTRS = (
+    "Bool",
+    "String",
+    "Int",
+    "Decimal",
+    "Float",
+    "Date",
+    "DateTimeOffset",
+    "TimeOfDay",
+    "Duration",
+    "Guid",
+    "EnumMember",
+    "Path",
+)
+
+
+def _edm(tag: str) -> str:
+    return f"{{{EDM_NAMESPACE}}}{tag}"
+
+
+def _annotation_value(annotation: ET.Element) -> str:
+    """The annotation's constant as written; ``''`` when it has none (a bare tag term,
+    which CSDL reads as ``true`` for a Boolean term)."""
+    return next(
+        (v for a in _ANNOTATION_VALUE_ATTRS if (v := annotation.get(a)) is not None), ""
+    )
+
+
+def _types_by_qualified_name(root: ET.Element) -> dict[str, ET.Element]:
+    """Every entity type under each name it can be referred by: ``Namespace.Name`` and,
+    when its schema declares one, ``Alias.Name``."""
+    types: dict[str, ET.Element] = {}
+    for schema in root.iter(_edm("Schema")):
+        prefixes = [p for p in (schema.get("Namespace"), schema.get("Alias")) if p]
+        for entity_type in schema.findall(_edm("EntityType")):
+            for prefix in prefixes:
+                types[f"{prefix}.{entity_type.get('Name')}"] = entity_type
+    return types
+
+
+def _entity_type_of(
+    root: ET.Element, entity: str, types: dict[str, ET.Element]
+) -> ET.Element:
+    sets = {s.get("Name"): s.get("EntityType") for s in root.iter(_edm("EntitySet"))}
+    wanted = sets.get(entity) or entity
+    if wanted in types:
+        return types[wanted]
+    by_short_name = {
+        id(t): t for q, t in types.items() if q.rsplit(".", 1)[-1] == wanted
+    }
+    if len(by_short_name) == 1:
+        return next(iter(by_short_name.values()))
+    if by_short_name:
+        raise ODataError(
+            f"several schemas declare an entity type {wanted!r}; name it qualified: "
+            + ", ".join(sorted(q for q in types if q.rsplit(".", 1)[-1] == wanted))
+        )
+    raise ODataError(
+        f"$metadata declares no entity {entity!r}; its entity sets are: "
+        + (", ".join(sorted(n for n in sets if n)) or "none")
+    )
+
+
+def _type_chain(
+    entity_type: ET.Element, types: dict[str, ET.Element]
+) -> list[ET.Element]:
+    """``entity_type`` and its ``BaseType`` ancestors, the root ancestor first."""
+    chain = [entity_type]
+    while (base := chain[-1].get("BaseType")) is not None:
+        if base not in types or types[base] in chain:
+            raise ODataError(
+                f"entity type {chain[-1].get('Name')!r} derives from {base!r}, which "
+                "$metadata "
+                + ("does not declare" if base not in types else "makes a cycle")
+            )
+        chain.append(types[base])
+    return chain[::-1]
+
+
+def _external_annotations(
+    root: ET.Element, types: dict[str, ET.Element]
+) -> dict[tuple[int, str], dict[str, str]]:
+    """Out-of-line ``<Annotations Target="NS.Type/Property">``, keyed by
+    ``(id(type element), property)``: how many services publish annotations."""
+    found: dict[tuple[int, str], dict[str, str]] = {}
+    for block in root.iter(_edm("Annotations")):
+        target, _, prop = (block.get("Target") or "").partition("/")
+        if prop and target in types:
+            found.setdefault((id(types[target]), prop), {}).update(
+                (a.get("Term", ""), _annotation_value(a))
+                for a in block.findall(_edm("Annotation"))
+            )
+    return found
+
+
+def parse_entity_metadata(body: bytes | str, entity: str) -> dict[str, dict[str, Any]]:
+    """The properties of ``entity`` in a CSDL document, inherited ones first.
+
+    ``entity`` may be an entity set, a qualified type (``Namespace.Name`` or
+    ``Alias.Name``) or, when only one schema declares it, a type's short name.
+    Annotations are merged from inline ``<Annotation>`` and out-of-line
+    ``<Annotations Target=...>`` elements.
+    """
+    root = ET.fromstring(body)
+    types = _types_by_qualified_name(root)
+    external = _external_annotations(root, types)
+    properties: dict[str, dict[str, Any]] = {}
+    for entity_type in _type_chain(_entity_type_of(root, entity, types), types):
+        for prop in entity_type.findall(_edm("Property")):
+            name = prop.get("Name")
+            annotations = {
+                **external.get((id(entity_type), name), {}),
+                **{
+                    a.get("Term", ""): _annotation_value(a)
+                    for a in prop.findall(_edm("Annotation"))
+                },
+            }
+            properties[name] = {
+                "type": prop.get("Type"),
+                "nullable": prop.get("Nullable", "true") != "false",
+                **({"annotations": annotations} if annotations else {}),
+            }
+    return properties
+
+
+def entity_metadata(
+    service: str,
+    entity: str,
+    *,
+    credentials: Credentials = None,
+    fetch: Fetch | None = None,
+) -> dict[str, dict[str, Any]]:
+    """The declared properties of one entity set's type, in declaration order.
+
+    ``entity`` may name the entity set (resolved through the container) or the type.
+
+    Raises:
+        ODataError: If the service declares no such entity, naming the ones it has.
+    """
+    body = odata_get(
+        f"{service.rstrip('/')}/$metadata",
+        credentials=credentials,
+        accept="application/xml",
+        fetch=fetch,
+    )
+    return parse_entity_metadata(body, entity)

odatadol/read.py

@@ -0,0 +1,365 @@
+"""Reading an entity set: its URL, its count, its records page by page, and wide reads.
+
+The functions take the service root and the entity set name first, then keyword-only
+options, and share three seams: ``credentials=``, ``fetch=`` and, for anything returning
+records, ``parse_float=``.
+
+Server quirks met in practice, each with a default that copes:
+
+- ``$count`` is asked for with ``Accept: */*``: some servers refuse a bare
+  ``text/plain``.
+- A name that is not an OData identifier (``Unit Price``) is double-quoted in
+  ``$select``/``$orderby`` (:func:`quote_name`).
+- ``:`` travels bare in the query string: a percent-encoded ``:`` inside a ``$filter``
+  timestamp is a syntax error on some servers.
+- JSON is parsed with ``strict=False``: raw control characters (a bare CR line break)
+  inside strings are kept verbatim instead of failing the page.
+- A front end may refuse URLs past ~2k characters, so :func:`entity_columns` splits a
+  wide ``$select`` into column groups under ``max_url`` and joins them on a key.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.parse
+from collections.abc import Callable, Iterable, Iterator, Mapping
+from typing import Any
+
+from odatadol.transport import (
+    Credentials,
+    Fetch,
+    ODataError,
+    odata_get,
+    redact,
+    resolve_credentials,
+    same_origin,
+)
+
+__all__ = [
+    "DEFAULT_PAGE_SIZE",
+    "MAX_URL_CHARS",
+    "UnstableRead",
+    "entity_url",
+    "quote_name",
+    "entity_count",
+    "entity_records",
+    "entity_columns",
+]
+
+#: Records per request: small enough that one slow page is not a timeout, large enough
+#: that tens of thousands of records are a few dozen requests.
+DEFAULT_PAGE_SIZE = 1000
+#: The longest URL :func:`entity_columns` sends. IIS's default query-string ceiling is
+#: 2,048 characters, and past it IIS answers its own 404 as if the entity did not exist.
+MAX_URL_CHARS = 2000
+#: Characters left unescaped in the query string, as RFC 3986 allows.
+_QUERY_SAFE = "$,'():"
+#: A ``$skip`` wider than any real one, to size the longest URL a page can have.
+_WIDEST_SKIP = 10**9
+
+ParseFloat = Callable[[str], Any]
+
+
+class UnstableRead(ODataError):
+    """Reads of one entity set disagreed about which records exist, or their versions."""
+
+
+def entity_url(
+    service: str,
+    entity: str,
+    *,
+    suffix: str = "",
+    query: Mapping[str, Any] | None = None,
+) -> str:
+    """The URL of an entity set, optionally with a path suffix and query options.
+
+    ``None``-valued options are dropped, and ``:`` is not percent-encoded.
+
+    >>> entity_url('https://example.com/odata/', 'Products', suffix='/$count',
+    ...            query={'$filter': 'ModifiedAt gt 2026-01-01T00:00:00Z', '$top': None})
+    'https://example.com/odata/Products/$count?$filter=ModifiedAt%20gt%202026-01-01T00:00:00Z'
+    """
+    root, _, service_query = service.partition("?")  # a service may need its own
+    url = f"{root.rstrip('/')}/{urllib.parse.quote(entity)}{suffix}"
+    options = {k: v for k, v in (query or {}).items() if v is not None}
+    encoded = urllib.parse.urlencode(
+        options, quote_via=urllib.parse.quote, safe=_QUERY_SAFE
+    )
+    if joined := "&".join(q for q in (service_query, encoded) if q):
+        url += "?" + joined
+    return url
+
+
+def _is_plain(name: str) -> bool:
+    """An OData path as written: ``*``, identifiers (Unicode ones too), qualified
+    names and ``/`` navigation, so ``Category/Name`` and ``NS.Action`` stay bare."""
+    return all(
+        segment == "*" or all(part.isidentifier() for part in segment.split("."))
+        for segment in name.split("/")
+    )
+
+
+def quote_name(name: str) -> str:
+    """A property name as ``$select``/``$orderby`` accept it.
+
+    >>> quote_name('Status'), quote_name('Unit Price'), quote_name('Category/Name')
+    ('Status', '"Unit Price"', 'Category/Name')
+
+    A bare ``Unit Price`` is a syntax error at the space; servers that allow such names
+    (FileMaker's OData, for one) take them double-quoted. A standard OData name is never
+    quoted.
+
+    Raises:
+        ValueError: For a name holding a ``"``, which no quoting can carry.
+    """
+    if _is_plain(name):
+        return name
+    if '"' in name:
+        raise ValueError(f"cannot quote the name {name!r}: it holds a double quote")
+    return f'"{name}"'
+
+
+def _encoded_width(name: str) -> int:
+    """Characters ``name`` adds to a ``$select`` list, its comma included."""
+    return len(urllib.parse.quote(quote_name(name), safe=_QUERY_SAFE)) + 1
+
+
+def entity_count(
+    service: str,
+    entity: str,
+    *,
+    filter: str | None = None,
+    credentials: Credentials = None,
+    fetch: Fetch | None = None,
+) -> int:
+    """How many records the entity set holds now (or match ``filter``). One request."""
+    url = entity_url(service, entity, suffix="/$count", query={"$filter": filter})
+    body = odata_get(url, credentials=credentials, accept="*/*", fetch=fetch)
+    return int(body.decode("utf-8-sig").strip())
+
+
+def _is_annotation(name: str) -> bool:
+    """``@odata.etag``, ``Price@odata.type``, ``Name@Core.Description``: an OData
+    property name never holds ``@`` otherwise."""
+    return "@" in name
+
+
+def _parse(body: bytes, parse_float: ParseFloat | None) -> dict:
+    return json.loads(body, strict=False, parse_float=parse_float)
+
+
+def entity_records(
+    service: str,
+    entity: str,
+    *,
+    select: Iterable[str] | None = None,
+    filter: str | None = None,
+    orderby: str | None = None,
+    top: int | None = None,
+    page_size: int = DEFAULT_PAGE_SIZE,
+    credentials: Credentials = None,
+    fetch: Fetch | None = None,
+    parse_float: ParseFloat | None = None,
+) -> Iterator[dict[str, Any]]:
+    """Records of an entity set, lazily, page by page, with OData annotations removed.
+
+    Follows ``@odata.nextLink`` when the server sends one, and otherwise pages with
+    ``$top``/``$skip`` until an empty page (a short page may be a server capping its
+    pages without a nextLink, so it is not taken as the end). ``orderby`` should name a unique key: paging
+    without a stable order can skip or repeat a record that moves between pages.
+
+    Args:
+        select: Properties to fetch (quoted as needed). ``None`` fetches all of them.
+        filter: A ``$filter`` expression, sent as written.
+        orderby: A ``$orderby`` expression, sent as written (quote spaced names
+            yourself with :func:`quote_name`).
+        top: The most records to yield in all; ``None`` for every one.
+        page_size: Records asked for per request.
+        parse_float: Called on every JSON number with a fraction; ``decimal.Decimal``
+            keeps every digit. Defaults to ``float``.
+    """
+    if page_size < 1:
+        raise ValueError(f"page_size must be at least 1, not {page_size}")
+    select = None if select is None else list(select)
+    if select == []:
+        raise ValueError("select is empty: pass None to fetch every property")
+    query: dict[str, Any] = {
+        "$select": ",".join(map(quote_name, select)) if select is not None else None,
+        "$filter": filter,
+        "$orderby": orderby,
+    }
+    credentials = resolve_credentials(credentials)  # once per read, not per page
+
+    def page_url(skip: int, size: int) -> str:
+        return entity_url(service, entity, query={**query, "$top": size, "$skip": skip})
+
+    def next_size(yielded: int) -> int:
+        return page_size if top is None else min(page_size, top - yielded)
+
+    yielded, size, server_paged = 0, next_size(0), False
+    url: str | None = page_url(0, size) if size > 0 else None
+    asked: set[str] = set()
+    previous = None
+    while url:
+        if url in asked:
+            raise ODataError(
+                f"the server sent {redact(url)} twice: paging would not end"
+            )
+        asked.add(url)
+        page = _parse(odata_get(url, credentials=credentials, fetch=fetch), parse_float)
+        rows = page.get("value", [])
+        if rows and rows == previous:
+            raise ODataError(
+                f"the server answered {redact(url)} with the page before it: it ignores "
+                "$skip, so paging would not end"
+            )
+        previous = rows
+        for row in rows:
+            yield {k: v for k, v in row.items() if not _is_annotation(k)}
+            yielded += 1
+            if top is not None and yielded >= top:  # a server page may outrun $top
+                return
+        if next_link := page.get("@odata.nextLink"):
+            following = urllib.parse.urljoin(url, next_link)
+            if not same_origin(url, following):
+                raise ODataError(
+                    "refusing to follow a nextLink to another origin, "
+                    f"{redact(following)}: the credentials would go with it"
+                )
+            url, server_paged = following, True
+        elif server_paged or not rows:
+            url = None  # a server paging by nextLink ends by omitting it
+        else:
+            # A short page may be the end, or a server capping pages without saying
+            # so; only an empty page tells them apart, at the cost of one request.
+            size = next_size(yielded)
+            url = page_url(yielded, size)
+
+
+def _select_groups(select: Iterable[str], *, key: str, budget: int) -> list[list[str]]:
+    """``select`` cut into groups, each led by ``key``, whose names fit ``budget``."""
+    groups: list[list[str]] = []
+    current, width = [key], _encoded_width(key)
+    if width > budget:
+        raise ValueError(
+            "the key, version and filter alone exceed max_url: shorten the filter or "
+            "raise max_url"
+        )
+    for name in select:
+        if name == key:
+            continue
+        name_width = _encoded_width(name)
+        if width + name_width > budget:
+            if current == [key]:
+                raise ValueError(
+                    f"{name!r} alone does not fit a URL of max_url characters beside the "
+                    "key; raise max_url"
+                )
+            groups.append(current)
+            current, width = [key], _encoded_width(key)
+        current.append(name)
+        width += name_width
+    groups.append(current)
+    return groups
+
+
+def entity_columns(
+    service: str,
+    entity: str,
+    *,
+    select: Iterable[str],
+    key: str,
+    version: str | None = None,
+    filter: str | None = None,
+    max_url: int = MAX_URL_CHARS,
+    page_size: int = DEFAULT_PAGE_SIZE,
+    credentials: Credentials = None,
+    fetch: Fetch | None = None,
+    parse_float: ParseFloat | None = None,
+) -> list[dict[str, Any]]:
+    """Every record with every ``select`` field, however many fields that is, as one moment.
+
+    Splits ``select`` into groups whose URL fits ``max_url``, reads each group in
+    ``key`` order, and joins them on ``key``. Records come back in key order, each with
+    exactly the ``select`` fields, in ``select`` order.
+
+    **A join of reads is only one moment if every read saw the same records in the same
+    state.** So ``version`` (a property the server rewrites on every edit, such as a
+    modification timestamp) is read in every group and must agree per key; without it,
+    an edit landing between two groups yields a record that never existed: the old
+    name beside the new price. And there are always at least two reads: a single group
+    is followed by a narrow ``key``/``version`` read, because ``$skip`` paging over a
+    delete before the cursor and an insert after it skips a live record while the
+    count still balances.
+
+    Raises:
+        UnstableRead: If reads disagree about which keys exist, their order, or their
+            versions. The entity set changed during the read; retrying is safe.
+        ODataError: If ``key`` is empty or repeats: the join would silently merge
+            different records into one.
+        ValueError: If one field cannot fit under ``max_url`` at all.
+    """
+    select = list(select)
+    credentials = resolve_credentials(credentials)  # once for every group
+    probe = len(
+        entity_url(
+            service,
+            entity,
+            query={
+                "$select": "",
+                "$filter": filter,
+                "$orderby": quote_name(key),
+                "$top": page_size,
+                "$skip": _WIDEST_SKIP,
+            },
+        )
+    )
+    budget = max_url - probe - (_encoded_width(version) if version else 0)
+    groups = [
+        group + ([version] if version and version not in group else [])
+        for group in _select_groups(select, key=key, budget=budget)
+    ]
+    if len(groups) == 1:
+        groups.append([key] + ([version] if version and version != key else []))
+    wanted = set(select)
+    joined: dict[Any, dict[str, Any]] = {}
+    first: list[tuple[Any, Any]] = []
+    for i, group in enumerate(groups):
+        rows = entity_records(
+            service,
+            entity,
+            select=group,
+            filter=filter,
+            orderby=quote_name(key),
+            page_size=page_size,
+            credentials=credentials,
+            fetch=fetch,
+            parse_float=parse_float,
+        )
+        seen: list[tuple[Any, Any]] = []
+        for row in rows:
+            k = row.get(key)
+            if k is None or (i == 0 and k in joined):
+                raise ODataError(
+                    f"{entity!r} has {'an empty' if k is None else 'a repeated'} {key!r} "
+                    f"({k!r}): it cannot be joined on that key without merging different "
+                    "records into one."
+                )
+            seen.append((k, row.get(version) if version else None))
+            if i == 0:
+                joined[k] = row
+            elif k in joined:
+                joined[k].update({n: v for n, v in row.items() if n in wanted})
+        if i == 0:
+            first = seen
+        elif seen != first:
+            keys, others = {k for k, _ in first}, {k for k, _ in seen}
+            edited = sum(
+                1 for a, b in zip(first, seen) if a[0] == b[0] and a[1] != b[1]
+            )
+            raise UnstableRead(
+                f"read {i + 1} of {len(groups)} disagrees with the first: "
+                f"{len(others - keys)} new, {len(keys - others)} gone, {edited} edited, "
+                "or the order moved. The entity set changed during the read. Retry."
+            )
+    return [{name: joined[k].get(name) for name in select} for k, _ in first]