Office365-REST-Python-Client 3.1.0__py3-none-any.whl → 3.2.0__py3-none-any.whl

office365/runtime/converters/records.py

@@ -16,7 +16,7 @@ if TYPE_CHECKING:
     from office365.runtime.client_object import ClientObject
 
 
-def iter_records(collection: "ClientObjectCollection") -> List[Dict[str, Any]]:
+def iter_records(collection: "ClientObjectCollection", raw: bool = False) -> List[Dict[str, Any]]:
     """Project a loaded collection into plain dict records.
 
     Mirrors ``csv_writer.write_csv``: plain select fields become columns,
@@ -29,19 +29,33 @@ def iter_records(collection: "ClientObjectCollection") -> List[Dict[str, Any]]:
 
     Args:
         collection: A loaded collection (items populated after execute_query()).
+        raw: When True, keep native property values (skip ``serialize_value``
+            coercion) — used for lossless round-trips like the migration toolkit.
 
     Raises:
         ValueError: When dotted select fields reference more than one navigation
             property.
     """
-    items = list(collection)
+    return records_from_items(
+        list(collection), collection.query_options.select, collection.query_options.expand, raw=raw
+    )
+
+
+def records_from_items(
+    items: List["ClientObject"],
+    select: List[str],
+    expand: "set[str] | list[str]",
+    raw: bool = False,
+) -> List[Dict[str, Any]]:
+    """Project a list of loaded items into records (the page-level primitive).
+
+    Used by :func:`iter_records` and by the paged (streaming) exporter, so a
+    large collection can be exported page by page without materializing it all.
+    """
     if not items:
         return []
-
-    select = collection.query_options.select
-    if not select:
-        select = sorted({key for item in items for key in item.properties.keys()})
-    expand = set(collection.query_options.expand)
+    select = list(select) if select else sorted({key for item in items for key in item.properties.keys()})
+    expand = set(expand)
     dotted = [field.split("/", 1) for field in select if "/" in field]
     navs = {nav for nav, _ in dotted}
     if len(navs) > 1:
@@ -54,15 +68,20 @@ def iter_records(collection: "ClientObjectCollection") -> List[Dict[str, Any]]:
     records: List[Dict[str, Any]] = []
     for item in items:
         children = _resolve_property(item, nav, expand)
-        base: Dict[str, Any] = {key: serialize_value(item.properties.get(key)) for key in plain}
+        base: Dict[str, Any] = {key: _value(item.properties.get(key), raw) for key in plain}
         for child in children:
             record = dict(base)
             for _nav_prop, field_name in dotted:
-                record[f"{_nav_prop}/{field_name}"] = _property_value(child, field_name)
+                record[f"{_nav_prop}/{field_name}"] = _property_value(child, field_name, raw)
             records.append(record)
     return records
 
 
+def _value(value: Any, raw: bool) -> Any:
+    """Coerce a property value to its JSON-safe form unless ``raw`` is requested."""
+    return value if raw else serialize_value(value)
+
+
 def _resolve_property(item: "ClientObject", nav: str | None, expand: set[str]) -> list[Any]:
     """Resolve an expanded navigation property into a list of child items."""
     if nav is None or nav not in expand:
@@ -75,8 +94,8 @@ def _resolve_property(item: "ClientObject", nav: str | None, expand: set[str]) -
     return [raw]
 
 
-def _property_value(prop: "dict | ClientObject", field_name: str) -> Any:
+def _property_value(prop: "dict | ClientObject", field_name: str, raw: bool = False) -> Any:
     """Read a property value from an item that may be a ClientObject or dict."""
     if isinstance(prop, dict):
-        return serialize_value(prop.get(field_name))
-    return serialize_value(prop.properties.get(field_name))
+        return _value(prop.get(field_name), raw)
+    return _value(prop.properties.get(field_name), raw)

office365/runtime/converters/registry.py

@@ -0,0 +1,185 @@
+"""Format registry — maps a format name to its record reader/writer.
+
+Keeps the import/export surface of :class:`~office365.runtime.record_collection.RecordCollection`
+data-driven: adding a format is a registration, not a new method. Readers turn a
+source into plain dict records; writers persist a collection's records to a
+target. Both import their (optional) dependency lazily, so registering a format
+never pulls in pandas/openpyxl.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable, Dict, List
+
+if TYPE_CHECKING:
+    from office365.runtime.client_object_collection import ClientObjectCollection
+
+RecordReader = Callable[..., List[dict]]
+RecordWriter = Callable[..., None]
+
+_READERS: Dict[str, RecordReader] = {}
+_WRITERS: Dict[str, RecordWriter] = {}
+
+
+def register(name: str, *, reader: RecordReader, writer: RecordWriter) -> None:
+    """Register (or replace) a format's reader and writer."""
+    _READERS[name] = reader
+    _WRITERS[name] = writer
+
+
+def reader_for(name: str) -> RecordReader:
+    """The reader for a format, or a ``KeyError`` listing the known formats."""
+    try:
+        return _READERS[name]
+    except KeyError:
+        raise KeyError(f"Unknown import format {name!r}; known: {sorted(_READERS)}") from None
+
+
+def writer_for(name: str) -> RecordWriter:
+    """The writer for a format, or a ``KeyError`` listing the known formats."""
+    try:
+        return _WRITERS[name]
+    except KeyError:
+        raise KeyError(f"Unknown export format {name!r}; known: {sorted(_WRITERS)}") from None
+
+
+def formats() -> List[str]:
+    """The registered format names."""
+    return sorted(set(_READERS) | set(_WRITERS))
+
+
+# ── Built-in formats (lazy imports) ──────────────────────────────
+
+
+def _read_csv(source: Any, **opts: Any) -> List[dict]:
+    from office365.runtime.converters.csv_reader import read_csv_records
+    from office365.runtime.converters.streams import open_text
+
+    with open_text(source, "r") as f:
+        return read_csv_records(f, opts.get("delimiter", ","))
+
+
+def _write_csv(collection: "ClientObjectCollection", target: Any, **opts: Any) -> None:
+    from office365.runtime.converters.csv_writer import write_csv
+    from office365.runtime.converters.streams import open_text
+
+    with open_text(target, "w") as f:
+        write_csv(collection, f, delimiter=opts.get("delimiter", ","))
+
+
+def _read_tsv(source: Any, **opts: Any) -> List[dict]:
+    return _read_csv(source, delimiter="\t", **opts)
+
+
+def _write_tsv(collection: "ClientObjectCollection", target: Any, **opts: Any) -> None:
+    _write_csv(collection, target, delimiter="\t", **opts)
+
+
+def _read_ndjson(source: Any, **opts: Any) -> List[dict]:
+    from office365.runtime.converters.ndjson import read_ndjson
+    from office365.runtime.converters.streams import open_text
+
+    with open_text(source, "r") as f:
+        return read_ndjson(f)
+
+
+def _write_ndjson(collection: "ClientObjectCollection", target: Any, **opts: Any) -> None:
+    from office365.runtime.converters.ndjson import write_ndjson
+    from office365.runtime.converters.records import iter_records
+    from office365.runtime.converters.streams import open_text
+
+    with open_text(target, "w") as f:
+        write_ndjson(iter_records(collection), f)
+
+
+def _read_json_file(source: Any, **opts: Any) -> List[dict]:
+    from office365.runtime.converters.json_file import read_json
+    from office365.runtime.converters.streams import open_text
+
+    with open_text(source, "r") as f:
+        return read_json(f)
+
+
+def _write_json_file(collection: "ClientObjectCollection", target: Any, **opts: Any) -> None:
+    from office365.runtime.converters.json_file import write_json
+    from office365.runtime.converters.records import iter_records
+    from office365.runtime.converters.streams import open_text
+
+    with open_text(target, "w") as f:
+        write_json(iter_records(collection), f, indent=opts.get("indent"))
+
+
+def _read_excel(source: Any, **opts: Any) -> List[dict]:
+    from office365.runtime.converters.excel import read_excel
+
+    return read_excel(source)
+
+
+def _write_excel(collection: "ClientObjectCollection", target: Any, **opts: Any) -> None:
+    from office365.runtime.converters.excel import write_excel
+    from office365.runtime.converters.records import iter_records
+
+    write_excel(iter_records(collection), target)
+
+
+def _read_dataframe(df: Any, **opts: Any) -> List[dict]:
+    from office365.runtime.converters.dataframe import read_dataframe
+
+    return read_dataframe(df)
+
+
+def _write_dataframe(collection: "ClientObjectCollection", target: Any, **opts: Any) -> None:
+    from office365.runtime.converters.dataframe import write_dataframe
+
+    write_dataframe(collection, target)
+
+
+def _read_parquet(source: Any, **opts: Any) -> List[dict]:
+    from office365.runtime.converters.parquet import read_parquet
+
+    return read_parquet(source)
+
+
+def _write_parquet(collection: "ClientObjectCollection", target: Any, **opts: Any) -> None:
+    from office365.runtime.converters.parquet import write_parquet
+    from office365.runtime.converters.records import iter_records
+
+    write_parquet(iter_records(collection), target)
+
+
+def _read_orc(source: Any, **opts: Any) -> List[dict]:
+    from office365.runtime.converters.parquet import read_orc
+
+    return read_orc(source)
+
+
+def _write_orc(collection: "ClientObjectCollection", target: Any, **opts: Any) -> None:
+    from office365.runtime.converters.parquet import write_orc
+    from office365.runtime.converters.records import iter_records
+
+    write_orc(iter_records(collection), target)
+
+
+def _read_feather(source: Any, **opts: Any) -> List[dict]:
+    from office365.runtime.converters.parquet import read_feather
+
+    return read_feather(source)
+
+
+def _write_feather(collection: "ClientObjectCollection", target: Any, **opts: Any) -> None:
+    from office365.runtime.converters.parquet import write_feather
+    from office365.runtime.converters.records import iter_records
+
+    write_feather(iter_records(collection), target)
+
+
+register("csv", reader=_read_csv, writer=_write_csv)
+register("tsv", reader=_read_tsv, writer=_write_tsv)
+register("json", reader=_read_json_file, writer=_write_json_file)
+register("json_file", reader=_read_json_file, writer=_write_json_file)  # deprecated alias
+register("ndjson", reader=_read_ndjson, writer=_write_ndjson)
+register("excel", reader=_read_excel, writer=_write_excel)
+register("dataframe", reader=_read_dataframe, writer=_write_dataframe)
+register("parquet", reader=_read_parquet, writer=_write_parquet)
+register("orc", reader=_read_orc, writer=_write_orc)
+register("feather", reader=_read_feather, writer=_write_feather)

office365/runtime/converters/scalars.py

@@ -53,6 +53,14 @@ def parse_bool(value: Any) -> Any:
     return value
 
 
+def parse_int(value: Any) -> Optional[int]:
+    """Convert a value to an int, returning ``None`` when absent/invalid."""
+    try:
+        return int(value) if value is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
 def try_int(value: Any) -> Any:
     """Convert a value to an int, falling back to the raw value on failure."""
     try:
@@ -69,6 +77,29 @@ def try_float(value: Any) -> Any:
         return value
 
 
+def utc_now_iso() -> str:
+    """Current UTC time as an ISO-8601 string (second precision)."""
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def iso(value: Any) -> str:
+    """ISO-8601 string for a datetime (second precision); ``""`` for ``None``."""
+    if value is None:
+        return ""
+    return value.isoformat(timespec="seconds")
+
+
+def iso_or_none(value: Any) -> Optional[str]:
+    """ISO-8601 string for a datetime, or ``None`` when unset/absent.
+
+    Treats ``datetime.min`` (the "not loaded" sentinel used by some SharePoint
+    properties) as unset.
+    """
+    if value is None or value == datetime.min:
+        return None
+    return value.isoformat(timespec="seconds") if hasattr(value, "isoformat") else (str(value) or None)
+
+
 def _normalize_datetime_string(value: str) -> str:
     """Truncate or pad fractional seconds to exactly 6 digits for %f compatibility."""
     if "." not in value:

office365/runtime/converters/sql.py

@@ -0,0 +1,69 @@
+"""SQL/DuckDB record streaming — the optional-dependency boundary.
+
+``from_sql``/``from_duckdb`` accept a connection object (SQLAlchemy
+``Engine``/``Connection``, DuckDB connection) or a connection string, plus a
+query, and yield record batches (bounded memory). ``to_sql``/``to_duckdb`` write
+loaded records to a table. Dependencies are imported lazily:
+``[sql]`` (SQLAlchemy) and ``[duckdb]``.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterator, List
+
+
+def require_sqlalchemy():
+    """Import SQLAlchemy or raise an actionable error."""
+    try:
+        import sqlalchemy  # type: ignore[import-not-found]
+    except ImportError:
+        raise ImportError("pip install office365-rest-python-client[sql]") from None
+    return sqlalchemy
+
+
+def sql_chunks(source: Any, query: str, chunksize: int) -> Iterator[List[Dict[str, Any]]]:
+    """Yield record batches from a SQLAlchemy engine/connection (or URL string)."""
+    from office365.runtime.converters.dataframe import require_pandas
+
+    engine = source
+    if isinstance(source, str):
+        engine = require_sqlalchemy().create_engine(source)
+    for chunk in require_pandas().read_sql_query(query, engine, chunksize=chunksize):
+        yield chunk.to_dict("records")
+
+
+def write_sql(records: List[Dict[str, Any]], target: Any, *, table: str, **opts: Any) -> None:
+    """Write records to a SQL table (SQLAlchemy connection/engine or URL)."""
+    from office365.runtime.converters.dataframe import require_pandas
+
+    engine = target
+    if isinstance(target, str):
+        engine = require_sqlalchemy().create_engine(target)
+    require_pandas().DataFrame.from_records(records).to_sql(table, engine, **opts)
+
+
+def duckdb_chunks(source: Any, query: str, chunksize: int) -> Iterator[List[Dict[str, Any]]]:
+    """Yield record batches from a DuckDB connection (or database path)."""
+    import duckdb  # type: ignore[import-not-found]
+
+    connection = source if hasattr(source, "execute") else duckdb.connect(source)
+    result = connection.execute(query)
+    columns = [description[0] for description in result.description]
+    while True:
+        rows = result.fetchmany(chunksize)
+        if not rows:
+            break
+        yield [dict(zip(columns, row)) for row in rows]
+
+
+def write_duckdb(records: List[Dict[str, Any]], target: Any, *, table: str, **opts: Any) -> None:
+    """Write records to a DuckDB table (connection or database path)."""
+    import duckdb  # type: ignore[import-not-found]
+
+    from office365.runtime.converters.dataframe import require_pandas
+
+    connection = target if hasattr(target, "execute") else duckdb.connect(target)
+    df = require_pandas().DataFrame.from_records(records)
+    connection.register("_records_df", df)
+    connection.execute(f"CREATE OR REPLACE TABLE {table} AS SELECT * FROM _records_df")
+    connection.unregister("_records_df")

office365/runtime/converters/streamers.py

@@ -0,0 +1,105 @@
+"""Incremental (paged) writers for the streaming exporter.
+
+Only row-appendable formats support paging (CSV/TSV/NDJSON/JSON); the exporter
+falls back to a whole-collection write otherwise. Each streamer is a small
+object with ``write(records)`` and ``close()``.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Callable, Dict, List, Optional, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class RecordStreamWriter(Protocol):
+    """Append-only writer used by ``export_to(..., page_size=...)``."""
+
+    def write(self, records: List[Dict[str, Any]]) -> None:
+        """Append a page of records."""
+        ...
+
+    def close(self) -> None:
+        """Finalize and release the target."""
+        ...
+
+
+class CsvStreamWriter:
+    """Append CSV/TSV rows, writing the header on the first page."""
+
+    def __init__(self, target: Any, delimiter: str = ",") -> None:
+        from office365.runtime.converters.csv_writer import write_records, write_rows
+        from office365.runtime.converters.streams import open_text
+
+        self._write_records = write_records
+        self._write_rows = write_rows
+        self._delimiter = delimiter
+        self._columns: Optional[List[str]] = None
+        self._ctx = open_text(target, "w")
+        self._file = self._ctx.__enter__()
+
+    def write(self, records: List[Dict[str, Any]]) -> None:
+        if not records:
+            return
+        if self._columns is None:
+            self._columns = list(dict.fromkeys(key for record in records for key in record))
+            self._write_records(records, self._file, columns=self._columns, delimiter=self._delimiter)
+        else:
+            self._write_rows(records, self._file, self._columns, self._delimiter)
+
+    def close(self) -> None:
+        self._ctx.__exit__(None, None, None)
+
+
+class NdjsonStreamWriter:
+    """Append NDJSON lines."""
+
+    def __init__(self, target: Any) -> None:
+        from office365.runtime.converters.ndjson import write_ndjson
+        from office365.runtime.converters.streams import open_text
+
+        self._write_ndjson = write_ndjson
+        self._ctx = open_text(target, "w")
+        self._file = self._ctx.__enter__()
+
+    def write(self, records: List[Dict[str, Any]]) -> None:
+        self._write_ndjson(records, self._file)
+
+    def close(self) -> None:
+        self._ctx.__exit__(None, None, None)
+
+
+class JsonArrayStreamWriter:
+    """Append a single JSON array across pages."""
+
+    def __init__(self, target: Any) -> None:
+        from office365.runtime.converters.streams import open_text
+
+        self._ctx = open_text(target, "w")
+        self._file = self._ctx.__enter__()
+        self._file.write("[")
+        self._first = True
+
+    def write(self, records: List[Dict[str, Any]]) -> None:
+        for record in records:
+            if not self._first:
+                self._file.write(",")
+            json.dump(record, self._file, default=str)
+            self._first = False
+
+    def close(self) -> None:
+        self._file.write("]")
+        self._ctx.__exit__(None, None, None)
+
+
+_STREAMERS: Dict[str, Callable[..., RecordStreamWriter]] = {
+    "csv": lambda target, **opts: CsvStreamWriter(target, opts.get("delimiter", ",")),
+    "tsv": lambda target, **opts: CsvStreamWriter(target, "\t"),
+    "ndjson": lambda target, **opts: NdjsonStreamWriter(target),
+    "json": lambda target, **opts: JsonArrayStreamWriter(target),
+}
+
+
+def streamer_for(name: str) -> Optional[Callable[..., RecordStreamWriter]]:
+    """The incremental writer factory for ``name``, or ``None`` (not appendable)."""
+    return _STREAMERS.get(name)

office365/runtime/converters/streams.py

@@ -0,0 +1,42 @@
+"""Path | file-like resolution shared by the format readers/writers.
+
+Pandas-style: every reader/writer accepts a filesystem path, a ``PathLike`` or an
+already-open file object. Binary vs text mode is chosen by the format; an
+existing stream is passed through untouched.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from os import PathLike
+from typing import IO, Any, Iterator, Union, cast
+
+Source = Union[str, PathLike, IO[Any]]
+
+
+def is_stream(source: Any) -> bool:
+    """Whether ``source`` is an already-open file object (vs a path)."""
+    return hasattr(source, "read") or hasattr(source, "write")
+
+
+@contextmanager
+def open_text(source: Source, mode: str = "r", **kwargs: Any) -> Iterator[IO[Any]]:
+    """Yield a text stream for ``source`` (a path is opened, a stream passed through)."""
+    if is_stream(source):
+        yield source  # type: ignore[misc]
+        return
+    kwargs.setdefault("encoding", "utf-8")
+    if "b" not in mode:
+        kwargs.setdefault("newline", "")
+    with open(cast(Any, source), mode, **kwargs) as f:
+        yield f
+
+
+@contextmanager
+def open_binary(source: Source, mode: str = "rb", **kwargs: Any) -> Iterator[IO[Any]]:
+    """Yield a binary stream for ``source`` (a path is opened, a stream passed through)."""
+    if is_stream(source):
+        yield source  # type: ignore[misc]
+        return
+    with open(cast(Any, source), mode, **kwargs) as f:
+        yield f

office365/runtime/converters/upsert.py

@@ -0,0 +1,101 @@
+"""Key-based conflict resolution for record imports (destination-agnostic).
+
+A natural key is hashed into a stable value stored on the destination; a target
+implements :class:`UpsertTarget` to load existing keys and create/update records.
+The same :func:`keyed_queue` drives every collection (and the migration toolkit),
+so idempotent upsert is one implementation, not one per destination.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any, Iterable, Mapping, MutableMapping, Protocol, runtime_checkable
+
+ON_CONFLICT_MODES = ("skip", "upsert")
+
+
+def record_key(record: dict, key_columns: Iterable[str]) -> str:
+    """Deterministic hash key for a record's natural key column(s).
+
+    Canonicalizes the key values as JSON and hashes them (SHA-256), so the key is
+    stable across runs and safe to store in a text column regardless of the
+    source values' length or characters.
+    """
+    values = [record.get(column) for column in key_columns]
+    payload = json.dumps(values, default=str, ensure_ascii=False, separators=(",", ":"))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+@runtime_checkable
+class UpsertTarget(Protocol):
+    """Destination capability needed for keyed skip/upsert."""
+
+    @property
+    def key_field(self) -> str:
+        """The column storing the derived key hash."""
+        ...
+
+    def ensure_key_field(self) -> None:
+        """Queue creation of the key column if missing (does not execute)."""
+        ...
+
+    def load_keys(self) -> Mapping[str, Any]:
+        """Return the existing ``{key: item_id}`` map (executes)."""
+        ...
+
+    def create_records(self, records: list[dict]) -> None:
+        """Queue a create for each record."""
+        ...
+
+    def update_record(self, item_id: Any, record: dict) -> None:
+        """Queue an update of an existing item."""
+        ...
+
+
+def keyed_queue(
+    target: UpsertTarget,
+    records: list[dict],
+    *,
+    key_columns: Iterable[str],
+    existing: MutableMapping[str, Any],
+    on_conflict: str = "skip",
+    dry_run: bool = False,
+) -> tuple[int, int]:
+    """Queue a chunk with key-based conflict resolution.
+
+    Each record's natural key is hashed into ``target.key_field``. Records whose
+    key is already in ``existing`` are either skipped (``on_conflict="skip"``) or
+    queued as an update (``on_conflict="upsert"``); the rest are created.
+    ``existing`` is updated in place so duplicate keys within the same run are
+    deduplicated too. With ``dry_run`` the counts are computed without queuing
+    anything (a plan preview).
+
+    Returns:
+        ``(queued, skipped)`` — records queued (created or updated) and records
+        skipped as already present.
+    """
+    if on_conflict not in ON_CONFLICT_MODES:
+        raise ValueError(f"on_conflict must be one of {ON_CONFLICT_MODES}, got {on_conflict!r}")
+    key_field = target.key_field
+    creates: list[dict] = []
+    updates: list[tuple[Any, dict]] = []
+    skipped = 0
+    for record in records:
+        key = record_key(record, key_columns)
+        record[key_field] = key
+        if key in existing:
+            item_id = existing[key]
+            if item_id is not None and on_conflict == "upsert":
+                updates.append((item_id, record))
+            else:
+                skipped += 1
+            continue
+        existing[key] = None  # created this run (id unknown) — dedupe later duplicates
+        creates.append(record)
+    if not dry_run:
+        if creates:
+            target.create_records(creates)
+        for item_id, record in updates:
+            target.update_record(item_id, record)
+    return len(creates) + len(updates), skipped