snapdelta 0.1.0__py3-none-any.whl

snapdelta/__init__.py

@@ -0,0 +1,5 @@
+"""snapdelta: snapshot anything with diffsync models, then compare two snapshots."""
+
+from importlib.metadata import version
+
+__version__ = version("snapdelta")

snapdelta/about.py

@@ -0,0 +1,50 @@
+"""What a user or a support request needs to know: the versions, the plugins and the project links."""
+
+from __future__ import annotations
+
+import platform
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from importlib.metadata import PackageNotFoundError, version
+
+from .source import broken_sources, source_classes
+
+PROJECT_URL = "https://github.com/antoinekh/snapdelta"
+DOCS_URL = f"{PROJECT_URL}/tree/main/docs"
+CHANGELOG_URL = f"{PROJECT_URL}/blob/main/CHANGELOG.md"
+ISSUES_URL = f"{PROJECT_URL}/issues"
+
+
+@dataclass(frozen=True)
+class Component:
+    name: str
+    version: str
+    sources: list[str] = field(default_factory=list)
+
+
+def package_version(distribution: str) -> str:
+    try:
+        return version(distribution)
+    except PackageNotFoundError:
+        return "unknown"
+
+
+def components(extra: Iterable[str] = ()) -> list[Component]:
+    """snapdelta, the other given packages (the web UI), diffsync and Python."""
+    names = ["snapdelta", *extra, "diffsync"]
+    return [*(Component(name, package_version(name)) for name in names), Component("Python", platform.python_version())]
+
+
+def plugins() -> list[Component]:
+    """The installed plugins with their sources; a source with no distribution is listed under its own name."""
+    by_plugin: dict[str, list[str]] = {}
+    for name, cls in sorted(source_classes().items()):
+        by_plugin.setdefault(cls.distribution or name, []).append(name)
+    for name in sorted(broken_sources()):
+        by_plugin.setdefault(f"{name} (cannot load)", []).append(name)
+    return [Component(plugin, package_version(plugin), names) for plugin, names in sorted(by_plugin.items())]
+
+
+def version_line() -> str:
+    core, diffsync = package_version("snapdelta"), package_version("diffsync")
+    return f"snapdelta {core} (diffsync {diffsync}, Python {platform.python_version()})"

snapdelta/activity.py

@@ -0,0 +1,182 @@
+"""What snapdelta did: one run per command (snapshots, a diff), with its result and the log messages of its loads.
+
+A source plugin logs with the standard `logging` module. While a load runs, `LogCollector` adds each message to
+the log of that load, even from the threads of `run_parallel()`, which copy the context of their caller.
+"""
+
+from __future__ import annotations
+
+import getpass
+import logging
+import secrets
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, Field
+
+from .errors import SnapdeltaError
+
+if TYPE_CHECKING:
+    from .store import Store
+
+_log = logging.getLogger(__name__)
+# The log that receives the messages of the current context, and the target that they are about.
+_lines: ContextVar[list[LogLine] | None] = ContextVar("snapdelta_log_lines", default=None)
+_target: ContextVar[str | None] = ContextVar("snapdelta_log_target", default=None)
+# Only a load collects: the compare of a run logs too (diffsync), and that says nothing about the targets.
+_loading: ContextVar[bool] = ContextVar("snapdelta_loading", default=False)
+
+
+class LogLine(BaseModel):
+    at: datetime
+    level: str
+    logger: str
+    target: str | None = None
+    message: str
+
+
+class RunStatus(StrEnum):
+    OK = "ok"
+    PARTIAL = "partial"
+    FAILED = "failed"
+
+
+class Run(BaseModel):
+    """One command: what it took or compared, how it ended, and the log of its loads."""
+
+    id: str
+    kind: str = Field(description="snapshot or diff")
+    origin: str = Field(description="cli, web, api, mcp, or python for a direct call")
+    user: str | None = None
+    started_at: datetime
+    ended_at: datetime
+    status: RunStatus
+    targets: list[str] = []
+    saved: list[str] = Field(default=[], description="The references of the saved snapshots.")
+    output: str | None = Field(default=None, description="The name of the saved diff.")
+    errors: dict[str, str] = Field(default={}, description="The error of each failed target, or of the run under `*`.")
+    log: list[LogLine] = []
+
+    @property
+    def duration_s(self) -> float:
+        return (self.ended_at - self.started_at).total_seconds()
+
+
+class LogCollector(logging.Handler):
+    """Adds each message of a load to the log of the current context; any other message goes nowhere."""
+
+    def emit(self, record: logging.LogRecord) -> None:
+        lines = _lines.get()
+        if lines is None or not _loading.get():
+            return
+        try:
+            message = record.getMessage()
+        except (TypeError, ValueError):
+            message = str(record.msg)
+        lines.append(
+            LogLine(
+                at=datetime.fromtimestamp(record.created, UTC),
+                level=record.levelname.lower(),
+                logger=record.name,
+                target=_target.get(),
+                message=message,
+            )
+        )
+
+
+def configure_logging(level: str = "info") -> None:
+    """Collect the messages at `level` and above, and keep the console at warning; once per process."""
+    root = logging.getLogger()
+    wanted = logging.getLevelNamesMapping()[level.upper()]
+    for handler in root.handlers:
+        # The console handler of logging.basicConfig() shows warnings only, as before the collector.
+        if type(handler) is logging.StreamHandler and handler.level == logging.NOTSET:
+            handler.setLevel(logging.WARNING)
+    if not any(isinstance(handler, LogCollector) for handler in root.handlers):
+        root.addHandler(LogCollector(wanted))
+    else:
+        for handler in root.handlers:
+            if isinstance(handler, LogCollector):
+                handler.setLevel(wanted)
+    root.setLevel(min(root.level or logging.WARNING, wanted))
+
+
+@contextmanager
+def collecting(target: str | None = None, loading: bool = True) -> Iterator[list[LogLine]]:
+    """The log of the load in the block, about `target`; the lines also go to the log around it."""
+    outer = _lines.get()
+    lines: list[LogLine] = []
+    tokens = (_lines.set(lines), _target.set(target), _loading.set(loading))
+    try:
+        yield lines
+    finally:
+        _lines.reset(tokens[0])
+        _target.reset(tokens[1])
+        _loading.reset(tokens[2])
+        if outer is not None:
+            outer.extend(lines)
+
+
+class RunRecord:
+    """A run while it goes on; the caller fills `saved`, `output` and `errors`."""
+
+    def __init__(self, kind: str, origin: str, targets: list[str]) -> None:
+        self.kind = kind
+        self.origin = origin
+        self.targets = targets
+        self.started_at = datetime.now(UTC)
+        self.saved: list[str] = []
+        self.output: str | None = None
+        self.errors: dict[str, str] = {}
+        self.log: list[LogLine] = []
+
+    def finished(self) -> Run:
+        done = len(self.saved) + (1 if self.output else 0)
+        status = RunStatus.OK if not self.errors else RunStatus.PARTIAL if done else RunStatus.FAILED
+        # Microseconds, so that the ids of two runs of the same second sort in their order.
+        stamp = self.started_at.strftime("%Y%m%dT%H%M%S%fZ")
+        return Run(
+            id=f"{stamp}-{self.kind}-{secrets.token_hex(2)}",
+            kind=self.kind,
+            origin=self.origin,
+            user=_user() if self.origin == "cli" else None,
+            started_at=self.started_at,
+            ended_at=datetime.now(UTC),
+            status=status,
+            targets=self.targets,
+            saved=self.saved,
+            output=self.output,
+            errors=self.errors,
+            # The loads add their lines when they end: the time gives back the order of the events.
+            log=sorted(self.log, key=lambda line: line.at),
+        )
+
+
+@contextmanager
+def recorded(store: Store, kind: str, origin: str, targets: list[str]) -> Iterator[RunRecord]:
+    """Record the block as a run of the store; an error fails the run and goes on to the caller."""
+    record = RunRecord(kind, origin, targets)
+    try:
+        with collecting(loading=False) as lines:
+            record.log = lines
+            yield record
+    except Exception as error:
+        record.errors["*"] = str(error) if isinstance(error, SnapdeltaError) else f"{type(error).__name__}: {error}"
+        raise
+    finally:
+        try:
+            store.save_run(record.finished())
+        except (OSError, SnapdeltaError) as error:
+            # The command itself went well: a run that cannot be saved must not turn it into an error.
+            _log.warning("the run was not saved: %s", error)
+
+
+def _user() -> str | None:
+    try:
+        return getpass.getuser()
+    except (KeyError, OSError):
+        return None

snapdelta/catalog.py

@@ -0,0 +1,160 @@
+"""The models of the installed sources, read from their classes: what a profile can name and which sources compare."""
+
+from __future__ import annotations
+
+import inspect
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any, get_args
+
+from diffsync import DiffSyncModel
+
+from .source import Source
+
+# diffsync joins the identifiers of an object with this into its unique id.
+ID_SEPARATOR = "__"
+MODULE_PATH_RE = re.compile(r"\b(?:\w+\.)+(\w+)")
+
+
+@dataclass(frozen=True)
+class ModelField:
+    name: str
+    type: str
+
+
+@dataclass(frozen=True)
+class ModelInfo:
+    name: str
+    doc: str | None
+    identifiers: list[ModelField]
+    attributes: list[ModelField]
+    children: list[str]
+    parent: str | None
+    # Kept in the snapshot, never compared.
+    context: list[ModelField] = field(default_factory=list)
+
+    @property
+    def unique_id(self) -> str:
+        """The form of the unique id of an object, for example `interface__index`."""
+        return ID_SEPARATOR.join(item.name for item in self.identifiers)
+
+    @property
+    def shape(self) -> tuple[tuple[str, ...], tuple[str, ...]]:
+        """What two sources must share for this model to compare: the names of the identifiers and attributes."""
+        return tuple(item.name for item in self.identifiers), tuple(item.name for item in self.attributes)
+
+
+@dataclass(frozen=True)
+class SourceModels:
+    source: str
+    distribution: str | None
+    fetches: bool
+    doc: str | None
+    models: list[ModelInfo]
+    top_level: list[str]
+
+
+@dataclass
+class ModelSet:
+    """Sources with the same models: one table of models serves them all."""
+
+    sources: list[SourceModels]
+    # The sources of other sets that share top-level models with this one, so that a compare between them works.
+    compares_with: list[str] = field(default_factory=list)
+    # One message per model that another set names the same way with other fields.
+    conflicts: list[str] = field(default_factory=list)
+
+    @property
+    def models(self) -> list[ModelInfo]:
+        return self.sources[0].models
+
+    @property
+    def names(self) -> list[str]:
+        return [item.source for item in self.sources]
+
+    @property
+    def top_level(self) -> list[str]:
+        return self.sources[0].top_level
+
+
+def describe(cls: type[Source], source: str | None = None) -> SourceModels:
+    """The models of one source class, top-level models first, each followed by its children."""
+    order = cls.model_order()
+    parents = {child: name for name in order for child in _model(cls, name).get_children_mapping()}
+    models = [_model_info(name, _model(cls, name), parents.get(name)) for name in order]
+    return SourceModels(
+        source=source or cls.source,
+        distribution=cls.distribution,
+        fetches=cls.fetches,
+        doc=inspect.getdoc(cls),
+        models=models,
+        top_level=list(cls.top_level),
+    )
+
+
+def catalog(classes: Mapping[str, type[Source]]) -> list[ModelSet]:
+    """The installed sources grouped by the same models, with the other sets that they compare with or conflict with."""
+    groups: dict[tuple[Any, ...], list[SourceModels]] = {}
+    for name, cls in sorted(classes.items()):
+        item = describe(cls, name)
+        key = tuple((model.name, model.shape, tuple(model.children)) for model in item.models)
+        groups.setdefault(key, []).append(item)
+    sets = [ModelSet(sources) for sources in groups.values()]
+    for found in sets:
+        own = {model.name: model for model in found.models}
+        for other in sets:
+            if other is found:
+                continue
+            if any(name in other.top_level for name in found.top_level):
+                found.compares_with += other.names
+            others = {model.name: model for model in other.models}
+            found.conflicts += [
+                _conflict(own[name], others[name], ", ".join(other.names))
+                for name in own
+                if name in others and own[name].shape != others[name].shape
+            ]
+    return sets
+
+
+def _model(cls: type[Source], name: str) -> type[DiffSyncModel]:
+    return getattr(cls, name)
+
+
+def context_fields(model: type[DiffSyncModel]) -> list[str]:
+    """The fields that a snapshot keeps but that diffsync never compares: not an identifier, attribute or child."""
+    compared = {*model._identifiers, *model._attributes, *model._children.values(), *DiffSyncModel.model_fields}
+    return [name for name in model.model_fields if name not in compared]
+
+
+def _model_info(name: str, model: type[DiffSyncModel], parent: str | None) -> ModelInfo:
+    fields = model.model_fields
+    return ModelInfo(
+        name=name,
+        doc=inspect.getdoc(model) if model.__doc__ else None,
+        identifiers=[ModelField(key, _type_name(fields[key].annotation)) for key in model._identifiers],
+        attributes=[ModelField(key, _type_name(fields[key].annotation)) for key in model._attributes],
+        children=list(model.get_children_mapping()),
+        parent=parent,
+        context=[ModelField(key, _type_name(fields[key].annotation)) for key in context_fields(model)],
+    )
+
+
+def _type_name(annotation: Any) -> str:
+    """`int`, `str | None`, `list[str]`, `InterfaceType | None`: the type as a reader writes it, without modules."""
+    if isinstance(annotation, type) and not get_args(annotation):
+        return annotation.__name__
+    return MODULE_PATH_RE.sub(r"\1", str(annotation).replace("typing.", ""))
+
+
+def _conflict(own: ModelInfo, other: ModelInfo, source: str) -> str:
+    own_fields = set(own.shape[0]) | set(own.shape[1])
+    other_fields = set(other.shape[0]) | set(other.shape[1])
+    parts = []
+    if own.shape[0] != other.shape[0]:
+        parts.append(f"identifiers {', '.join(own.shape[0])} here, {', '.join(other.shape[0])} in {source}")
+    if only_here := sorted(own_fields - other_fields):
+        parts.append(f"only here: {', '.join(only_here)}")
+    if only_there := sorted(other_fields - own_fields):
+        parts.append(f"only in {source}: {', '.join(only_there)}")
+    return f"{own.name}: not the same fields as in {source} ({'; '.join(parts) or 'other order'})"