csvledger 0.0.2__py3-none-any.whl

csvledger/__init__.py

@@ -0,0 +1,326 @@
+"""Long-form CSV tables that merge idempotently: a git-friendly history store.
+
+A table is described by a :class:`TableSpec` (columns, key, merge policy,
+partitioning). Rows are merged into CSV files, one per table (or one per year
+of its time column), sorted so new rows land at the end of a file, through any
+``MutableMapping[str, str]`` (relative path to text): a local folder
+(:class:`Files`, the default), a git checkout, or any ``dol`` store.
+
+Merge policies are chosen so that re-running a collection never changes the
+result (idempotent), and the order of runs does not matter for daily series:
+
+- ``max``: upsert by key; integer fields keep the larger value. A day's count
+  only grows as the day completes, so the fullest count wins whichever run saw it.
+- ``replace_group``: rows of a snapshot (``group`` columns, e.g. one repo and
+  one fetch date) replace any earlier rows of the same group.
+- ``change_only``: a row is stored only when its values differ from the
+  previous row of the same ``entity``; the key is the entity columns plus one
+  date column. For slowly changing values, where daily copies would be noise.
+- ``upsert``: upsert by key, the new row wins (current metadata).
+
+>>> spec = TableSpec("views", ("date", "repo", "count"), key=("date", "repo"),
+...                  merge="max", int_columns=("count",), time_column="date")
+>>> store = {}
+>>> write_rows(store, spec, [{"date": "2026-01-01", "repo": "a/b", "count": 3}])
+1
+>>> write_rows(store, spec, [{"date": "2026-01-01", "repo": "a/b", "count": 2}])
+0
+>>> print(store["views/2026.csv"], end="")
+date,repo,count
+2026-01-01,a/b,3
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import os
+from collections.abc import Iterable, Iterator, Mapping, MutableMapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+__all__ = [
+    "Files",
+    "MANIFEST_KEY",
+    "MergePolicy",
+    "TableSpec",
+    "dump_csv",
+    "manifest",
+    "merge_rows",
+    "parse_csv",
+    "read_table",
+    "table_files",
+    "write_manifest",
+    "write_rows",
+]
+
+MergePolicy = Literal["max", "replace_group", "change_only", "upsert"]
+
+
+@dataclass(frozen=True)
+class TableSpec:
+    """How one table is laid out on disk and merged."""
+
+    name: str
+    columns: tuple[str, ...]
+    key: tuple[str, ...]
+    merge: MergePolicy
+    int_columns: tuple[str, ...] = ()
+    time_column: str | None = None  # partition by its year when set
+    group: tuple[str, ...] = ()  # for 'replace_group'
+    entity: tuple[str, ...] = ()  # for 'change_only': what a row tracks
+    description: str = ""
+    sort: tuple[str, ...] = field(default=())
+
+    def __post_init__(self):
+        if self.merge == "change_only":
+            if self.time_column is not None:
+                raise ValueError("A 'change_only' table cannot be partitioned by time.")
+            if len([c for c in self.key if c not in self.entity]) != 1:
+                raise ValueError(
+                    "A 'change_only' table's key must be its entity columns "
+                    f"plus one date column; got key={self.key}, entity={self.entity}."
+                )
+
+    @property
+    def change_column(self) -> str:
+        """For 'change_only': the key column that orders an entity's rows (a date)."""
+        return next(c for c in self.key if c not in self.entity)
+
+    @property
+    def sort_columns(self) -> tuple[str, ...]:
+        """Sort order on disk: time first, so new rows land at the end of a file."""
+        return self.sort or self.key
+
+    def path_for(self, row: dict) -> str:
+        """The relative path of the file holding ``row``."""
+        if self.time_column is None:
+            return f"{self.name}.csv"
+        return f"{self.name}/{str(row[self.time_column])[:4]}.csv"
+
+
+# --------------------------------------------------------------------------- store
+
+
+class Files(MutableMapping):
+    """Text files under ``rootdir``, keyed by relative POSIX path."""
+
+    def __init__(self, rootdir: str | os.PathLike):
+        self.rootdir = Path(rootdir).expanduser()
+
+    def _path(self, k: str) -> Path:
+        return self.rootdir / k
+
+    def __getitem__(self, k: str) -> str:
+        try:
+            return self._path(k).read_text(encoding="utf-8")
+        except FileNotFoundError:
+            raise KeyError(k) from None
+
+    def __setitem__(self, k: str, v: str) -> None:
+        p = self._path(k)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        tmp = p.with_name(p.name + ".tmp")
+        tmp.write_text(v, encoding="utf-8", newline="\n")
+        os.replace(tmp, p)  # atomic: a crash never leaves half a file
+
+    def __delitem__(self, k: str) -> None:
+        try:
+            self._path(k).unlink()
+        except FileNotFoundError:
+            raise KeyError(k) from None
+
+    def __iter__(self) -> Iterator[str]:
+        if not self.rootdir.exists():
+            return
+        for p in sorted(self.rootdir.rglob("*")):
+            if p.is_file() and ".git" not in p.parts:
+                yield p.relative_to(self.rootdir).as_posix()
+
+    def __len__(self) -> int:
+        return sum(1 for _ in self)
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}({str(self.rootdir)!r})"
+
+
+# ------------------------------------------------------------------ csv in and out
+
+
+def _coerce(spec: TableSpec, row: dict) -> dict:
+    out = {c: row.get(c, "") for c in spec.columns}
+    for c in spec.int_columns:
+        v = out[c]
+        out[c] = int(v) if v not in ("", None) else ""
+    for c, v in out.items():
+        if isinstance(v, bool):
+            out[c] = "true" if v else "false"
+    return out
+
+
+def parse_csv(spec: TableSpec, text: str) -> list[dict]:
+    """Rows of a table file, with integer columns as ``int``."""
+    return [_coerce(spec, r) for r in csv.DictReader(io.StringIO(text))]
+
+
+def _sort_key(spec: TableSpec):
+    return lambda r: tuple(str(r[c]) for c in spec.sort_columns)
+
+
+def dump_csv(spec: TableSpec, rows: Iterable[dict]) -> str:
+    """CSV text of ``rows``, sorted by the spec's sort columns, ``\\n`` line ends."""
+    buf = io.StringIO()
+    w = csv.DictWriter(buf, fieldnames=spec.columns, lineterminator="\n")
+    w.writeheader()
+    w.writerows(sorted((_coerce(spec, r) for r in rows), key=_sort_key(spec)))
+    return buf.getvalue()
+
+
+# -------------------------------------------------------------------------- merging
+
+
+def _values(spec: TableSpec, row: dict) -> tuple:
+    return tuple(row[c] for c in spec.columns if c not in spec.key)
+
+
+def merge_rows(spec: TableSpec, old: list[dict], new: list[dict]) -> list[dict]:
+    """Merge ``new`` rows into ``old`` according to ``spec.merge``.
+
+    >>> v = TableSpec("views", ("date", "repo", "count", "uniques"),
+    ...               key=("date", "repo"), merge="max",
+    ...               int_columns=("count", "uniques"), time_column="date")
+    >>> old = [{"date": "2026-01-01", "repo": "a/b", "count": 3, "uniques": 1}]
+    >>> new = [{"date": "2026-01-01", "repo": "a/b", "count": 5, "uniques": 1},
+    ...        {"date": "2026-01-02", "repo": "a/b", "count": 1, "uniques": 1}]
+    >>> [r["count"] for r in merge_rows(v, old, new)]
+    [5, 1]
+    >>> merge_rows(v, merge_rows(v, old, new), new) == merge_rows(v, old, new)
+    True
+    """
+    old = [_coerce(spec, r) for r in old]
+    new = [_coerce(spec, r) for r in new]
+
+    def key(r):
+        return tuple(str(r[c]) for c in spec.key)
+
+    if spec.merge in ("max", "upsert"):
+        merged = {key(r): r for r in old}
+        for r in new:
+            k = key(r)
+            if spec.merge == "max" and k in merged:
+                prev = merged[k]
+                r = {
+                    c: (
+                        max(prev[c], r[c])
+                        if c in spec.int_columns and prev[c] != "" and r[c] != ""
+                        else r[c]
+                        if r[c] != ""
+                        else prev[c]
+                    )
+                    for c in spec.columns
+                }
+            merged[k] = r
+        return list(merged.values())
+
+    if spec.merge == "replace_group":
+        groups = {tuple(str(r[c]) for c in spec.group) for r in new}
+        kept = [r for r in old if tuple(str(r[c]) for c in spec.group) not in groups]
+        return kept + new
+
+    if spec.merge == "change_only":
+        merged: dict[tuple, dict] = {}
+        for r in old + new:  # same key: field-wise, a non-empty newer value wins
+            prev = merged.get(key(r), {})
+            merged[key(r)] = {
+                c: r[c] if r[c] != "" else prev.get(c, "") for c in spec.columns
+            }
+
+        def entity(r):
+            return tuple(str(r[c]) for c in spec.entity)
+
+        kept, last_values = [], {}
+        when = spec.change_column
+        for r in sorted(merged.values(), key=lambda r: (entity(r), str(r[when]))):
+            if last_values.get(entity(r)) != _values(spec, r):
+                kept.append(r)
+                last_values[entity(r)] = _values(spec, r)
+        return kept
+
+    raise ValueError(f"Unknown merge policy {spec.merge!r} for table {spec.name!r}")
+
+
+def read_table(
+    store: MutableMapping, spec: TableSpec, *, years: Iterable[str] | None = None
+) -> list[dict]:
+    """All rows of a table (optionally only some year partitions)."""
+    keys = table_files(store, spec)
+    if years is not None:
+        wanted = {f"{spec.name}/{y}.csv" for y in years}
+        keys = [k for k in keys if k in wanted]
+    return [r for k in keys for r in parse_csv(spec, store[k])]
+
+
+def table_files(store: MutableMapping, spec: TableSpec) -> list[str]:
+    """The relative paths of a table's files present in ``store``, sorted."""
+    if spec.time_column is None:
+        return [f"{spec.name}.csv"] if f"{spec.name}.csv" in store else []
+    prefix = f"{spec.name}/"
+    return sorted(k for k in store if k.startswith(prefix) and k.endswith(".csv"))
+
+
+def write_rows(store: MutableMapping, spec: TableSpec, rows: Iterable[dict]) -> int:
+    """Merge ``rows`` into the table's files; return how many files changed."""
+    by_path: dict[str, list[dict]] = {}
+    for r in rows:
+        by_path.setdefault(spec.path_for(r), []).append(r)
+    changed = 0
+    for path, new in by_path.items():
+        old_text = store.get(path)
+        old = parse_csv(spec, old_text) if old_text else []
+        text = dump_csv(spec, merge_rows(spec, old, new))
+        if text != old_text:
+            store[path] = text
+            changed += 1
+    return changed
+
+
+# ------------------------------------------------------------------------ manifest
+
+MANIFEST_KEY = "manifest.json"
+
+
+def manifest(store: MutableMapping, tables: Mapping[str, TableSpec], **extra) -> dict:
+    """A description of the tables in ``store``: files, columns, keys.
+
+    ``extra`` keys (a schema version, an update date) come first. A reader (a
+    dashboard) can discover every file of every table from it in one request.
+    """
+    return {
+        **extra,
+        "tables": {
+            name: {
+                "files": table_files(store, spec),
+                "columns": list(spec.columns),
+                "key": list(spec.key),
+                "description": spec.description,
+            }
+            for name, spec in tables.items()
+        },
+    }
+
+
+def write_manifest(
+    store: MutableMapping,
+    tables: Mapping[str, TableSpec],
+    *,
+    key: str = MANIFEST_KEY,
+    **extra,
+) -> dict:
+    """Write :func:`manifest` as JSON under ``key`` (only if it changed)."""
+    m = manifest(store, tables, **extra)
+    text = json.dumps(m, indent=2) + "\n"
+    if store.get(key) != text:
+        store[key] = text
+    return m

csvledger-0.0.2.dist-info/METADATA

@@ -0,0 +1,57 @@
+Metadata-Version: 2.5
+Name: csvledger
+Version: 0.0.2
+Summary: Long-form CSV tables that merge idempotently (max, replace-group, change-only, upsert), partitioned by year, in any MutableMapping store: a git-friendly history store with no dependencies.
+Project-URL: Homepage, https://github.com/thorwhalen/csvledger
+Project-URL: Repository, https://github.com/thorwhalen/csvledger
+Author: Thor Whalen
+License-Expression: MIT
+License-File: LICENSE
+Keywords: csv,dol,git,history,idempotent,merge,time-series
+Requires-Python: >=3.11
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: sphinx-rtd-theme>=1.0; extra == 'docs'
+Requires-Dist: sphinx>=6.0; extra == 'docs'
+Description-Content-Type: text/markdown
+
+# csvledger
+
+Long-form CSV tables that merge idempotently: a git-friendly history store with no dependencies.
+
+You collect numbers on a schedule (traffic, downloads, prices), each run sees an overlapping window, and you want one growing history in plain files: re-runs harmless, the order of runs irrelevant, small diffs, past years never rewritten. `csvledger` is that store.
+
+```python
+from csvledger import TableSpec, Files, write_rows, read_table
+
+views = TableSpec(
+    "views", ("date", "repo", "count"),
+    key=("date", "repo"), merge="max", int_columns=("count",), time_column="date",
+)
+store = Files("history")                      # or any MutableMapping[str, str]
+write_rows(store, views, [{"date": "2026-01-01", "repo": "a/b", "count": 3}])
+write_rows(store, views, [{"date": "2026-01-01", "repo": "a/b", "count": 2}])  # no change
+read_table(store, views)   # [{'date': '2026-01-01', 'repo': 'a/b', 'count': 3}]
+```
+
+This writes `history/views/2026.csv`, sorted by date, so each run appends at the end of the current year's file.
+
+## Merge policies
+
+| `merge=` | per key | for |
+|---|---|---|
+| `max` | integer fields keep the larger value | daily counts seen by overlapping windows (a partial day only grows) |
+| `replace_group` | rows of a `group` (e.g. one repo, one fetch date) replace that group's earlier rows | top-N snapshots |
+| `change_only` | a row is kept only when its values differ from the previous row of its `entity` (key = entity + one date column) | slowly changing values (stars, versions) |
+| `upsert` | the new row wins | current metadata |
+
+All four are idempotent: merging the same rows again changes nothing.
+
+## The store
+
+Any `MutableMapping[str, str]` (relative path to text). `Files(root)` is a folder with atomic writes; a [dol](https://github.com/i2mint/dol) store puts the same tables in S3 or a database. `write_manifest(store, tables, **extra)` writes a `manifest.json` listing every table's files and columns, so a static dashboard can discover the data in one request.
+
+Used by [ghtraffic](https://github.com/thorwhalen/ghtraffic) (GitHub traffic) and [pkgreach](https://github.com/thorwhalen/pkgreach) (package downloads), where it was extracted from.

csvledger-0.0.2.dist-info/RECORD

@@ -0,0 +1,5 @@
+csvledger/__init__.py,sha256=USpq4Aj2KMxMr_HPUGMAaAvDv8NQ79Anq-JcqjZ8ijw,11640
+csvledger-0.0.2.dist-info/METADATA,sha256=j3QVjcnQK8guQ6IZji3QRJVXF7jCfYs9Rvj9cbD8sro,3044
+csvledger-0.0.2.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+csvledger-0.0.2.dist-info/licenses/LICENSE,sha256=ekE13yCE9fWWEJfP1NPnUJabZ_Wwrc6coKJV2ZZx27w,1068
+csvledger-0.0.2.dist-info/RECORD,,

csvledger-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

csvledger-0.0.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Thor Whalen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.