steam-data-reader 1.0.0.dev1__py3-none-any.whl

steam_data_reader/__init__.py

@@ -0,0 +1,8 @@
+"""Independent offline Steam catalog reader."""
+
+from steam_data_schema import READER_VERSION as __version__
+
+from .catalog import Catalog
+from .sync import DEFAULT_SOURCE_URL, ReaderUpgradeRequired, SyncError, sync
+
+__all__ = ["DEFAULT_SOURCE_URL", "Catalog", "ReaderUpgradeRequired", "SyncError", "__version__", "sync"]

steam_data_reader/catalog.py

@@ -0,0 +1,364 @@
+"""Pinned, read-only query API and the one shared serialization implementation."""
+
+import csv
+import hashlib
+import math
+import os
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+from steam_data_schema import (
+    APP_FIELDS,
+    INFO_COLUMNS,
+    MINIMAL_FIELDS,
+    SPECIAL_FIELDS,
+    TABLES,
+    app_id,
+    canonical,
+    check_compatibility,
+    json_default,
+    normalize_text,
+)
+
+from .database import connect, decoded
+
+
+@contextmanager
+def dataset_lock(directory, *, exclusive=False):
+    path = Path(directory)
+    path.mkdir(parents=True, exist_ok=True)
+    with (path / ".catalog.lock").open("a+b") as f:
+        try:
+            if os.name == "nt":
+                import msvcrt
+
+                f.seek(0)
+                f.write(b"0")
+                f.flush()
+                f.seek(0)
+                msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK if exclusive else msvcrt.LK_NBRLCK, 1)
+            else:
+                import fcntl
+
+                fcntl.flock(f, (fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) | fcntl.LOCK_NB)
+        except OSError:
+            raise OSError("Catalog is in use; close readers or another sync and retry") from None
+        try:
+            yield
+        finally:
+            if os.name == "nt":
+                f.seek(0)
+                msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1)
+            else:
+                fcntl.flock(f, fcntl.LOCK_UN)
+
+
+@contextmanager
+def staged_text(path):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=".export-")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8", newline="") as stream:
+            yield stream
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, path)
+    finally:
+        Path(temporary).unlink(missing_ok=True)
+
+
+class Result:
+    def __init__(self, catalog, sql, parameters, columns, missing=None):
+        self.catalog, self.sql, self.parameters, self.columns = catalog, sql, parameters, columns
+        self.missing_appids = missing or []
+
+    def __iter__(self):
+        cursor = self.catalog.connection.cursor()
+        try:
+            cursor.execute(self.sql, self.parameters)
+            while batch := cursor.fetchmany(128):
+                rows = [decoded(row, self.columns) for row in batch]
+                special = set(self.columns) & set(SPECIAL_FIELDS)
+                if special:
+                    by_id = {row["steam_appid"]: row for row in rows}
+                    for field in special:
+                        table = {
+                            "relationships": "app_relationships",
+                            "provenance": "field_provenance",
+                        }.get(field, field)
+                        for row in rows:
+                            row[field] = []
+                        # Avoid a whole-catalog list aggregation for a small requested set.
+                        with self.catalog.connection.cursor() as nested:
+                            nested.execute(
+                                f"SELECT * FROM steam.{table} WHERE steam_appid=ANY(?::BIGINT[]) ORDER BY "
+                                + ",".join(TABLES[table]["key"]),
+                                [list(by_id)],
+                            )
+                            while values := nested.fetchmany(2000):
+                                for value in values:
+                                    record = decoded(value, TABLES[table]["columns"])
+                                    by_id[record["steam_appid"]][field].append(record)
+                yield from rows
+        finally:
+            cursor.close()
+
+    def to_list(self):
+        return list(self)
+
+    def metadata(self):
+        return {k: self.catalog.info[k] for k in ("dataset_version", "schema_version")} | {
+            "missing_appids": self.missing_appids
+        }
+
+    def json_to(self, stream):
+        stream.write(canonical(self.metadata()).decode()[:-1] + ',"rows":[')
+        for i, row in enumerate(self):
+            stream.write(("," if i else "") + canonical(row).decode())
+        stream.write("]}\n")
+
+    def tsv_to(self, stream):
+        writer = csv.writer(stream, delimiter="\t", lineterminator="\n")
+        writer.writerow(self.columns)
+        count = 0
+        for row in self:
+            cells = []
+            for value in row.values():
+                if value is None:
+                    cell = "\\N"
+                elif isinstance(value, (dict, list)):
+                    cell = canonical(value).decode()
+                elif isinstance(value, bool):
+                    cell = "true" if value else "false"
+                elif isinstance(value, str):
+                    cell = value.replace("\\", "\\\\")
+                else:
+                    cell = str(json_default(value)) if hasattr(value, "tzinfo") else str(value)
+                cells.append(cell)
+            writer.writerow(cells)
+            count += 1
+        return count
+
+    def write_json(self, path):
+        with staged_text(path) as stream:
+            self.json_to(stream)
+
+    def write_tsv(self, path):
+        path = Path(path)
+        with staged_text(path) as stream:
+            count = self.tsv_to(stream)
+        digest = hashlib.sha256()
+        with path.open("rb") as stream:
+            while block := stream.read(1024 * 1024):
+                digest.update(block)
+        meta = {
+            **self.metadata(),
+            "columns": self.columns,
+            "rows": count,
+            "bytes": path.stat().st_size,
+            "sha256": digest.hexdigest(),
+            "encoding": "utf-8",
+            "null": "\\N",
+            "scalar_string_escaping": "double backslashes before CSV quoting",
+            "delimiter": "\t",
+        }
+        with staged_text(str(path) + ".metadata.json") as stream:
+            stream.write(canonical(meta).decode() + "\n")
+
+
+def sequence(values, validator=lambda v: v):
+    if isinstance(values, (str, bytes)):
+        raise TypeError("Expected a sequence, not a string")
+    return list(dict.fromkeys(validator(v) for v in values))
+
+
+class Catalog:
+    @classmethod
+    def open(cls, directory):
+        return cls(directory)
+
+    def __init__(self, directory):
+        self.lock = dataset_lock(directory)
+        self.lock.__enter__()
+        self.connection = None
+        try:
+            self.connection = connect(Path(directory) / "catalog.duckdb", read_only=True)
+            row = self.connection.execute("SELECT * FROM steam.snapshot_info").fetchone()
+            self.info = decoded(row, INFO_COLUMNS)
+            try:
+                check_compatibility({**self.info, "protocol_version": 1})
+            except ValueError:
+                from .sync import ReaderUpgradeRequired
+
+                raise ReaderUpgradeRequired(
+                    "Installed schema needs a newer reader",
+                    installed_version=self.info["dataset_version"],
+                ) from None
+        except Exception:
+            self.close()
+            raise
+
+    def close(self):
+        if self.connection:
+            self.connection.close()
+            self.connection = None
+        if self.lock:
+            self.lock.__exit__(None, None, None)
+            self.lock = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    def dataset_info(self):
+        return dict(self.info)
+
+    def list_genres(self):
+        return Result(
+            self,
+            "SELECT genre_id,label FROM steam.genres ORDER BY genre_id",
+            [],
+            TABLES["genres"]["columns"],
+        )
+
+    def _fields(self, fields, *, full=False):
+        fields = (
+            list(APP_FIELDS)
+            if full and fields is None
+            else MINIMAL_FIELDS
+            if fields is None
+            else sequence(fields)
+        )
+        if any(f not in APP_FIELDS for f in fields):
+            raise ValueError("Unknown app field")
+        names = ["steam_appid"] + [f for f in fields if f != "steam_appid"]
+        expr = []
+        for f in names:
+            if f in SPECIAL_FIELDS:
+                expr.append(f"NULL::JSON AS {f}")
+            else:
+                expr.append(f'a."{f}"')
+        return ",".join(expr), {f: APP_FIELDS[f] for f in names}
+
+    def get_apps(self, appids, *, fields=None):
+        ids = sequence(appids, app_id)
+        expr, columns = self._fields(fields, full=True)
+        # Scalar parameters let DuckDB push the ID filter into the view's
+        # component scans. An ANY(array) semi-join alone can retain a wide
+        # whole-catalog hash join even for a handful of requested apps.
+        selected = ",".join("?" for _ in ids) or "NULL"
+        found = {
+            r[0]
+            for r in self.connection.execute(
+                "SELECT steam_appid FROM steam.apps WHERE steam_appid=ANY(?::BIGINT[])", [ids]
+            ).fetchall()
+        }
+        return Result(
+            self,
+            f"SELECT {expr} FROM steam.apps a JOIN unnest(?::BIGINT[]) WITH ORDINALITY requested(steam_appid,position) USING(steam_appid) WHERE a.steam_appid IN ({selected}) ORDER BY requested.position",
+            [ids, *ids],
+            columns,
+            [i for i in ids if i not in found],
+        )
+
+    def list_apps(
+        self,
+        *,
+        appids=None,
+        name_prefix=None,
+        name_contains=None,
+        app_types=None,
+        genre_ids=None,
+        genre_match="any",
+        developer_names=None,
+        publisher_names=None,
+        min_reviews=None,
+        max_reviews=None,
+        min_positive_percentage=None,
+        max_positive_percentage=None,
+        fields=None,
+        order_by="steam_appid",
+        limit=None,
+        after_appid=None,
+    ):
+        expr, columns = self._fields(fields)
+        clauses, params = [], []
+
+        def add(sql, value):
+            clauses.append(sql)
+            params.append(value)
+
+        if genre_match not in ("any", "all") or order_by not in ("steam_appid", "name"):
+            raise ValueError("Invalid genre match/order")
+        if appids is not None:
+            ids = sequence(appids, app_id)
+            clauses.append("a.steam_appid IN (" + (",".join("?" for _ in ids) or "NULL") + ")")
+            params.extend(ids)
+        for value, function in ((name_prefix, "starts_with"), (name_contains, "contains")):
+            if value is not None:
+                add(
+                    f"{function}((SELECT name_key FROM transport.app_search s WHERE s.steam_appid=a.steam_appid),?)",
+                    normalize_text(value),
+                )
+        if app_types is not None:
+            values = sequence(
+                app_types, lambda v: v if isinstance(v, str) and v else normalize_text(v)
+            )
+            add("a.app_type=ANY(?::VARCHAR[])", values)
+        if genre_ids is not None:
+            values = sequence(
+                genre_ids, lambda v: v if isinstance(v, str) and v else normalize_text(v)
+            )
+            if not values and genre_match == "all":
+                raise ValueError("Empty all-genres filter")
+            add(
+                "(SELECT count(*) FROM steam.app_genres g WHERE g.steam_appid=a.steam_appid AND g.genre_id=ANY(?::VARCHAR[])) "
+                + (f"={len(values)}" if genre_match == "all" else ">0"),
+                values,
+            )
+        for values, role in ((developer_names, "developer"), (publisher_names, "publisher")):
+            if values is not None:
+                add(
+                    f"list_has_any((SELECT {role}_keys FROM transport.app_search s WHERE s.steam_appid=a.steam_appid), ?::VARCHAR[])",
+                    sequence(values, normalize_text),
+                )
+        for lower, upper, column, percentage in (
+            (min_reviews, max_reviews, "total_reviews", False),
+            (min_positive_percentage, max_positive_percentage, "positive_percentage", True),
+        ):
+            for value, op in ((lower, ">="), (upper, "<=")):
+                if value is not None:
+                    if (
+                        isinstance(value, bool)
+                        or not isinstance(value, (int, float))
+                        or not math.isfinite(value)
+                        or value < 0
+                        or (percentage and value > 100)
+                        or (not percentage and type(value) is not int)
+                    ):
+                        raise ValueError("Invalid numeric bound")
+                    add(f"a.{column}{op}?", value)
+            if lower is not None and upper is not None and lower > upper:
+                raise ValueError("Minimum exceeds maximum")
+        if after_appid is not None:
+            if order_by != "steam_appid":
+                raise ValueError("after_appid requires appID ordering")
+            add("a.steam_appid>?", app_id(after_appid))
+        if limit is not None and (type(limit) is not int or limit < 0):
+            raise ValueError("Limit must be a nonnegative integer")
+        sql = f"SELECT {expr} FROM steam.apps a" + (
+            " WHERE " + " AND ".join(clauses) if clauses else ""
+        )
+        sql += " ORDER BY " + (
+            "(SELECT name_key FROM transport.app_search s WHERE s.steam_appid=a.steam_appid),a.steam_appid"
+            if order_by == "name"
+            else "a.steam_appid"
+        )
+        if limit is not None:
+            sql += " LIMIT ?"
+            params.append(limit)
+        return Result(self, sql, params, columns)

steam_data_reader/cli.py

@@ -0,0 +1,94 @@
+"""Thin CLI over the independent reader API; diagnostics never enter data output."""
+
+import argparse
+import sys
+from importlib.metadata import metadata
+
+import duckdb
+from steam_data_schema import READER_VERSION, canonical
+
+from .catalog import Catalog
+from .sync import ReaderUpgradeRequired, SyncError, sync
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="steam-snapshot")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {READER_VERSION}")
+    commands = parser.add_subparsers(dest="command", required=True)
+    commands.add_parser("readme", help="Print the installed package's full README (offline)")
+    for name in ("sync", "info", "list-apps", "get-apps", "list-genres"):
+        p = commands.add_parser(name)
+        p.add_argument("--directory", required=True)
+        if name == "sync":
+            p.add_argument("--source", dest="source_url",
+                           help="Override the saved source or built-in public R2 preview URL")
+            p.add_argument(
+                "--allow-local-http",
+                action="store_true",
+                help="Explicit loopback-only private test transport",
+            )
+        if name in ("list-apps", "get-apps", "list-genres"):
+            p.add_argument("--format", choices=("json", "tsv"), default="json")
+            p.add_argument("--output")
+        if name in ("list-apps", "get-apps"):
+            p.add_argument("--fields", type=lambda value: value.split(",") if value else [])
+        if name == "get-apps":
+            p.add_argument("appids", type=int, nargs="+")
+        if name == "list-apps":
+            for flag, dest, kind in (
+                ("appid", "appids", int),
+                ("app-type", "app_types", str),
+                ("genre-id", "genre_ids", str),
+                ("developer-name", "developer_names", str),
+                ("publisher-name", "publisher_names", str),
+            ):
+                p.add_argument("--" + flag, dest=dest, type=kind, action="append")
+            for flag in ("name-prefix", "name-contains"):
+                p.add_argument("--" + flag)
+            p.add_argument("--genre-match", choices=("any", "all"), default="any")
+            p.add_argument("--order-by", choices=("steam_appid", "name"), default="steam_appid")
+            for flag in ("min-reviews", "max-reviews", "limit", "after-appid"):
+                p.add_argument("--" + flag, type=int)
+            for flag in ("min-positive-percentage", "max-positive-percentage"):
+                p.add_argument("--" + flag, type=float)
+    args = vars(parser.parse_args(argv))
+    command, directory = args.pop("command"), args.pop("directory", None)
+    try:
+        if command == "readme":
+            print(metadata("steam-data-reader").get_payload())
+            return 0
+        if command == "sync":
+            print(canonical(sync(directory, **args).to_dict()).decode())
+            return 0
+        with Catalog.open(directory) as catalog:
+            if command == "info":
+                print(canonical(catalog.dataset_info()).decode())
+                return 0
+            format_, output = args.pop("format"), args.pop("output")
+            result = getattr(catalog, command.replace("-", "_"))(**args)
+            if output:
+                getattr(result, "write_" + format_)(output)
+                print(canonical({**result.metadata(), "output": output}).decode(), file=sys.stderr)
+            elif format_ == "json":
+                result.json_to(sys.stdout)
+            else:
+                result.tsv_to(sys.stdout)
+                print(canonical(result.metadata()).decode(), file=sys.stderr)
+        return 0
+    except ReaderUpgradeRequired as error:
+        print(str(error), file=sys.stderr)
+        return 3
+    except (ValueError, TypeError) as error:
+        print(str(error), file=sys.stderr)
+        return 2
+    except (SyncError, OSError, RuntimeError, duckdb.Error) as error:
+        print(str(error), file=sys.stderr)
+        return 1
+
+
+def entrypoint():
+    raise SystemExit(main())
+
+
+if __name__ == "__main__":
+    entrypoint()