sqlitexplorer 1.1.1__tar.gz → 1.1.2__tar.gz

{sqlitexplorer-1.1.1/sqlitexplorer.egg-info → sqlitexplorer-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlitexplorer
-Version: 1.1.1
+Version: 1.1.2
 Summary: Explore SQLite databases from the terminal: tables, schema, stats, search, queries, charts, export/import and an interactive shell.
 Author-email: "Carlos A. Planchón" <carlosandresplanchonprestes@gmail.com>
 License-Expression: MIT
@@ -128,9 +128,9 @@ their values wrapped so that every column and label stays visible. Use
 Big results do not need to fit in memory: `--page` fetches only the requested
 page (in SQL for `show`, from the cursor for `query`), and the csv, tsv, json
 and markdown formats, `export` and `dump` are written row by row. The table
-format is one exception, since it needs every row to size its columns, and
-`chart` is the other: it reads the whole result before reducing it to what the
-canvas can show.
+format is the exception, since it needs every row to size its columns.
+`chart` reduces as it reads, so the memory it needs does not grow with the
+table; `chart --no-resample` is the one that holds every row.
 
 ## Queries
 
@@ -163,11 +163,13 @@ stderr. `--kind` selects `line` (default), `scatter` or `hist` (first column
 only, `--bins`). `--height`, `--width`, `--x-label`, `--y-label` and `--color`
 adjust the drawing.
 
-Line and scatter charts are reduced to what the canvas can show before they
-reach plotille, keeping the minimum and the maximum of every column of braille
-dots, so spikes survive and the drawing no longer costs more as the query grows;
-the reduction is reported on stderr. `--no-resample` sends every row instead.
-Histograms are never reduced, since dropping rows would change the distribution.
+Line and scatter charts are reduced to what the canvas can show, keeping the
+minimum and the maximum of every column of braille dots, so spikes survive and
+the true extremes keep the X of their own row. The rows are reduced as they are
+read, in chunks, so a chart over millions of rows needs no more memory than one
+over a thousand; the reduction is reported on stderr. `--no-resample` reads and
+plots every row instead. Histograms are never reduced, since dropping rows would
+change the distribution.
 
 ## Export, import, dump and diff
 

{sqlitexplorer-1.1.1 → sqlitexplorer-1.1.2}/README.md

@@ -98,9 +98,9 @@ their values wrapped so that every column and label stays visible. Use
 Big results do not need to fit in memory: `--page` fetches only the requested
 page (in SQL for `show`, from the cursor for `query`), and the csv, tsv, json
 and markdown formats, `export` and `dump` are written row by row. The table
-format is one exception, since it needs every row to size its columns, and
-`chart` is the other: it reads the whole result before reducing it to what the
-canvas can show.
+format is the exception, since it needs every row to size its columns.
+`chart` reduces as it reads, so the memory it needs does not grow with the
+table; `chart --no-resample` is the one that holds every row.
 
 ## Queries
 
@@ -133,11 +133,13 @@ stderr. `--kind` selects `line` (default), `scatter` or `hist` (first column
 only, `--bins`). `--height`, `--width`, `--x-label`, `--y-label` and `--color`
 adjust the drawing.
 
-Line and scatter charts are reduced to what the canvas can show before they
-reach plotille, keeping the minimum and the maximum of every column of braille
-dots, so spikes survive and the drawing no longer costs more as the query grows;
-the reduction is reported on stderr. `--no-resample` sends every row instead.
-Histograms are never reduced, since dropping rows would change the distribution.
+Line and scatter charts are reduced to what the canvas can show, keeping the
+minimum and the maximum of every column of braille dots, so spikes survive and
+the true extremes keep the X of their own row. The rows are reduced as they are
+read, in chunks, so a chart over millions of rows needs no more memory than one
+over a thousand; the reduction is reported on stderr. `--no-resample` reads and
+plots every row instead. Histograms are never reduced, since dropping rows would
+change the distribution.
 
 ## Export, import, dump and diff
 

{sqlitexplorer-1.1.1 → sqlitexplorer-1.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sqlitexplorer"
-version = "1.1.1"
+version = "1.1.2"
 description = "Explore SQLite databases from the terminal: tables, schema, stats, search, queries, charts, export/import and an interactive shell."
 readme = "README.md"
 license = "MIT"

{sqlitexplorer-1.1.1 → sqlitexplorer-1.1.2}/sqlitexplorer/charts.py

@@ -12,11 +12,12 @@ from contextlib import contextmanager
 from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
+from itertools import islice
 
 import plotille
 import plotilleresample
 
-from sqlitexplorer.core import ExplorerError, ResultSet
+from sqlitexplorer.core import ExplorerError, ResultSet, RowStream
 
 __all__ = [
     "ChartKind",
@@ -26,6 +27,7 @@ __all__ = [
     "render_histogram",
     "resample_series",
     "series_from_result",
+    "stream_series",
 ]
 
 PALETTE = ("red", "green", "yellow", "blue", "magenta", "cyan")
@@ -36,6 +38,10 @@ AXIS_WIDTH = 12
 # A tuple, not bool | int | float: the union would be rebuilt on every call,
 # and this runs once per value of the result.
 _NUMERIC = (bool, int, float)
+# Rows read at a time when streaming, and how many reduced points may pile
+# up before they are reduced again.
+_CHUNK = 65536
+_PILE = 8
 
 
 class ChartKind(str, Enum):
@@ -74,11 +80,13 @@ def _x_value(value: object) -> float | datetime | None:
     return None
 
 
-def series_from_result(result: ResultSet) -> tuple[list[Series], int]:
+def series_from_result(result: ResultSet, *, require_rows: bool = True) -> tuple[list[Series], int]:
     """Split *result* into one series per numeric column after the first.
 
     Rows with a NULL in the X column or in any series are skipped; the second
-    item of the returned tuple counts them.
+    item of the returned tuple counts them. With *require_rows* false an empty
+    result yields empty series instead of raising, which is what the streaming
+    reader needs for a chunk that holds nothing usable.
     """
     if len(result.columns) < 2:
         raise ExplorerError("need an X column and at least one numeric column")
@@ -106,7 +114,7 @@ def series_from_result(result: ResultSet) -> tuple[list[Series], int]:
                 raise ExplorerError(f"column {name} is not numeric: {value!r}")
             append(number)
         keep_x(x)
-    if not xs:
+    if require_rows and not xs:
         raise ExplorerError("no rows to plot")
     return [
         Series(label=name, x=xs, y=bucket) for name, bucket in zip(y_names, ys, strict=True)
@@ -134,6 +142,50 @@ def resample_series(
     return reduced
 
 
+def stream_series(
+    stream: RowStream, *, kind: ChartKind, width: int, height: int
+) -> tuple[list[Series], int, int]:
+    """Reduce *stream* to the canvas without ever holding every row.
+
+    Rows are read in chunks, each chunk is reduced on its own and the reduced
+    points are reduced again as they pile up, so the memory a chart needs stops
+    growing with the size of the table. Returns the series, how many rows were
+    skipped for their NULLs and how many were read.
+    """
+    reduced: list[Series] = []
+    x_type: type | None = None
+    skipped = rows_read = 0
+    while True:
+        rows = list(islice(stream.rows, _CHUNK))
+        if not rows:
+            break
+        rows_read += len(rows)
+        chunk, chunk_skipped = series_from_result(
+            ResultSet(columns=stream.columns, rows=rows), require_rows=False
+        )
+        skipped += chunk_skipped
+        if not chunk[0].x:
+            continue
+        if x_type is None:
+            x_type = type(chunk[0].x[0])
+        elif not isinstance(chunk[0].x[0], x_type):
+            raise ExplorerError(f"column {stream.columns[0]} mixes numbers and dates")
+        chunk = resample_series(chunk, kind=kind, width=width, height=height)
+        reduced = (
+            [
+                Series(label=old.label, x=old.x + new.x, y=old.y + new.y)
+                for old, new in zip(reduced, chunk, strict=True)
+            ]
+            if reduced
+            else chunk
+        )
+        if len(reduced[0].x) > _PILE * len(chunk[0].x):
+            reduced = resample_series(reduced, kind=kind, width=width, height=height)
+    if not reduced or not reduced[0].x:
+        raise ExplorerError("no rows to plot")
+    return resample_series(reduced, kind=kind, width=width, height=height), skipped, rows_read
+
+
 def histogram_values(result: ResultSet) -> tuple[list[float], int]:
     """Numeric values of the first column of *result*, and how many NULLs were skipped."""
     if not result.columns:

{sqlitexplorer-1.1.1 → sqlitexplorer-1.1.2}/sqlitexplorer/cli.py

@@ -28,8 +28,8 @@ from sqlitexplorer.charts import (
     histogram_values,
     render_chart,
     render_histogram,
-    resample_series,
     series_from_result,
+    stream_series,
 )
 from sqlitexplorer.completion import complete_table
 from sqlitexplorer.core import (
@@ -666,12 +666,13 @@ def chart(
     parameters = _parameters(params)
     with _reporting_errors(), open_database(database) as db:
         text = sys.stdin.read() if sql == "-" else sql
-        result = db.execute(text, parameters if parameters else ())
-        if not result.returns_rows:
-            _fail("the statement returned no rows")
+        bound = parameters if parameters else ()
         use_color = resolve_color(color)
         screen = width if width is not None else shutil.get_terminal_size().columns
         if kind is ChartKind.HIST:
+            result = db.execute(text, bound)
+            if not result.returns_rows:
+                _fail("the statement returned no rows")
             values, skipped = histogram_values(result)
             drawing = render_histogram(
                 values,
@@ -683,13 +684,23 @@ def chart(
                 y_label=y_label or "count",
             )
         else:
-            series, skipped = series_from_result(result)
             if resample:
-                rows = len(series[0].x)
-                series = resample_series(series, kind=kind, width=screen, height=height)
-                points = len(series[0].x)
-                if points < rows:
-                    typer.echo(f"resampled {rows} rows to {points} points", err=True)
+                # Reduces as it reads, so the rows never pile up in memory.
+                stream = db.stream(text, bound)
+                if not stream.returns_rows:
+                    _fail("the statement returned no rows")
+                columns = stream.columns
+                series, skipped, read = stream_series(
+                    stream, kind=kind, width=screen, height=height
+                )
+                if len(series[0].x) < read:
+                    typer.echo(f"resampled {read} rows to {len(series[0].x)} points", err=True)
+            else:
+                result = db.execute(text, bound)
+                if not result.returns_rows:
+                    _fail("the statement returned no rows")
+                columns = result.columns
+                series, skipped = series_from_result(result)
             default_y = series[0].label if len(series) == 1 else "value"
             drawing = render_chart(
                 series,
@@ -697,7 +708,7 @@ def chart(
                 width=screen,
                 height=height,
                 color=use_color,
-                x_label=x_label or result.columns[0],
+                x_label=x_label or columns[0],
                 y_label=y_label or default_y,
             )
         if skipped:

{sqlitexplorer-1.1.1 → sqlitexplorer-1.1.2/sqlitexplorer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlitexplorer
-Version: 1.1.1
+Version: 1.1.2
 Summary: Explore SQLite databases from the terminal: tables, schema, stats, search, queries, charts, export/import and an interactive shell.
 Author-email: "Carlos A. Planchón" <carlosandresplanchonprestes@gmail.com>
 License-Expression: MIT
@@ -128,9 +128,9 @@ their values wrapped so that every column and label stays visible. Use
 Big results do not need to fit in memory: `--page` fetches only the requested
 page (in SQL for `show`, from the cursor for `query`), and the csv, tsv, json
 and markdown formats, `export` and `dump` are written row by row. The table
-format is one exception, since it needs every row to size its columns, and
-`chart` is the other: it reads the whole result before reducing it to what the
-canvas can show.
+format is the exception, since it needs every row to size its columns.
+`chart` reduces as it reads, so the memory it needs does not grow with the
+table; `chart --no-resample` is the one that holds every row.
 
 ## Queries
 
@@ -163,11 +163,13 @@ stderr. `--kind` selects `line` (default), `scatter` or `hist` (first column
 only, `--bins`). `--height`, `--width`, `--x-label`, `--y-label` and `--color`
 adjust the drawing.
 
-Line and scatter charts are reduced to what the canvas can show before they
-reach plotille, keeping the minimum and the maximum of every column of braille
-dots, so spikes survive and the drawing no longer costs more as the query grows;
-the reduction is reported on stderr. `--no-resample` sends every row instead.
-Histograms are never reduced, since dropping rows would change the distribution.
+Line and scatter charts are reduced to what the canvas can show, keeping the
+minimum and the maximum of every column of braille dots, so spikes survive and
+the true extremes keep the X of their own row. The rows are reduced as they are
+read, in chunks, so a chart over millions of rows needs no more memory than one
+over a thousand; the reduction is reported on stderr. `--no-resample` reads and
+plots every row instead. Histograms are never reduced, since dropping rows would
+change the distribution.
 
 ## Export, import, dump and diff
 

{sqlitexplorer-1.1.1 → sqlitexplorer-1.1.2}/tests/test_charts.py

@@ -13,8 +13,9 @@ from sqlitexplorer.charts import (
     render_histogram,
     resample_series,
     series_from_result,
+    stream_series,
 )
-from sqlitexplorer.core import ExplorerError, ResultSet
+from sqlitexplorer.core import ExplorerError, ResultSet, RowStream
 
 
 def braille(text: str) -> bool:
@@ -152,3 +153,77 @@ def test_resample_series_scatter_keeps_more_points_than_a_line() -> None:
     line = resample_series(series, kind=ChartKind.LINE, width=80, height=15)
     scatter = resample_series(series, kind=ChartKind.SCATTER, width=80, height=15)
     assert len(scatter[0].x) > len(line[0].x)
+
+
+def _stream(columns: tuple[str, ...], rows: list[tuple]) -> RowStream:
+    return RowStream(columns=columns, rows=iter(rows))
+
+
+def test_stream_series_keeps_the_envelope_and_the_real_pairs(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr("sqlitexplorer.charts._CHUNK", 50)
+    rows: list[tuple] = [(i, 0.0) for i in range(5000)]
+    rows[1234] = (1234, 999.0)
+    rows[4321] = (4321, -999.0)
+    series, skipped, read = stream_series(
+        _stream(("x", "y"), rows), kind=ChartKind.LINE, width=80, height=15
+    )
+    assert read == 5000
+    assert skipped == 0
+    assert len(series[0].x) < 5000
+    assert max(series[0].y) == 999.0
+    assert min(series[0].y) == -999.0
+    # The extremes keep the X of their own row, not a neighbour's.
+    assert series[0].x[series[0].y.index(999.0)] == 1234
+    assert series[0].x[series[0].y.index(-999.0)] == 4321
+    assert series[0].x == sorted(series[0].x)
+
+
+def test_stream_series_matches_the_collected_path_on_one_chunk() -> None:
+    rows = [(i, float(i % 97)) for i in range(4000)]
+    collected, _ = series_from_result(ResultSet(columns=("x", "y"), rows=rows))
+    collected = resample_series(collected, kind=ChartKind.LINE, width=80, height=15)
+    streamed, _, read = stream_series(
+        _stream(("x", "y"), rows), kind=ChartKind.LINE, width=80, height=15
+    )
+    assert read == 4000
+    assert streamed[0].x == collected[0].x
+    assert streamed[0].y == collected[0].y
+
+
+def test_stream_series_skips_nulls_across_chunks(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr("sqlitexplorer.charts._CHUNK", 10)
+    rows: list[tuple] = [(i, None) if 10 <= i < 20 else (i, float(i)) for i in range(100)]
+    series, skipped, read = stream_series(
+        _stream(("x", "y"), rows), kind=ChartKind.LINE, width=80, height=15
+    )
+    assert (read, skipped) == (100, 10)
+    assert min(series[0].y) == 0.0
+    assert max(series[0].y) == 99.0
+
+
+def test_stream_series_handles_dates_and_rejects_a_mix(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr("sqlitexplorer.charts._CHUNK", 10)
+    dates = [(f"2020-01-01T00:00:{i:02d}", float(i)) for i in range(60)]
+    series, _, read = stream_series(
+        _stream(("t", "v"), dates), kind=ChartKind.LINE, width=80, height=15
+    )
+    assert read == 60
+    assert all(isinstance(value, datetime) for value in series[0].x)
+    mixed = dates + [(5, 1.0)] * 10
+    with pytest.raises(ExplorerError, match="mixes numbers and dates"):
+        stream_series(_stream(("t", "v"), mixed), kind=ChartKind.LINE, width=80, height=15)
+
+
+def test_stream_series_rejects_an_empty_stream() -> None:
+    with pytest.raises(ExplorerError, match="no rows to plot"):
+        stream_series(_stream(("x", "y"), []), kind=ChartKind.LINE, width=80, height=15)
+
+
+def test_series_from_result_can_allow_an_empty_result() -> None:
+    series, skipped = series_from_result(ResultSet(columns=("x", "y")), require_rows=False)
+    assert [item.label for item in series] == ["y"]
+    assert series[0].x == [] and skipped == 0