sqlitexplorer 1.1.3__tar.gz → 1.2.0__tar.gz

{sqlitexplorer-1.1.3/sqlitexplorer.egg-info → sqlitexplorer-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlitexplorer
-Version: 1.1.3
+Version: 1.2.0
 Summary: Explore SQLite databases from the terminal: tables, schema, stats, search, queries, charts, export/import and an interactive shell.
 Author-email: "Carlos A. Planchón" <carlosandresplanchonprestes@gmail.com>
 License-Expression: MIT
@@ -21,7 +21,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: outfancy>=0.11
 Requires-Dist: plotille>=6
-Requires-Dist: plotilleresample>=1.0
+Requires-Dist: plotilleresample>=1.2
 Requires-Dist: typer>=0.12
 Provides-Extra: dev
 Requires-Dist: pytest>=8; extra == "dev"
@@ -50,10 +50,11 @@ with [plotille](https://github.com/tammoippen/plotille), resampled by
 Requires Python 3.10 or newer.
 
 ```sh
-# From a clone of this repository:
-uv tool install .
+uv tool install sqlitexplorer
 # or with pip:
-pip install .
+pip install sqlitexplorer
+# or from a clone of this repository:
+uv tool install .
 ```
 
 ## Quick tour
@@ -129,8 +130,9 @@ Big results do not need to fit in memory: `--page` fetches only the requested
 page (in SQL for `show`, from the cursor for `query`), and the csv, tsv, json
 and markdown formats, `export` and `dump` are written row by row. The table
 format is the exception, since it needs every row to size its columns.
-`chart` reduces as it reads, so the memory it needs does not grow with the
-table; `chart --no-resample` is the one that holds every row.
+`chart` reduces as it reads, and a histogram counts as it reads, so the memory
+they need does not grow with the table; `chart --no-resample` is the one that
+holds every row.
 
 ## Queries
 
@@ -161,15 +163,18 @@ The first column is the X axis (numbers or ISO dates), every other column is a
 series named after the column; rows with NULLs are skipped and counted on
 stderr. `--kind` selects `line` (default), `scatter` or `hist` (first column
 only, `--bins`). `--height`, `--width`, `--x-label`, `--y-label` and `--color`
-adjust the drawing.
+adjust the drawing; the Y label is cut to eight characters, the room plotille
+gives it.
 
 Line and scatter charts are reduced to what the canvas can show, keeping the
 minimum and the maximum of every column of braille dots, so spikes survive and
-the true extremes keep the X of their own row. The rows are reduced as they are
+the true extremes keep the X of their own row; a scatter plot keeps a uniform
+sample of its points as well, for its density. The rows are reduced as they are
 read, in chunks, so a chart over millions of rows needs no more memory than one
 over a thousand; the reduction is reported on stderr. `--no-resample` reads and
 plots every row instead. Histograms are never reduced, since dropping rows would
-change the distribution.
+change the distribution: the query runs twice, once for the range and once for
+the counts, so they do not hold the rows either.
 
 ## Export, import, dump and diff
 
@@ -183,8 +188,10 @@ sqlitexplorer diff app.db backup.db                      # exit status 1 when th
 
 `import` creates the table when it does not exist, inferring INTEGER, REAL or
 TEXT for each column; values with leading zeros and integers too large for
-SQLite stay TEXT, so codes such as `007` keep their digits. Empty CSV cells
-become NULL. The format comes from the file extension unless `--format` is
+SQLite stay TEXT, so codes such as `007` keep their digits. From JSON the types
+are the JSON types: numbers become INTEGER or REAL and strings stay TEXT even
+when they look like numbers, so a json export imports back as it was. Empty CSV
+cells become NULL. The format comes from the file extension unless `--format` is
 given, and `--encoding` reads a file that is not UTF-8.
 
 `export --all` writes one file per table and view inside the directory: a name

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/README.md

@@ -20,10 +20,11 @@ with [plotille](https://github.com/tammoippen/plotille), resampled by
 Requires Python 3.10 or newer.
 
 ```sh
-# From a clone of this repository:
-uv tool install .
+uv tool install sqlitexplorer
 # or with pip:
-pip install .
+pip install sqlitexplorer
+# or from a clone of this repository:
+uv tool install .
 ```
 
 ## Quick tour
@@ -99,8 +100,9 @@ Big results do not need to fit in memory: `--page` fetches only the requested
 page (in SQL for `show`, from the cursor for `query`), and the csv, tsv, json
 and markdown formats, `export` and `dump` are written row by row. The table
 format is the exception, since it needs every row to size its columns.
-`chart` reduces as it reads, so the memory it needs does not grow with the
-table; `chart --no-resample` is the one that holds every row.
+`chart` reduces as it reads, and a histogram counts as it reads, so the memory
+they need does not grow with the table; `chart --no-resample` is the one that
+holds every row.
 
 ## Queries
 
@@ -131,15 +133,18 @@ The first column is the X axis (numbers or ISO dates), every other column is a
 series named after the column; rows with NULLs are skipped and counted on
 stderr. `--kind` selects `line` (default), `scatter` or `hist` (first column
 only, `--bins`). `--height`, `--width`, `--x-label`, `--y-label` and `--color`
-adjust the drawing.
+adjust the drawing; the Y label is cut to eight characters, the room plotille
+gives it.
 
 Line and scatter charts are reduced to what the canvas can show, keeping the
 minimum and the maximum of every column of braille dots, so spikes survive and
-the true extremes keep the X of their own row. The rows are reduced as they are
+the true extremes keep the X of their own row; a scatter plot keeps a uniform
+sample of its points as well, for its density. The rows are reduced as they are
 read, in chunks, so a chart over millions of rows needs no more memory than one
 over a thousand; the reduction is reported on stderr. `--no-resample` reads and
 plots every row instead. Histograms are never reduced, since dropping rows would
-change the distribution.
+change the distribution: the query runs twice, once for the range and once for
+the counts, so they do not hold the rows either.
 
 ## Export, import, dump and diff
 
@@ -153,8 +158,10 @@ sqlitexplorer diff app.db backup.db                      # exit status 1 when th
 
 `import` creates the table when it does not exist, inferring INTEGER, REAL or
 TEXT for each column; values with leading zeros and integers too large for
-SQLite stay TEXT, so codes such as `007` keep their digits. Empty CSV cells
-become NULL. The format comes from the file extension unless `--format` is
+SQLite stay TEXT, so codes such as `007` keep their digits. From JSON the types
+are the JSON types: numbers become INTEGER or REAL and strings stay TEXT even
+when they look like numbers, so a json export imports back as it was. Empty CSV
+cells become NULL. The format comes from the file extension unless `--format` is
 given, and `--encoding` reads a file that is not UTF-8.
 
 `export --all` writes one file per table and view inside the directory: a name

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sqlitexplorer"
-version = "1.1.3"
+version = "1.2.0"
 description = "Explore SQLite databases from the terminal: tables, schema, stats, search, queries, charts, export/import and an interactive shell."
 readme = "README.md"
 license = "MIT"
@@ -29,7 +29,7 @@ classifiers = [
 dependencies = [
     "outfancy>=0.11",
     "plotille>=6",
-    "plotilleresample>=1.0",
+    "plotilleresample>=1.2",
     "typer>=0.12",
 ]
 

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/sqlitexplorer/charts.py

@@ -6,6 +6,7 @@ other column is a numeric series. Histograms use the first column only.
 
 from __future__ import annotations
 
+import functools
 import os
 from collections.abc import Callable, Iterator, Sequence
 from contextlib import contextmanager
@@ -13,6 +14,7 @@ from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
 from itertools import islice
+from typing import NamedTuple
 
 import plotille
 import plotilleresample
@@ -22,12 +24,13 @@ from sqlitexplorer.render import strip_ansi
 
 __all__ = [
     "ChartKind",
+    "Histogram",
     "Series",
-    "histogram_values",
     "render_chart",
     "render_histogram",
     "resample_series",
     "series_from_result",
+    "stream_histogram",
     "stream_series",
 ]
 
@@ -39,9 +42,9 @@ AXIS_LABEL_WIDTH = 8
 AXIS_WIDTH = 13
 # Narrowest canvas worth drawing on when the labels ask for too much room.
 MIN_CANVAS = 10
-# A tuple, not bool | int | float: the union would be rebuilt on every call,
-# and this runs once per value of the result.
-_NUMERIC = (bool, int, float)
+# A tuple, not int | float: the union would be rebuilt on every call, and
+# this runs once per value of the result. bool is an int.
+_NUMERIC = (int, float)
 # Rows read at a time when streaming, and how many reduced points may pile
 # up before they are reduced again.
 _CHUNK = 65536
@@ -61,6 +64,15 @@ class Series:
     y: list[float]
 
 
+class Histogram(NamedTuple):
+    """Counts of the first column of a query, in the bins plotille would draw."""
+
+    counts: list[int]
+    edges: list[float]
+    column: str
+    skipped: int
+
+
 def _number(value: object) -> float | None:
     if isinstance(value, _NUMERIC):
         return float(value)
@@ -131,11 +143,12 @@ def resample_series(
     """Reduce every series to the points the canvas can actually draw.
 
     min/max keeps the extremes of every bucket, so spikes survive, and it only
-    indexes X, which the LTTB resamplers cannot do when X is a date.
+    indexes X, which the LTTB resamplers cannot do when X is a date. A scatter
+    plot gets a uniform stride for its density plus those extremes.
     """
     budget = _canvas_width(width)
     reduce = (
-        plotilleresample.resample_scatter
+        plotilleresample.resample_scatter_minmax
         if kind is ChartKind.SCATTER
         else plotilleresample.resample_plot_minmax
     )
@@ -190,24 +203,51 @@ def stream_series(
     return resample_series(reduced, kind=kind, width=width, height=height), skipped, rows_read
 
 
-def histogram_values(result: ResultSet) -> tuple[list[float], int]:
-    """Numeric values of the first column of *result*, and how many NULLs were skipped."""
-    if not result.columns:
-        raise ExplorerError("need a numeric column to plot")
-    name = result.columns[0]
-    values: list[float] = []
+def stream_histogram(open_stream: Callable[[], RowStream], *, bins: int) -> Histogram:
+    """Count the first column of a query into *bins* without holding its values.
+
+    The query runs twice: the first pass validates the values and finds their
+    range, the second counts them, so only the counts stay in memory. The
+    bins are the ones plotille computes from the raw values: equal widths
+    from the minimum to the maximum, the last one closed. ``skipped`` counts
+    the rows whose first column was NULL.
+    """
+    first = open_stream()
+    if not first.returns_rows:
+        raise ExplorerError("the statement returned no rows")
+    column = first.columns[0]
+    low = high = None
     skipped = 0
-    for row in result.rows:
-        if row[0] is None:
-            skipped += 1
-            continue
-        number = _number(row[0])
+    for row in first.rows:
+        number = _histogram_value(row[0], column)
         if number is None:
-            raise ExplorerError(f"column {name} is not numeric: {row[0]!r}")
-        values.append(number)
-    if not values:
+            skipped += 1
+        elif low is None or high is None:
+            low = high = number
+        else:
+            low, high = min(low, number), max(high, number)
+    if low is None or high is None:
         raise ExplorerError("no rows to plot")
-    return values, skipped
+    if low == high:
+        low, high = low - 0.5, high + 0.5
+    step = (high - low) / bins
+    counts = [0] * bins
+    for row in open_stream().rows:
+        number = _histogram_value(row[0], column)
+        if number is not None:
+            # Clamped: a query that is not deterministic may not repeat its range.
+            counts[max(0, min(bins - 1, int((number - low) // step)))] += 1
+    edges = [low + index * step for index in range(bins + 1)]
+    return Histogram(counts, edges, column, skipped)
+
+
+def _histogram_value(value: object, column: str) -> float | None:
+    if value is None:
+        return None
+    number = _number(value)
+    if number is None:
+        raise ExplorerError(f"column {column} is not numeric: {value!r}")
+    return number
 
 
 @contextmanager
@@ -233,19 +273,31 @@ def _canvas_width(width: int) -> int:
     return max(MIN_CANVAS, width - AXIS_WIDTH)
 
 
-def _fit(draw: Callable[[int], str], width: int) -> str:
+def _fit(
+    draw: Callable[[int], str], width: int, *, probe: Callable[[int], str] | None = None
+) -> str:
     """Draw on the widest canvas whose longest line still fits in *width*.
 
     plotille writes the X label and the tick numbers past the end of the
     canvas, by an amount that depends on both, so the only way to know the
-    room they take is to draw and measure.
+    room they take is to draw and measure. That room does not depend on the
+    number of points, so *probe*, a drawing with the same labels and the same
+    ranges but two points per series, finds the canvas cheaply and *draw*
+    then runs once.
     """
     canvas = _canvas_width(width)
+    if probe is not None:
+        canvas = _narrow(probe, width, canvas)[1]
+    return _narrow(draw, width, canvas)[0]
+
+
+def _narrow(draw: Callable[[int], str], width: int, canvas: int) -> tuple[str, int]:
+    """Shrink *canvas* until the drawing fits in *width*; return the drawing and its canvas."""
     while True:
         drawing = draw(canvas)
         excess = max(len(strip_ansi(line)) for line in drawing.splitlines()) - width
         if excess <= 0 or canvas <= MIN_CANVAS:
-            return drawing
+            return drawing, canvas
         canvas = max(MIN_CANVAS, canvas - excess)
 
 
@@ -260,50 +312,81 @@ def render_chart(
     y_label: str,
 ) -> str:
     """Draw *series* as a line chart or scatter plot, with a legend when there are several."""
+    draw = functools.partial(
+        _draw_series, kind=kind, height=height, color=color, x_label=x_label, y_label=y_label
+    )
+    return _fit(
+        lambda canvas: draw(series, canvas=canvas),
+        width,
+        probe=lambda canvas: draw(_extremes(series), canvas=canvas),
+    )
 
-    def draw(canvas: int) -> str:
-        figure = plotille.Figure()
-        figure.width = canvas
-        figure.height = max(3, height)
-        figure.with_colors = color
-        figure.color_mode = "names"
-        figure.x_label = x_label
-        figure.y_label = y_label[:AXIS_LABEL_WIDTH]
-        for index, item in enumerate(series):
-            line_color = PALETTE[index % len(PALETTE)] if color else None
-            if kind is ChartKind.SCATTER:
-                figure.scatter(item.x, item.y, lc=line_color, label=item.label)
-            else:
-                figure.plot(item.x, item.y, lc=line_color, label=item.label)
-        with _color_environment(color):
-            return figure.show(legend=len(series) > 1)
 
-    return _fit(draw, width)
+def _figure(
+    *, canvas: int, height: int, color: bool, x_label: str, y_label: str
+) -> plotille.Figure:
+    figure = plotille.Figure()
+    figure.width = canvas
+    figure.height = max(3, height)
+    figure.with_colors = color
+    figure.color_mode = "names"
+    figure.x_label = x_label
+    figure.y_label = y_label[:AXIS_LABEL_WIDTH]
+    return figure
 
 
-def render_histogram(
-    values: Sequence[float],
+def _draw_series(
+    series: Sequence[Series],
     *,
-    bins: int,
-    width: int,
+    kind: ChartKind,
+    canvas: int,
     height: int,
     color: bool,
     x_label: str,
     y_label: str,
 ) -> str:
-    """Draw the distribution of *values*."""
-    numbers = list(values)
-
-    def draw(canvas: int) -> str:
-        with _color_environment(color):
-            return plotille.histogram(
-                numbers,
-                bins=bins,
-                width=canvas,
-                height=max(3, height),
-                X_label=x_label,
-                Y_label=y_label[:AXIS_LABEL_WIDTH],
-                lc=PALETTE[0] if color else None,
-            )
-
-    return _fit(draw, width)
+    figure = _figure(canvas=canvas, height=height, color=color, x_label=x_label, y_label=y_label)
+    for index, item in enumerate(series):
+        line_color = PALETTE[index % len(PALETTE)] if color else None
+        if kind is ChartKind.SCATTER:
+            figure.scatter(item.x, item.y, lc=line_color, label=item.label)
+        else:
+            figure.plot(item.x, item.y, lc=line_color, label=item.label)
+    with _color_environment(color):
+        return figure.show(legend=len(series) > 1)
+
+
+def _extremes(series: Sequence[Series]) -> list[Series]:
+    """Two points per series, on the same ranges: enough to lay the axes out."""
+    return [
+        Series(label=item.label, x=[min(item.x), max(item.x)], y=[min(item.y), max(item.y)])
+        for item in series
+    ]
+
+
+def render_histogram(
+    histogram: Histogram, *, width: int, height: int, color: bool, x_label: str, y_label: str
+) -> str:
+    """Draw *histogram* the way plotille draws one from the raw values."""
+    draw = functools.partial(
+        _draw_histogram, histogram, height=height, color=color, x_label=x_label, y_label=y_label
+    )
+    return _fit(lambda canvas: draw(canvas=canvas), width)
+
+
+def _draw_histogram(
+    histogram: Histogram, *, canvas: int, height: int, color: bool, x_label: str, y_label: str
+) -> str:
+    figure = _figure(canvas=canvas, height=height, color=color, x_label=x_label, y_label=y_label)
+    # plotille only bins raw values, but its Histogram plot keeps the counts
+    # apart from them and draws from the counts alone: the two edges give it
+    # the same range, hence the same bins, and the real counts then replace
+    # the ones it took from those two values.
+    figure.histogram(
+        [histogram.edges[0], histogram.edges[-1]],
+        bins=len(histogram.counts),
+        lc=PALETTE[0] if color else None,
+    )
+    figure._plots[-1].frequencies = list(histogram.counts)
+    with _color_environment(color):
+        return figure.show()

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/sqlitexplorer/cli.py

@@ -25,10 +25,10 @@ import typer
 from sqlitexplorer import __version__
 from sqlitexplorer.charts import (
     ChartKind,
-    histogram_values,
     render_chart,
     render_histogram,
     series_from_result,
+    stream_histogram,
     stream_series,
 )
 from sqlitexplorer.completion import complete_table
@@ -203,9 +203,15 @@ def _fail(message: str) -> NoReturn:
 
 @contextmanager
 def _reporting_errors() -> Iterator[None]:
-    """Turn ExplorerError and OSError into a message on stderr and exit status 1."""
+    """Turn ExplorerError and OSError into a message on stderr and exit status 1.
+
+    A closed pipe (``| head`` stopped reading) is not an error to report: it
+    is left to Typer, which ends the command quietly.
+    """
     try:
         yield
+    except BrokenPipeError:
+        raise
     except ExplorerError as error:
         _fail(_error_message(error))
     except OSError as error:
@@ -657,7 +663,12 @@ def chart(
         ),
     ] = True,
     x_label: Annotated[str | None, typer.Option("--x-label", help="Label of the X axis.")] = None,
-    y_label: Annotated[str | None, typer.Option("--y-label", help="Label of the Y axis.")] = None,
+    y_label: Annotated[
+        str | None,
+        typer.Option(
+            "--y-label", help="Label of the Y axis, at most 8 characters (plotille's limit)."
+        ),
+    ] = None,
     params: ParamOption = None,
     width: WidthOption = None,
     color: ColorOption = None,
@@ -670,17 +681,16 @@ def chart(
         use_color = resolve_color(color)
         screen = width if width is not None else shutil.get_terminal_size().columns
         if kind is ChartKind.HIST:
-            result = db.execute(text, bound)
-            if not result.returns_rows:
-                _fail("the statement returned no rows")
-            values, skipped = histogram_values(result)
+            # Two passes over the query, the range first and then the counts,
+            # so the values never pile up in memory.
+            histogram = stream_histogram(lambda: db.stream(text, bound), bins=bins)
+            skipped = histogram.skipped
             drawing = render_histogram(
-                values,
-                bins=bins,
+                histogram,
                 width=screen,
                 height=height,
                 color=use_color,
-                x_label=x_label or result.columns[0],
+                x_label=x_label or histogram.column,
                 y_label=y_label or "count",
             )
         else:
@@ -693,8 +703,9 @@ def chart(
                 series, skipped, read = stream_series(
                     stream, kind=kind, width=screen, height=height
                 )
-                if len(series[0].x) < read:
-                    typer.echo(f"resampled {read} rows to {len(series[0].x)} points", err=True)
+                plotted, points = read - skipped, len(series[0].x)
+                if points < plotted:
+                    typer.echo(f"resampled {plotted} rows to {points} points", err=True)
             else:
                 result = db.execute(text, bound)
                 if not result.returns_rows:
@@ -831,7 +842,10 @@ def import_(
     text = _read_text(file, encoding)
     with _reporting_errors(), open_database(database, write=True) as db:
         headers, raw_rows = parse_rows(text, input_format, delimiter=delimiter)
-        types = infer_types(raw_rows, len(headers))
+        # A JSON file brings its own types; a CSV only has text to go by.
+        types = infer_types(
+            raw_rows, len(headers), parse_text=input_format is not OutputFormat.JSON
+        )
         count = db.import_rows(
             table, list(zip(headers, types, strict=True)), coerce_rows(raw_rows, types)
         )

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/sqlitexplorer/core.py

@@ -8,9 +8,10 @@ that should reach the user as a plain message is raised as
 
 from __future__ import annotations
 
+import re
 import sqlite3
 from collections.abc import Iterable, Iterator, Mapping, Sequence
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from dataclasses import dataclass, field
 from itertools import islice
 from pathlib import Path
@@ -100,10 +101,16 @@ class StatsReport(NamedTuple):
 
 
 def _rows_of(cursor: sqlite3.Cursor) -> Iterator[tuple]:
+    # Not ``yield from``: closing the generator would then close the cursor
+    # itself, outside the ``finally`` below.
     try:
-        yield from cursor
+        for row in cursor:  # noqa: UP028
+            yield row
     finally:
-        cursor.close()
+        # A stream cut short by an error, Ctrl-C or a closed pipe is finalised
+        # after the connection is closed, which already released the cursor.
+        with suppress(sqlite3.ProgrammingError):
+            cursor.close()
 
 
 def _window(stream: RowStream, page: Page) -> ResultSet:
@@ -129,7 +136,8 @@ def split_statements(sql: str) -> list[str]:
 
     Uses :func:`sqlite3.complete_statement`, so semicolons inside strings,
     comments and ``CREATE TRIGGER ... END`` blocks do not split. Trailing text
-    without a semicolon is returned as a final statement.
+    without a semicolon is returned as a final statement; text that holds
+    only whitespace, semicolons and comments is not a statement.
     """
     statements: list[str] = []
     buffer = ""
@@ -147,7 +155,8 @@ def split_statements(sql: str) -> list[str]:
 
 
 def _has_content(text: str) -> bool:
-    return bool(text.strip().rstrip(";").strip())
+    """Whether *text* holds anything besides whitespace, semicolons and comments."""
+    return bool(_COMMENT.sub("", text).strip().rstrip(";").strip())
 
 
 def plan_tree(result: ResultSet) -> ResultSet:
@@ -695,6 +704,8 @@ class Explorer:
             cursor.close()
 
 
+# SQL comments, removed only to decide whether a statement is empty.
+_COMMENT = re.compile(r"--[^\n]*|/\*.*?\*/", re.DOTALL)
 _SEARCH_COLUMNS = ("table", "column", "rowid", "value")
 _STATS_COLUMNS = ("column", "type", "nulls", "distinct", "min", "max", "top")
 # Columns per aggregate query: five expressions each, well below SQLITE_MAX_COLUMN.

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/sqlitexplorer/render.py

@@ -51,11 +51,11 @@ __all__ = [
 
 ELLIPSIS = "…"
 _ANSI = re.compile(r"\x1b\[[0-9;]*m")
-_INTEGER = re.compile(r"^[+-]?\d+$")
-_REAL = re.compile(r"^[+-]?(\d+\.\d*|\.\d+|\d+)([eE][+-]?\d+)?$")
-_LEADING_ZERO = re.compile(r"^[+-]?0\d")
-# A tuple, not bool | int: the union would be rebuilt on every call.
-_INTEGRAL = (bool, int)
+# ASCII digits only: \d also matches the digits of other scripts, which
+# int() would happily convert.
+_INTEGER = re.compile(r"^[+-]?[0-9]+$")
+_REAL = re.compile(r"^[+-]?([0-9]+\.[0-9]*|\.[0-9]+|[0-9]+)([eE][+-]?[0-9]+)?$")
+_LEADING_ZERO = re.compile(r"^[+-]?0[0-9]")
 # Integers outside this range do not fit in a SQLite INTEGER column.
 _INT64 = range(-(2**63), 2**63)
 
@@ -195,7 +195,9 @@ def render_table(
     columns = list(result.columns)
 
     screen = width if width is not None else shutil.get_terminal_size().columns
-    # outfancy keeps a two-column margin and one separator per column.
+    # One separator per column, plus two columns of margin: outfancy needs
+    # none when the widths are given, but its own corrector leaves the same
+    # two, and a console that wraps on the last column would add blank lines.
     available = screen - 2 - len(columns)
     widths = fit_widths(columns, _natural_widths(columns, rows), available)
     return table.render(data=rows, label_list=columns, width=widths, screen_x=width)
@@ -480,11 +482,13 @@ def _plain_json_value(value: object) -> object:
     return value
 
 
-def _classify(value: object) -> str:
-    if isinstance(value, _INTEGRAL):
+def _classify(value: object, *, parse_text: bool) -> str:
+    if isinstance(value, int):  # bool is an int
         return "INTEGER"
     if isinstance(value, float):
         return "REAL"
+    if not parse_text:
+        return "TEXT"
     text = str(value).strip()
     if _LEADING_ZERO.match(text):
         # 007 is a code, not a number: storing it as one would drop the zeros.
@@ -496,8 +500,15 @@ def _classify(value: object) -> str:
     return "TEXT"
 
 
-def infer_types(rows: Sequence[Sequence[object]], count: int) -> list[str]:
-    """Pick INTEGER, REAL or TEXT for each of the *count* columns from the values seen."""
+def infer_types(
+    rows: Sequence[Sequence[object]], count: int, *, parse_text: bool = True
+) -> list[str]:
+    """Pick INTEGER, REAL or TEXT for each of the *count* columns from the values seen.
+
+    With *parse_text*, textual values that look like numbers count as numbers,
+    which is what a CSV needs. Without it they stay TEXT: a JSON string is a
+    string whatever it holds, and its numbers already come as numbers.
+    """
     rank = {"INTEGER": 0, "REAL": 1, "TEXT": 2}
     types = ["INTEGER"] * count
     seen = [False] * count
@@ -507,7 +518,7 @@ def infer_types(rows: Sequence[Sequence[object]], count: int) -> list[str]:
             if value is None:
                 continue
             seen[index] = True
-            kind = _classify(value)
+            kind = _classify(value, parse_text=parse_text)
             if rank[kind] > rank[types[index]]:
                 types[index] = kind
     return [kind if was_seen else "TEXT" for kind, was_seen in zip(types, seen, strict=True)]

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0/sqlitexplorer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlitexplorer
-Version: 1.1.3
+Version: 1.2.0
 Summary: Explore SQLite databases from the terminal: tables, schema, stats, search, queries, charts, export/import and an interactive shell.
 Author-email: "Carlos A. Planchón" <carlosandresplanchonprestes@gmail.com>
 License-Expression: MIT
@@ -21,7 +21,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: outfancy>=0.11
 Requires-Dist: plotille>=6
-Requires-Dist: plotilleresample>=1.0
+Requires-Dist: plotilleresample>=1.2
 Requires-Dist: typer>=0.12
 Provides-Extra: dev
 Requires-Dist: pytest>=8; extra == "dev"
@@ -50,10 +50,11 @@ with [plotille](https://github.com/tammoippen/plotille), resampled by
 Requires Python 3.10 or newer.
 
 ```sh
-# From a clone of this repository:
-uv tool install .
+uv tool install sqlitexplorer
 # or with pip:
-pip install .
+pip install sqlitexplorer
+# or from a clone of this repository:
+uv tool install .
 ```
 
 ## Quick tour
@@ -129,8 +130,9 @@ Big results do not need to fit in memory: `--page` fetches only the requested
 page (in SQL for `show`, from the cursor for `query`), and the csv, tsv, json
 and markdown formats, `export` and `dump` are written row by row. The table
 format is the exception, since it needs every row to size its columns.
-`chart` reduces as it reads, so the memory it needs does not grow with the
-table; `chart --no-resample` is the one that holds every row.
+`chart` reduces as it reads, and a histogram counts as it reads, so the memory
+they need does not grow with the table; `chart --no-resample` is the one that
+holds every row.
 
 ## Queries
 
@@ -161,15 +163,18 @@ The first column is the X axis (numbers or ISO dates), every other column is a
 series named after the column; rows with NULLs are skipped and counted on
 stderr. `--kind` selects `line` (default), `scatter` or `hist` (first column
 only, `--bins`). `--height`, `--width`, `--x-label`, `--y-label` and `--color`
-adjust the drawing.
+adjust the drawing; the Y label is cut to eight characters, the room plotille
+gives it.
 
 Line and scatter charts are reduced to what the canvas can show, keeping the
 minimum and the maximum of every column of braille dots, so spikes survive and
-the true extremes keep the X of their own row. The rows are reduced as they are
+the true extremes keep the X of their own row; a scatter plot keeps a uniform
+sample of its points as well, for its density. The rows are reduced as they are
 read, in chunks, so a chart over millions of rows needs no more memory than one
 over a thousand; the reduction is reported on stderr. `--no-resample` reads and
 plots every row instead. Histograms are never reduced, since dropping rows would
-change the distribution.
+change the distribution: the query runs twice, once for the range and once for
+the counts, so they do not hold the rows either.
 
 ## Export, import, dump and diff
 
@@ -183,8 +188,10 @@ sqlitexplorer diff app.db backup.db                      # exit status 1 when th
 
 `import` creates the table when it does not exist, inferring INTEGER, REAL or
 TEXT for each column; values with leading zeros and integers too large for
-SQLite stay TEXT, so codes such as `007` keep their digits. Empty CSV cells
-become NULL. The format comes from the file extension unless `--format` is
+SQLite stay TEXT, so codes such as `007` keep their digits. From JSON the types
+are the JSON types: numbers become INTEGER or REAL and strings stay TEXT even
+when they look like numbers, so a json export imports back as it was. Empty CSV
+cells become NULL. The format comes from the file extension unless `--format` is
 given, and `--encoding` reads a file that is not UTF-8.
 
 `export --all` writes one file per table and view inside the directory: a name

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/sqlitexplorer.egg-info/requires.txt

@@ -1,6 +1,6 @@
 outfancy>=0.11
 plotille>=6
-plotilleresample>=1.0
+plotilleresample>=1.2
 typer>=0.12
 
 [dev]

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/tests/test_charts.py

@@ -4,18 +4,21 @@ from __future__ import annotations
 
 from datetime import datetime
 
+import plotille
 import pytest
 
 from sqlitexplorer.charts import (
     AXIS_WIDTH,
     MIN_CANVAS,
     ChartKind,
+    Histogram,
     Series,
-    histogram_values,
+    _draw_histogram,
     render_chart,
     render_histogram,
     resample_series,
     series_from_result,
+    stream_histogram,
     stream_series,
 )
 from sqlitexplorer.core import ExplorerError, ResultSet, RowStream
@@ -67,20 +70,6 @@ def test_series_requires_two_columns() -> None:
         series_from_result(ResultSet(columns=("x",), rows=[(1,)]))
 
 
-def test_histogram_values_uses_first_column() -> None:
-    result = ResultSet(columns=("v", "other"), rows=[(1, "a"), (None, "b"), (2.5, "c")])
-    values, skipped = histogram_values(result)
-    assert values == [1.0, 2.5]
-    assert skipped == 1
-    with pytest.raises(ExplorerError, match="not numeric"):
-        histogram_values(ResultSet(columns=("v",), rows=[("x",)]))
-
-
-def test_histogram_values_requires_a_column() -> None:
-    with pytest.raises(ExplorerError, match="need a numeric column"):
-        histogram_values(ResultSet())
-
-
 def test_render_chart_prints_braille_without_colors() -> None:
     series, _ = series_from_result(ResultSet(columns=("x", "y"), rows=[(1, 1), (2, 3), (3, 2)]))
     text = render_chart(
@@ -110,15 +99,6 @@ def test_render_chart_single_point_does_not_crash() -> None:
     assert braille(text)
 
 
-def test_render_histogram() -> None:
-    text = render_histogram(
-        [1.0, 2.0, 2.0, 3.0], bins=3, width=50, height=6, color=False, x_label="v", y_label="count"
-    )
-    assert braille(text)
-    assert "(v)" in text
-    assert "\x1b[" not in text
-
-
 def test_resample_series_reduces_and_keeps_the_extremes() -> None:
     rows: list[tuple] = [(i, 0.0) for i in range(5000)]
     rows[1234] = (1234, 999.0)
@@ -163,6 +143,68 @@ def _stream(columns: tuple[str, ...], rows: list[tuple]) -> RowStream:
     return RowStream(columns=columns, rows=iter(rows))
 
 
+def _histogram(values: list[object], bins: int = 3) -> Histogram:
+    return stream_histogram(lambda: _stream(("v", "other"), [(v, "x") for v in values]), bins=bins)
+
+
+def test_stream_histogram_counts_the_first_column_in_two_passes() -> None:
+    opened = 0
+
+    def open_stream() -> RowStream:
+        nonlocal opened
+        opened += 1
+        return _stream(("v", "other"), [(1, "a"), (None, "b"), (2.5, "c"), (4, "d")])
+
+    histogram = stream_histogram(open_stream, bins=3)
+    assert opened == 2
+    assert histogram.column == "v"
+    assert histogram.skipped == 1
+    assert histogram.edges == [1.0, 2.0, 3.0, 4.0]
+    assert histogram.counts == [1, 1, 1]  # the last bin is closed: 4 lands in it
+
+
+def test_stream_histogram_single_value_gets_a_unit_range() -> None:
+    histogram = _histogram([7, 7, 7], bins=2)
+    assert histogram.edges == [6.5, 7.0, 7.5]
+    assert histogram.counts == [0, 3]
+
+
+def test_stream_histogram_rejects_bad_input() -> None:
+    with pytest.raises(ExplorerError, match="not numeric"):
+        _histogram(["x"])
+    with pytest.raises(ExplorerError, match="no rows to plot"):
+        _histogram([None, None])
+    with pytest.raises(ExplorerError, match="returned no rows"):
+        stream_histogram(lambda: RowStream(), bins=3)
+
+
+def test_render_histogram() -> None:
+    histogram = _histogram([1.0, 2.0, 2.0, 3.0])
+    text = render_histogram(
+        histogram, width=50, height=6, color=False, x_label="v", y_label="count"
+    )
+    assert braille(text)
+    assert "(v)" in text
+    assert "\x1b[" not in text
+    colored = render_histogram(
+        histogram, width=50, height=6, color=True, x_label="v", y_label="count"
+    )
+    assert "\x1b[" in colored
+
+
+def test_render_histogram_matches_plotille(monkeypatch: pytest.MonkeyPatch) -> None:
+    # The counts are fed to plotille's own Histogram plot, so a plotille
+    # release that changes how it keeps them would show up here.
+    monkeypatch.setenv("NO_COLOR", "1")
+    values = [float(i % 97) for i in range(2000)] + [3.5, 3.5, 96.0]
+    ours = _draw_histogram(
+        _histogram(values, bins=12), canvas=50, height=8, color=False, x_label="v", y_label="count"
+    )
+    theirs = plotille.histogram(values, bins=12, width=50, height=8, X_label="v", Y_label="count")
+    assert ours == theirs
+    assert braille(ours)
+
+
 def test_stream_series_keeps_the_envelope_and_the_real_pairs(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -262,8 +304,7 @@ def test_render_chart_never_draws_wider_than_the_terminal(width: int, x_label: s
 @pytest.mark.parametrize("width", [40, 60, 80, 96, 120, 200])
 def test_render_histogram_never_draws_wider_than_the_terminal(width: int) -> None:
     drawing = render_histogram(
-        [float(i % 97) for i in range(2000)],
-        bins=10,
+        _histogram([float(i % 97) for i in range(2000)], bins=10),
         width=width,
         height=6,
         color=False,
@@ -289,3 +330,37 @@ def test_render_chart_stops_narrowing_at_the_floor() -> None:
     body = [line for line in drawing.splitlines() if braille(line)]
     assert body, drawing
     assert max(len(line) for line in body) == MIN_CANVAS + AXIS_WIDTH
+
+
+def test_render_chart_draws_the_full_series_once(monkeypatch: pytest.MonkeyPatch) -> None:
+    # The canvas is found on a two-point stand-in; the real points are drawn once.
+    lengths: list[int] = []
+    original = plotille.Figure.plot
+
+    def recording(self: plotille.Figure, X: list, Y: list, **kwargs: object) -> None:
+        lengths.append(len(X))
+        original(self, X, Y, **kwargs)
+
+    monkeypatch.setattr(plotille.Figure, "plot", recording)
+    xs = [float(i) for i in range(1000)]
+    render_chart(
+        [Series(label="v", x=xs, y=[i % 7 for i in range(1000)])],
+        kind=ChartKind.LINE,
+        width=80,
+        height=9,
+        color=False,
+        x_label="t",
+        y_label="v",
+    )
+    assert lengths.count(1000) == 1
+    assert set(lengths) == {2, 1000}
+
+
+def test_resample_series_scatter_keeps_an_isolated_spike() -> None:
+    rows: list[tuple] = [(i, 0.0) for i in range(50_000)]
+    rows[12_347] = (12_347, 999.0)
+    series, _ = series_from_result(ResultSet(columns=("x", "y"), rows=rows))
+    reduced = resample_series(series, kind=ChartKind.SCATTER, width=80, height=15)
+    assert len(reduced[0].x) < 50_000
+    assert 999.0 in reduced[0].y
+    assert reduced[0].x[reduced[0].y.index(999.0)] == 12_347

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/tests/test_cli.py

@@ -3,14 +3,18 @@
 from __future__ import annotations
 
 import json
+import os
 import re
 import sqlite3
+import sys
 from collections.abc import Callable
+from contextlib import suppress
 from pathlib import Path
 
 import pytest
 
 from sqlitexplorer import __version__
+from sqlitexplorer.cli import main
 
 # --- Basics -------------------------------------------------------------------
 
@@ -35,6 +39,26 @@ def test_missing_database_is_a_usage_error(invoke: Callable, tmp_path: Path) ->
     assert not missing.exists()
 
 
+def test_broken_pipe_exits_quietly(
+    database: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    # ``sqlitexplorer show ... | head``: the reader goes away, the first
+    # write fails with EPIPE and the command must end without a word.
+    read_end, write_end = os.pipe()
+    os.close(read_end)
+    sink = os.fdopen(write_end, "w", encoding="utf-8")
+    monkeypatch.setattr(sys, "stdout", sink)
+    monkeypatch.setattr(sys, "argv", ["sqlitexplorer", "show", str(database), "users", "-f", "csv"])
+    try:
+        with pytest.raises(SystemExit) as exit_info:
+            main()
+    finally:
+        with suppress(OSError):
+            sink.close()
+    assert exit_info.value.code == 1
+    assert capsys.readouterr().err == ""
+
+
 # --- tables / schema / describe ----------------------------------------------
 
 
@@ -463,6 +487,7 @@ def test_query_file_runs_every_statement(
         "INSERT INTO users (name) VALUES ('Zoe');\n"
         "-- a comment; with a semicolon\n"
         "SELECT COUNT(*) AS total FROM users;\n"
+        "-- a trailing comment is not a statement\n"
     )
     result = invoke("query", database, "--file", script, "--write", "-f", "csv")
     assert result.exit_code == 0
@@ -568,6 +593,7 @@ def test_chart_line_prints_braille(invoke: Callable, database: Path) -> None:
     assert braille(result.stdout)
     assert "\x1b[" not in result.stdout
     assert "skipped 1 row with NULL values" in result.stderr
+    assert "resampled" not in result.stderr  # two rows do not need reducing
 
 
 def test_chart_hist_uses_first_column(invoke: Callable, database: Path) -> None:
@@ -678,6 +704,19 @@ def test_import_into_existing_table_and_json(
     ]
 
 
+def test_import_json_keeps_strings_as_text(
+    invoke: Callable, database: Path, tmp_path: Path
+) -> None:
+    source = tmp_path / "typed.json"
+    source.write_text('[{"zip": "12345", "n": 7}, {"zip": "67890", "n": 8}]')
+    result = invoke("import", database, "typed", source)
+    assert result.exit_code == 0
+    described = invoke("describe", database, "typed", "-f", "csv").output.splitlines()
+    assert [line.split(",")[2] for line in described[1:]] == ["TEXT", "INTEGER"]
+    rows = invoke("query", database, "SELECT typeof(zip), zip FROM typed ORDER BY zip", "-f", "csv")
+    assert rows.output.splitlines() == ["typeof(zip),zip", "text,12345", "text,67890"]
+
+
 def test_import_rolls_back_on_error(invoke: Callable, database: Path, tmp_path: Path) -> None:
     source = tmp_path / "bad.csv"
     source.write_text("x,y\n1,2\n")
@@ -830,13 +869,16 @@ def test_chart_resamples_lines_by_default_but_never_histograms(
         connection.executemany(
             "INSERT INTO m VALUES (?, ?)", [(i, float(i % 100)) for i in range(5000)]
         )
+        connection.executemany("INSERT INTO m VALUES (?, NULL)", [(i,) for i in range(5000, 5010)])
         connection.commit()
     finally:
         connection.close()
     args = ("chart", source, "SELECT t, v FROM m", "--width", "80", "--no-color")
     result = invoke(*args)
     assert result.exit_code == 0
+    # Rows skipped for their NULLs are reported apart, not as resampled.
     assert "resampled 5000 rows to" in result.stderr
+    assert "skipped 10 rows with NULL values" in result.stderr
     plain = invoke(*args, "--no-resample")
     assert plain.exit_code == 0
     assert "resampled" not in plain.stderr

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/tests/test_core.py

@@ -49,6 +49,10 @@ def test_split_statements_handles_strings_comments_and_triggers() -> None:
     ]
     assert split_statements("  ;; \n") == []
     assert split_statements("SELECT 1;") == ["SELECT 1;"]
+    # Comments after the last statement are not a statement of their own.
+    assert split_statements("SELECT 1; -- done") == ["SELECT 1;"]
+    assert split_statements("/* nothing */ ; -- at all\n") == []
+    assert split_statements("SELECT '--'; SELECT '/*'") == ["SELECT '--';", "SELECT '/*'"]
 
 
 def test_plan_tree_indents_by_depth() -> None:
@@ -155,6 +159,15 @@ def test_stream_yields_rows_lazily(database: Path) -> None:
         assert streamed.rows == db.rows("users", columns=["name"], order_by="id", limit=2).rows
 
 
+def test_abandoned_stream_is_quiet_after_the_connection_closed(database: Path) -> None:
+    # A stream cut short by an error, Ctrl-C or a closed pipe is finalised
+    # once the connection is gone; the cursor is already released by then.
+    with open_database(database) as db:
+        stream = db.stream("SELECT id FROM users ORDER BY id")
+        assert next(stream.rows) == (1,)
+    stream.rows.close()
+
+
 def test_explain_returns_an_indented_plan(database: Path) -> None:
     with open_database(database) as db:
         result = db.explain("SELECT * FROM users WHERE name = 'Marie'")

{sqlitexplorer-1.1.3 → sqlitexplorer-1.2.0}/tests/test_render.py

@@ -167,6 +167,7 @@ def test_parse_number() -> None:
     assert parse_number("007") == 7
     assert parse_number("abc") == "abc"
     assert parse_number("1_000") == "1_000"
+    assert parse_number("١٢٣") == "١٢٣"  # digits of other scripts are not numbers
 
 
 def test_parse_rows_csv_and_json() -> None:
@@ -200,6 +201,14 @@ def test_infer_types_and_coerce_rows() -> None:
     types = infer_types(rows, 4)
     assert types == ["INTEGER", "REAL", "TEXT", "TEXT"]
     assert coerce_rows(rows, types) == [[1, 1.5, "x", None], [2, 2.0, "3", None]]
+    # JSON brings its own types: a string stays TEXT whatever it looks like.
+    assert infer_types([["1", 2.5, True, None]], 4, parse_text=False) == [
+        "TEXT",
+        "REAL",
+        "INTEGER",
+        "TEXT",
+    ]
+    assert infer_types([["١٢٣"]], 1) == ["TEXT"]
 
 
 def test_infer_types_keeps_codes_and_huge_integers_as_text() -> None: