chartbook 0.2.0__tar.gz → 0.3.0__tar.gz

{chartbook-0.2.0 → chartbook-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: chartbook
-Version: 0.2.0
+Version: 0.3.0
 Summary: A tool for generating chart documentation websites
 Project-URL: Documentation, https://backofficedev.github.io/chartbook
 Project-URL: Issues, https://github.com/backofficedev/chartbook/issues
@@ -22,7 +22,7 @@ Requires-Dist: ablog>=0.11.11
 Requires-Dist: click>=8.1.7
 Requires-Dist: cookiecutter>=2.2
 Requires-Dist: cruft
-Requires-Dist: duckdb>=1.4
+Requires-Dist: duckdb>=1.5
 Requires-Dist: fredapi>=0.5.0
 Requires-Dist: jinja2>=3.1.4
 Requires-Dist: kaleido>=0.2.1
@@ -36,8 +36,8 @@ Requires-Dist: pandas-datareader>=0.10.0
 Requires-Dist: pandas>=2.1.4
 Requires-Dist: plotly>=5.24.0
 Requires-Dist: pluggy>=1.3.0
-Requires-Dist: polars>=1.20
-Requires-Dist: pyarrow
+Requires-Dist: polars>=1.33
+Requires-Dist: pyarrow>=16
 Requires-Dist: pydata-sphinx-theme>=0.15.4
 Requires-Dist: python-decouple>=3.8
 Requires-Dist: sphinx-autodoc2>=0.5.0
@@ -55,7 +55,7 @@ Requires-Dist: mkdocs-jupyter>=0.25; extra == 'mkdocs'
 Requires-Dist: mkdocs-material>=9.5; extra == 'mkdocs'
 Requires-Dist: mkdocs>=1.6; extra == 'mkdocs'
 Provides-Extra: s3
-Requires-Dist: boto3>=1.28; extra == 's3'
+Requires-Dist: boto3>=1.35; extra == 's3'
 Description-Content-Type: text/markdown
 
 # ChartBook
@@ -129,6 +129,21 @@ from chartbook import models
 joblib.dump(model, models.path("rate_forecaster"))  # the model gets a page: scores, lineage
 ```
 
+### Keep files that are not tables
+
+```toml
+[files.filings]
+path = "./_data/filings/**/*.htm"     # a file, a folder, or a glob: text, PDFs, images, archives
+license = "SEC public data"
+```
+
+```python
+from chartbook import assets
+
+for path in assets.files("file/filings"):  # the files, sorted; another pipeline's as pinned
+    ...
+```
+
 ### Browse your catalog
 
 ```bash

{chartbook-0.2.0 → chartbook-0.3.0}/README.md

@@ -69,6 +69,21 @@ from chartbook import models
 joblib.dump(model, models.path("rate_forecaster"))  # the model gets a page: scores, lineage
 ```
 
+### Keep files that are not tables
+
+```toml
+[files.filings]
+path = "./_data/filings/**/*.htm"     # a file, a folder, or a glob: text, PDFs, images, archives
+license = "SEC public data"
+```
+
+```python
+from chartbook import assets
+
+for path in assets.files("file/filings"):  # the files, sorted; another pipeline's as pinned
+    ...
+```
+
 ### Browse your catalog
 
 ```bash

{chartbook-0.2.0 → chartbook-0.3.0}/pyproject.toml

@@ -31,11 +31,14 @@ dependencies = [
     "tomli-w>=1.1.0",
     # Data loading
     "packaging",
-    # 1.20 has the IO-plugin seam that DuckDB's lazy frames are built on
-    "polars>=1.20",
-    "pyarrow",
-    # The local SQL engine: [sources] with backend = "duckdb", and data.sql()
-    "duckdb>=1.4",  # .pl(lazy=True) arrived in 1.4
+    # DuckDB's lazy frames are built on polars' IO-plugin seam: DuckDB 1.5's
+    # ask it for is_pure, which polars has from 1.33
+    "polars>=1.33",
+    # polars writes text in an Arrow IPC file as string_view, which pyarrow reads from 16
+    "pyarrow>=16",
+    # The local SQL engine: [sources] with backend = "duckdb", and data.sql().
+    # 1.4's lazy frames (.pl(lazy=True)) close their connection under polars 1.33 and later
+    "duckdb>=1.5",
     "python-decouple>=3.8",
     # Plotting
     "matplotlib>=3.7.0",
@@ -79,7 +82,7 @@ dependencies = [
 #
 # The store in an S3 bucket: [store] url = "s3://bucket/prefix"
 s3 = [
-    "boto3>=1.28",
+    "boto3>=1.35",
 ]
 # A source in a Databricks SQL warehouse: [sources.x] backend = "databricks"
 databricks = [
@@ -113,7 +116,7 @@ dev = [
     "openpyxl",
     "requests",
     "seaborn",
-    "boto3>=1.28",
+    "boto3>=1.35",
     "moto[s3]>=5",
     "mkdocs>=1.6",
     "mkdocs-material>=9.5",
@@ -195,7 +198,7 @@ extra-dependencies = [
     "requests",
     "seaborn",
     # The S3 store, against a mocked bucket
-    "boto3>=1.28",
+    "boto3>=1.35",
     "moto[s3]>=5",
     # The MkDocs writer's build test
     "mkdocs>=1.6",

chartbook-0.3.0/src/chartbook/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{chartbook-0.2.0 → chartbook-0.3.0}/src/chartbook/__init__.py

@@ -10,6 +10,10 @@ For the path of anything a pipeline declares:
     assets.path("chart/yield_curve")
     models.path("rate_forecaster")
 
+For the files of an asset, and a table of them:
+    assets.files("file/filings")
+    assets.index("file/filings")
+
 For plotting:
     from chartbook import plotting
     plotting.line(df, chart_id="my_chart", x="date", y="value")
@@ -21,7 +25,7 @@ For CLI (install via pipx):
 
 import importlib
 
-from . import assets, data, env, models
+from . import assets, data, env, files, models
 from .__about__ import __version__
 
 # Lazy import for plotting to avoid requiring plotting dependencies
@@ -43,6 +47,7 @@ __all__ = [
     "assets",
     "data",
     "models",
+    "files",
     "env",
     "plotting",
 ]

chartbook-0.3.0/src/chartbook/assets.py

@@ -0,0 +1,344 @@
+"""Assets: everything a pipeline declares, addressed one way.
+
+An asset is something ``chartbook.toml`` declares: a dataframe, a chart, a
+notebook, a note, a model or a files asset. Each is addressed by a
+*reference*. Within the pipeline you are in, that is the asset's name, with
+its kind in front unless it is a dataframe::
+
+    from chartbook import assets
+
+    assets.path("treasury_rates")        # [dataframes.treasury_rates]
+    assets.path("chart/yield_curve")     # [charts.yield_curve]
+    assets.path("notebook/analysis")     # [notebooks.analysis]
+    assets.path("note/methodology")      # [notes.methodology]
+    assets.path("model/rate_forecaster") # [models.rate_forecaster]
+    assets.path("file/filings")          # [files.filings]
+
+An asset's path may be a folder or a glob. :func:`files` gives the files
+themselves, sorted, and :func:`index` a table of them, with their sizes and
+hashes, to choose from before opening any::
+
+    assets.files("file/filings")         # [Path(".../2024/10-K.htm"), ...]
+    assets.index("file/filings")         # a LazyFrame: path, size_bytes, sha256
+
+The kind may be given as an argument instead, which reads better when the
+name is in a variable. It says what you expect: a reference that names no
+kind takes it, and one that names a different kind is an error::
+
+    assets.path("yield_curve", kind="chart")       # the same chart
+    assets.path("chart/yield_curve", kind="chart") # the same chart
+    assets.path("yield_curve")                     # a *dataframe* called yield_curve
+
+Data is the common case, so it gets the short spelling here and its own
+front door: :func:`chartbook.data.path` is this function for dataframes
+only, and :func:`chartbook.models.path` is this function for models only.
+
+A reference can also be built and passed as an object, :class:`AssetRef`,
+which is what :func:`parse` returns and what ChartBook carries internally::
+
+    ref = assets.parse("chart/yield_curve")
+    ref.kind, ref.name                   # ("chart", "yield_curve")
+    assets.path(ref)
+
+An asset of another pipeline has that pipeline's name in front::
+
+    assets.path("fed_yield_curve:treasury_rates")
+    assets.path("fed_yield_curve:chart/curve_today")
+
+The pipeline must be one that ``[dependencies]`` declares, and the path is
+that of the copy ``chartbook.lock`` pins, in the store. It is not the other
+pipeline's working directory, so what it holds does not change until you
+run ``chartbook update``. Read it; do not write to it.
+
+The full grammar, ``[registry/][scope/]pipeline:[kind/]name[#part][@rev]``,
+is specified in ``docs_src/design/references.md``. A part, a revision and a
+registry parse, and are refused as not yet supported.
+"""
+
+import hashlib
+import os
+import sys
+from pathlib import Path
+from typing import Optional, Union
+
+from chartbook.refs import (
+    DEFAULT_KIND,
+    KINDS,
+    AssetRef,
+    KindMismatchError,
+)
+from chartbook.refs import parse_asset_ref as parse
+from chartbook.resolve import FileLocation, caller_start_dirs, require_file, resolve
+
+__all__ = [
+    "DEFAULT_KIND",
+    "KINDS",
+    "AssetRef",
+    "KindMismatchError",
+    "files",
+    "index",
+    "parse",
+    "path",
+]
+
+#: The columns of :func:`index` that every row has; a ``key=value`` folder's key with one of these names is left out.
+INDEX_COLUMNS = ("path", "size_bytes", "sha256")
+
+
+def _path(ref: Union[str, AssetRef], kind: Optional[str], start_dirs: list) -> Path:
+    """Resolve a reference to a file path, from given starting points.
+
+    The public functions are thin shells over this one, because each must
+    read *its own* caller's stack frame to find the calling file.
+    """
+    asset = parse(ref, kind=kind)
+    location = resolve(asset, start_dirs=start_dirs)
+    return require_file(location, f"The {asset.kind} {asset.name!r}")
+
+
+def path(
+    ref: Union[str, AssetRef],
+    kind: Optional[str] = None,
+    project_dir: Optional[Union[str, Path]] = None,
+) -> Path:
+    """Get the path of an asset: this pipeline's own, or a pinned copy of another's.
+
+    The enclosing ``chartbook.toml`` is the single source of truth for where
+    an asset lives, so code can ask for the path rather than repeat it:
+
+    >>> from chartbook import assets
+    >>> fig.write_html(assets.path("chart/yield_curve"))  # doctest: +SKIP
+
+    and a build definition can use the same call for what it reads and what
+    it writes. The path is whatever ``path`` the entry declares, made
+    absolute; nothing checks that the file exists, so a target can be named
+    before it is built.
+
+    Each kind of asset has its own names: ``[dataframes.gdp]`` and
+    ``[charts.gdp]`` are two assets, ``gdp`` and ``chart/gdp``. Asking for a
+    name its table does not declare raises immediately, naming the table to
+    add, or saying what kind of asset the name is declared as.
+
+    This is a convenience, never a requirement. A script that hardcodes the
+    same path, in any language, works just as well; ``chartbook path`` on the
+    command line gives the same answer to a shell, Make, R or Stata.
+
+    **Another pipeline's asset** is named ``"pipeline:name"``, and resolves
+    differently on purpose. The pipeline must be declared in
+    ``[dependencies]``, and the path returned is in the store: the copy that
+    ``chartbook.lock`` pins, not the other pipeline's working directory.
+
+    >>> assets.path("fed_yield_curve:treasury_rates")  # doctest: +SKIP
+
+    So it does not change while the other pipeline is being worked on, and
+    moves only when you run ``chartbook update``. It must have been pinned
+    (``chartbook pin``), and is for reading: never write to it.
+
+    :param ref: A reference to the asset: ``"name"`` for a dataframe,
+        ``"kind/name"`` for anything else, ``"pipeline:name"`` for another
+        pipeline's, or an :class:`AssetRef`.
+    :type ref: Union[str, AssetRef]
+    :param kind: The kind of asset expected: ``"dataframe"``, ``"chart"``,
+        ``"notebook"``, ``"note"``, ``"model"`` or ``"file"``. A reference that names no kind takes
+        this one. By default, a reference that names no kind is a dataframe.
+    :type kind: Optional[str]
+    :param project_dir: Where to look for the pipeline. The enclosing
+        ``chartbook.toml`` is found by walking up from here. By default the
+        search starts at the calling file's directory, then falls back to the
+        current working directory.
+    :type project_dir: Optional[Union[str, Path]]
+    :returns: The absolute path of the asset (it may contain glob
+        characters for partitioned data; :func:`files` gives the files).
+    :rtype: Path
+    :raises AssetNotDeclaredError: If the manifest declares no such asset.
+    :raises DependencyNotDeclaredError: If the reference names a pipeline
+        that ``[dependencies]`` does not declare.
+    :raises AssetNotPinnedError: If another pipeline's asset has no pinned
+        copy in the store.
+    :raises ManifestNotFoundError: If no ``chartbook.toml`` encloses the
+        starting directory.
+    :raises KindMismatchError: If the reference names a kind other than *kind*.
+    :raises ValueError: If the reference does not follow the grammar, uses
+        syntax that is reserved and not yet supported, or the entry has no
+        ``path``.
+    """
+    return _path(ref, kind, caller_start_dirs(project_dir, sys._getframe(1)))
+
+
+def _located(ref: Union[str, AssetRef], kind: Optional[str], start_dirs: list) -> tuple[AssetRef, FileLocation]:
+    """A reference and where its files are; a table or a query has none."""
+    asset = parse(ref, kind=kind)
+    location = resolve(asset, start_dirs=start_dirs)
+    if not isinstance(location, FileLocation):
+        raise TypeError(
+            f"The {asset.kind} {asset.name!r} is {location.kind}-backed, so it has no files. "
+            f"Use data.load() to read it, data.sql() to query it, or data.ref() for its name in SQL."
+        )
+    return asset, location
+
+
+def _not_there(asset: AssetRef, location: FileLocation) -> FileNotFoundError:
+    return FileNotFoundError(
+        f"The {asset.kind} {str(asset)!r} has no files: no file is at {location.path}. "
+        f"If it is this pipeline's own, its build makes them."
+    )
+
+
+def _found(asset: AssetRef, location: FileLocation) -> list[tuple[str, Path]]:
+    """Each file of an asset, by its path in the asset, sorted as a hash lists them."""
+    from chartbook import hashing
+    from chartbook.store import on_disk, plain_parts
+
+    if location.listed:
+        # A folder from the store: the files its record lists, and nothing else the folder may hold
+        listed = [(relative, plain_parts(relative)) for relative in location.listed]
+        return sorted((relative, on_disk(Path(location.path), parts)) for relative, parts in listed if parts)
+    found = hashing.find_files(location.path)
+    if found is None or not found.files:
+        raise _not_there(asset, location)
+    if found.single:
+        return [(found.files[0].name, Path(found.files[0]))]
+    return sorted((Path(os.path.relpath(file, found.root)).as_posix(), Path(file)) for file in found.files)
+
+
+def files(
+    ref: Union[str, AssetRef],
+    kind: Optional[str] = None,
+    project_dir: Optional[Union[str, Path]] = None,
+) -> list[Path]:
+    """Get the files of an asset: the file, a folder's files, or a glob's matches, sorted.
+
+    >>> from chartbook import assets
+    >>> for path in assets.files("filings:file/earnings_releases"):  # doctest: +SKIP
+    ...     records = read_one(path)
+
+    The asset is found as :func:`path` finds it, and its files are the
+    ones its content hash is made of (:mod:`chartbook.hashing`): every file
+    under a folder or matched by a glob, but ``.DS_Store`` and the few
+    others operating systems write by themselves. Of another pipeline's
+    asset they are the files of the pinned copy, as its record lists them.
+    They are sorted by their path inside the asset, so a dataset of a file
+    a year is read in order of the years.
+
+    It works for every asset with files: a files asset, a dataframe of
+    many parts (to read one at a time), a model that is a folder.
+
+    :param ref: A reference to the asset, as for :func:`path`.
+    :param kind: The kind of asset expected, as for :func:`path`.
+    :param project_dir: Where to look for the pipeline, as for :func:`path`.
+    :returns: The absolute path of each file.
+    :raises FileNotFoundError: If nothing is there: the asset is not built.
+    :raises TypeError: If the asset is a table or a query in a source.
+    :raises AssetNotDeclaredError: As for :func:`path`, and the rest of its errors.
+    """
+    asset, location = _located(ref, kind, caller_start_dirs(project_dir, sys._getframe(1)))
+    return [file for _, file in _found(asset, location)]
+
+
+def _hash_cache(declared: Path):
+    """The hash cache for one asset's files, kept in this machine's cache: an asset hashed again reads only what changed."""
+    from chartbook import hashing
+    from chartbook.config import get_cache_path
+
+    name = hashlib.sha256(str(declared).encode("utf-8", errors="surrogateescape")).hexdigest()[:32]
+    return hashing.HashCache(get_cache_path() / "hashes" / f"{name}.json")
+
+
+def _rows(asset: AssetRef, location: FileLocation) -> list[tuple[Path, str, int, str]]:
+    """Each file of an asset with its path in the asset, its size and its hash.
+
+    From the store's record when the store holds what is read: nothing is
+    read but the record. Otherwise the files are hashed, each read only if
+    it has changed since it was last hashed.
+    """
+    from chartbook import hashing
+    from chartbook.files import listing_root
+    from chartbook.store import default_store, on_disk, plain_parts
+    from chartbook.utils import is_glob_pattern
+
+    declared = Path(location.path)
+    single = not is_glob_pattern(str(declared)) and declared.is_file()
+    listed = default_store().listing(location.content_hash, single) if location.content_hash else None
+    if listed is not None:
+        if single:
+            # One file, read by the name its asset gives it: the same bytes may be two assets'
+            return [(declared, declared.name, listed[0].size_bytes, listed[0].content_hash)]
+        root = listing_root(declared)
+        return [
+            (on_disk(root, plain_parts(entry.path)), entry.path, entry.size_bytes, entry.content_hash)
+            for entry in listed
+            if plain_parts(entry.path)
+        ]
+    cache = _hash_cache(declared)
+    try:
+        content = hashing.hash_path(declared, cache)
+    except OSError as error:
+        raise FileNotFoundError(f"The {asset.kind} {str(asset)!r} could not be read: {error}") from None
+    if content is None:
+        raise _not_there(asset, location)
+    try:
+        cache.save()
+    except OSError:
+        pass  # a cache that cannot be written is only slower
+    if not content.files and not single:
+        raise _not_there(asset, location)  # a folder with no file in it
+    if not content.files:
+        return [(declared, declared.name, content.size_bytes, content.content_hash)]
+    root = listing_root(declared)
+    return [(root.joinpath(*entry.path.split("/")), entry.path, entry.size_bytes, entry.content_hash) for entry in content.files]
+
+
+def index(
+    ref: Union[str, AssetRef],
+    kind: Optional[str] = None,
+    project_dir: Optional[Union[str, Path]] = None,
+):
+    """Get a table of an asset's files, one row per file, to choose from before opening any.
+
+    >>> import polars as pl
+    >>> from chartbook import assets
+    >>> chosen = (
+    ...     assets.index("filings:file/earnings_releases")
+    ...     .filter(pl.col("year") >= "2020")
+    ...     .collect()
+    ... )  # doctest: +SKIP
+
+    The columns are ``path`` (the file, as :func:`files` gives it),
+    ``size_bytes``, ``sha256`` (the file's hash, in hex), and then the key
+    of each ``key=value`` folder the files are in, as text:
+    ``filings/year=2024/q=1/x.htm`` has ``year`` ``"2024"`` and ``q``
+    ``"1"``. The rows are in the order :func:`files` gives.
+
+    Of another pipeline's asset, every file's size and hash come from the
+    record the store keeps of it, and nothing is read. Of this pipeline's,
+    the files are hashed, and a file is read again only when it has changed
+    (a hash cache in this machine's cache, ``~/.chartbook/cache/hashes``).
+
+    :param ref: A reference to the asset, as for :func:`path`.
+    :param kind: The kind of asset expected, as for :func:`path`.
+    :param project_dir: Where to look for the pipeline, as for :func:`path`.
+    :returns: A ``polars.LazyFrame``.
+    :raises FileNotFoundError: If nothing is there: the asset is not built.
+    :raises TypeError: If the asset is a table or a query in a source.
+    :raises AssetNotDeclaredError: As for :func:`path`, and the rest of its errors.
+    """
+    import polars as pl
+
+    from chartbook.files import partitions
+
+    asset, location = _located(ref, kind, caller_start_dirs(project_dir, sys._getframe(1)))
+    rows = sorted(_rows(asset, location), key=lambda row: row[1])
+    keys: dict = {}
+    for _, relative, _, _ in rows:
+        for key in partitions(relative):
+            if key not in INDEX_COLUMNS:
+                keys.setdefault(key, None)
+    columns: dict = {
+        "path": [str(file) for file, _, _, _ in rows],
+        "size_bytes": [size for _, _, size, _ in rows],
+        "sha256": [content_hash.split(":", 1)[-1] for _, _, _, content_hash in rows],
+    }
+    for key in keys:
+        columns[key] = [partitions(relative).get(key) for _, relative, _, _ in rows]
+    schema = {"path": pl.String, "size_bytes": pl.Int64, "sha256": pl.String, **{key: pl.String for key in keys}}
+    return pl.DataFrame(columns, schema=schema).lazy()

{chartbook-0.2.0 → chartbook-0.3.0}/src/chartbook/backends/__init__.py

@@ -18,9 +18,11 @@ and which dialect lives there, and it substitutes identifiers. The SQL is
 the user's (:func:`chartbook.data.sql`).
 
 Three backends are built. ``parquet`` is what a file-backed dataframe has
-always been: it reads the file, and in SQL it is ``read_parquet('…')``.
+always been: it reads the file, parquet, CSV, JSON lines or Arrow IPC
+(:mod:`chartbook.formats`), and in SQL it is DuckDB's reader of its format,
+``read_parquet('…')``, ``read_csv('…', …)`` or ``read_json('…', …)``.
 ``duckdb`` is the local engine: a file, or ``:memory:``, that holds tables
-and runs any SQL that mentions them, and reads parquet files too, so a
+and runs any SQL that mentions them, and reads those files too, so a
 query can join a table of one source with a file of this pipeline.
 ``databricks`` is a SQL warehouse (:mod:`chartbook.backends.databricks`),
 whose driver ``pip install chartbook[databricks]`` installs. A source's
@@ -391,7 +393,7 @@ class Backend:
     :class:`BackendError` itself, by Polars version.
 
     ``reads_files`` says whether SQL run in it may name a file of this
-    machine as :class:`ParquetBackend` writes one, ``read_parquet('…')``:
+    machine as :class:`ParquetBackend` writes one, ``read_parquet('…')`` say:
     so whether :func:`chartbook.data.sql` may join this engine's tables
     with a pipeline's files in one query.
     """
@@ -428,11 +430,16 @@ class Backend:
         """A table as a quoted identifier; a query as a sub-select: what stands for it in SQL here."""
         raise NotImplementedError
 
-    def sql(self, query: str):
+    def sql(self, query: str, arrow: Optional[dict] = None):
         """Run SQL in this engine and return a Polars LazyFrame.
 
         The engine binds the query here, so a name it does not know is
-        refused at once; collecting the frame runs it.
+        refused at once; collecting the frame runs it. *arrow* is Arrow
+        data the query names, by the name it goes by, for an engine that
+        reads files (``reads_files``) and can scan Arrow data: an Arrow IPC
+        file, which DuckDB has no reader of its own for. A backend whose
+        ``sql()`` takes no such argument is handed none, and a query naming
+        such a file is refused before it reaches it.
         """
         raise NotImplementedError
 
@@ -450,10 +457,12 @@ class Backend:
 
 
 class ParquetBackend:
-    """What a file-backed dataframe has always been.
+    """What a file-backed dataframe has always been, in any format it is stored in.
 
-    In SQL a file is ``read_parquet('…')``, which DuckDB reads as a table,
-    partitions and all.
+    It reads through :mod:`chartbook.formats`, as everything that reads a
+    file does. In SQL a file is DuckDB's reader of its format:
+    ``read_parquet('…')``, which DuckDB reads as a table, partitions and
+    all; ``read_csv('…', …)``; or ``read_json('…', …)``.
     """
 
     name = "parquet"
@@ -461,34 +470,15 @@ class ParquetBackend:
     @staticmethod
     def ref(location: "FileLocation") -> str:
         """The file as a DuckDB table expression."""
-        from chartbook.utils import is_glob_pattern
-
-        from chartbook.resolve import nothing_to_read
-
-        said = nothing_to_read(location)
-        if said:
-            raise ValueError(said)
-        # With / on every system: DuckDB reads it so on Windows too, and SQL
-        # with a path in it reads the same wherever it is pasted
-        text = Path(location.path).as_posix()
-        if location.files:
-            # A directory pinned from another pipeline: the files its record lists
-            listed = ", ".join(quote_literal(Path(file).as_posix()) for file in location.files)
-            return f"read_parquet([{listed}], hive_partitioning = true)"
-        if is_glob_pattern(text):
-            return f"read_parquet({quote_literal(text)}, hive_partitioning = true)"
-        return f"read_parquet({quote_literal(text)})"
+        from chartbook import formats
+
+        return formats.sql(location)
 
     @staticmethod
     def scan(location: "FileLocation"):
-        import polars as pl
-
-        from chartbook.resolve import nothing_to_read
+        from chartbook import formats
 
-        said = nothing_to_read(location)
-        if said:
-            raise ValueError(said)
-        return pl.scan_parquet([str(file) for file in location.files] or location.path, hive_partitioning=True)
+        return formats.scan_location(location)
 
 
 class DuckDBBackend(Backend):
@@ -564,7 +554,7 @@ class DuckDBBackend(Backend):
             return subselect(location.sql)
         return ParquetBackend.ref(location)
 
-    def sql(self, query: str):
+    def sql(self, query: str, arrow: Optional[dict] = None):
         """Run SQL in this engine and return a Polars LazyFrame.
 
         The engine binds the query here, so a name it does not know or SQL
@@ -573,13 +563,20 @@ class DuckDBBackend(Backend):
         a value, is raised then with DuckDB's message: as Polars'
         ``ComputeError`` or as DuckDB's own exception, by Polars version.
 
+        :param arrow: Arrow data the query names, by the name it goes by:
+            an Arrow IPC file, which DuckDB has no reader of its own for
+            (:func:`chartbook.formats.arrow_dataset`). It is registered with
+            the connection the query runs in, for as long as that lives.
         :raises chartbook.backends.BackendError: If DuckDB refuses the query. The message is
             DuckDB's own; the SQL is the user's.
         """
         import duckdb
 
         try:
-            return self.connect().sql(query).pl(lazy=True)
+            connection = self.connect()
+            for name, table in (arrow or {}).items():
+                connection.register(name, table)
+            return connection.sql(query).pl(lazy=True)
         except duckdb.Error as error:
             raise BackendError(str(error)) from None
 
@@ -589,8 +586,9 @@ class DuckDBBackend(Backend):
 
         try:
             connection = self.connect()
+            # Fetched whole: a relation has no fetchone from DuckDB 2.0, and a result read only in part keeps the connection busy
             described = connection.sql(f"describe select * from {self.ref(location)}").fetchall()
-            (rows,) = connection.sql(f"select count(*) from {self.ref(location)}").fetchone()
+            ((rows,),) = connection.sql(f"select count(*) from {self.ref(location)}").fetchall()
         except duckdb.Error as error:
             raise BackendError(str(error)) from None
         return [{"name": row[0], "dtype": row[1]} for row in described], int(rows)
@@ -608,7 +606,7 @@ class DuckDBBackend(Backend):
             return self.connect().sql(
                 f"select min(try_cast({name} as timestamp)), max(try_cast({name} as timestamp)) "
                 f"from {self.ref(location)}"
-            ).fetchone()
+            ).fetchall()[0]
         except duckdb.Error as error:
             raise BackendError(str(error)) from None
 

{chartbook-0.2.0 → chartbook-0.3.0}/src/chartbook/build_docs.py

@@ -362,7 +362,7 @@ def _refuse_unreadable(built: dict) -> None:
     unreadable = [
         record
         for record in built.get("records", [])
-        if record.get("kind") in ("Dataframe", "Chart", "Notebook", "Model")
+        if record.get("kind") in ("Dataframe", "Chart", "Notebook", "Model", "File")
         and (record.get("location") or {}).get("kind") == "file"
         and (record.get("location") or {}).get("resolved_path")
         and "content_hash" not in record

chartbook-0.3.0/src/chartbook/built_from.txt

@@ -0,0 +1 @@
+8fc26a3d96b1b6f62651edd08ac1d0f88b3330e2