rb-dataio 0.1.34__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rb_dataio/__init__.py +16 -0
- rb_dataio/_distribution.py +24 -0
- rb_dataio/_pyproject.toml +157 -0
- rb_dataio/cli.py +76 -0
- rb_dataio/commands/__init__.py +1 -0
- rb_dataio/commands/_parameters.py +79 -0
- rb_dataio/commands/auth.py +66 -0
- rb_dataio/commands/backups.py +30 -0
- rb_dataio/commands/config.py +87 -0
- rb_dataio/commands/convert.py +165 -0
- rb_dataio/commands/describe.py +34 -0
- rb_dataio/commands/doctor.py +43 -0
- rb_dataio/commands/formats.py +33 -0
- rb_dataio/commands/info.py +79 -0
- rb_dataio/commands/init.py +48 -0
- rb_dataio/commands/join.py +126 -0
- rb_dataio/commands/login.py +83 -0
- rb_dataio/commands/project.py +55 -0
- rb_dataio/commands/restore.py +31 -0
- rb_dataio/commands/skill.py +138 -0
- rb_dataio/commands/status.py +35 -0
- rb_dataio/core/__init__.py +1 -0
- rb_dataio/core/data_files.py +28 -0
- rb_dataio/core/dataio/__init__.py +4 -0
- rb_dataio/core/dataio/address.py +71 -0
- rb_dataio/core/dataio/api.py +256 -0
- rb_dataio/core/dataio/backup.py +161 -0
- rb_dataio/core/dataio/batch.py +98 -0
- rb_dataio/core/dataio/columns.py +57 -0
- rb_dataio/core/dataio/containers.py +64 -0
- rb_dataio/core/dataio/joins.py +180 -0
- rb_dataio/core/dataio/objects.py +67 -0
- rb_dataio/core/dataio/options.py +163 -0
- rb_dataio/core/dataio/pipeline.py +434 -0
- rb_dataio/core/dataio/protection.py +79 -0
- rb_dataio/core/dataio/query.py +72 -0
- rb_dataio/core/dataio/registry.py +75 -0
- rb_dataio/core/dataio/report.py +139 -0
- rb_dataio/core/dataio/settings.py +64 -0
- rb_dataio/core/dataio/sources.py +128 -0
- rb_dataio/core/dataio/stdio.py +67 -0
- rb_dataio/core/dataio/targets.py +145 -0
- rb_dataio/core/dataio/values.py +150 -0
- rb_dataio/core/deps.py +165 -0
- rb_dataio/core/errors.py +418 -0
- rb_dataio/core/models.py +25 -0
- rb_dataio/core/normalize.py +52 -0
- rb_dataio/core/opencli.py +285 -0
- rb_dataio/core/services.py +82 -0
- rb_dataio/core/skills.py +64 -0
- rb_dataio/core/version.py +90 -0
- rb_dataio/credentials/__init__.py +1 -0
- rb_dataio/credentials/health.py +215 -0
- rb_dataio/credentials/manager.py +304 -0
- rb_dataio/data/SKILL.md +106 -0
- rb_dataio/data/__init__.py +1 -0
- rb_dataio/data/settings.toml +26 -0
- rb_dataio/formats/__init__.py +269 -0
- rb_dataio/formats/_excel.py +48 -0
- rb_dataio/formats/csv.py +131 -0
- rb_dataio/formats/duckdb.py +148 -0
- rb_dataio/formats/gsheet.py +195 -0
- rb_dataio/formats/http.py +79 -0
- rb_dataio/formats/json.py +82 -0
- rb_dataio/formats/jsonl.py +34 -0
- rb_dataio/formats/ods.py +15 -0
- rb_dataio/formats/parquet.py +47 -0
- rb_dataio/formats/sqlite.py +151 -0
- rb_dataio/formats/tsv.py +18 -0
- rb_dataio/formats/txt.py +47 -0
- rb_dataio/formats/xls.py +15 -0
- rb_dataio/formats/xlsb.py +15 -0
- rb_dataio/formats/xlsx.py +129 -0
- rb_dataio/output/__init__.py +285 -0
- rb_dataio/output/human.py +34 -0
- rb_dataio/output/json.py +11 -0
- rb_dataio/providers/__init__.py +29 -0
- rb_dataio/providers/base.py +144 -0
- rb_dataio/providers/google.py +144 -0
- rb_dataio/providers/http.py +136 -0
- rb_dataio-0.1.34.dist-info/METADATA +156 -0
- rb_dataio-0.1.34.dist-info/RECORD +85 -0
- rb_dataio-0.1.34.dist-info/WHEEL +4 -0
- rb_dataio-0.1.34.dist-info/entry_points.txt +2 -0
- rb_dataio-0.1.34.dist-info/licenses/LICENSE +21 -0
rb_dataio/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""dataio: read any source into a pandas DataFrame and write a DataFrame to any target.
|
|
2
|
+
|
|
3
|
+
df = read("phrases.csv")
|
|
4
|
+
report = write(df, "out.csv")
|
|
5
|
+
report = convert("a.csv", "b.csv")
|
|
6
|
+
report = join("main.csv", "out.csv", lookups=["prices.xlsx#Prices"], on=["sku"])
|
|
7
|
+
|
|
8
|
+
Every failure is a `DataioError` with `code`, `message` and `action`.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from .core.dataio.api import backups, convert, formats, info, join, read, restore, write
|
|
14
|
+
from .core.errors import DataioError
|
|
15
|
+
|
|
16
|
+
__all__ = ["DataioError", "backups", "convert", "formats", "info", "join", "read", "restore", "write"]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""The distribution name and the tool namespace copier writes from its answers, because an editable install cannot map
|
|
2
|
+
this import package to its distribution at runtime.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
# Rendered by copier from the `dist_name` answer: the [project] name in
|
|
6
|
+
# pyproject.toml (`rb-<slug>`, the Standard, Technology), which may differ from this
|
|
7
|
+
# import package's name. importlib.metadata finds this package's version,
|
|
8
|
+
# summary and console script by it; an editable install (`uv sync`) can't be
|
|
9
|
+
# mapped from package to distribution at runtime, so it is written down here.
|
|
10
|
+
# tests/test_cli.py fails if it drifts from pyproject.toml.
|
|
11
|
+
DIST_NAME = "rb-dataio"
|
|
12
|
+
# The previous distribution name during a rename: an install made
|
|
13
|
+
# under it (not yet reinstalled) still finds its own version. Drop it once
|
|
14
|
+
# every machine has the new name.
|
|
15
|
+
DIST_NAME_LEGACY: str | None = None
|
|
16
|
+
# The command name (`cli_name`) as an identifier: the namespace of this tool's
|
|
17
|
+
# shared-database names (core/db.py) — table prefix `<TOOL>_`, env var
|
|
18
|
+
# `<TOOL>_DATABASE_URL`, Alembic table `alembic_version_<TOOL>`. Written down,
|
|
19
|
+
# not derived at runtime: a bare source run falls back to the package name,
|
|
20
|
+
# and a table name must never depend on how the tool was started.
|
|
21
|
+
TOOL = "dataio"
|
|
22
|
+
# The cli-generator release this project was generated or last updated from
|
|
23
|
+
# (copier's `_commit`), for `describe --json`; None for a project not made by copier.
|
|
24
|
+
TEMPLATE_VERSION: str | None = "v1.27.3"
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "rb-dataio"
|
|
3
|
+
# Version comes from the git tag vX.Y.Z (uv-dynamic-versioning) — never add `version =`.
|
|
4
|
+
dynamic = ["version"]
|
|
5
|
+
description = "Read any table into a pandas DataFrame and write it anywhere: csv, tsv, xlsx, json, jsonl, parquet, Google Sheets, stdin."
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
authors = [{ name = "net.paradox" }]
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"typer>=0.27,<0.29",
|
|
13
|
+
"pydantic>=2.6",
|
|
14
|
+
"rich>=13.7",
|
|
15
|
+
"platformdirs>=4.2",
|
|
16
|
+
"tomli-w>=1.0",
|
|
17
|
+
"httpx>=0.27",
|
|
18
|
+
"packaging>=24.0",
|
|
19
|
+
"keyring>=25.0",
|
|
20
|
+
"pandas>=3.0",
|
|
21
|
+
"duckdb>=1.1",
|
|
22
|
+
"python-calamine>=0.9.0",
|
|
23
|
+
"openpyxl>=3.1.5",
|
|
24
|
+
"google-auth[requests]>=2.20",
|
|
25
|
+
"gspread>=6.1",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.scripts]
|
|
29
|
+
dataio = "rb_dataio.cli:main"
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Repository = "https://github.com/bolandeu/rb-dataio"
|
|
33
|
+
|
|
34
|
+
[build-system]
|
|
35
|
+
requires = ["hatchling", "uv-dynamic-versioning"]
|
|
36
|
+
build-backend = "hatchling.build"
|
|
37
|
+
|
|
38
|
+
[tool.hatch.version]
|
|
39
|
+
source = "uv-dynamic-versioning"
|
|
40
|
+
|
|
41
|
+
[tool.uv-dynamic-versioning]
|
|
42
|
+
vcs = "git"
|
|
43
|
+
style = "pep440"
|
|
44
|
+
fallback-version = "0.0.0"
|
|
45
|
+
|
|
46
|
+
[tool.uv]
|
|
47
|
+
# Rebuild the editable install when a commit or tag moves, so `--version` follows the tag.
|
|
48
|
+
cache-keys = [{ file = "pyproject.toml" }, { git = { commit = true, tags = true } }]
|
|
49
|
+
|
|
50
|
+
# Our packages come only from the Gitea registry: a version range in
|
|
51
|
+
# `dependencies` ("rb-serp>=0.3") and a source line per package, never a git
|
|
52
|
+
# tag. explicit = true: only the packages named in [tool.uv.sources] are looked
|
|
53
|
+
# up there, and they are never taken from PyPI (an unrelated package of the
|
|
54
|
+
# same name may live there).
|
|
55
|
+
# [tool.uv.sources]
|
|
56
|
+
# rb-serp = { index = "gitea" }
|
|
57
|
+
[tool.hatch.build.targets.wheel]
|
|
58
|
+
packages = ["src/rb_dataio"]
|
|
59
|
+
|
|
60
|
+
[tool.hatch.build.targets.wheel.force-include]
|
|
61
|
+
# The installed CLI reads [[tool.rb.external]] from here (core/deps.py).
|
|
62
|
+
"pyproject.toml" = "rb_dataio/_pyproject.toml"
|
|
63
|
+
|
|
64
|
+
[dependency-groups]
|
|
65
|
+
# Mutation testing, run now and then and before big releases — not in the gate:
|
|
66
|
+
# `uv run --group mutation mutmut run`, then `mutmut results` / `mutmut browse`.
|
|
67
|
+
# Needs fork(): macOS/Linux (WSL on Windows). Settings: [tool.mutmut] below.
|
|
68
|
+
mutation = ["mutmut>=3.3"]
|
|
69
|
+
dev = [
|
|
70
|
+
"jsonschema>=4.23",
|
|
71
|
+
"pytest>=8.0",
|
|
72
|
+
"pyyaml>=6.0", # .copier-answers.yml, the CI workflow, tests/contracts/*.yaml
|
|
73
|
+
"pytest-cov>=5.0",
|
|
74
|
+
"ruff>=0.6",
|
|
75
|
+
"syrupy>=4.8",
|
|
76
|
+
"ty>=0.0.80",
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
# External (non-Python) dependencies — every CLI this tool runs (require_cli,
|
|
80
|
+
# shutil.which, subprocess with a literal program, npx). The gate checks this
|
|
81
|
+
# list against the code both ways; doctor and require_cli read it at runtime;
|
|
82
|
+
# the tools catalog reads it (the Standard, Technology).
|
|
83
|
+
# [[tool.rb.external]]
|
|
84
|
+
# name = "ffmpeg"
|
|
85
|
+
# required = true # false: the tool works without it (doctor stays healthy)
|
|
86
|
+
# min_version = "6.0" # optional
|
|
87
|
+
# check = "ffmpeg -version" # prints the version; default "<name> --version"
|
|
88
|
+
# why = "transcodes the downloaded videos"
|
|
89
|
+
# detect = false # run via an injected runner / absolute path the gate can't see
|
|
90
|
+
# dist = "rb-serp" # for one of our CLIs: its distribution (default rb-<name>)
|
|
91
|
+
# url = "https://github.com/bolandeu/serp" # its repo: DEPENDENCY_TOO_OLD then prints a pinned install
|
|
92
|
+
|
|
93
|
+
# Where sysadmin's tool-updater installs this tool after its first tag. The
|
|
94
|
+
# installation is sysadmin's; this section only says where. homeserver: users
|
|
95
|
+
# to install for (empty = build and publish only); mac: install on the Mac.
|
|
96
|
+
# extras (optional): the extras to install with, e.g. extras = ["browser"].
|
|
97
|
+
[tool.rb.install]
|
|
98
|
+
homeserver = ["rbtools"]
|
|
99
|
+
mac = true
|
|
100
|
+
|
|
101
|
+
[tool.cli-generator]
|
|
102
|
+
# Where scripts/check_command_docstrings.py (called by the project's checks)
|
|
103
|
+
# looks for commands: a glob or a list of globs, relative to the repo root.
|
|
104
|
+
commands = ["src/rb_dataio/commands/*.py"]
|
|
105
|
+
# gate check (the gate): extras to install before ruff/ty/pytest.
|
|
106
|
+
# gate_extras = ["browser"]
|
|
107
|
+
|
|
108
|
+
[tool.pytest.ini_options]
|
|
109
|
+
# Only tests/: mutmut leaves a copy of the tree in mutants/.
|
|
110
|
+
testpaths = ["tests"]
|
|
111
|
+
addopts = "--cov --cov-report=term-missing:skip-covered"
|
|
112
|
+
# tests/test_help.py keeps help snapshots at tests/snapshots/help/<command>.txt
|
|
113
|
+
# (the Standard, Commands), not in syrupy's own `__snapshots__/<test file>/` layout, so
|
|
114
|
+
# syrupy warns it "can not relate" each file to its test. It can — by name.
|
|
115
|
+
filterwarnings = ["ignore:(?s).*Can not relate snapshot:UserWarning"]
|
|
116
|
+
|
|
117
|
+
[tool.mutmut]
|
|
118
|
+
source_paths = ["src/rb_dataio"]
|
|
119
|
+
also_copy = ["scripts/", "pyproject.toml"] # tests read them (test_scripts.py, the distribution checks)
|
|
120
|
+
pytest_add_cli_args = ["--no-cov"] # fail_under would kill every mutant
|
|
121
|
+
# only_mutate = ["*/core/normalize.py"] # one module at a time on a big tool
|
|
122
|
+
|
|
123
|
+
[tool.coverage.run]
|
|
124
|
+
source = ["src"]
|
|
125
|
+
# sysmon (default on 3.14) keeps a record per recompiled code object; Typer recompiles annotations on every invoke, so memory grows.
|
|
126
|
+
core = "ctrace"
|
|
127
|
+
|
|
128
|
+
[tool.coverage.report]
|
|
129
|
+
# The template ships at ~94%; keep the bar where it is instead of letting
|
|
130
|
+
# new commands land untested. Raise it, never lower it.
|
|
131
|
+
fail_under = 90
|
|
132
|
+
show_missing = true
|
|
133
|
+
|
|
134
|
+
[tool.ty.terminal]
|
|
135
|
+
# Only errors fail the gate; recent ty (>= 0.0.84) otherwise exits 1 on warnings too.
|
|
136
|
+
# Legacy code that can't be fixed yet: downgrade specific rules instead of
|
|
137
|
+
# ignoring them, e.g. [tool.ty.rules] possibly-unbound-attribute = "warn".
|
|
138
|
+
error-on-warning = false
|
|
139
|
+
|
|
140
|
+
[tool.ruff]
|
|
141
|
+
line-length = 120
|
|
142
|
+
|
|
143
|
+
[tool.ruff.lint]
|
|
144
|
+
select = ["E4", "E7", "E9", "F", "I", "UP", "B", "SIM", "RUF", "S", "T20", "TID251"]
|
|
145
|
+
|
|
146
|
+
[tool.ruff.lint.per-file-ignores]
|
|
147
|
+
"tests/*" = ["S101", "T201", "TID251", "RUF001"] # assert is the point of a test; print in code strings; httpx for mocks; Cyrillic test data
|
|
148
|
+
"scripts/*" = ["S603", "S607", "T201"] # subprocess with fixed argv lists (git, uv), no shell; scripts print to the console
|
|
149
|
+
"src/*/providers/*" = ["TID251"] # external APIs live here
|
|
150
|
+
|
|
151
|
+
[tool.ruff.lint.flake8-tidy-imports.banned-api]
|
|
152
|
+
"httpx".msg = "Call external APIs through providers/ (subclass providers.HttpProvider)."
|
|
153
|
+
|
|
154
|
+
[tool.ruff.lint.flake8-bugbear]
|
|
155
|
+
# Typer's `param: T = typer.Option(...)` style is a call in a default on purpose.
|
|
156
|
+
# commands/_parameters.option is typer.Option with the vocabulary's help text.
|
|
157
|
+
extend-immutable-calls = ["typer.Option", "typer.Argument", "rb_dataio.commands._parameters.option"]
|
rb_dataio/cli.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""The Typer app: the global options and the registration of every command.
|
|
2
|
+
|
|
3
|
+
A bare `<cli>` prints the app help, the entry point for humans and agents.
|
|
4
|
+
Each command lives in `commands/`; this module only names it.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
|
|
11
|
+
from . import output
|
|
12
|
+
from .commands import auth, backups, convert, doctor, formats, info, join, restore
|
|
13
|
+
from .core.version import DEV_VERSION, cli_name, installed_version, optional_module, package_summary
|
|
14
|
+
|
|
15
|
+
_PACKAGE_NAME = __name__.split(".")[0]
|
|
16
|
+
# Optional parts (copier questions, dropped by _exclude): each one ships a module that registers its
|
|
17
|
+
# commands and global options. Probed at runtime, so this file needs no template logic.
|
|
18
|
+
PARTS = ("project", "skill", "describe", "db")
|
|
19
|
+
|
|
20
|
+
# A bare `<cli>` is the discovery entry point for humans and agents alike:
|
|
21
|
+
# name + version + description, then every command with the first line of
|
|
22
|
+
# its docstring. No `no_args_is_help` (that exits 2) — main_options() below
|
|
23
|
+
# prints this and exits 0. Shell completion stays enabled.
|
|
24
|
+
app = typer.Typer(
|
|
25
|
+
cls=output.RootGroup, # a usage error under --json is the error envelope
|
|
26
|
+
help=f"{cli_name()} {installed_version() or DEV_VERSION} — {package_summary() or 'project-aware automation CLI'}",
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
app.add_typer(auth.app, name="auth") # `auth status` in every CLI; login/logout only with credentials
|
|
30
|
+
app.command(name="doctor")(doctor.doctor)
|
|
31
|
+
app.command(name="convert")(convert.convert)
|
|
32
|
+
app.command(name="info")(info.info)
|
|
33
|
+
app.command(name="join")(join.join)
|
|
34
|
+
app.command(name="formats")(formats.formats)
|
|
35
|
+
app.command(name="backups")(backups.backups)
|
|
36
|
+
app.command(name="restore")(restore.restore)
|
|
37
|
+
for part in PARTS:
|
|
38
|
+
if module := optional_module(f"{_PACKAGE_NAME}.commands.{part}"):
|
|
39
|
+
module.register(app)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _version_callback(value: bool) -> None:
|
|
43
|
+
if not value:
|
|
44
|
+
return
|
|
45
|
+
typer.echo(installed_version() or DEV_VERSION)
|
|
46
|
+
raise typer.Exit()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@app.callback(invoke_without_command=True)
|
|
50
|
+
def main_options(
|
|
51
|
+
ctx: typer.Context,
|
|
52
|
+
json_output: bool = typer.Option(False, "--json", help="Emit machine-readable JSON output."),
|
|
53
|
+
quiet: bool = typer.Option(False, "--quiet", help="Suppress non-essential output."),
|
|
54
|
+
verbose: bool = typer.Option(False, "--verbose", help="Emit extra diagnostic output."),
|
|
55
|
+
non_interactive: bool = typer.Option(False, "--non-interactive", help="Never prompt; fail instead."),
|
|
56
|
+
dry_run: bool = typer.Option(False, "--dry-run", help="Show what would happen without making changes."),
|
|
57
|
+
version: bool = typer.Option(
|
|
58
|
+
False, "--version", callback=_version_callback, is_eager=True, help="Show the version and exit."
|
|
59
|
+
),
|
|
60
|
+
) -> None:
|
|
61
|
+
ctx.ensure_object(dict) # a part's global option (--project) may have put its value here already
|
|
62
|
+
ctx.obj["json"] = json_output
|
|
63
|
+
ctx.obj["quiet"] = quiet
|
|
64
|
+
ctx.obj["verbose"] = verbose
|
|
65
|
+
ctx.obj["non_interactive"] = non_interactive
|
|
66
|
+
ctx.obj["dry_run"] = dry_run
|
|
67
|
+
output.configure_logging(verbose)
|
|
68
|
+
output.help_if_no_command(ctx)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def main() -> None:
|
|
72
|
+
app()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
if __name__ == "__main__":
|
|
76
|
+
main()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The commands of the CLI, one module per command or command group."""
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""The CLI flags of the parameter vocabulary (core/dataio/options.py): one help text per word.
|
|
2
|
+
|
|
3
|
+
Typer names a flag after its Python parameter (`in_encoding` -> `--in-encoding`,
|
|
4
|
+
a boolean -> `--x/--no-x`), so a command parameter named like the library keyword
|
|
5
|
+
is the flag tests/test_parity.py expects. `side` is `in` or `out` in `convert` and `join`.
|
|
6
|
+
`pairs`, `names` and `renames` turn the repeatable flag values (`COLUMN=TYPE`, comma lists,
|
|
7
|
+
`old=new`) into the dict or list the library takes.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
|
|
16
|
+
from ..core.errors import InvalidArguments
|
|
17
|
+
|
|
18
|
+
_HELP = {
|
|
19
|
+
"format": "Format name; wins over the extension (see the formats command).",
|
|
20
|
+
"sheet": "Sheet name; wins over the part of the address after #.",
|
|
21
|
+
"table": "Table name; wins over the part of the address after #.",
|
|
22
|
+
"encoding": "Text encoding, e.g. utf-8 or cp1251.",
|
|
23
|
+
"sep": "Field delimiter, e.g. ';' or '|'.",
|
|
24
|
+
"header": "The first row holds the column names.",
|
|
25
|
+
"as_text": "Read every value as text.",
|
|
26
|
+
"dtypes": "Type of one column as COLUMN=TYPE, e.g. freq=int; repeatable.",
|
|
27
|
+
"limit": "Read at most this many rows.",
|
|
28
|
+
"query": "SQL whose result is read: inside a base its tables, over any other source the table data.",
|
|
29
|
+
"source_column": "Column that names the source of each row when an address names many sources.",
|
|
30
|
+
"strict": "Sources with different columns fail with SCHEMA_MISMATCH instead of merging as a union.",
|
|
31
|
+
"split_by": "Column whose values split the table into one target each, in a target with {column}.",
|
|
32
|
+
"chunk_rows": "Rows per part, in a target with {n}.",
|
|
33
|
+
"columns": "Source columns the target keeps, in this order; a comma list, repeatable.",
|
|
34
|
+
"rename": "New name of a kept column as old=new, applied after --columns; a comma list, repeatable.",
|
|
35
|
+
"drop_split_column": "Leave the --split-by column out of the targets.",
|
|
36
|
+
"keep_going": "Go on after a failed target; the report lists the failures in errors.",
|
|
37
|
+
"mode": "Write mode: what to do with the data the target already has.",
|
|
38
|
+
"profile": "Write profile, e.g. excel.",
|
|
39
|
+
"credentials": "Key file (JSON) of the service, e.g. a Google service-account key; default: the standard order.",
|
|
40
|
+
"start": "Top-left cell of the table in a sheet of a service, e.g. A5 (default A1).",
|
|
41
|
+
"value_input": "How a sheet of a service takes the values: raw as they are, user as if typed.",
|
|
42
|
+
"overwrite": "Allow a write over the source itself, or a replace of a sheet with rows that asks first.",
|
|
43
|
+
"backup": "Keep a backup of the target before it is replaced.",
|
|
44
|
+
"lookups": "Address of a lookup joined to the source, e.g. prices.xlsx#Prices; repeatable, joined in order.",
|
|
45
|
+
"on": "Key column: a name, or main_name=lookup_name; repeat for several keys.",
|
|
46
|
+
"how": "Rows kept: left every source row, right every lookup row (one lookup), inner the matched, full all.",
|
|
47
|
+
}
|
|
48
|
+
_SIDE = {None: "", "in": "Source: ", "out": "Target: "}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def option(word: str, default: Any, side: str | None = None, *names_: str, **extra: Any) -> Any:
|
|
52
|
+
"""`names_` are flag names that differ from the parameter's (`--lookup` for `lookups`)."""
|
|
53
|
+
return typer.Option(default, *names_, help=_SIDE[side] + _HELP[word], **extra)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def pairs(values: list[str] | None) -> dict[str, str] | None:
|
|
57
|
+
"""`["freq=int"]` -> `{"freq": "int"}`, the shape of a dict flag."""
|
|
58
|
+
if not values:
|
|
59
|
+
return None
|
|
60
|
+
if bad := [value for value in values if "=" not in value]:
|
|
61
|
+
raise InvalidArguments(f"Expected COLUMN=TYPE, got: {', '.join(bad)}.")
|
|
62
|
+
return dict(value.split("=", 1) for value in values)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def names(values: list[str] | None) -> list[str] | None:
|
|
66
|
+
"""`["a,b", "c"]` -> `["a", "b", "c"]`, the shape of a comma-list flag."""
|
|
67
|
+
if not values:
|
|
68
|
+
return None
|
|
69
|
+
return [name for value in values for name in value.split(",")]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def renames(values: list[str] | None) -> dict[str, str] | None:
|
|
73
|
+
"""`["a=x,b=y"]` -> `{"a": "x", "b": "y"}`; a pair splits on its first `=`."""
|
|
74
|
+
if not values:
|
|
75
|
+
return None
|
|
76
|
+
items = names(values) or []
|
|
77
|
+
if bad := [item for item in items if "=" not in item]:
|
|
78
|
+
raise InvalidArguments(f"Expected old=new, got: {', '.join(bad)}.")
|
|
79
|
+
return dict(item.split("=", 1) for item in items)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""The `auth` command group: `status` reports the health of the credentials of every provider, or an empty report when
|
|
2
|
+
the package has none.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import importlib
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
|
|
12
|
+
from .. import output
|
|
13
|
+
from ..core.errors import InvalidArguments
|
|
14
|
+
from ..core.version import DEV_VERSION, cli_name, installed_version, module_exists, optional_module
|
|
15
|
+
|
|
16
|
+
# `auth status` exists in every CLI (the Standard, Health); a CLI
|
|
17
|
+
# generated without credentials (no credentials/ package, no commands/login.py)
|
|
18
|
+
# reports an empty `credentials` list and exits 0. Detected at runtime so this
|
|
19
|
+
# file needs no template logic — same approach as doctor.py.
|
|
20
|
+
_PACKAGE_NAME = __name__.split(".")[0]
|
|
21
|
+
_HAS_CREDENTIALS = module_exists(f"{_PACKAGE_NAME}.credentials.manager")
|
|
22
|
+
|
|
23
|
+
app = output.group("Credentials: report their health (and store them, if this CLI has any).")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@app.command("status")
|
|
27
|
+
@output.guard
|
|
28
|
+
def auth_status(
|
|
29
|
+
ctx: typer.Context,
|
|
30
|
+
check: bool = typer.Option(
|
|
31
|
+
False, "--check", help="Also make one cheap read-only call per provider (10 s timeout) to prove each works."
|
|
32
|
+
),
|
|
33
|
+
provider: str | None = typer.Option(
|
|
34
|
+
None, "--provider", help="Report only this provider, or one target as <kind>:<target> (default: all)."
|
|
35
|
+
),
|
|
36
|
+
json_output: bool = output.JsonOption,
|
|
37
|
+
quiet: bool = output.QuietOption,
|
|
38
|
+
) -> None:
|
|
39
|
+
"""Report every provider's credential: where it comes from and whether it works. Never prints values.
|
|
40
|
+
|
|
41
|
+
Without --check only presence is reported (status `unchecked` or `missing`).
|
|
42
|
+
Exits 0 when every required provider is ok/unchecked (an optional one may be
|
|
43
|
+
missing), 3 when any required provider has a problem or any credential is
|
|
44
|
+
invalid/expired, 2 on a usage error. A per-target provider (one credential per
|
|
45
|
+
site/host/account) gets one entry per stored target, named `<kind>:<target>`. The --json shape is a cross-tool contract:
|
|
46
|
+
`tool`, `version`, `checked_at`, `credentials[]` with `provider`, `required`,
|
|
47
|
+
`fields`, `source`, `source_ref`, `status`, `expires_at`, `quota`, `message`.
|
|
48
|
+
|
|
49
|
+
Example: {cli} auth status --check --json
|
|
50
|
+
"""
|
|
51
|
+
tool, version = cli_name(), installed_version() or DEV_VERSION
|
|
52
|
+
if not _HAS_CREDENTIALS:
|
|
53
|
+
if provider is not None:
|
|
54
|
+
raise InvalidArguments(f"Unknown provider '{provider}' ({tool} stores no credentials).")
|
|
55
|
+
checked_at = datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
56
|
+
payload = {"tool": tool, "version": version, "checked_at": checked_at, "credentials": []}
|
|
57
|
+
output.emit_report(ctx, payload, lambda _: output.human.echo(f"{tool} stores no credentials."), None)
|
|
58
|
+
return
|
|
59
|
+
health = importlib.import_module(f"{_PACKAGE_NAME}.credentials.health")
|
|
60
|
+
payload, problem = health.status(tool, version, check=check, only=provider)
|
|
61
|
+
output.emit_report(ctx, payload, health.render, problem)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
if _login := optional_module(f"{_PACKAGE_NAME}.commands.login"):
|
|
65
|
+
app.command("login")(_login.login)
|
|
66
|
+
app.command("logout")(_login.logout)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""The `backups` command: the list of kept backups."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from .. import output
|
|
8
|
+
from ..core.dataio import api
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _render(data: dict) -> None:
|
|
12
|
+
if not data["backups"]:
|
|
13
|
+
output.human.echo("No backups.")
|
|
14
|
+
for entry in data["backups"]:
|
|
15
|
+
rows = "?" if entry["rows"] is None else entry["rows"] # None: dataio could not read the old content
|
|
16
|
+
target = entry["target"] if entry["table"] is None else f"{entry['target']}#{entry['table']}"
|
|
17
|
+
output.human.echo(f"{entry['id']} {entry['time']} {target} {rows} rows")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@output.guard
|
|
21
|
+
def backups(
|
|
22
|
+
ctx: typer.Context,
|
|
23
|
+
json_output: bool = output.JsonOption,
|
|
24
|
+
quiet: bool = output.QuietOption,
|
|
25
|
+
) -> None:
|
|
26
|
+
"""List the kept backups, newest first: id, time, target (with #table for a table of a base) and rows.
|
|
27
|
+
|
|
28
|
+
Example: {cli} backups --json
|
|
29
|
+
"""
|
|
30
|
+
output.emit_success(ctx, {"backups": [entry.to_dict() for entry in api.backups()]}, render_human=_render)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""The `config` command group: show and change the project's config file."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from .. import output
|
|
10
|
+
from ..core import services
|
|
11
|
+
from ..core.errors import InvalidArguments
|
|
12
|
+
from ..core.models import ProjectConfig
|
|
13
|
+
|
|
14
|
+
app = output.group("Manage project configuration.")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@app.command("list")
|
|
18
|
+
@output.guard
|
|
19
|
+
def list_config(
|
|
20
|
+
ctx: typer.Context,
|
|
21
|
+
json_output: bool = output.JsonOption,
|
|
22
|
+
quiet: bool = output.QuietOption,
|
|
23
|
+
) -> None:
|
|
24
|
+
"""List all configuration values.
|
|
25
|
+
|
|
26
|
+
Example: {cli} config list --json
|
|
27
|
+
"""
|
|
28
|
+
start = Path(ctx.obj.get("project") or Path.cwd())
|
|
29
|
+
root = services.require_project_root(start)
|
|
30
|
+
config = services.load_config(root)
|
|
31
|
+
data = config.model_dump()
|
|
32
|
+
output.emit_success(ctx, data, render_human=lambda d: output.human.key_values("Config", d))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@app.command("get")
|
|
36
|
+
@output.guard
|
|
37
|
+
def get_config(
|
|
38
|
+
ctx: typer.Context,
|
|
39
|
+
key: str = typer.Argument(..., help="Dotted config key, e.g. cli.min_version (see `config list`)."),
|
|
40
|
+
json_output: bool = output.JsonOption,
|
|
41
|
+
quiet: bool = output.QuietOption,
|
|
42
|
+
) -> None:
|
|
43
|
+
"""Get a single configuration value.
|
|
44
|
+
|
|
45
|
+
Example: {cli} config get cli.min_version
|
|
46
|
+
"""
|
|
47
|
+
start = Path(ctx.obj.get("project") or Path.cwd())
|
|
48
|
+
root = services.require_project_root(start)
|
|
49
|
+
value: object = services.load_config(root).model_dump()
|
|
50
|
+
for part in key.split("."):
|
|
51
|
+
if not isinstance(value, dict) or part not in value:
|
|
52
|
+
raise InvalidArguments(f"Unknown config key '{key}'.")
|
|
53
|
+
value = value[part]
|
|
54
|
+
output.emit_success(
|
|
55
|
+
ctx, {"key": key, "value": value}, render_human=lambda d: output.human.echo(f"{d['key']} = {d['value']}")
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@app.command("set")
|
|
60
|
+
@output.guard
|
|
61
|
+
def set_config(
|
|
62
|
+
ctx: typer.Context,
|
|
63
|
+
key: str = typer.Argument(..., help="Dotted config key, e.g. cli.min_version (see `config list`)."),
|
|
64
|
+
value: str = typer.Argument(..., help="New value (stored as a string)."),
|
|
65
|
+
json_output: bool = output.JsonOption,
|
|
66
|
+
quiet: bool = output.QuietOption,
|
|
67
|
+
) -> None:
|
|
68
|
+
"""Set a single, non-secret configuration value.
|
|
69
|
+
|
|
70
|
+
Example: {cli} config set cli.min_version 0.2.0
|
|
71
|
+
"""
|
|
72
|
+
start = Path(ctx.obj.get("project") or Path.cwd())
|
|
73
|
+
root = services.require_project_root(start)
|
|
74
|
+
data = services.load_config(root).model_dump()
|
|
75
|
+
parts = key.split(".")
|
|
76
|
+
target = data
|
|
77
|
+
for part in parts[:-1]:
|
|
78
|
+
if not isinstance(target, dict) or part not in target:
|
|
79
|
+
raise InvalidArguments(f"Unknown config key '{key}'.")
|
|
80
|
+
target = target[part]
|
|
81
|
+
if not isinstance(target, dict) or parts[-1] not in target:
|
|
82
|
+
raise InvalidArguments(f"Unknown config key '{key}'.")
|
|
83
|
+
target[parts[-1]] = value
|
|
84
|
+
services.save_config(root, ProjectConfig.model_validate(data))
|
|
85
|
+
output.emit_success(
|
|
86
|
+
ctx, {"key": key, "value": value}, render_human=lambda d: output.human.echo(f"Set {d['key']} = {d['value']}")
|
|
87
|
+
)
|