queria 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- queria/__init__.py +7 -0
- queria/cli.py +175 -0
- queria/core.py +254 -0
- queria/mcp.py +132 -0
- queria-0.1.0.dist-info/METADATA +77 -0
- queria-0.1.0.dist-info/RECORD +9 -0
- queria-0.1.0.dist-info/WHEEL +4 -0
- queria-0.1.0.dist-info/entry_points.txt +2 -0
- queria-0.1.0.dist-info/licenses/LICENSE +21 -0
queria/__init__.py
ADDED
queria/cli.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Command-line interface for exploring Queria's public open data."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import csv
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import sys
|
|
10
|
+
from collections.abc import Sequence
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from queria import core
|
|
14
|
+
|
|
15
|
+
FORMATS = ("table", "csv", "json", "jsonl")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _jsonable(value: Any) -> Any:
|
|
19
|
+
if value is None or isinstance(value, (bool, int, float, str)):
|
|
20
|
+
return value
|
|
21
|
+
return str(value)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _print_table(columns: Sequence[str], rows: list[tuple]) -> None:
|
|
25
|
+
text_rows = [["" if v is None else str(v) for v in row] for row in rows]
|
|
26
|
+
widths = [len(c) for c in columns]
|
|
27
|
+
for row in text_rows:
|
|
28
|
+
for i, v in enumerate(row):
|
|
29
|
+
widths[i] = max(widths[i], len(v))
|
|
30
|
+
print(" | ".join(c.ljust(w) for c, w in zip(columns, widths)))
|
|
31
|
+
print("-+-".join("-" * w for w in widths))
|
|
32
|
+
for row in text_rows:
|
|
33
|
+
print(" | ".join(v.ljust(w) for v, w in zip(row, widths)))
|
|
34
|
+
print(f"\n({len(text_rows)} rows)", file=sys.stderr)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _emit(conn: core.Connection, sql: str, fmt: str, out: str | None) -> None:
|
|
38
|
+
if out:
|
|
39
|
+
ext = os.path.splitext(out)[1].lower()
|
|
40
|
+
if ext not in (".csv", ".parquet"):
|
|
41
|
+
sys.exit(f"--out expects a .csv or .parquet path (got {out!r})")
|
|
42
|
+
copy_fmt = "PARQUET" if ext == ".parquet" else "CSV, HEADER"
|
|
43
|
+
# Run once first so referenced datasets get attached before COPY.
|
|
44
|
+
conn.sql(sql)
|
|
45
|
+
conn.execute(f"COPY ({sql}) TO '{out}' (FORMAT {copy_fmt})")
|
|
46
|
+
print(f"Wrote {out}", file=sys.stderr)
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
rel = conn.sql(sql)
|
|
50
|
+
columns = rel.columns
|
|
51
|
+
rows = rel.fetchall()
|
|
52
|
+
if fmt == "table":
|
|
53
|
+
_print_table(columns, rows)
|
|
54
|
+
elif fmt == "csv":
|
|
55
|
+
writer = csv.writer(sys.stdout)
|
|
56
|
+
writer.writerow(columns)
|
|
57
|
+
writer.writerows(rows)
|
|
58
|
+
elif fmt == "json":
|
|
59
|
+
records = [
|
|
60
|
+
{c: _jsonable(v) for c, v in zip(columns, row)} for row in rows
|
|
61
|
+
]
|
|
62
|
+
json.dump(records, sys.stdout, ensure_ascii=False, indent=2)
|
|
63
|
+
print()
|
|
64
|
+
elif fmt == "jsonl":
|
|
65
|
+
for row in rows:
|
|
66
|
+
record = {c: _jsonable(v) for c, v in zip(columns, row)}
|
|
67
|
+
print(json.dumps(record, ensure_ascii=False))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _add_output_args(parser: argparse.ArgumentParser) -> None:
|
|
71
|
+
parser.add_argument(
|
|
72
|
+
"--format",
|
|
73
|
+
choices=FORMATS,
|
|
74
|
+
default="table",
|
|
75
|
+
help="stdout format (default: table; ignored when --out is set)",
|
|
76
|
+
)
|
|
77
|
+
parser.add_argument("--out", help="write the result to a .csv or .parquet file")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
81
|
+
parser = argparse.ArgumentParser(
|
|
82
|
+
prog="queria",
|
|
83
|
+
description="Explore Queria public open data (data.queria.io, read-only).",
|
|
84
|
+
)
|
|
85
|
+
parser.add_argument(
|
|
86
|
+
"--version", action="version", version=f"queria {core.version()}"
|
|
87
|
+
)
|
|
88
|
+
parser.add_argument(
|
|
89
|
+
"--storage-url",
|
|
90
|
+
dest="storage",
|
|
91
|
+
default=core.DEFAULT_STORAGE,
|
|
92
|
+
help=f"catalog base URL (default: {core.DEFAULT_STORAGE})",
|
|
93
|
+
)
|
|
94
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
95
|
+
|
|
96
|
+
p_list = sub.add_parser("list", help="list available datasets")
|
|
97
|
+
_add_output_args(p_list)
|
|
98
|
+
|
|
99
|
+
p_search = sub.add_parser("search", help="search datasets by keyword")
|
|
100
|
+
p_search.add_argument("keyword")
|
|
101
|
+
_add_output_args(p_search)
|
|
102
|
+
|
|
103
|
+
p_schema = sub.add_parser("schema", help="list a dataset's tables")
|
|
104
|
+
p_schema.add_argument("dataset")
|
|
105
|
+
_add_output_args(p_schema)
|
|
106
|
+
|
|
107
|
+
p_columns = sub.add_parser("columns", help="list a dataset's columns")
|
|
108
|
+
p_columns.add_argument("dataset")
|
|
109
|
+
p_columns.add_argument("table", nargs="?", help="filter to one table")
|
|
110
|
+
_add_output_args(p_columns)
|
|
111
|
+
|
|
112
|
+
p_sql = sub.add_parser("sql", help="run a read-only SQL query")
|
|
113
|
+
p_sql.add_argument("query")
|
|
114
|
+
p_sql.add_argument(
|
|
115
|
+
"--datasets",
|
|
116
|
+
help="comma-separated datasets to pre-attach (usually auto-detected)",
|
|
117
|
+
)
|
|
118
|
+
_add_output_args(p_sql)
|
|
119
|
+
|
|
120
|
+
sub.add_parser("mcp", help="run the stdio MCP server")
|
|
121
|
+
|
|
122
|
+
return parser
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def main(argv: Sequence[str] | None = None) -> None:
|
|
126
|
+
args = build_parser().parse_args(argv)
|
|
127
|
+
|
|
128
|
+
if args.command == "mcp":
|
|
129
|
+
from queria import mcp
|
|
130
|
+
|
|
131
|
+
mcp.serve(args.storage)
|
|
132
|
+
return
|
|
133
|
+
|
|
134
|
+
try:
|
|
135
|
+
conn = core.connect(
|
|
136
|
+
args.storage, user_agent=f"queria-cli/{core.version()}"
|
|
137
|
+
)
|
|
138
|
+
except RuntimeError as exc:
|
|
139
|
+
sys.exit(str(exc))
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
if args.command == "list":
|
|
143
|
+
_emit(conn, core.list_datasets_sql(), args.format, args.out)
|
|
144
|
+
elif args.command == "search":
|
|
145
|
+
_emit(
|
|
146
|
+
conn, core.search_datasets_sql(args.keyword), args.format, args.out
|
|
147
|
+
)
|
|
148
|
+
elif args.command == "schema":
|
|
149
|
+
_emit(conn, core.schema_sql(args.dataset), args.format, args.out)
|
|
150
|
+
elif args.command == "columns":
|
|
151
|
+
_emit(
|
|
152
|
+
conn,
|
|
153
|
+
core.columns_sql(args.dataset, args.table),
|
|
154
|
+
args.format,
|
|
155
|
+
args.out,
|
|
156
|
+
)
|
|
157
|
+
elif args.command == "sql":
|
|
158
|
+
if not core.is_read_only(args.query):
|
|
159
|
+
sys.exit(
|
|
160
|
+
"Only read-only queries are allowed "
|
|
161
|
+
"(SELECT/WITH/DESCRIBE/SHOW/PRAGMA/EXPLAIN/SUMMARIZE)."
|
|
162
|
+
)
|
|
163
|
+
if args.datasets:
|
|
164
|
+
for ds in args.datasets.split(","):
|
|
165
|
+
if ds.strip():
|
|
166
|
+
conn.attach(ds.strip())
|
|
167
|
+
_emit(conn, args.query, args.format, args.out)
|
|
168
|
+
except ValueError as exc:
|
|
169
|
+
sys.exit(str(exc))
|
|
170
|
+
finally:
|
|
171
|
+
conn.close()
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
if __name__ == "__main__":
|
|
175
|
+
main()
|
queria/core.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""Core connection and catalog helpers for Queria's public DuckLake catalogs.
|
|
2
|
+
|
|
3
|
+
Queria (https://data.queria.io) publishes Japanese open data as read-only
|
|
4
|
+
DuckLake catalogs. This module owns the connection lifecycle: attaching the
|
|
5
|
+
``catalog`` metadata dataset, auto-attaching other datasets referenced in
|
|
6
|
+
queries, and building the catalog queries shared by the CLI and MCP server.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import urllib.request
|
|
14
|
+
|
|
15
|
+
import duckdb
|
|
16
|
+
|
|
17
|
+
DEFAULT_STORAGE = "https://data.queria.io"
|
|
18
|
+
CATALOG_ALIAS = "catalog"
|
|
19
|
+
MAX_AUTO_ATTACH = 8
|
|
20
|
+
|
|
21
|
+
# duckdb version requirement, coupled to Queria's published DuckLake v1 format.
|
|
22
|
+
# The catalog is DuckLake format 1.0; only duckdb >= 1.5.4 ships a ducklake
|
|
23
|
+
# extension new enough to read it. Bump together with the pin in pyproject.toml
|
|
24
|
+
# if Queria's catalog format changes.
|
|
25
|
+
MIN_DUCKDB = (1, 5, 4)
|
|
26
|
+
|
|
27
|
+
_IDENT_RE = re.compile(r"^[A-Za-z0-9_]+$")
|
|
28
|
+
_MISSING_CATALOG_RE = re.compile(r'Catalog "?([A-Za-z0-9_]+)"? does not exist')
|
|
29
|
+
|
|
30
|
+
# Advisory guard for user-supplied SQL. The real protection is the READ_ONLY
|
|
31
|
+
# attach: writes to the catalogs fail at the engine level regardless. This
|
|
32
|
+
# regex only catches obvious accidents early with a clearer message.
|
|
33
|
+
_READONLY_RE = re.compile(
|
|
34
|
+
r"^\s*(with|select|describe|show|pragma|explain|summarize|values|table|from)\b",
|
|
35
|
+
re.IGNORECASE,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def version() -> str:
|
|
40
|
+
"""Return the installed queria package version."""
|
|
41
|
+
try:
|
|
42
|
+
from importlib.metadata import version as _pkg_version
|
|
43
|
+
|
|
44
|
+
return _pkg_version("queria")
|
|
45
|
+
except Exception:
|
|
46
|
+
return "0.0.0"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def is_read_only(sql: str) -> bool:
|
|
50
|
+
"""Advisory check that ``sql`` looks like a read-only statement."""
|
|
51
|
+
return bool(_READONLY_RE.match(sql))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _check_duckdb_version() -> None:
|
|
55
|
+
parts = []
|
|
56
|
+
for chunk in duckdb.__version__.split(".")[:3]:
|
|
57
|
+
m = re.match(r"\d+", chunk)
|
|
58
|
+
parts.append(int(m.group()) if m else 0)
|
|
59
|
+
if tuple(parts) < MIN_DUCKDB:
|
|
60
|
+
want = ".".join(str(n) for n in MIN_DUCKDB)
|
|
61
|
+
raise RuntimeError(
|
|
62
|
+
f"duckdb >= {want} is required to read Queria's DuckLake catalogs "
|
|
63
|
+
f"(found {duckdb.__version__}). Upgrade with: "
|
|
64
|
+
f"pip install 'duckdb>={want}'"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _compat_hint(storage: str, exc: Exception) -> str | None:
|
|
69
|
+
"""Build an actionable message from the storage compatibility manifest.
|
|
70
|
+
|
|
71
|
+
Queria publishes ``{storage}/meta.json`` describing the catalog format and
|
|
72
|
+
the minimum client versions able to read it. The manifest is consulted
|
|
73
|
+
only after an ATTACH failure, purely to improve the error message; its
|
|
74
|
+
absence (or any fetch error) is never itself an error.
|
|
75
|
+
"""
|
|
76
|
+
if not storage.startswith(("http://", "https://")):
|
|
77
|
+
return None
|
|
78
|
+
try:
|
|
79
|
+
with urllib.request.urlopen(f"{storage}/meta.json", timeout=3) as res:
|
|
80
|
+
meta = json.load(res)
|
|
81
|
+
except Exception:
|
|
82
|
+
return None
|
|
83
|
+
if not isinstance(meta, dict):
|
|
84
|
+
return None
|
|
85
|
+
lines = [f"Failed to attach the Queria catalog at {storage}: {exc}"]
|
|
86
|
+
if "ducklake_format" in meta:
|
|
87
|
+
lines.append(f"The catalog uses DuckLake format {meta['ducklake_format']}.")
|
|
88
|
+
if "min_duckdb" in meta:
|
|
89
|
+
lines.append(
|
|
90
|
+
f"It requires duckdb >= {meta['min_duckdb']} "
|
|
91
|
+
f"(you have {duckdb.__version__})."
|
|
92
|
+
)
|
|
93
|
+
if "min_cli" in meta:
|
|
94
|
+
lines.append(
|
|
95
|
+
f"It requires queria >= {meta['min_cli']} (you have {version()}). "
|
|
96
|
+
"Upgrade with: pip install -U queria (or clear the uvx cache)."
|
|
97
|
+
)
|
|
98
|
+
return "\n".join(lines)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class Connection:
|
|
102
|
+
"""Read-only connection to Queria's public DuckLake catalogs.
|
|
103
|
+
|
|
104
|
+
On construction the ``catalog`` metadata dataset is attached. Other
|
|
105
|
+
datasets are attached lazily: when a query references a dataset that is
|
|
106
|
+
not attached yet, :meth:`sql` catches the missing-catalog error, attaches
|
|
107
|
+
the dataset, and retries.
|
|
108
|
+
|
|
109
|
+
Use as a context manager to make sure the underlying duckdb connection is
|
|
110
|
+
closed::
|
|
111
|
+
|
|
112
|
+
with queria.connect() as conn:
|
|
113
|
+
conn.sql("SELECT * FROM catalog.main.mart_datasets").show()
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
def __init__(
|
|
117
|
+
self, storage: str = DEFAULT_STORAGE, *, user_agent: str | None = None
|
|
118
|
+
) -> None:
|
|
119
|
+
_check_duckdb_version()
|
|
120
|
+
self.storage = storage.rstrip("/")
|
|
121
|
+
config = {"custom_user_agent": user_agent or f"queria-python/{version()}"}
|
|
122
|
+
self._con = duckdb.connect(config=config)
|
|
123
|
+
self._con.execute("INSTALL ducklake; LOAD ducklake;")
|
|
124
|
+
self._con.execute("INSTALL httpfs; LOAD httpfs;")
|
|
125
|
+
# spatial: several datasets (nlftp, address_br, e_stat boundaries)
|
|
126
|
+
# expose GEOMETRY columns and ST_* functions.
|
|
127
|
+
self._con.execute("INSTALL spatial; LOAD spatial;")
|
|
128
|
+
self._attached: set[str] = set()
|
|
129
|
+
try:
|
|
130
|
+
self.attach(CATALOG_ALIAS)
|
|
131
|
+
except duckdb.Error as exc:
|
|
132
|
+
hint = _compat_hint(self.storage, exc)
|
|
133
|
+
if hint:
|
|
134
|
+
raise RuntimeError(hint) from exc
|
|
135
|
+
raise
|
|
136
|
+
|
|
137
|
+
def attach(self, dataset: str) -> None:
|
|
138
|
+
"""Attach a dataset by name (idempotent)."""
|
|
139
|
+
if dataset in self._attached:
|
|
140
|
+
return
|
|
141
|
+
_validate_ident(dataset)
|
|
142
|
+
url = f"{self.storage}/{dataset}/ducklake.duckdb"
|
|
143
|
+
data_path = f"{self.storage}/{dataset}/ducklake.duckdb.files/"
|
|
144
|
+
self._con.execute(
|
|
145
|
+
f"ATTACH 'ducklake:{url}' AS {dataset} "
|
|
146
|
+
f"(READ_ONLY, DATA_PATH '{data_path}', OVERRIDE_DATA_PATH true)"
|
|
147
|
+
)
|
|
148
|
+
self._attached.add(dataset)
|
|
149
|
+
|
|
150
|
+
def sql(self, query: str) -> duckdb.DuckDBPyRelation:
|
|
151
|
+
"""Run a query, auto-attaching datasets it references."""
|
|
152
|
+
for _ in range(MAX_AUTO_ATTACH):
|
|
153
|
+
try:
|
|
154
|
+
return self._con.sql(query)
|
|
155
|
+
except duckdb.Error as exc:
|
|
156
|
+
self._attach_missing(exc)
|
|
157
|
+
return self._con.sql(query)
|
|
158
|
+
|
|
159
|
+
def execute(self, query: str) -> None:
|
|
160
|
+
"""Execute a statement (e.g. COPY), auto-attaching referenced datasets."""
|
|
161
|
+
for _ in range(MAX_AUTO_ATTACH):
|
|
162
|
+
try:
|
|
163
|
+
self._con.execute(query)
|
|
164
|
+
return
|
|
165
|
+
except duckdb.Error as exc:
|
|
166
|
+
self._attach_missing(exc)
|
|
167
|
+
self._con.execute(query)
|
|
168
|
+
|
|
169
|
+
def _attach_missing(self, exc: duckdb.Error) -> None:
|
|
170
|
+
"""Attach the dataset named in a missing-catalog error, or re-raise."""
|
|
171
|
+
m = _MISSING_CATALOG_RE.search(str(exc))
|
|
172
|
+
if not m or m.group(1) in self._attached:
|
|
173
|
+
raise exc
|
|
174
|
+
self.attach(m.group(1))
|
|
175
|
+
|
|
176
|
+
def close(self) -> None:
|
|
177
|
+
"""Close the underlying duckdb connection."""
|
|
178
|
+
self._con.close()
|
|
179
|
+
|
|
180
|
+
def __enter__(self) -> Connection:
|
|
181
|
+
return self
|
|
182
|
+
|
|
183
|
+
def __exit__(self, *exc_info: object) -> None:
|
|
184
|
+
self.close()
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def connect(
|
|
188
|
+
storage: str = DEFAULT_STORAGE, *, user_agent: str | None = None
|
|
189
|
+
) -> Connection:
|
|
190
|
+
"""Open a read-only connection to Queria's public catalogs.
|
|
191
|
+
|
|
192
|
+
Args:
|
|
193
|
+
storage: Base URL (or local path) of the catalog storage.
|
|
194
|
+
user_agent: HTTP user agent reported to the storage. Defaults to
|
|
195
|
+
``queria-python/<version>``.
|
|
196
|
+
"""
|
|
197
|
+
return Connection(storage, user_agent=user_agent)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
# ---- catalog queries shared by the CLI and the MCP server -------------------
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _validate_ident(name: str) -> None:
|
|
204
|
+
if not _IDENT_RE.match(name):
|
|
205
|
+
raise ValueError(f"invalid dataset/table name: {name!r}")
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _quote(value: str) -> str:
|
|
209
|
+
return value.replace("'", "''")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def list_datasets_sql() -> str:
|
|
213
|
+
"""SQL listing all published datasets."""
|
|
214
|
+
return (
|
|
215
|
+
"SELECT datasource, title, description "
|
|
216
|
+
f"FROM {CATALOG_ALIAS}.main.mart_datasets ORDER BY datasource"
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def search_datasets_sql(keyword: str) -> str:
|
|
221
|
+
"""SQL searching datasets by keyword over title and description."""
|
|
222
|
+
kw = _quote(keyword)
|
|
223
|
+
return f"""
|
|
224
|
+
SELECT datasource, title, description
|
|
225
|
+
FROM {CATALOG_ALIAS}.main.mart_datasets
|
|
226
|
+
WHERE lower(title || ' ' || COALESCE(description, '')) LIKE lower('%{kw}%')
|
|
227
|
+
ORDER BY datasource
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def schema_sql(dataset: str) -> str:
|
|
232
|
+
"""SQL listing a dataset's tables and views."""
|
|
233
|
+
_validate_ident(dataset)
|
|
234
|
+
return f"""
|
|
235
|
+
SELECT schema_name, name AS table_name, description
|
|
236
|
+
FROM {CATALOG_ALIAS}.main.mart_nodes
|
|
237
|
+
WHERE datasource = '{dataset}' AND resource_type = 'model'
|
|
238
|
+
ORDER BY schema_name, name
|
|
239
|
+
"""
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def columns_sql(dataset: str, table: str | None = None) -> str:
|
|
243
|
+
"""SQL listing a dataset's columns, optionally filtered to one table."""
|
|
244
|
+
_validate_ident(dataset)
|
|
245
|
+
table_filter = ""
|
|
246
|
+
if table:
|
|
247
|
+
_validate_ident(table)
|
|
248
|
+
table_filter = f"AND table_name = '{table}'"
|
|
249
|
+
return f"""
|
|
250
|
+
SELECT table_name, column_name, data_type, description
|
|
251
|
+
FROM {CATALOG_ALIAS}.main.mart_columns
|
|
252
|
+
WHERE datasource = '{dataset}' {table_filter}
|
|
253
|
+
ORDER BY table_name, column_name
|
|
254
|
+
"""
|
queria/mcp.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Stdio MCP server exposing Queria's public open data to agents.
|
|
2
|
+
|
|
3
|
+
Run with ``queria mcp`` (requires the ``mcp`` extra: ``pip install
|
|
4
|
+
'queria[mcp]'`` or ``uvx --from 'queria[mcp]' queria mcp``).
|
|
5
|
+
|
|
6
|
+
The ``query`` tool enforces a row and payload-size cap so a single tool call
|
|
7
|
+
cannot flood an agent's context. Bulk extraction should go through the CLI's
|
|
8
|
+
``--out`` option instead.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import sys
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from queria import core
|
|
18
|
+
|
|
19
|
+
DEFAULT_MAX_ROWS = 100
|
|
20
|
+
MAX_ROWS_LIMIT = 1000
|
|
21
|
+
MAX_PAYLOAD_BYTES = 1_000_000
|
|
22
|
+
|
|
23
|
+
_BULK_HINT = (
|
|
24
|
+
"Result truncated. Narrow the query (LIMIT / WHERE / aggregate), or use "
|
|
25
|
+
"the CLI for bulk export: queria sql '<query>' --out result.parquet"
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _jsonable(value: Any) -> Any:
|
|
30
|
+
if value is None or isinstance(value, (bool, int, float, str)):
|
|
31
|
+
return value
|
|
32
|
+
return str(value)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _relation_payload(
|
|
36
|
+
rel: Any, max_rows: int | None = None
|
|
37
|
+
) -> dict[str, Any]:
|
|
38
|
+
"""Convert a duckdb relation into a JSON-safe payload with caps applied.
|
|
39
|
+
|
|
40
|
+
Rows are capped at ``max_rows`` first, then dropped from the tail until
|
|
41
|
+
the serialized payload fits within ``MAX_PAYLOAD_BYTES``.
|
|
42
|
+
"""
|
|
43
|
+
columns = rel.columns
|
|
44
|
+
if max_rows is None:
|
|
45
|
+
raw = rel.fetchall()
|
|
46
|
+
truncated = False
|
|
47
|
+
else:
|
|
48
|
+
raw = rel.fetchmany(max_rows + 1)
|
|
49
|
+
truncated = len(raw) > max_rows
|
|
50
|
+
raw = raw[:max_rows]
|
|
51
|
+
rows = [[_jsonable(v) for v in row] for row in raw]
|
|
52
|
+
|
|
53
|
+
payload = {"columns": columns, "rows": rows, "truncated": truncated}
|
|
54
|
+
while (
|
|
55
|
+
len(json.dumps(payload, ensure_ascii=False)) > MAX_PAYLOAD_BYTES
|
|
56
|
+
and payload["rows"]
|
|
57
|
+
):
|
|
58
|
+
payload["rows"] = payload["rows"][: max(1, len(payload["rows"]) // 2)]
|
|
59
|
+
payload["truncated"] = True
|
|
60
|
+
if len(payload["rows"]) == 1:
|
|
61
|
+
break
|
|
62
|
+
payload["row_count"] = len(payload["rows"])
|
|
63
|
+
if payload["truncated"]:
|
|
64
|
+
payload["hint"] = _BULK_HINT
|
|
65
|
+
return payload
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def build_server(storage: str = core.DEFAULT_STORAGE) -> Any:
|
|
69
|
+
"""Build the FastMCP server with all Queria tools registered."""
|
|
70
|
+
try:
|
|
71
|
+
from mcp.server.fastmcp import FastMCP
|
|
72
|
+
except ImportError:
|
|
73
|
+
sys.exit(
|
|
74
|
+
"The MCP server requires the 'mcp' extra:\n"
|
|
75
|
+
" pip install 'queria[mcp]'\n"
|
|
76
|
+
" # or: uvx --from 'queria[mcp]' queria mcp"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
server = FastMCP(
|
|
80
|
+
"queria",
|
|
81
|
+
instructions=(
|
|
82
|
+
"Read-only access to Queria (data.queria.io), a catalog of "
|
|
83
|
+
"Japanese open data published as DuckLake. Start with "
|
|
84
|
+
"list_datasets or search_datasets, inspect tables with "
|
|
85
|
+
"get_schema / get_columns, then run DuckDB SQL with query. "
|
|
86
|
+
"Reference tables as <dataset>.<schema>.<table>."
|
|
87
|
+
),
|
|
88
|
+
)
|
|
89
|
+
conn = core.connect(storage, user_agent=f"queria-mcp/{core.version()}")
|
|
90
|
+
|
|
91
|
+
@server.tool()
|
|
92
|
+
def list_datasets() -> dict:
|
|
93
|
+
"""List all datasets published on Queria."""
|
|
94
|
+
return _relation_payload(conn.sql(core.list_datasets_sql()))
|
|
95
|
+
|
|
96
|
+
@server.tool()
|
|
97
|
+
def search_datasets(keyword: str) -> dict:
|
|
98
|
+
"""Search datasets by keyword over their title and description."""
|
|
99
|
+
return _relation_payload(conn.sql(core.search_datasets_sql(keyword)))
|
|
100
|
+
|
|
101
|
+
@server.tool()
|
|
102
|
+
def get_schema(dataset: str) -> dict:
|
|
103
|
+
"""List a dataset's tables and views with descriptions."""
|
|
104
|
+
return _relation_payload(conn.sql(core.schema_sql(dataset)))
|
|
105
|
+
|
|
106
|
+
@server.tool()
|
|
107
|
+
def get_columns(dataset: str, table: str | None = None) -> dict:
|
|
108
|
+
"""List a dataset's columns (optionally for a single table)."""
|
|
109
|
+
return _relation_payload(conn.sql(core.columns_sql(dataset, table)))
|
|
110
|
+
|
|
111
|
+
@server.tool()
|
|
112
|
+
def query(sql: str, max_rows: int = DEFAULT_MAX_ROWS) -> dict:
|
|
113
|
+
"""Run a read-only DuckDB SQL query against Queria datasets.
|
|
114
|
+
|
|
115
|
+
Reference tables as <dataset>.<schema>.<table>; datasets attach
|
|
116
|
+
automatically. Results are capped at max_rows (up to 1000) and ~1MB;
|
|
117
|
+
use the queria CLI with --out for bulk extraction.
|
|
118
|
+
"""
|
|
119
|
+
if not core.is_read_only(sql):
|
|
120
|
+
raise ValueError(
|
|
121
|
+
"Only read-only queries are allowed "
|
|
122
|
+
"(SELECT/WITH/DESCRIBE/SHOW/PRAGMA/EXPLAIN/SUMMARIZE)."
|
|
123
|
+
)
|
|
124
|
+
capped = max(1, min(max_rows, MAX_ROWS_LIMIT))
|
|
125
|
+
return _relation_payload(conn.sql(sql), max_rows=capped)
|
|
126
|
+
|
|
127
|
+
return server
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def serve(storage: str = core.DEFAULT_STORAGE) -> None:
|
|
131
|
+
"""Run the stdio MCP server (blocks until the client disconnects)."""
|
|
132
|
+
build_server(storage).run()
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: queria
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Query Japanese open data on Queria (data.queria.io) from the terminal, Python, and MCP
|
|
5
|
+
Project-URL: Homepage, https://docs.queria.io/
|
|
6
|
+
Project-URL: Documentation, https://docs.queria.io/
|
|
7
|
+
Project-URL: Repository, https://github.com/queria-io/queria-cli
|
|
8
|
+
Project-URL: Changelog, https://docs.queria.io/resources/changelog/
|
|
9
|
+
Author: flo8s
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: duckdb,ducklake,japan,mcp,opendata
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Requires-Dist: duckdb<2.0,>=1.5.4
|
|
15
|
+
Provides-Extra: mcp
|
|
16
|
+
Requires-Dist: mcp>=1.2; extra == 'mcp'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# queria
|
|
20
|
+
|
|
21
|
+
Query Japanese open data on [Queria](https://data.queria.io) from the terminal, Python, and MCP.
|
|
22
|
+
|
|
23
|
+
[Queria](https://data.queria.io) が公開する日本のオープンデータ(e-Stat、国土数値情報、EDINET、気象庁ほか)を、ターミナル・Python・MCP から read-only で探索するクライアントです。データは DuckLake 形式で公開されており、計算はすべて手元の DuckDB で行われます。
|
|
24
|
+
|
|
25
|
+
## インストール
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
uvx queria list # インストール不要で実行
|
|
29
|
+
pip install queria # または通常インストール
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Python 3.10+ が必要です。
|
|
33
|
+
|
|
34
|
+
## 使い方
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
queria list # データセット一覧
|
|
38
|
+
queria search 人口 # キーワード検索
|
|
39
|
+
queria schema e_stat # テーブル一覧
|
|
40
|
+
queria columns e_stat mart_population # カラム一覧
|
|
41
|
+
queria sql "SELECT * FROM zipcode.main.zipcodes LIMIT 10"
|
|
42
|
+
queria sql "SELECT * FROM zipcode.main.zipcodes" --out zipcodes.parquet
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
テーブルは `<dataset>.<schema>.<table>` で参照します。参照したデータセットは自動的に ATTACH されます。
|
|
46
|
+
|
|
47
|
+
## Python API
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
import queria
|
|
51
|
+
|
|
52
|
+
with queria.connect() as conn:
|
|
53
|
+
conn.sql("SELECT * FROM catalog.main.mart_datasets").show()
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## MCP サーバー
|
|
57
|
+
|
|
58
|
+
Claude Code / Claude Desktop / Cursor などの MCP クライアントから使えます:
|
|
59
|
+
|
|
60
|
+
```json
|
|
61
|
+
{
|
|
62
|
+
"mcpServers": {
|
|
63
|
+
"queria": {
|
|
64
|
+
"command": "uvx",
|
|
65
|
+
"args": ["--from", "queria[mcp]", "queria", "mcp"]
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## ドキュメント
|
|
72
|
+
|
|
73
|
+
https://docs.queria.io/
|
|
74
|
+
|
|
75
|
+
## License
|
|
76
|
+
|
|
77
|
+
MIT
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
queria/__init__.py,sha256=miQ0gR1Hc-pcuZrt06tzlC_zrg89lxobJ2FE9HlnA-A,251
|
|
2
|
+
queria/cli.py,sha256=HDYCDOYCn-G-GG1IXxnokAKl3rCToDNCQaxOyvrCEk0,5642
|
|
3
|
+
queria/core.py,sha256=2nYXJ-3prsAlkxROtK79bBbKep7_XwyW1gzHM7MOdLs,8960
|
|
4
|
+
queria/mcp.py,sha256=Dt7BpnGh5br0nF9OUp-udsHNRKjRRHRmsskDL6uZRr4,4595
|
|
5
|
+
queria-0.1.0.dist-info/METADATA,sha256=Qk2AOjcwjXTN1t_EEJAOAS3mwJ9IS2dx6XOrML2WE8s,2296
|
|
6
|
+
queria-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
7
|
+
queria-0.1.0.dist-info/entry_points.txt,sha256=zQssl32dXfYpvj1iWKrJTgj5TLaCAv1uR0pOiy4t1L4,43
|
|
8
|
+
queria-0.1.0.dist-info/licenses/LICENSE,sha256=t0Of_GryPUMuBvGrtopnujMcxjgVBWMbUIyL7cCZMvk,1062
|
|
9
|
+
queria-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 flo8s
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|