datajoins-nybble 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datajoins_nybble-1.0.0.dist-info/METADATA +695 -0
- datajoins_nybble-1.0.0.dist-info/RECORD +32 -0
- datajoins_nybble-1.0.0.dist-info/WHEEL +5 -0
- datajoins_nybble-1.0.0.dist-info/entry_points.txt +3 -0
- datajoins_nybble-1.0.0.dist-info/licenses/LICENSE +193 -0
- datajoins_nybble-1.0.0.dist-info/top_level.txt +1 -0
- nybble/__init__.py +133 -0
- nybble/arena.py +607 -0
- nybble/browse.py +534 -0
- nybble/cli.py +1803 -0
- nybble/config.py +368 -0
- nybble/core/__init__.py +12 -0
- nybble/core/detect_format.py +349 -0
- nybble/core/erd_data.py +245 -0
- nybble/core/import_to_sqlite.py +551 -0
- nybble/core/infer_fk.py +458 -0
- nybble/core/kg_builder.py +367 -0
- nybble/core/schema.py +85 -0
- nybble/diff.py +139 -0
- nybble/erd.py +263 -0
- nybble/highlight_store.py +112 -0
- nybble/infer_fk.py +228 -0
- nybble/kg.py +1392 -0
- nybble/lineage.py +1013 -0
- nybble/mcp_server.py +2478 -0
- nybble/profiler.py +180 -0
- nybble/report.py +754 -0
- nybble/resolver.py +1450 -0
- nybble/serve.py +416 -0
- nybble/share.py +404 -0
- nybble/templates/erd.html.jinja +1052 -0
- nybble/templates/kg.html.jinja +561 -0
|
@@ -0,0 +1,695 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datajoins-nybble
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A nybble is 4 bits. That's all you need. Browse, query, diagram, and graph any data source - one pip install, zero infrastructure.
|
|
5
|
+
Author-email: DataJoins <hello@datajoins.com>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://datajoins.com
|
|
8
|
+
Project-URL: Repository, https://github.com/datajoins/nybble
|
|
9
|
+
Project-URL: Documentation, https://nybble.datajoins.com/documentation.html
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/datajoins/nybble/issues
|
|
11
|
+
Project-URL: Contributing, https://github.com/datajoins/nybble/blob/main/CONTRIBUTING.md
|
|
12
|
+
Keywords: data,sql,erd,knowledge-graph,duckdb,etl,schema-inference,data-exploration,data-catalog,graphml,json-ld,parquet,graph,visualization
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Database
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Visualization
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
|
+
Classifier: Environment :: Console
|
|
25
|
+
Classifier: Framework :: Jupyter
|
|
26
|
+
Classifier: Operating System :: OS Independent
|
|
27
|
+
Requires-Python: >=3.10
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Requires-Dist: duckdb>=0.10
|
|
31
|
+
Requires-Dist: polars>=0.20
|
|
32
|
+
Requires-Dist: networkx>=3.0
|
|
33
|
+
Requires-Dist: rich>=13.0
|
|
34
|
+
Requires-Dist: click>=8.1
|
|
35
|
+
Requires-Dist: requests>=2.31
|
|
36
|
+
Requires-Dist: jinja2>=3.1
|
|
37
|
+
Requires-Dist: pyvis>=0.3
|
|
38
|
+
Requires-Dist: openpyxl>=3.1
|
|
39
|
+
Requires-Dist: pyarrow>=14.0
|
|
40
|
+
Requires-Dist: pygments>=2.17
|
|
41
|
+
Requires-Dist: pyyaml>=6.0
|
|
42
|
+
Requires-Dist: sqlglot>=20.0
|
|
43
|
+
Provides-Extra: postgres
|
|
44
|
+
Requires-Dist: psycopg2-binary; extra == "postgres"
|
|
45
|
+
Requires-Dist: connectorx<0.4,>=0.3; extra == "postgres"
|
|
46
|
+
Provides-Extra: mysql
|
|
47
|
+
Requires-Dist: pymysql; extra == "mysql"
|
|
48
|
+
Provides-Extra: mssql
|
|
49
|
+
Requires-Dist: pyodbc; extra == "mssql"
|
|
50
|
+
Provides-Extra: oracle
|
|
51
|
+
Requires-Dist: oracledb; extra == "oracle"
|
|
52
|
+
Provides-Extra: mongo
|
|
53
|
+
Requires-Dist: pymongo; extra == "mongo"
|
|
54
|
+
Provides-Extra: redis
|
|
55
|
+
Requires-Dist: redis; extra == "redis"
|
|
56
|
+
Provides-Extra: cassandra
|
|
57
|
+
Requires-Dist: cassandra-driver; extra == "cassandra"
|
|
58
|
+
Provides-Extra: elastic
|
|
59
|
+
Requires-Dist: elasticsearch; extra == "elastic"
|
|
60
|
+
Provides-Extra: kafka
|
|
61
|
+
Requires-Dist: confluent-kafka; extra == "kafka"
|
|
62
|
+
Provides-Extra: dynamo
|
|
63
|
+
Requires-Dist: boto3; extra == "dynamo"
|
|
64
|
+
Provides-Extra: textual
|
|
65
|
+
Requires-Dist: textual>=0.50; extra == "textual"
|
|
66
|
+
Provides-Extra: kuzu
|
|
67
|
+
Requires-Dist: kuzu>=0.3; extra == "kuzu"
|
|
68
|
+
Provides-Extra: rdf
|
|
69
|
+
Requires-Dist: rdflib>=6.0; extra == "rdf"
|
|
70
|
+
Provides-Extra: mcp
|
|
71
|
+
Requires-Dist: mcp>=1.0; extra == "mcp"
|
|
72
|
+
Provides-Extra: semantic
|
|
73
|
+
Requires-Dist: sentence-transformers>=2.0; extra == "semantic"
|
|
74
|
+
Provides-Extra: serve
|
|
75
|
+
Requires-Dist: waitress>=2.1; extra == "serve"
|
|
76
|
+
Provides-Extra: dev
|
|
77
|
+
Requires-Dist: pytest>=8.1; extra == "dev"
|
|
78
|
+
Requires-Dist: pytest-cov>=4.1; extra == "dev"
|
|
79
|
+
Requires-Dist: flask>=3.0; extra == "dev"
|
|
80
|
+
Provides-Extra: all
|
|
81
|
+
Requires-Dist: psycopg2-binary; extra == "all"
|
|
82
|
+
Requires-Dist: connectorx<0.4,>=0.3; extra == "all"
|
|
83
|
+
Requires-Dist: pymysql; extra == "all"
|
|
84
|
+
Requires-Dist: mcp>=1.0; extra == "all"
|
|
85
|
+
Requires-Dist: rdflib>=6.0; extra == "all"
|
|
86
|
+
Requires-Dist: waitress>=2.1; extra == "all"
|
|
87
|
+
Dynamic: license-file
|
|
88
|
+
|
|
89
|
+
<div align="center">
|
|
90
|
+
|
|
91
|
+
# Nybble 🍱
|
|
92
|
+
|
|
93
|
+
### *You don't need a Googol of infrastructure just to see your data.*
|
|
94
|
+
|
|
95
|
+
[](https://github.com/datajoins/nybble/actions/workflows/ci.yml)
|
|
96
|
+
[](https://pypi.org/project/datajoins-nybble/)
|
|
97
|
+
[](https://pypi.org/project/datajoins-nybble/)
|
|
98
|
+
[](LICENSE)
|
|
99
|
+
[](https://github.com/datajoins/nybble)
|
|
100
|
+
|
|
101
|
+
**[datajoins.com](https://datajoins.com) · [PyPI](https://pypi.org/project/datajoins-nybble/) · [Changelog](CHANGELOG.md) · [Documentation](https://nybble.datajoins.com/documentation.html) · [Issues](https://github.com/datajoins/nybble/issues)**
|
|
102
|
+
|
|
103
|
+
</div>
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## Why Nybble?
|
|
108
|
+
|
|
109
|
+
In computer science, a **nybble** is exactly 4 bits - half a byte, one of the smallest
|
|
110
|
+
functional units of digital storage.
|
|
111
|
+
|
|
112
|
+
A **googol** is 10¹⁰⁰ - a number larger than the estimated count of elementary particles
|
|
113
|
+
in the entire observable universe.
|
|
114
|
+
|
|
115
|
+
Every developer working with data eventually ends up wrestling with a googol of
|
|
116
|
+
complexity: heavy enterprise BI platforms, sprawling cloud data warehouses, multi-service
|
|
117
|
+
ETL pipelines, and schema documentation that's always out of date. Tools that charge
|
|
118
|
+
tens of thousands of dollars a year for capabilities that should take ten seconds to
|
|
119
|
+
invoke.
|
|
120
|
+
|
|
121
|
+
**Nybble exists because you don't need any of that just to understand your own data.**
|
|
122
|
+
|
|
123
|
+
You need something the exact opposite size - something small enough to install in one
|
|
124
|
+
command, fast enough to produce results before you finish your coffee, and honest enough
|
|
125
|
+
to show you exactly what's in your database without a sales call, a YAML sprawl, or a
|
|
126
|
+
Node.js build pipeline.
|
|
127
|
+
|
|
128
|
+
> *A nybble is half a byte. It is also exactly enough.*
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## What Nybble Does
|
|
133
|
+
|
|
134
|
+
Nybble is an ultra-lightweight Python engine that takes a single, effortless bite out of
|
|
135
|
+
any data source and gives you back four things that normally require four separate tools:
|
|
136
|
+
|
|
137
|
+
| Capability | What you get | Replaces |
|
|
138
|
+
|---|---|---|
|
|
139
|
+
| **Browse** | Full-terminal paginated data grid, live filter, column sort | TablePlus, DBeaver (desktop) |
|
|
140
|
+
| **SQL Arena** | Cross-source SQL via DuckDB - join a CSV to a Parquet to a live DB | Redash, DataGrip |
|
|
141
|
+
| **ERD** | Auto-generated interactive Entity-Relationship Diagram with FK inference | DbDiagram.io (paid), SchemaSpy |
|
|
142
|
+
| **Knowledge Graph** | Semantic entity graph with GraphML, JSON-LD, and HTML export | DataHub, Neo4j (enterprise) |
|
|
143
|
+
|
|
144
|
+
All four capabilities work from the same `pip install`. No Docker required. No server to
|
|
145
|
+
run. No Node.js. No API keys.
|
|
146
|
+
|
|
147
|
+
---
|
|
148
|
+
|
|
149
|
+
## Installation
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
pip install datajoins-nybble
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
**With optional connectors and extras:**
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
# PostgreSQL
|
|
159
|
+
pip install "datajoins-nybble[postgres]"
|
|
160
|
+
|
|
161
|
+
# MySQL / MariaDB
|
|
162
|
+
pip install "datajoins-nybble[mysql]"
|
|
163
|
+
|
|
164
|
+
# MCP server (AI agent integration - Claude, Cursor, VS Code)
|
|
165
|
+
pip install "datajoins-nybble[mcp]"
|
|
166
|
+
|
|
167
|
+
# Semantic node search (sentence-transformers)
|
|
168
|
+
pip install "datajoins-nybble[semantic]"
|
|
169
|
+
|
|
170
|
+
# Everything (postgres + mysql + mcp + rdflib)
|
|
171
|
+
pip install "datajoins-nybble[all]"
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
**Requirements:** Python ≥ 3.9
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## 30-Second Smoke Test
|
|
179
|
+
|
|
180
|
+
Copy and paste this block — it uses the demo CSV files in `.nybble-test-fixtures/`.
|
|
181
|
+
No database required, no configuration needed.
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
# 1. Install
|
|
185
|
+
pip install datajoins-nybble
|
|
186
|
+
|
|
187
|
+
# 2. Use the demo fixtures (copy from .nybble-test-fixtures/ in the repo)
|
|
188
|
+
# Or create them: customers.csv, orders.csv, products.csv - see documentation.html
|
|
189
|
+
|
|
190
|
+
# 3. Inspect a source
|
|
191
|
+
nybble info customers.csv
|
|
192
|
+
|
|
193
|
+
# 4. Profile column statistics
|
|
194
|
+
nybble profile customers.csv
|
|
195
|
+
|
|
196
|
+
# 5. Browse a file in the terminal (arrow keys, / to filter, q to quit)
|
|
197
|
+
nybble browse customers.csv
|
|
198
|
+
|
|
199
|
+
# 6. Run a cross-source SQL query
|
|
200
|
+
nybble query "SELECT c.name, COUNT(o.order_id) AS orders, SUM(o.amount) AS revenue
|
|
201
|
+
FROM customers c JOIN orders o ON c.customer_id = o.customer_id
|
|
202
|
+
GROUP BY c.name ORDER BY revenue DESC" \
|
|
203
|
+
--source customers.csv \
|
|
204
|
+
--source orders.csv
|
|
205
|
+
|
|
206
|
+
# 7. Diff two schemas
|
|
207
|
+
nybble diff customers.csv orders.csv
|
|
208
|
+
|
|
209
|
+
# 8. Generate a self-contained ERD HTML (opens in browser automatically)
|
|
210
|
+
nybble erd --source customers.csv --source orders.csv --source products.csv --no-browser
|
|
211
|
+
|
|
212
|
+
# 9. Build a Knowledge Graph
|
|
213
|
+
nybble kg --source customers.csv --source orders.csv --source products.csv --no-browser
|
|
214
|
+
|
|
215
|
+
# 10. Search the Knowledge Graph
|
|
216
|
+
nybble kg search "Alice" --source customers.csv
|
|
217
|
+
nybble kg search "shipped" --source orders.csv
|
|
218
|
+
|
|
219
|
+
# 11. Generate DuckDB SQL/PGQ DDL
|
|
220
|
+
nybble kg pgq --source customers.csv --source orders.csv --source products.csv
|
|
221
|
+
|
|
222
|
+
# 12. Print KG summary JSON
|
|
223
|
+
nybble kg --summary-only --source customers.csv --source orders.csv --source products.csv
|
|
224
|
+
|
|
225
|
+
# 13. Generate a Markdown data report
|
|
226
|
+
nybble report --source customers.csv --output customers_report.md
|
|
227
|
+
|
|
228
|
+
# 14. View query history
|
|
229
|
+
nybble history --limit 5
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Every command exits cleanly. No side effects outside the `.nybble/` directory.
|
|
233
|
+
|
|
234
|
+
---
|
|
235
|
+
|
|
236
|
+
## Workspace Config (`nybble.yaml`)
|
|
237
|
+
|
|
238
|
+
Instead of repeating `--source` flags on every command, declare your sources once in a
|
|
239
|
+
`nybble.yaml` file. Any `nybble` command run in the same directory will load them
|
|
240
|
+
automatically.
|
|
241
|
+
|
|
242
|
+
```yaml
|
|
243
|
+
# nybble.yaml
|
|
244
|
+
version: "1.0"
|
|
245
|
+
|
|
246
|
+
workspace:
|
|
247
|
+
name: "my-project"
|
|
248
|
+
|
|
249
|
+
files:
|
|
250
|
+
- id: customers
|
|
251
|
+
type: csv
|
|
252
|
+
path: ./data/customers.csv
|
|
253
|
+
|
|
254
|
+
- id: orders
|
|
255
|
+
type: parquet
|
|
256
|
+
path: ./data/orders.parquet
|
|
257
|
+
|
|
258
|
+
databases:
|
|
259
|
+
- id: production
|
|
260
|
+
type: postgresql
|
|
261
|
+
connection_string: "postgresql://${DB_USER}:${DB_PASS}@localhost:5432/mydb"
|
|
262
|
+
|
|
263
|
+
streams:
|
|
264
|
+
- id: github-issues
|
|
265
|
+
type: api
|
|
266
|
+
url: "https://api.github.com/repos/myorg/myrepo/issues"
|
|
267
|
+
headers:
|
|
268
|
+
Authorization: "Bearer ${GITHUB_TOKEN}"
|
|
269
|
+
format: json
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
```bash
|
|
273
|
+
# All sources from nybble.yaml are loaded automatically - no --source flags needed
|
|
274
|
+
nybble erd
|
|
275
|
+
nybble kg
|
|
276
|
+
nybble query "SELECT * FROM customers LIMIT 10"
|
|
277
|
+
nybble repl
|
|
278
|
+
|
|
279
|
+
# Or point to an explicit config file path
|
|
280
|
+
nybble erd --config ./configs/staging.yaml
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
- **`${ENV_VAR}`** tokens in the config are substituted from your shell environment at
|
|
284
|
+
load time. Nybble raises a clear error if a referenced variable is not set.
|
|
285
|
+
- The config file is discovered by searching for `nybble.yaml`, `nybble.yml`,
|
|
286
|
+
`.nybble.yaml`, or `.nybble.yml` in the current working directory.
|
|
287
|
+
- See [`nybble.yaml.example`](nybble.yaml.example) for a fully annotated reference.
|
|
288
|
+
|
|
289
|
+
---
|
|
290
|
+
|
|
291
|
+
## MCP Server - AI Agent Integration
|
|
292
|
+
|
|
293
|
+
Nybble ships a full [Model Context Protocol](https://modelcontextprotocol.io/) server
|
|
294
|
+
that exposes the complete Nybble API to any MCP-compatible AI agent - Claude Desktop,
|
|
295
|
+
Cursor, VS Code Copilot, Raycast, and others.
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
pip install "datajoins-nybble[mcp]"
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
**Register in your AI client config** (e.g. `~/.config/claude/claude_desktop_config.json`):
|
|
302
|
+
|
|
303
|
+
```json
|
|
304
|
+
{
|
|
305
|
+
"mcpServers": {
|
|
306
|
+
"nybble": { "command": "nybble-mcp" }
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
```
|
|
310
|
+
|
|
311
|
+
**Or start manually:**
|
|
312
|
+
|
|
313
|
+
```bash
|
|
314
|
+
nybble-mcp
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
**Available MCP tools:**
|
|
318
|
+
|
|
319
|
+
| Tool | Description |
|
|
320
|
+
|---|---|
|
|
321
|
+
| `nybble_load` | Load any source: file path, DB URI, S3/GCS URL, HTTP API |
|
|
322
|
+
| `nybble_list_tables` | List all loaded sources |
|
|
323
|
+
| `nybble_schema` | Column names and types for a loaded table |
|
|
324
|
+
| `nybble_query` | Execute DuckDB SQL across all sources |
|
|
325
|
+
| `nybble_explain` | Show DuckDB query execution plan |
|
|
326
|
+
| `nybble_profile` | Column statistics (null%, cardinality, min/max/top value) |
|
|
327
|
+
| `nybble_infer_schema` | Infer FK relationships - deterministic, zero LLM calls |
|
|
328
|
+
| `nybble_kg_summary` | Knowledge-graph summary (node/edge counts, entity types) |
|
|
329
|
+
| `nybble_diff` | Schema diff between two sources |
|
|
330
|
+
| `nybble_erd` | Generate interactive ERD HTML, return path |
|
|
331
|
+
| `nybble_kg` | Build and export Knowledge Graph, return paths |
|
|
332
|
+
| `nybble_search_history` | Search the persistent query history log |
|
|
333
|
+
|
|
334
|
+
**How it works - no LLM calls on the Nybble side:**
|
|
335
|
+
|
|
336
|
+
```
|
|
337
|
+
AI agent (LLM decides what to call)
|
|
338
|
+
└─ nybble_load("sales.csv") → resolver._load_file() ← DuckDB CREATE VIEW
|
|
339
|
+
└─ nybble_query("SELECT ...") → arena.query() ← duckdb.execute()
|
|
340
|
+
└─ nybble_infer_schema() → infer_fk.py Tier 1-4 ← column matching + stats
|
|
341
|
+
└─ nybble_kg_summary() → kg._build_graph() ← NetworkX + PageRank
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
The MCP extra adds ~6 transitive packages (`mcp`, `anyio`, `httpx`, `starlette`,
|
|
345
|
+
`uvicorn`, `pydantic`). It is always optional - `pip install datajoins-nybble` stays lean.
|
|
346
|
+
|
|
347
|
+
---
|
|
348
|
+
|
|
349
|
+
## Quick Start
|
|
350
|
+
|
|
351
|
+
### Browse any data source in the terminal
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
nybble browse ./sales_data.csv
|
|
355
|
+
nybble browse ./warehouse.parquet
|
|
356
|
+
nybble browse "postgresql://user:pass@localhost:5432/mydb"
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
### Run SQL across multiple sources at once
|
|
360
|
+
|
|
361
|
+
```bash
|
|
362
|
+
nybble query "SELECT o.id, c.name, o.total
|
|
363
|
+
FROM orders o JOIN customers c ON o.customer_id = c.id
|
|
364
|
+
ORDER BY o.total DESC LIMIT 20" \
|
|
365
|
+
--source orders=orders.csv \
|
|
366
|
+
--source customers=customers.parquet
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
### Generate an ERD from a live database
|
|
370
|
+
|
|
371
|
+
```bash
|
|
372
|
+
# Produces a self-contained HTML file and opens it in your browser
|
|
373
|
+
nybble erd --source "postgresql://user:pass@localhost/mydb"
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
### Build a Knowledge Graph
|
|
377
|
+
|
|
378
|
+
```bash
|
|
379
|
+
# Entity graph from two related files, exported as GraphML + JSON-LD + HTML
|
|
380
|
+
nybble kg \
|
|
381
|
+
--source customers=customers.csv \
|
|
382
|
+
--source orders=orders.parquet
|
|
383
|
+
```
|
|
384
|
+
|
|
385
|
+
### Profile column statistics
|
|
386
|
+
|
|
387
|
+
```bash
|
|
388
|
+
nybble profile orders.csv
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
### Compare two schema versions
|
|
392
|
+
|
|
393
|
+
```bash
|
|
394
|
+
nybble diff schema_v1.csv schema_v2.csv
|
|
395
|
+
nybble diff "sqlite:///old.db" "sqlite:///new.db"
|
|
396
|
+
```
|
|
397
|
+
|
|
398
|
+
### Generate a Markdown data report
|
|
399
|
+
|
|
400
|
+
```bash
|
|
401
|
+
nybble report --source orders.csv --output report.md
|
|
402
|
+
nybble report --source mydb.sqlite --output docs/schema.html
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
### Python API
|
|
406
|
+
|
|
407
|
+
```python
|
|
408
|
+
from nybble import load, query, erd, kg, kg_summary, profile_relation, schema_diff
|
|
409
|
+
|
|
410
|
+
# Load any source - file, JSON, database URI, or HTTP URL
|
|
411
|
+
customers = load("customers.csv")
|
|
412
|
+
orders = load("orders.parquet")
|
|
413
|
+
|
|
414
|
+
# Query across both sources using DuckDB SQL
|
|
415
|
+
result = query("""
|
|
416
|
+
SELECT c.name, COUNT(o.id) AS order_count, SUM(o.total) AS revenue
|
|
417
|
+
FROM customers c
|
|
418
|
+
JOIN orders o ON c.id = o.customer_id
|
|
419
|
+
GROUP BY c.name
|
|
420
|
+
ORDER BY revenue DESC
|
|
421
|
+
""")
|
|
422
|
+
print(result.to_polars())
|
|
423
|
+
|
|
424
|
+
# Profile a table
|
|
425
|
+
for col in profile_relation("customers"):
|
|
426
|
+
print(col.name, col.null_pct, col.distinct)
|
|
427
|
+
|
|
428
|
+
# Compare two schemas
|
|
429
|
+
diff = schema_diff("v1.csv", "v2.csv")
|
|
430
|
+
print(diff.added, diff.removed, diff.changed)
|
|
431
|
+
|
|
432
|
+
# Generate a self-contained ERD HTML file
|
|
433
|
+
erd(output=".nybble/schema.html")
|
|
434
|
+
|
|
435
|
+
# Build a semantic Knowledge Graph
|
|
436
|
+
result = kg()
|
|
437
|
+
print(kg_summary())
|
|
438
|
+
# {'node_count': 1840, 'edge_count': 4122, 'avg_degree': 4.48, ...}
|
|
439
|
+
|
|
440
|
+
# Load from a nybble.yaml config file
|
|
441
|
+
from nybble import load_config, apply_config
|
|
442
|
+
cfg = load_config("nybble.yaml")
|
|
443
|
+
apply_config(cfg) # all sources in config are now loaded
|
|
444
|
+
```
|
|
445
|
+
|
|
446
|
+
---
|
|
447
|
+
|
|
448
|
+
## Features
|
|
449
|
+
|
|
450
|
+
### Universal source loading
|
|
451
|
+
|
|
452
|
+
Nybble's resolver accepts anything you throw at it without configuration:
|
|
453
|
+
|
|
454
|
+
| Source type | Examples |
|
|
455
|
+
|---|---|
|
|
456
|
+
| CSV / TSV | `customers.csv`, `data.tsv` |
|
|
457
|
+
| Parquet | `events.parquet` |
|
|
458
|
+
| JSON / JSONL | `records.json`, `stream.jsonl` |
|
|
459
|
+
| Excel | `report.xlsx`, `legacy.xls` |
|
|
460
|
+
| Avro | `kafka-export.avro` |
|
|
461
|
+
| SQLite | `local.db`, `sqlite:///path/to/db` |
|
|
462
|
+
| PostgreSQL | `postgresql://user:pass@host/db` |
|
|
463
|
+
| MySQL / MariaDB | `mysql://user:pass@host/db` |
|
|
464
|
+
| S3 / GCS / R2 | `s3://bucket/data.parquet`, `gs://bucket/file.csv` |
|
|
465
|
+
| HTTP URL | `https://api.example.com/data.json` |
|
|
466
|
+
| Inline Python | `load([{"id": 1, "name": "Alice"}, ...])` |
|
|
467
|
+
|
|
468
|
+
All sources register as virtual DuckDB tables in a shared, thread-safe, in-process
|
|
469
|
+
session - no intermediate files, no network calls, no copies.
|
|
470
|
+
|
|
471
|
+
### ERD generation with automatic FK inference - *Nybble Sense*
|
|
472
|
+
|
|
473
|
+
When a schema has no declared foreign-key constraints (common in CSV exports, ORMs that
|
|
474
|
+
enforce at the application layer, and cloud-sync snapshots), Nybble's four-tier inference
|
|
475
|
+
engine fills the gap:
|
|
476
|
+
|
|
477
|
+
1. **Tier 1** - Explicit constraints from database introspection (short-circuits here when found)
|
|
478
|
+
2. **Tier 2** - Column name suffix matching (`_id`, `_fk`, `_key`, `_ref`, `_uuid`, `_code`)
|
|
479
|
+
3. **Tier 3** - Column name equality across tables
|
|
480
|
+
4. **Tier 4** - Statistical uniqueness sampling (≥95% unique source → candidate FK)
|
|
481
|
+
|
|
482
|
+
Every inferred FK carries a confidence score. The ERD output is a single self-contained
|
|
483
|
+
HTML file - no CDN dependencies, no server required to view it. Deep-linkable via URL
|
|
484
|
+
parameters (`?highlight=<table>`, `?embed=1` for iframe embedding).
|
|
485
|
+
|
|
486
|
+
### Knowledge Graph - *Nybble GraphAI*
|
|
487
|
+
|
|
488
|
+
Nybble builds a semantic `MultiDiGraph` (NetworkX) from any set of loaded relations.
|
|
489
|
+
Every row becomes a node. Every FK relationship becomes a directed edge. Co-occurring
|
|
490
|
+
values across unlinked tables become inferred cross-source edges.
|
|
491
|
+
|
|
492
|
+
**Export formats:**
|
|
493
|
+
- **GraphML** - importable by Gephi, yEd, NetworkX
|
|
494
|
+
- **JSON-LD** - schema.org vocabulary, consumable by AI/ML pipelines and graph databases
|
|
495
|
+
- **Self-contained HTML** - interactive force-directed visualisation, no server required
|
|
496
|
+
- **SQL/PGQ DDL** - `nybble kg --export pgq` emits DuckDB `CREATE PROPERTY GRAPH` DDL
|
|
497
|
+
|
|
498
|
+
Node-level graph metrics are computed for every entity: PageRank, degree centrality,
|
|
499
|
+
betweenness centrality, and community membership (Louvain algorithm).
|
|
500
|
+
|
|
501
|
+
### SQL Arena
|
|
502
|
+
|
|
503
|
+
Every source loaded by Nybble becomes a virtual DuckDB table, joinable with any other
|
|
504
|
+
source in a single SQL statement. The `query()` function returns a `QueryResult` with
|
|
505
|
+
conversion methods to Polars, Pandas, PyArrow, and plain Python dicts. Queries are
|
|
506
|
+
persisted to `.nybble/queries.json` with `save_query()`; query history is persisted to
|
|
507
|
+
`.nybble/history.jsonl`.
|
|
508
|
+
|
|
509
|
+
The interactive REPL (`nybble repl`) supports meta-commands: `.tables` to list sources,
|
|
510
|
+
`.save <name> <sql>` to persist a query, `.run <name>` to replay it.
|
|
511
|
+
|
|
512
|
+
### Terminal browser
|
|
513
|
+
|
|
514
|
+
`nybble browse` renders any source as a full-terminal data grid powered by Rich:
|
|
515
|
+
keyboard navigation (`↑↓`, `PgUp/PgDn`, `g`/`G`), a live filter bar (`/`), column
|
|
516
|
+
sorting, and a status bar showing source metadata, row count, active page, and current
|
|
517
|
+
filters.
|
|
518
|
+
|
|
519
|
+
---
|
|
520
|
+
|
|
521
|
+
## Repository Layout
|
|
522
|
+
|
|
523
|
+
```
|
|
524
|
+
nybble/ ← repository root
|
|
525
|
+
│
|
|
526
|
+
├── nybble/ ← pip package (pip install datajoins-nybble)
|
|
527
|
+
│ ├── resolver.py ← universal source loader, DuckDB session
|
|
528
|
+
│ ├── browse.py ← Rich terminal browser
|
|
529
|
+
│ ├── arena.py ← SQL REPL, query API, history
|
|
530
|
+
│ ├── erd.py ← ERD HTML generator
|
|
531
|
+
│ ├── infer_fk.py ← Nybble Sense: FK inference engine
|
|
532
|
+
│ ├── kg.py ← Nybble GraphAI: Knowledge Graph
|
|
533
|
+
│ ├── config.py ← nybble.yaml declarative config loader
|
|
534
|
+
│ ├── diff.py ← schema diff between two sources
|
|
535
|
+
│ ├── profiler.py ← column-level statistics profiler
|
|
536
|
+
│ ├── report.py ← Markdown report builder
|
|
537
|
+
│ ├── highlight_store.py ← persistent cell-highlight rules
|
|
538
|
+
│ ├── mcp_server.py ← MCP server (AI agent integration)
|
|
539
|
+
│ ├── cli.py ← `nybble` CLI entry point
|
|
540
|
+
│ └── templates/ ← self-contained Jinja2 HTML templates
|
|
541
|
+
│
|
|
542
|
+
├── deploy/ ← Dockerfile, docker-compose.yml, nginx.conf
|
|
543
|
+
├── tests/ ← pytest suite
|
|
544
|
+
│
|
|
545
|
+
├── nybble.yaml.example ← fully annotated workspace config reference
|
|
546
|
+
├── CHANGELOG.md
|
|
547
|
+
├── LICENSE
|
|
548
|
+
├── pyproject.toml
|
|
549
|
+
└── README.md
|
|
550
|
+
```
|
|
551
|
+
|
|
552
|
+
---
|
|
553
|
+
|
|
554
|
+
## CLI Reference
|
|
555
|
+
|
|
556
|
+
```
|
|
557
|
+
nybble [OPTIONS] COMMAND [ARGS]...
|
|
558
|
+
|
|
559
|
+
Commands:
|
|
560
|
+
browse Launch interactive terminal browser for any data source
|
|
561
|
+
diff Compare schemas of two sources (added / removed / type-changed columns)
|
|
562
|
+
erd Generate a self-contained interactive ERD HTML file
|
|
563
|
+
explain Show DuckDB query execution plan for a SQL statement
|
|
564
|
+
history Show recent query history, optionally filtered by search term
|
|
565
|
+
info Print schema, row count, source type, and sample rows for a source
|
|
566
|
+
kg Build and export a semantic Knowledge Graph
|
|
567
|
+
lineage Build an SQL query lineage DAG and render it as an HTML graph
|
|
568
|
+
profile Print column statistics (null%, cardinality, min, max, top value)
|
|
569
|
+
query Execute SQL across all loaded sources
|
|
570
|
+
repl Launch the interactive SQL REPL
|
|
571
|
+
report Generate a Markdown or HTML data report: schema, samples, FK joins
|
|
572
|
+
serve Start the Nybble web UI (Flask) without Docker
|
|
573
|
+
share Share an ERD or Knowledge Graph as a public URL via GitHub Gist
|
|
574
|
+
|
|
575
|
+
Options:
|
|
576
|
+
--version Show version and exit.
|
|
577
|
+
--help Show help and exit.
|
|
578
|
+
```
|
|
579
|
+
|
|
580
|
+
Every multi-source command (`query`, `erd`, `kg`, `explain`, `report`, `repl`, `serve`, `lineage`)
|
|
581
|
+
accepts:
|
|
582
|
+
|
|
583
|
+
- `--source name=path` - pre-load a source (may be repeated)
|
|
584
|
+
- `--config / -c PATH` - path to a `nybble.yaml` workspace config file; if omitted,
|
|
585
|
+
`nybble.yaml` in the current directory is loaded automatically when present
|
|
586
|
+
|
|
587
|
+
Run `nybble <command> --help` for full option details.
|
|
588
|
+
|
|
589
|
+
---
|
|
590
|
+
|
|
591
|
+
## Self-Hosting with Docker
|
|
592
|
+
|
|
593
|
+
Nybble ships a full Flask/nginx server for browser-based access:
|
|
594
|
+
|
|
595
|
+
```bash
|
|
596
|
+
# Mode C - upload files and connect databases via the browser UI (default)
|
|
597
|
+
docker compose -f deploy/docker-compose.yml up
|
|
598
|
+
|
|
599
|
+
# Mode B - pre-load a database at startup
|
|
600
|
+
DB_FILE=/absolute/path/to/my.db docker compose -f deploy/docker-compose.yml up
|
|
601
|
+
|
|
602
|
+
# Mode A - bake a database into a custom image at build time
|
|
603
|
+
# Place input.db in the repository root, then:
|
|
604
|
+
docker build -f deploy/Dockerfile -t my-org/nybble-analytics .
|
|
605
|
+
docker run -p 7700:80 my-org/nybble-analytics
|
|
606
|
+
```
|
|
607
|
+
|
|
608
|
+
Open [http://localhost:7700](http://localhost:7700) for the full browser UI.
|
|
609
|
+
|
|
610
|
+
Or run without Docker after `pip install "datajoins-nybble[dev]"`:
|
|
611
|
+
|
|
612
|
+
```bash
|
|
613
|
+
nybble serve --port 7700 --source mydb.sqlite
|
|
614
|
+
```
|
|
615
|
+
|
|
616
|
+
---
|
|
617
|
+
|
|
618
|
+
## Contributing
|
|
619
|
+
|
|
620
|
+
Contributions are welcome. Before opening a pull request for a significant change,
|
|
621
|
+
please open an issue first so the approach can be discussed.
|
|
622
|
+
|
|
623
|
+
```bash
|
|
624
|
+
# Fork and clone
|
|
625
|
+
git clone https://github.com/your-username/nybble.git
|
|
626
|
+
cd nybble
|
|
627
|
+
|
|
628
|
+
# Install in editable mode with dev dependencies
|
|
629
|
+
pip install -e ".[dev]"
|
|
630
|
+
|
|
631
|
+
# Run the test suite
|
|
632
|
+
pytest tests/
|
|
633
|
+
|
|
634
|
+
# Run linting
|
|
635
|
+
pip install ruff
|
|
636
|
+
ruff check nybble/ tests/
|
|
637
|
+
```
|
|
638
|
+
|
|
639
|
+
**Commit message format:** `type(scope): description`
|
|
640
|
+
Types: `feat`, `fix`, `docs`, `refactor`, `test`, `chore`
|
|
641
|
+
|
|
642
|
+
Please ensure:
|
|
643
|
+
- [ ] All existing tests pass (`pytest tests/`)
|
|
644
|
+
- [ ] New behaviour is covered by tests
|
|
645
|
+
- [ ] `ruff check` passes with no new warnings
|
|
646
|
+
- [ ] Public API changes are reflected in `nybble/__init__.py`
|
|
647
|
+
|
|
648
|
+
This project follows the [Contributor Covenant](https://www.contributor-covenant.org/)
|
|
649
|
+
Code of Conduct.
|
|
650
|
+
|
|
651
|
+
---
|
|
652
|
+
|
|
653
|
+
## Roadmap
|
|
654
|
+
|
|
655
|
+
**Shipped (v1.0.0):**
|
|
656
|
+
- [x] `nybble browse` - full-terminal interactive data grid
|
|
657
|
+
- [x] `nybble query` - cross-source DuckDB SQL with `table` / `csv` / `json` output
|
|
658
|
+
- [x] `nybble erd` - self-contained interactive ERD HTML with 4-tier FK inference
|
|
659
|
+
- [x] `nybble kg` - semantic Knowledge Graph with GraphML, JSON-LD, HTML, SQL/PGQ export
|
|
660
|
+
- [x] `nybble kg search` - substring and semantic node search (sentence-transformers)
|
|
661
|
+
- [x] `nybble kg pgq` - DuckDB `CREATE PROPERTY GRAPH` DDL export
|
|
662
|
+
- [x] `nybble diff` - structural schema diff between two sources
|
|
663
|
+
- [x] `nybble report` - Markdown + HTML data report builder
|
|
664
|
+
- [x] `nybble serve` - start the Flask UI directly from the CLI
|
|
665
|
+
- [x] `nybble profile` - column statistics panel (null%, cardinality, min/max/top)
|
|
666
|
+
- [x] `nybble info` - schema, row count, source type, and sample rows
|
|
667
|
+
- [x] `nybble explain` - DuckDB query execution plan
|
|
668
|
+
- [x] `nybble history` - persistent query history log
|
|
669
|
+
- [x] `nybble lineage` - SQL lineage DAG rendered as HTML graph
|
|
670
|
+
- [x] `nybble repl` - interactive SQL REPL with `.save` / `.run` meta-commands
|
|
671
|
+
- [x] `nybble share` - publish ERD/KG as a public URL via GitHub Gist
|
|
672
|
+
- [x] `nybble.yaml` declarative workspace config (files, databases, HTTP APIs)
|
|
673
|
+
- [x] MCP server (`nybble-mcp`) - AI agent integration (Claude, Cursor, VS Code)
|
|
674
|
+
- [x] Python API - `load()`, `query()`, `erd()`, `kg()`, `kg_summary()`, `schema_diff()`
|
|
675
|
+
|
|
676
|
+
**Upcoming:**
|
|
677
|
+
- [ ] Native Snowflake, BigQuery, and ClickHouse source support
|
|
678
|
+
- [ ] `nybble dashboard` - pin saved queries as auto-refreshing KPI tiles
|
|
679
|
+
- [ ] DuckDB-WASM in-browser query execution for the web UI
|
|
680
|
+
- [ ] `nybble-sense` and `nybble-graphai` as independently versioned sub-packages
|
|
681
|
+
|
|
682
|
+
---
|
|
683
|
+
|
|
684
|
+
## DataJoins Ecosystem
|
|
685
|
+
|
|
686
|
+
Nybble is part of the [DataJoins](https://datajoins.com) open-source portfolio.
|
|
687
|
+
Visit [datajoins.com](https://datajoins.com) for the full ecosystem.
|
|
688
|
+
|
|
689
|
+
---
|
|
690
|
+
|
|
691
|
+
## License
|
|
692
|
+
|
|
693
|
+
Apache License 2.0 - Copyright © 2026 [DataJoins](https://datajoins.com)
|
|
694
|
+
|
|
695
|
+
See [LICENSE](LICENSE) for the full text.
|