datajoins-nybble 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. datajoins_nybble-1.0.0/CHANGELOG.md +369 -0
  2. datajoins_nybble-1.0.0/LICENSE +193 -0
  3. datajoins_nybble-1.0.0/MANIFEST.in +27 -0
  4. datajoins_nybble-1.0.0/PKG-INFO +695 -0
  5. datajoins_nybble-1.0.0/README.md +607 -0
  6. datajoins_nybble-1.0.0/datajoins_nybble.egg-info/SOURCES.txt +48 -0
  7. datajoins_nybble-1.0.0/nybble/__init__.py +133 -0
  8. datajoins_nybble-1.0.0/nybble/arena.py +607 -0
  9. datajoins_nybble-1.0.0/nybble/browse.py +534 -0
  10. datajoins_nybble-1.0.0/nybble/cli.py +1803 -0
  11. datajoins_nybble-1.0.0/nybble/config.py +368 -0
  12. datajoins_nybble-1.0.0/nybble/core/__init__.py +12 -0
  13. datajoins_nybble-1.0.0/nybble/core/detect_format.py +349 -0
  14. datajoins_nybble-1.0.0/nybble/core/erd_data.py +245 -0
  15. datajoins_nybble-1.0.0/nybble/core/import_to_sqlite.py +551 -0
  16. datajoins_nybble-1.0.0/nybble/core/infer_fk.py +458 -0
  17. datajoins_nybble-1.0.0/nybble/core/kg_builder.py +367 -0
  18. datajoins_nybble-1.0.0/nybble/core/schema.py +85 -0
  19. datajoins_nybble-1.0.0/nybble/diff.py +139 -0
  20. datajoins_nybble-1.0.0/nybble/erd.py +263 -0
  21. datajoins_nybble-1.0.0/nybble/highlight_store.py +112 -0
  22. datajoins_nybble-1.0.0/nybble/infer_fk.py +228 -0
  23. datajoins_nybble-1.0.0/nybble/kg.py +1392 -0
  24. datajoins_nybble-1.0.0/nybble/lineage.py +1013 -0
  25. datajoins_nybble-1.0.0/nybble/mcp_server.py +2478 -0
  26. datajoins_nybble-1.0.0/nybble/profiler.py +180 -0
  27. datajoins_nybble-1.0.0/nybble/report.py +754 -0
  28. datajoins_nybble-1.0.0/nybble/resolver.py +1450 -0
  29. datajoins_nybble-1.0.0/nybble/serve.py +416 -0
  30. datajoins_nybble-1.0.0/nybble/share.py +404 -0
  31. datajoins_nybble-1.0.0/nybble/templates/erd.html.jinja +1052 -0
  32. datajoins_nybble-1.0.0/nybble/templates/kg.html.jinja +561 -0
  33. datajoins_nybble-1.0.0/pyproject.toml +139 -0
  34. datajoins_nybble-1.0.0/setup.cfg +4 -0
  35. datajoins_nybble-1.0.0/tests/test_api_server_security.py +200 -0
  36. datajoins_nybble-1.0.0/tests/test_arena.py +192 -0
  37. datajoins_nybble-1.0.0/tests/test_config_limitations.py +440 -0
  38. datajoins_nybble-1.0.0/tests/test_core.py +482 -0
  39. datajoins_nybble-1.0.0/tests/test_erd.py +342 -0
  40. datajoins_nybble-1.0.0/tests/test_kg.py +4289 -0
  41. datajoins_nybble-1.0.0/tests/test_lineage.py +557 -0
  42. datajoins_nybble-1.0.0/tests/test_new_features.py +917 -0
  43. datajoins_nybble-1.0.0/tests/test_p3_2_connectors.py +536 -0
  44. datajoins_nybble-1.0.0/tests/test_pgq_ddl.py +552 -0
  45. datajoins_nybble-1.0.0/tests/test_report.py +542 -0
  46. datajoins_nybble-1.0.0/tests/test_resolver.py +448 -0
  47. datajoins_nybble-1.0.0/tests/test_semantic_search.py +641 -0
  48. datajoins_nybble-1.0.0/tests/test_serve.py +632 -0
  49. datajoins_nybble-1.0.0/tests/test_serve_waitress.py +148 -0
  50. datajoins_nybble-1.0.0/tests/test_share.py +693 -0
  51. datajoins_nybble-1.0.0/tests/test_ui_hints.py +364 -0
@@ -0,0 +1,369 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ---
9
+
10
+ ## [4.0.0] - 2026-07-11
11
+
12
+ ### Added
13
+
14
+ #### Phase 5a - Row counts on ERD nodes
15
+
16
+ - **`api_server.py`** - new `GET /api/rowcounts/<key>` endpoint: runs `SELECT COUNT(*)`
17
+ for every table in the schema's SQLite database in a single connection pass and returns
18
+ a `{tableName: count}` map. Returns `{}` gracefully if no queryable DB exists yet (e.g.
19
+ live-connection schemas where the pipeline has not created a local SQLite file).
20
+ - **`templates/erd.html`** - row counts loaded **lazily** via a `fetch()` call after the
21
+ layout completes (non-blocking, never slows layout). Two display surfaces updated:
22
+ - **Node header cards** - count rendered in the top-right corner of each card header
23
+ in dimmed monospace text; numbers ≥1 000 are formatted as `1.2k`, ≥1 000 000 as `1.4M`.
24
+ - **Sidebar table list** - column-count badge replaced with `N rows` once counts arrive;
25
+ full count shown as `title` tooltip on hover. Falls back silently if the fetch fails.
26
+
27
+ #### Phase 5b - Column profiling panel in Browse view
28
+
29
+ - **`api_server.py`** - new `GET /api/profile/<key>/<table>/<col>` endpoint: executes two
30
+ SQLite aggregation queries per call and returns `{total, null_count, distinct_count, min,
31
+ max, avg, top_values[{val,count}×10]}`. Validates table + column existence before
32
+ querying; read-only SQLite URI connection. Zero new dependencies.
33
+ - **`templates/browse.html`** - **Column profile pane** (slide-in from right, z-index 310,
34
+ above the cell pane):
35
+ - Triggered from the `⠿ Column profile` item in the column header ⠿ popover menu.
36
+ - Displays a 2×3 stat grid: Total rows, Null count (red if > 0), Distinct, Avg, Min, Max.
37
+ - Null-presence bar: a thin red bar showing the null percentage at a glance.
38
+ - Top-values frequency chart: pure CSS horizontal bars (no Chart.js) for the top 10
39
+ values ordered by row count. Values truncated to 22 chars; NULL shown as a pill.
40
+ - Loading spinner while the API call is in flight; inline error message on failure.
41
+ - Closed by `✕` button or `Escape` key (checked before cell pane on Escape).
42
+
43
+ #### Phase 5c - Server-side saved queries
44
+
45
+ - **`api_server.py`** - four new REST endpoints for persistent saved-query storage:
46
+ - `GET /api/queries/<key>` - list all saved queries (array of `{id, name, sql, created}`)
47
+ - `POST /api/queries/<key>` - create a new saved query; body: `{name, sql}`
48
+ - `PATCH /api/queries/<key>/<id>` - update `name` and/or `sql` of an existing query
49
+ - `DELETE /api/queries/<key>/<id>` - delete a saved query
50
+ Storage: a simple JSON array in `/output/erd/<key>/queries.json` alongside `schema.json`.
51
+ Helper functions `_queries_path / _load_queries / _save_queries` added.
52
+ - **`templates/query.html`** - saved-queries sidebar upgraded:
53
+ - **Server is the source of truth**: `_fetchSaved()` loads from `GET /api/queries/<key>`
54
+ on page load and keeps `localStorage` in sync as an offline read cache.
55
+ - **Backward-compatible migration**: existing localStorage entries (pre-Phase-5c, which
56
+ had no `id` field) are automatically shaped into the new `{id, name, sql}` format on
57
+ first load so no user data is lost.
58
+ - **Optimistic deletion**: `deleteSaved(qid)` removes from local state immediately, then
59
+ fires `DELETE /api/queries/<key>/<id>` asynchronously (best-effort).
60
+ - **Offline fallback**: `saveCurrentQuery()` posts to the server; if the request fails,
61
+ saves to localStorage only and shows `"Query saved (offline)"` toast.
62
+
63
+ ### Tests
64
+
65
+ - **`_smoke_test.py`** - 6 new Phase 5 tests added; total now **30/30 passing**:
66
+ - `api_server: /api/rowcounts route present`
67
+ - `api_server: /api/profile route present`
68
+ - `api_server: saved queries CRUD routes present`
69
+ - `templates/erd.html: row counts JS present`
70
+ - `templates/browse.html: column profile pane present`
71
+ - `templates/query.html: server-side saved queries present`
72
+
73
+ ---
74
+
75
+ ## [3.1.0] - 2026-07-09
76
+
77
+ ### Added
78
+
79
+ #### Phase 4b - Flat-file → SQLite data loader (CSV, JSON, JSONL, XLSX, Parquet)
80
+
81
+ - **`flat_file_to_sqlite.py`** *(new)* - zero-extra-dependency data loader for all five
82
+ flat-file formats. Runs after schema extraction in the pipeline and writes a queryable
83
+ `.db` file into the output dir, making Browse and Query available for every file type:
84
+
85
+ | Format | Source | Loader behaviour |
86
+ |---|---|---|
87
+ | **CSV** | stdlib `csv` | Auto-detects delimiter; empty cells → NULL |
88
+ | **JSON** | stdlib `json` | Handles `[…]` array and `{key:[…]}` envelope; list/dict values → JSON text |
89
+ | **JSONL** | stdlib `json` | One record per line; schema widened on first encounter of new keys |
90
+ | **XLSX** | `openpyxl` (already in requirements) | All sheets → individual tables; `datetime/date/time` objects coerced to ISO strings |
91
+ | **Parquet** | `pyarrow` (already in requirements) | Streams row batches; Arrow list/struct → JSON text |
92
+
93
+ All loaders: 500-row `executemany` batches (memory-bounded); `WAL` mode for write safety;
94
+ empty-value strings from CSV coerced to NULL; bool → 0/1; nested objects → JSON text.
95
+
96
+ - **`run_pipeline.py`** - Step 2c: after schema extraction for `csv`, `json`, `jsonl`,
97
+ `xlsx`, `parquet` formats, calls `flat_file_to_sqlite.load_flat_file()` and logs row
98
+ counts to the live pipeline log stream; gracefully no-ops on errors without failing the
99
+ schema step.
100
+
101
+ ### Result
102
+
103
+ Every file type uploadable to the ERD system now produces a fully queryable SQLite
104
+ database automatically:
105
+
106
+ | Uploaded file | ERD | Browse rows | SQL Query |
107
+ |---|---|---|---|
108
+ | `.db` / `.sqlite` | ✅ | ✅ | ✅ |
109
+ | `.sql` / `.sql.gz` (PG or MySQL) | ✅ | ✅ (Phase 4a) | ✅ (Phase 4a) |
110
+ | `.csv` | ✅ | ✅ **(Phase 4b)** | ✅ **(Phase 4b)** |
111
+ | `.json` / `.jsonl` | ✅ | ✅ **(Phase 4b)** | ✅ **(Phase 4b)** |
112
+ | `.xlsx` | ✅ | ✅ **(Phase 4b)** | ✅ **(Phase 4b)** |
113
+ | `.parquet` | ✅ | ✅ **(Phase 4b)** | ✅ **(Phase 4b)** |
114
+
115
+ ---
116
+
117
+ ## [3.0.0] - 2026-07-09
118
+
119
+ ### Added
120
+
121
+ #### Phase 4a - SQL dump → SQLite auto-import (Browse & Query now work for `.sql` / `.sql.gz`)
122
+
123
+ - **`import_to_sqlite.py`** *(new)* - zero-dependency (stdlib-only) SQL dump importer:
124
+ - Reads PostgreSQL and MySQL dumps, optionally gzip-compressed
125
+ - **DDL translation layer**: maps 50+ PostgreSQL/MySQL types to their SQLite equivalents
126
+ (`SERIAL/BIGSERIAL → INTEGER`, `DOUBLE PRECISION/NUMERIC → REAL`,
127
+ `BOOLEAN/BOOL → INTEGER`, `TIMESTAMP/DATE/TIME → TEXT`, `BYTEA → BLOB`,
128
+ `CHARACTER VARYING/VARCHAR → TEXT`, `JSONB/JSON/UUID/INET/… → TEXT`,
129
+ MySQL `ENUM/SET → TEXT`, `DECIMAL → REAL`)
130
+ - Strips SQLite-unsupported syntax: `schema.` prefixes, `COLLATE`, `TABLESPACE`,
131
+ `STORAGE`, `WITH (…)`, `INHERITS`, `DEFERRABLE`, `AUTO_INCREMENT`, `ENGINE=`,
132
+ `DEFAULT CHARSET=`, PostgreSQL type casts (`::type`), `GENERATED … AS IDENTITY`,
133
+ `DEFAULT nextval(…)`
134
+ - **PostgreSQL `COPY … FROM stdin` parser**: loads tab-delimited data blocks including
135
+ `\N` → NULL and backslash escape sequences (`\t`, `\n`, `\\`)
136
+ - **`INSERT … VALUES` parser**: handles multi-row `(…),(…)` syntax, quoted strings
137
+ with `''` escapes, NULL, integer and float literals - works for both PG and MySQL
138
+ - Writes a queryable `.db` file into the output dir alongside `schema.json`
139
+ - **`run_pipeline.py`** - after extracting the schema from a `postgresql` or `mysql` dump,
140
+ automatically calls `import_to_sqlite.import_dump()` and logs row counts to the live
141
+ pipeline log stream; gracefully skips if the dump is schema-only (no data rows)
142
+ - **`api_server.py`** - new `_find_sqlite_db(key)` helper searches in two locations:
143
+ 1. `UPLOAD_DIR/<key>/` for an uploaded `.db`/`.sqlite`/`.sqlite3` file (existing behaviour)
144
+ 2. `OUTPUT_DIR/<key>/` for a pipeline-generated `.db` file *(new - Phase 4a)*
145
+ Both `api_rows()` and `api_query()` now use this helper, so **Browse and Query work
146
+ automatically for any SQL dump uploaded via the UI - no manual conversion needed**
147
+
148
+ ### Fixed
149
+
150
+ - `api_rows()`: improved "no queryable source" error message - now explains that flat files
151
+ (CSV/JSON/XLSX/Parquet) have no row data and directs the user to re-process SQL dumps
152
+ - `api_query()`: same error path now uses `_find_sqlite_db()` and reports a consistent message
153
+
154
+ ---
155
+
156
+ ## [2.2.0] - 2026-07-09
157
+
158
+ ### Added
159
+
160
+ #### Phase 3 - Semantic badges, cell detail pane, CTE builder, Parquet extractor
161
+
162
+ - **`extract_parquet.py`** *(new)* - schema extraction from `.parquet` files via `pyarrow`:
163
+ Arrow type → SQL type mapping (`INT8`-`INT64` → `INTEGER`, `FLOAT`/`DOUBLE` → `REAL`,
164
+ `DATE32` → `DATE`, `TIMESTAMP` → `TIMESTAMP`, `TIME` → `TIME`, `BOOL` → `BOOLEAN`,
165
+ `BINARY`/`LARGE_BINARY` → `BLOB`, all others → `TEXT`); first integer column heuristic
166
+ marks the primary key.
167
+ - **`run_pipeline.py`** - `.parquet` format now dispatched to `extract_parquet.extract()`.
168
+ - **`requirements.txt`** - `pyarrow>=14.0.0` added for Parquet support.
169
+ - **`templates/browse.html`** - **semantic type badges** appear inline for recognised cell
170
+ patterns: `@ email`, `🔗 url`, `🖼 img`, `📞 phone`, `TRUE/FALSE` (bool), `uuid`, `ip`;
171
+ **cell detail pane** slides in from the right on any cell click - shows full value,
172
+ JSON pretty-print for objects, character count, semantic hint, Copy/Filter/Query actions;
173
+ **Filter action** sets `col = value` filter instantly; **Query action** drills through
174
+ to the Query view with a pre-filled `SELECT … WHERE col = 'val'` statement.
175
+ - **`templates/query.html`** - same **semantic type badges** and **cell detail pane**
176
+ as browse view (Copy + Query actions; no Filter since there is no table state);
177
+ **CTE builder** - "✛ CTE" button in results toolbar wraps the current SQL as
178
+ `WITH _result AS (…) SELECT * FROM _result` so the user can immediately layer a
179
+ second query on top; **Escape** key closes the cell pane before clearing other panels.
180
+
181
+ ---
182
+
183
+ ## [2.1.0] - 2026-07-08
184
+
185
+ ### Added
186
+
187
+ #### Phase 2 - Excel support, viz tabs, stale indicator, history timestamps, date presets
188
+
189
+ - **`extract_xlsx.py`** - schema extraction from `.xlsx` workbooks via `openpyxl`:
190
+ each sheet becomes a table, column types inferred from cell values
191
+ (`BOOLEAN / TIMESTAMP / DATE / TIME / INTEGER / REAL / TEXT`), first integer column
192
+ marked as primary key, sanitized sheet names used as table names.
193
+ - **`run_pipeline.py`** - `.xlsx` format now dispatched to `extract_xlsx.extract()`;
194
+ Parquet error message updated to suggest `pip install pyarrow`.
195
+ - **`requirements.txt`** - `openpyxl>=3.1.0` added for Phase 2 Excel support.
196
+ - **`api_server.py`** - `/api/rows?_format=xlsx` returns a true `.xlsx` file via
197
+ `openpyxl` write-only workbook; graceful `501` if openpyxl is not installed.
198
+ - **`templates/query.html`** - multiple viz tabs (Bar / Line / Scatter / Pivot);
199
+ stale-result amber banner when SQL changes after last run; history entries now
200
+ include timestamps with human-relative display ("just now", "5m ago", "3h ago",
201
+ "Jan 5"); "Clear all" button in history panel; `↓ XLSX` export button in results toolbar.
202
+ - **`templates/browse.html`** - relative date filter preset chips (Today / Yesterday /
203
+ Last 7 days / Last 30 days / This month / This year / Clear) auto-shown when a date
204
+ column is detected; `↓ XLSX` export button in topbar routed through `/api/rows?_format=xlsx`.
205
+
206
+ ### Changed
207
+
208
+ - **`templates/query.html`** - "Quick chart" `<details>` replaced with persistent tabbed
209
+ viz panel; chart instances keyed by type so all four tabs retain their own state.
210
+ - History deduplication now compares `sql` field instead of full entry; old plain-string
211
+ history entries (from v2.0) are still rendered gracefully.
212
+
213
+ ---
214
+
215
+ ## [2.0.0] - 2026-07-02
216
+
217
+ ### Added
218
+
219
+ #### Phase 0 - New format extractors (stdlib only, no new pip deps)
220
+ - **`extract_csv.py`** - schema inference from CSV / TSV files: BOM-safe (`utf-8-sig`),
221
+ auto-delimiter detection via `csv.Sniffer`, samples up to 1 000 rows, infers
222
+ `INTEGER / REAL / DATE / TEXT` per column, marks first non-empty column as primary key.
223
+ - **`extract_json.py`** - schema inference from JSON and JSONL/NDJSON files: handles list root,
224
+ dict-with-list root, and plain dict; JSONL streams first 500 lines; same type-inference rules
225
+ as CSV extractor; uses `utf-8` with `errors="replace"`.
226
+ - **`detect_format.py`** - extended with five new format tokens: `"csv"`, `"json"`, `"jsonl"`,
227
+ `"xlsx"` (ZIP magic `PK\x03\x04`), `"parquet"` (`PAR1` magic); content-sniff fallback for
228
+ JSON files without a recognised extension; all existing `sqlite / postgresql / mysql / unknown`
229
+ tokens unchanged.
230
+ - **`run_pipeline.py`** - dispatch for `csv` and `json/jsonl` tokens wired to new extractors;
231
+ graceful `sys.exit(1)` with an install hint for `xlsx` / `parquet` (requires optional deps);
232
+ updated `--help` text for `input` argument.
233
+
234
+ #### Phase 1 - Table browser, SQL editor, search, metadata
235
+
236
+ ##### API routes (`api_server.py`)
237
+ - **`GET /browse/<key>`** - serves the new table browser page.
238
+ - **`GET /query/<key>`** - serves the new SQL query editor page.
239
+ - **`GET /api/search?q=`** - cross-schema full-text search over table and column names
240
+ (returns up to 100 results, restricted to `status == "ready"` schemas).
241
+ - **`GET /api/metadata/<key>`** - returns the JSON metadata sidecar for a schema.
242
+ - **`PATCH /api/metadata/<key>`** - deep-merges provided fields into the metadata sidecar.
243
+ - **`GET /api/rows/<key>/<table>`** - paginated row data with offset pagination (`page` / `page_size`),
244
+ sort (`_sort` / `-col`), column filters (`col__op=val`, ops: `exact contains gt gte lt lte neq`),
245
+ and CSV download (`_format=csv`); supports both live connstring and SQLite file sources.
246
+ - **`POST /api/query/<key>`** - read-only SQL execution (`SELECT / WITH / EXPLAIN / PRAGMA`);
247
+ wraps queries in a LIMIT subquery; sets PostgreSQL `statement_timeout = 10s`; returns
248
+ `columns`, `rows`, `row_count`, `truncated`, `elapsed_ms`.
249
+ - **Metadata sidecar** - `_finalize()` now creates `/output/erd/<key>/metadata.json` on first
250
+ pipeline completion; helper functions `_meta_path / _load_meta / _save_meta` added.
251
+ - **`GET /erd/<key>`** - now passes `key=key` to the Jinja template (required for new nav links).
252
+
253
+ ##### Table browser (`templates/browse.html`) - NEW FILE
254
+ - Full-page dark-theme table browser matching the ERD viewer design language.
255
+ - Sidebar table list with column counts; click to load data.
256
+ - Paginated data grid fetched from `/api/rows/`; sticky header; right-aligned numerics;
257
+ NULL pills; JSON cell expand via `<details>`.
258
+ - Column header sort buttons (↑/↓) and popover menu (sort / focus filter).
259
+ - Filter row with debounced inputs; operator prefixes: `>`, `<`, `>=`, `<=`, `=`, `!=`, bare text → `contains`.
260
+ - Auto value formatting: USD amounts, percentages, ISO dates.
261
+ - CSV download and Copy-as-Markdown-table export.
262
+ - Quick chart (`<details>` section): Chart.js 4 from CDN, X/Y column pickers, bar/line/scatter.
263
+ - Row permalink (copies `/browse/<key>?table=…&col=val` to clipboard).
264
+ - Keyboard: `Escape` clears all filters.
265
+
266
+ ##### SQL query editor (`templates/query.html`) - NEW FILE
267
+ - Full-page dark-theme SQL editor; CodeMirror 5 from CDN with `monokai` theme, SQL mode,
268
+ `Ctrl+Enter` / `Cmd+Enter` run binding, and SQL hint autocomplete (`sql-hint` addon).
269
+ - Schema sidebar: table list with column tree (click table → inserts `SELECT * … LIMIT 100`;
270
+ click column → inserts at cursor).
271
+ - Resizable split pane (editor top / results bottom); ratio persisted in `localStorage`.
272
+ - Parameter widgets: scans SQL for `{{varname}}` and renders `<input>` fields inline.
273
+ - EXPLAIN button: prepends `EXPLAIN` and shows result in a collapsible panel.
274
+ - Error panel with Escape-to-dismiss.
275
+ - Result grid: sticky header, NULL pills, JSON cell expand, conditional row highlighting
276
+ (hsl scale on first numeric column), column type badges.
277
+ - Export: CSV Blob download, JSON Blob download, Copy as Markdown.
278
+ - Query history: `localStorage` ring buffer (50 entries, keyed per schema); clickable history panel.
279
+ - Saved queries: `localStorage` save/load/delete; listed in sidebar.
280
+ - Share URL: encodes SQL as `btoa()` in `?sql=` param; restores on page load.
281
+ - Quick chart: Chart.js, auto-update column pickers after each query run.
282
+ - Keyboard shortcuts: `Ctrl/Cmd+Enter` run, `Escape` dismiss, `?` shortcuts dialog, `Ctrl+H` history.
283
+
284
+ ##### Index UI (`templates/index.html`) - EXTENDED
285
+ - **Cmd+K / Ctrl+K command palette**: full-screen overlay; searches schema cards, dispatches
286
+ to `GET /api/search` for table/column results; arrow-key navigation; Enter activates.
287
+ - **Global search bar** in topbar: clicking opens the palette.
288
+ - **Cross-view nav buttons** on schema cards (server-rendered and JS-generated):
289
+ "Browse →" and "Query →" alongside existing "View ERD →" (visible when `status == "ready"`).
290
+ - **Toast notification system**: fixed bottom-right `showToast(msg, kind)` global function;
291
+ auto-dismisses at 3 s; `kind ∈ {success, error, info}`; triggered on upload completion.
292
+ - **File type hints** updated: upload zone now lists `.csv · .json · .jsonl`;
293
+ `<input accept>` extended to include `.csv, .json, .jsonl, .ndjson`.
294
+
295
+ ##### ERD viewer (`templates/erd.html`) - EXTENDED
296
+ - **PNG export button** ("⬇ PNG") in topbar: `cy.png({output:'blob', scale:2, bg:'#141616'})`.
297
+ - **SVG export button** ("⬇ SVG") in topbar: `cy.svg({scale:1, bg:'#141616', full:true})`.
298
+ - **Cross-view nav links** ("Browse →" / "Query →") in topbar, before zoom controls.
299
+ - **Deep-link hash URL**: `focusTable()` now calls `history.replaceState` to update
300
+ `#table=<name>`; `boot()` reads `location.hash` on startup and auto-focuses the named table.
301
+ - **`e` keyboard shortcut**: pressing `e` (not in an input) triggers PNG export.
302
+
303
+ ### Changed
304
+ - `run_pipeline.py` help text for `input` argument updated to list all supported extensions.
305
+ - `api_server.py` route comment block updated to document new routes and storage paths.
306
+
307
+ ### Notes
308
+ - **No new pip dependencies** for Phase 0 or Phase 1. All CSV/JSON extraction uses Python
309
+ stdlib (`csv`, `json`, `re`, `pathlib`). Chart.js and CodeMirror 5 are loaded via CDN
310
+ `<script>` tags. No Node.js, no npm, no build step.
311
+
312
+ ---
313
+
314
+ ## [1.0.0] - 2026-06-07
315
+
316
+ ### Added
317
+
318
+ #### Core pipeline
319
+ - Automatic schema introspection for **SQLite** (via `PRAGMA` - zero extra deps),
320
+ **PostgreSQL** dumps (streaming DDL parser, plain or gzip-compressed),
321
+ and **MySQL** dumps (streaming DDL parser, plain or gzip-compressed).
322
+ - Live database introspection for PostgreSQL and MySQL via SQLAlchemy `inspect()`.
323
+ - Foreign-key inference from column-naming conventions (e.g. `txn_firm_id` → `kb_firms.firm_id`)
324
+ for schemas that declare no explicit FK constraints; ambiguous matches are silently skipped.
325
+ - Magic-byte + content-heuristic format detection (`detect_format.py`) - no file extension required.
326
+ - Normalised schema dataclass model (`schema_model.py`) shared by all extractors and the renderer.
327
+
328
+ #### Interactive ERD viewer (`templates/erd.html`)
329
+ - **Cytoscape.js** graph rendering - no Node.js, no npm, no build step; loaded from CDN at runtime.
330
+ - **ELK layered** hierarchical layout (jsDelivr CDN): direction `RIGHT`, `BRANDES_KOEPF` node placement,
331
+ `LAYER_SWEEP` crossing minimisation - source tables on the left, terminal tables on the right.
332
+ - **Topological-sort fallback** layout (pure JavaScript, no CDN dependency): Kahn's algorithm over the FK
333
+ graph; assigns each table to a depth column and spaces columns evenly across the canvas.
334
+ - **Smooth curved edges**: Cytoscape `unbundled-bezier` curve style with Canvas `bezierCurveTo` glow
335
+ animation for pixel-perfect match.
336
+ - Animated FK edge particles (`requestAnimationFrame`) with colour-coded glow (purple `#a78bfa`).
337
+ - Three **view modes**: All fields / Keys only / Name - toggled from the top bar.
338
+ - **Sidebar search**: live filter by table name or column name; press `/` to focus, `Escape` to clear.
339
+ - **Detail panel**: click any node to open a slide-in column list with FK jump links.
340
+ - **Minimap**: toggleable overview (bottom-right corner).
341
+ - Fully **shareable URLs**: each processed schema gets a stable `GET /erd/<key>` route.
342
+
343
+ #### Web UI and API (`templates/index.html`, `api_server.py`)
344
+ - Selector UI with four upload tabs: File Upload, Connection String, DB Form, Remote URL.
345
+ - Schema cards with colour-coded state borders and per-card View ERD / Retry / Delete / Logs actions.
346
+ - Real-time log panel (800 ms polling during processing) with `[HH:MM:SS]` timestamps.
347
+ - REST API:
348
+ - `GET /api/files` - list all registered schemas
349
+ - `POST /api/upload` - upload a file (`multipart/form-data`, field `file`)
350
+ - `POST /api/connect` - register live DB or remote URL
351
+ - `POST /api/process/<key>` - trigger or re-trigger the pipeline
352
+ - `GET /api/status/<key>` - poll status; returns registry record with log tail
353
+ - `GET /api/log/<key>` - full pipeline log (supports `?stream=1` chunked response)
354
+ - `DELETE /api/delete/<key>` - remove schema from registry and disk
355
+ - `GET /erd/<key>` - full-page interactive ERD viewer
356
+
357
+ #### Docker
358
+ - Two-stage Dockerfile: `python:3.12-slim` builder + `nginx:1.27-alpine` server.
359
+ - Three operating modes:
360
+ - **Mode A** - bake a database into the image at build time.
361
+ - **Mode B** - volume-mount a file processed automatically on first container start.
362
+ - **Mode C** - upload files at runtime via the browser UI (default).
363
+ - Published to Docker Hub as `datajoins/nybble:1.0.0`.
364
+ - Named Docker volumes `erd-uploads` and `erd-output` for persistent data across restarts.
365
+ - Health check: `wget -q -O /dev/null http://127.0.0.1:80/`.
366
+
367
+ #### Standalone demo data seeder (`../scripts/seed_demo_standalone.py`)
368
+ - Generates a realistic synthetic Vyapar-style Indian trading database (12 tables, 17 FK edges)
369
+ without any Django dependency; targets SQLite (default) or PostgreSQL.
@@ -0,0 +1,193 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship made available under
36
+ the License, as indicated by a copyright notice that is included in
37
+ or attached to the work (an example is provided in the Appendix below).
38
+
39
+ "Derivative Works" shall mean any work, whether in Source or Object
40
+ form, that is based on (or derived from) the Work and for which the
41
+ editorial revisions, annotations, elaborations, or other modifications
42
+ represent, as a whole, an original work of authorship. For the purposes
43
+ of this License, Derivative Works shall not include works that remain
44
+ separable from, or merely link (or bind by name) to the interfaces of,
45
+ the Work and Derivative Works thereof.
46
+
47
+ "Contribution" shall mean, as submitted to the Licensor for inclusion
48
+ in the Work by the copyright owner or by an individual or Legal Entity
49
+ authorized to submit on behalf of the copyright owner. For the purposes
50
+ of this definition, "submitted" means any form of electronic, verbal,
51
+ or written communication sent to the Licensor or its representatives,
52
+ including but not limited to communication on electronic mailing lists,
53
+ source code control systems, and issue tracking systems that are managed
54
+ by, or on behalf of, the Licensor for the purpose of discussing and
55
+ improving the Work, but excluding communication that is conspicuously
56
+ marked or otherwise designated in writing by the copyright owner as
57
+ "Not a Contribution."
58
+
59
+ "Contributor" shall mean Licensor and any Legal Entity on behalf of
60
+ whom a Contribution has been received by the Licensor and included
61
+ within the Work.
62
+
63
+ 2. Grant of Copyright License. Subject to the terms and conditions of
64
+ this License, each Contributor hereby grants to You a perpetual,
65
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
66
+ copyright license to reproduce, prepare Derivative Works of,
67
+ publicly display, publicly perform, sublicense, and distribute the
68
+ Work and such Derivative Works in Source or Object form.
69
+
70
+ 3. Grant of Patent License. Subject to the terms and conditions of
71
+ this License, each Contributor hereby grants to You a perpetual,
72
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
73
+ (except as stated in this section) patent license to make, have made,
74
+ use, offer to sell, sell, import, and otherwise transfer the Work,
75
+ where such license applies only to those patent claims licensable
76
+ by such Contributor that are necessarily infringed by their
77
+ Contribution(s) alone or by combination of their Contribution(s)
78
+ with the Work to which such Contribution(s) was submitted. If You
79
+ institute patent litigation against any entity (including a
80
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
81
+ or a Contribution incorporated within the Work constitutes direct
82
+ or contributory patent infringement, then any patent licenses
83
+ granted to You under this License for that Work shall terminate
84
+ as of the date such litigation is filed.
85
+
86
+ 4. Redistribution. You may reproduce and distribute copies of the
87
+ Work or Derivative Works thereof in any medium, with or without
88
+ modifications, and in Source or Object form, provided that You
89
+ meet the following conditions:
90
+
91
+ (a) You must give any other recipients of the Work or Derivative
92
+ Works a copy of this License; and
93
+
94
+ (b) You must cause any modified files to carry prominent notices
95
+ stating that You changed the files; and
96
+
97
+ (c) You must retain, in the Source form of any Derivative Works
98
+ that You distribute, all copyright, patent, trademark, and
99
+ attribution notices from the Source form of the Work,
100
+ excluding those notices that do not pertain to any part of
101
+ the Derivative Works; and
102
+
103
+ (d) If the Work includes a "NOTICE" text file as part of its
104
+ distribution, You must include a readable copy of the
105
+ attribution notices contained within such NOTICE file, in
106
+ at least one of the following places: within a NOTICE text
107
+ file distributed as part of the Derivative Works; within
108
+ the Source form or documentation, if provided along with the
109
+ Derivative Works; or, within a display generated by the
110
+ Derivative Works, if and wherever such third-party notices
111
+ normally appear. The contents of the NOTICE file are for
112
+ informational purposes only and do not modify the License.
113
+ You may add Your own attribution notices within Derivative
114
+ Works that You distribute, alongside or as an addition to
115
+ the NOTICE text from the Work, provided that such additional
116
+ attribution notices cannot be construed as modifying the License.
117
+
118
+ You may add Your own license statement for Your modifications and
119
+ may provide additional grant of rights to use, copy, modify, merge,
120
+ publish, distribute, sublicense, and/or sell copies of the
121
+ Derivative Works, and to permit persons to whom the Derivative Works
122
+ is furnished to do so, subject to the following conditions above.
123
+
124
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
125
+ any Contribution intentionally submitted for inclusion in the Work
126
+ by You to the Licensor shall be under the terms and conditions of
127
+ this License, without any additional terms or conditions.
128
+ Notwithstanding the above, nothing herein shall supersede or modify
129
+ the terms of any separate license agreement you may have executed
130
+ with Licensor regarding such Contributions.
131
+
132
+ 6. Trademarks. This License does not grant permission to use the trade
133
+ names, trademarks, service marks, or product names of the Licensor,
134
+ except as required for reasonable and customary use in describing the
135
+ origin of the Work and reproducing the content of the NOTICE file.
136
+
137
+ 7. Disclaimer of Warranty. Unless required by applicable law or
138
+ agreed to in writing, Licensor provides the Work (and each
139
+ Contributor provides its Contributions) on an "AS IS" BASIS,
140
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
141
+ implied, including, without limitation, any warranties or conditions
142
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
143
+ PARTICULAR PURPOSE. You are solely responsible for determining the
144
+ appropriateness of using or reproducing the Work and assume any
145
+ risks associated with Your exercise of permissions under this License.
146
+
147
+ 8. Limitation of Liability. In no event and under no legal theory,
148
+ whether in tort (including negligence), contract, or otherwise,
149
+ unless required by applicable law (such as deliberate and grossly
150
+ negligent acts) or agreed to in writing, shall any Contributor be
151
+ liable to You for damages, including any direct, indirect, special,
152
+ incidental, or exemplary damages of any character arising as a
153
+ result of this License or out of the use or inability to use the
154
+ Work (including but not limited to damages for loss of goodwill,
155
+ work stoppage, computer failure or malfunction, or all other
156
+ commercial damages or losses), even if such Contributor has been
157
+ advised of the possibility of such damages.
158
+
159
+ 9. Accepting Warranty or Liability. While redistributing the Work or
160
+ Derivative Works thereof, You may choose to offer, and charge a fee
161
+ for, acceptance of support, warranty, indemnity, or other liability
162
+ obligations and rights consistent with this License. However, in
163
+ accepting such obligations, You may act only on Your own behalf and
164
+ on Your sole responsibility, not on behalf of any other Contributor,
165
+ and only if You agree to indemnify, defend, and hold each Contributor
166
+ harmless for any liability incurred by, or claims asserted against,
167
+ such Contributor by reason of your accepting any such warranty or
168
+ additional liability.
169
+
170
+ END OF TERMS AND CONDITIONS
171
+
172
+ APPENDIX: How to apply the Apache License to your work.
173
+
174
+ To apply the Apache License to your work, attach the following
175
+ boilerplate notice, with the fields enclosed by brackets "[]"
176
+ replaced with your own identifying information. (Don't include
177
+ the brackets!) The text should be enclosed in the appropriate
178
+ comment syntax for the syntax of the file. You may also add a
179
+ secondary license notice below the primary license if desired.
180
+
181
+ Copyright 2026 DataJoins (https://datajoins.com)
182
+
183
+ Licensed under the Apache License, Version 2.0 (the "License");
184
+ you may not use this file except in compliance with the License.
185
+ You may obtain a copy of the License at
186
+
187
+ http://www.apache.org/licenses/LICENSE-2.0
188
+
189
+ Unless required by applicable law or agreed to in writing, software
190
+ distributed under the License is distributed on an "AS IS" BASIS,
191
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
192
+ implied. See the License for the specific language governing
193
+ permissions and limitations under the License.
@@ -0,0 +1,27 @@
1
+ # MANIFEST.in - controls what goes into the sdist (source distribution)
2
+ # The wheel uses pyproject.toml package-data instead; this is a safety net.
3
+
4
+ # License and metadata
5
+ include LICENSE
6
+ include README.md
7
+ include CHANGELOG.md
8
+ include pyproject.toml
9
+
10
+ # Jinja2 templates bundled with the package
11
+ recursive-include nybble/templates *.jinja
12
+
13
+ # Exclude everything that should not be in the sdist
14
+ exclude .gitignore
15
+ recursive-exclude server *
16
+ recursive-exclude deploy *
17
+ recursive-exclude docs *
18
+ recursive-exclude website *
19
+ recursive-exclude data *
20
+ recursive-exclude .github *
21
+ recursive-exclude .bob *
22
+ recursive-exclude .nybble *
23
+ recursive-exclude .nybble-test-fixtures *
24
+ prune __pycache__
25
+ prune *.egg-info
26
+ prune dist
27
+ prune build