fina-olap 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fina_olap-0.2.0/PKG-INFO +188 -0
- fina_olap-0.2.0/README.md +164 -0
- fina_olap-0.2.0/pyproject.toml +99 -0
- fina_olap-0.2.0/pyproject.toml.orig +70 -0
- fina_olap-0.2.0/src/fina_olap/__init__.py +40 -0
- fina_olap-0.2.0/src/fina_olap/__main__.py +4 -0
- fina_olap-0.2.0/src/fina_olap/builder.py +485 -0
- fina_olap-0.2.0/src/fina_olap/cli.py +72 -0
- fina_olap-0.2.0/src/fina_olap/engine.py +378 -0
- fina_olap-0.2.0/src/fina_olap/export.py +122 -0
- fina_olap-0.2.0/src/fina_olap/fixture.py +99 -0
- fina_olap-0.2.0/src/fina_olap/gcs.py +131 -0
- fina_olap-0.2.0/src/fina_olap/mcp_server.py +275 -0
- fina_olap-0.2.0/src/fina_olap/schema.py +261 -0
- fina_olap-0.2.0/src/fina_olap/server.py +106 -0
- fina_olap-0.2.0/src/fina_olap/storage.py +213 -0
- fina_olap-0.2.0/src/fina_olap/upsert.py +366 -0
- fina_olap-0.2.0/src/fina_olap/vercel.py +42 -0
fina_olap-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: fina-olap
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: ag-grid Server-Side Row Model OLAP engine over Parquet (S3/GCS/local) powered by DuckDB — with a lean HTTP server, an MCP v2 tool server and a Vercel serverless deployment target
|
|
5
|
+
Keywords: ag-grid,ssrm,olap,duckdb,parquet,s3,gcs,mcp
|
|
6
|
+
Author: FinA
|
|
7
|
+
License: MIT
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Topic :: Database :: Front-Ends
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Visualization
|
|
15
|
+
Requires-Dist: duckdb>=1.1,<2
|
|
16
|
+
Requires-Dist: mcp>=1.6,<2
|
|
17
|
+
Requires-Dist: fastapi>=0.110,<1
|
|
18
|
+
Requires-Dist: pandas>=2.0,<3
|
|
19
|
+
Requires-Dist: pyarrow>=15,<22
|
|
20
|
+
Requires-Dist: pydantic>=2.8,<3
|
|
21
|
+
Requires-Dist: starlette>=0.40,<1
|
|
22
|
+
Requires-Python: >=3.12
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# fina-olap
|
|
26
|
+
|
|
27
|
+
ag-grid server-side row model (SSRM) OLAP engine over Parquet, DuckDB-powered.
|
|
28
|
+
|
|
29
|
+
`fina-olap` turns a (nearly) stock ag-grid SSRM request — row groups, group keys,
|
|
30
|
+
pivots, value columns, sorting, filtering, pagination — into a single DuckDB SQL
|
|
31
|
+
query over a Parquet store on S3 / GCS / local disk, and returns rows with the
|
|
32
|
+
exact SSRM contract (`rows`, `lastRow`, `pivotResultFields`, ...). It ships three
|
|
33
|
+
surfaces backed by the same engine:
|
|
34
|
+
|
|
35
|
+
| Surface | Where | Typical use |
|
|
36
|
+
| --- | --- | --- |
|
|
37
|
+
| **REST SSRM** | `POST /api/getRows`, `POST /api/getSchema`, `GET /api/health` | ag-grid SSRM datasource (`fina-table`) |
|
|
38
|
+
| **MCP v2** | stdio (`fina-olap-mcp`) or Streamable HTTP (`/mcp`, Vercel) | agents generating fixtures, inspecting schemas, ad-hoc OLAP |
|
|
39
|
+
| **CLI** | `fina-olap` | serve HTTP, run stdio MCP, generate fixtures, run a payload |
|
|
40
|
+
|
|
41
|
+
## OLAP extensions
|
|
42
|
+
|
|
43
|
+
Beyond the stock ag-grid model, `fina-olap` supports level-aware analytics used
|
|
44
|
+
by `fina-table`:
|
|
45
|
+
|
|
46
|
+
- **Custom grouping level** — `valueCols[].aggFuncsByLevel` selects the
|
|
47
|
+
aggregation per grouping depth: `{"leg": "first"}` (keyed by level column),
|
|
48
|
+
`{"0": "avg"}` (keyed by 0-based level index) or `["sum", null, "avg"]`.
|
|
49
|
+
- **Per-level metric visibility** — `valueCols[].visibleLevels` (e.g. `[0]`)
|
|
50
|
+
suppresses a metric at levels not listed: aggregated group levels render
|
|
51
|
+
`NULL`, and fully-drilled leaf rows omit the column (`SELECT * EXCLUDE (...)`)
|
|
52
|
+
so a measure can be hidden on raw leaves while remaining aggregate-only.
|
|
53
|
+
- **Grand total** — `includeGrandTotal: true` prepends a leading row with all
|
|
54
|
+
group keys `NULL` and an aggregate per measure over all filtered records.
|
|
55
|
+
`grandTotalAggFunc` selects that aggregate (default `"sum"`); it is independent
|
|
56
|
+
of `aggFuncsByLevel` / `visibleLevels`, so `"none"` at a grouping level never
|
|
57
|
+
blanks the grand total.
|
|
58
|
+
- **LOD configuration** — `lodConfig { type: fixed|include|exclude, groupKeys,
|
|
59
|
+
metrics: {measure: fn}, prefix }` joins a secondary level-of-detail aggregate
|
|
60
|
+
(a tableau-style LOD expression) onto every row via a `LEFT JOIN` on the
|
|
61
|
+
dimension columns, so each row can compare `sum(delta)` against
|
|
62
|
+
portfolio-level `_lod_delta`.
|
|
63
|
+
|
|
64
|
+
## Quick start
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
uv sync
|
|
68
|
+
uv run fina-olap --http --port 8000 # SSRM REST + MCP HTTP on one port
|
|
69
|
+
uv run fina-olap-mcp # MCP stdio tool server
|
|
70
|
+
curl -X POST localhost:8000/api/getRows \
|
|
71
|
+
-H 'content-type: application/json' \
|
|
72
|
+
-d '{"rowGroupCols":[{"id":"p","field":"portfolio"}],
|
|
73
|
+
"valueCols":[{"id":"d","aggFunc":"sum","field":"delta"}],
|
|
74
|
+
"groupKeys":[]}'
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
With no Parquet configured, requests run against a deterministic sample fixture
|
|
78
|
+
(`data/sample.parquet`, generated on first use / via `fina-olap gen-fixture`).
|
|
79
|
+
|
|
80
|
+
## Data sources
|
|
81
|
+
|
|
82
|
+
The backing store is selected by the `FINA_OLAP_STORE` switch (`local` | `s3` |
|
|
83
|
+
`gcs` | `auto`), and the Parquet partition layout by `FINA_OLAP_PARTITION_GLOB`
|
|
84
|
+
+ `FINA_OLAP_HIVE_PARTITIONING`. Resolution order for the Parquet backing:
|
|
85
|
+
|
|
86
|
+
1. `dataSource.uri` — explicit `s3://`, `gs://`, or local path
|
|
87
|
+
2. `dataSource.bucket` / `path(glob)` — bucket-constructed source
|
|
88
|
+
3. `FINA_OLAP_STORE=s3|gcs` (forced) — `S3_PATH_TEMPLATE`, or
|
|
89
|
+
`FINA_OLAP_BUCKET` (+ `FINA_OLAP_PATH`) and the partition glob. Fails loudly
|
|
90
|
+
when the store is unconfigured — no silent fixture fallback.
|
|
91
|
+
4. `FINA_OLAP_STORE=local` (forced) — `FINA_OLAP_PARQUET_ROOT` (aliases
|
|
92
|
+
`OLAP_PARQUET_ROOT` / `DATA_DIR`) + partition glob, then the generated fixture.
|
|
93
|
+
5. `auto` (default) — legacy cascade: template → local root → generated fixture
|
|
94
|
+
(`FINA_OLAP_FIXTURE`, default `data/sample.parquet`).
|
|
95
|
+
|
|
96
|
+
Partition configuration:
|
|
97
|
+
|
|
98
|
+
- `FINA_OLAP_PARTITION_GLOB` — on-store layout with `{tableName}` templating,
|
|
99
|
+
e.g. `{tableName}/region=*/date=*/*.parquet`. Default: `{tableName}*.parquet`
|
|
100
|
+
(flat files directly under the root).
|
|
101
|
+
- `FINA_OLAP_HIVE_PARTITIONING` — `1|0`; when enabled, DuckDB exposes hive
|
|
102
|
+
partition columns (`region`, `date`, …) as normal columns. Default: on for
|
|
103
|
+
`s3`/`gcs`, off for local; may be overridden via `dataSource.hivePartitioning`
|
|
104
|
+
per payload.
|
|
105
|
+
|
|
106
|
+
Remote stores use DuckDB `httpfs` with an S3-compatible secret (GCS via
|
|
107
|
+
`S3_API_KEY`/`S3_API_SECRET` or AWS via `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`
|
|
108
|
+
+ `AWS_ENDPOINT_URL`), configured automatically when store env vars are present.
|
|
109
|
+
`GET /api/health` (and the MCP `status` tool) report the active store config.
|
|
110
|
+
|
|
111
|
+
### Runtime store configuration (MCP tools)
|
|
112
|
+
|
|
113
|
+
The same settings can be switched at runtime via MCP tools, so a UI can point
|
|
114
|
+
the engine at a different store without a restart. Overrides layer on top of
|
|
115
|
+
the env base and reset with `clear=true` (env vars stay the process default):
|
|
116
|
+
|
|
117
|
+
- `store_config` — read the effective config (fields + which are overridden).
|
|
118
|
+
- `store_configure` — set `store`, `parquet_root`, `bucket`, `path`,
|
|
119
|
+
`partition_glob`, `hive_partitioning`; `clear=true` resets all overrides.
|
|
120
|
+
- `store_resolve` — preview how a table resolves (`source`, hive flag,
|
|
121
|
+
partition columns) under the current config.
|
|
122
|
+
|
|
123
|
+
### Upsert (CSV / JSON / Parquet → store)
|
|
124
|
+
|
|
125
|
+
`upsert_store(table_name, key, rows[, file_path][, data_format][, appender][, chunk_size])`
|
|
126
|
+
merges rows (or a CSV/JSON/Parquet file) into the store's Parquet layout. `key`
|
|
127
|
+
is a column or comma-separated columns that identify a stored row: matching keys
|
|
128
|
+
update, unknown keys insert, and the full store is rewritten in place — flat
|
|
129
|
+
files are replaced atomically (temp + rename), hive-partitioned stores are
|
|
130
|
+
rewritten with `COPY ... PARTITION_BY`. The target table is created if absent.
|
|
131
|
+
Return value reports before/after row counts, matched/inserted split, the
|
|
132
|
+
written location, writer used and the resulting schema. Error cases: no key /
|
|
133
|
+
missing key column / unsupported `data_format` / incoming file missing a target
|
|
134
|
+
key column.
|
|
135
|
+
|
|
136
|
+
Appender mode: pass `appender=true` to switch the flat-store write from DuckDB
|
|
137
|
+
`COPY` to an Arrow `ParquetWriter` (pyarrow); each `write_table` call appends one
|
|
138
|
+
row group, and `chunk_size` splits the merged result so it is written chunk by
|
|
139
|
+
chunk (result reports `row_groups`). Appender mode requires a flat local store —
|
|
140
|
+
it is rejected for hive-partitioned globs and object-store targets.
|
|
141
|
+
|
|
142
|
+
### Export (store → CSV / JSON / JSONL / Parquet)
|
|
143
|
+
|
|
144
|
+
`store_export(table_name, out_path[, data_format][, columns][, filters][, limit])`
|
|
145
|
+
dumps a table through the same read path an SSRM query uses (`resolve_source` +
|
|
146
|
+
hive columns), applies the same ag-grid `filterModel` semantics as the live
|
|
147
|
+
query builder (so an export matches what the grid shows), then streams out via
|
|
148
|
+
DuckDB `COPY`:
|
|
149
|
+
|
|
150
|
+
- `csv` → `(FORMAT CSV, HEADER)`
|
|
151
|
+
- `json` → `(FORMAT JSON, ARRAY true)` — a JSON array of objects
|
|
152
|
+
- `jsonl`/`ndjson` → `(FORMAT JSON)` — one object per line
|
|
153
|
+
- `parquet` → a single Parquet file
|
|
154
|
+
|
|
155
|
+
`data_format` overrides extension sniffing on `out_path`; `columns` is a
|
|
156
|
+
comma-separated subset; `filters` is the ag-grid `filterModel` (text / number /
|
|
157
|
+
date / set / combined AND-OR conditions); `limit` caps the rows. Returns the
|
|
158
|
+
written path, format, exported row count and column list.
|
|
159
|
+
|
|
160
|
+
## Deployment
|
|
161
|
+
|
|
162
|
+
Vercel serverless: the function in `api/index.py` serves both `/api/*` (SSRM
|
|
163
|
+
REST, passthrough) and `/mcp` (rewritten to `/api/mcp`, restored to `/mcp` for
|
|
164
|
+
the FastAPI mount). Local: `uv run fina-olap --http` exposes `/api/*` and `/mcp`
|
|
165
|
+
on the same port.
|
|
166
|
+
|
|
167
|
+
## Tests
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
uv run pytest # builder SQL * engine E2E * schema * HTTP * MCP stdio
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
## Repo layout
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
api/ Vercel ASGI adapter
|
|
177
|
+
src/fina_olap/ sdist/wheel (PyPI: fina-olap)
|
|
178
|
+
builder.py SSRM -> DuckDB SQL compiler
|
|
179
|
+
engine.py source resolution + query execution
|
|
180
|
+
schema.py pydantic SSRM request/response model
|
|
181
|
+
fixture.py deterministic sample parquet generator
|
|
182
|
+
gcs.py DuckDB httpfs object-store configuration
|
|
183
|
+
mcp_server.py FastMCP stdio + Streamable HTTP tools
|
|
184
|
+
server.py FastAPI app (REST + /mcp mount)
|
|
185
|
+
vercel.py Vercel path-rewrite ASGI wrapper
|
|
186
|
+
fina-table/ React headless table lib (separate npm package)
|
|
187
|
+
demo/ Next.js sample app using fina-table
|
|
188
|
+
```
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
# fina-olap
|
|
2
|
+
|
|
3
|
+
ag-grid server-side row model (SSRM) OLAP engine over Parquet, DuckDB-powered.
|
|
4
|
+
|
|
5
|
+
`fina-olap` turns a (nearly) stock ag-grid SSRM request — row groups, group keys,
|
|
6
|
+
pivots, value columns, sorting, filtering, pagination — into a single DuckDB SQL
|
|
7
|
+
query over a Parquet store on S3 / GCS / local disk, and returns rows with the
|
|
8
|
+
exact SSRM contract (`rows`, `lastRow`, `pivotResultFields`, ...). It ships three
|
|
9
|
+
surfaces backed by the same engine:
|
|
10
|
+
|
|
11
|
+
| Surface | Where | Typical use |
|
|
12
|
+
| --- | --- | --- |
|
|
13
|
+
| **REST SSRM** | `POST /api/getRows`, `POST /api/getSchema`, `GET /api/health` | ag-grid SSRM datasource (`fina-table`) |
|
|
14
|
+
| **MCP v2** | stdio (`fina-olap-mcp`) or Streamable HTTP (`/mcp`, Vercel) | agents generating fixtures, inspecting schemas, ad-hoc OLAP |
|
|
15
|
+
| **CLI** | `fina-olap` | serve HTTP, run stdio MCP, generate fixtures, run a payload |
|
|
16
|
+
|
|
17
|
+
## OLAP extensions
|
|
18
|
+
|
|
19
|
+
Beyond the stock ag-grid model, `fina-olap` supports level-aware analytics used
|
|
20
|
+
by `fina-table`:
|
|
21
|
+
|
|
22
|
+
- **Custom grouping level** — `valueCols[].aggFuncsByLevel` selects the
|
|
23
|
+
aggregation per grouping depth: `{"leg": "first"}` (keyed by level column),
|
|
24
|
+
`{"0": "avg"}` (keyed by 0-based level index) or `["sum", null, "avg"]`.
|
|
25
|
+
- **Per-level metric visibility** — `valueCols[].visibleLevels` (e.g. `[0]`)
|
|
26
|
+
suppresses a metric at levels not listed: aggregated group levels render
|
|
27
|
+
`NULL`, and fully-drilled leaf rows omit the column (`SELECT * EXCLUDE (...)`)
|
|
28
|
+
so a measure can be hidden on raw leaves while remaining aggregate-only.
|
|
29
|
+
- **Grand total** — `includeGrandTotal: true` prepends a leading row with all
|
|
30
|
+
group keys `NULL` and an aggregate per measure over all filtered records.
|
|
31
|
+
`grandTotalAggFunc` selects that aggregate (default `"sum"`); it is independent
|
|
32
|
+
of `aggFuncsByLevel` / `visibleLevels`, so `"none"` at a grouping level never
|
|
33
|
+
blanks the grand total.
|
|
34
|
+
- **LOD configuration** — `lodConfig { type: fixed|include|exclude, groupKeys,
|
|
35
|
+
metrics: {measure: fn}, prefix }` joins a secondary level-of-detail aggregate
|
|
36
|
+
(a tableau-style LOD expression) onto every row via a `LEFT JOIN` on the
|
|
37
|
+
dimension columns, so each row can compare `sum(delta)` against
|
|
38
|
+
portfolio-level `_lod_delta`.
|
|
39
|
+
|
|
40
|
+
## Quick start
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
uv sync
|
|
44
|
+
uv run fina-olap --http --port 8000 # SSRM REST + MCP HTTP on one port
|
|
45
|
+
uv run fina-olap-mcp # MCP stdio tool server
|
|
46
|
+
curl -X POST localhost:8000/api/getRows \
|
|
47
|
+
-H 'content-type: application/json' \
|
|
48
|
+
-d '{"rowGroupCols":[{"id":"p","field":"portfolio"}],
|
|
49
|
+
"valueCols":[{"id":"d","aggFunc":"sum","field":"delta"}],
|
|
50
|
+
"groupKeys":[]}'
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
With no Parquet configured, requests run against a deterministic sample fixture
|
|
54
|
+
(`data/sample.parquet`, generated on first use / via `fina-olap gen-fixture`).
|
|
55
|
+
|
|
56
|
+
## Data sources
|
|
57
|
+
|
|
58
|
+
The backing store is selected by the `FINA_OLAP_STORE` switch (`local` | `s3` |
|
|
59
|
+
`gcs` | `auto`), and the Parquet partition layout by `FINA_OLAP_PARTITION_GLOB`
|
|
60
|
+
+ `FINA_OLAP_HIVE_PARTITIONING`. Resolution order for the Parquet backing:
|
|
61
|
+
|
|
62
|
+
1. `dataSource.uri` — explicit `s3://`, `gs://`, or local path
|
|
63
|
+
2. `dataSource.bucket` / `path(glob)` — bucket-constructed source
|
|
64
|
+
3. `FINA_OLAP_STORE=s3|gcs` (forced) — `S3_PATH_TEMPLATE`, or
|
|
65
|
+
`FINA_OLAP_BUCKET` (+ `FINA_OLAP_PATH`) and the partition glob. Fails loudly
|
|
66
|
+
when the store is unconfigured — no silent fixture fallback.
|
|
67
|
+
4. `FINA_OLAP_STORE=local` (forced) — `FINA_OLAP_PARQUET_ROOT` (aliases
|
|
68
|
+
`OLAP_PARQUET_ROOT` / `DATA_DIR`) + partition glob, then the generated fixture.
|
|
69
|
+
5. `auto` (default) — legacy cascade: template → local root → generated fixture
|
|
70
|
+
(`FINA_OLAP_FIXTURE`, default `data/sample.parquet`).
|
|
71
|
+
|
|
72
|
+
Partition configuration:
|
|
73
|
+
|
|
74
|
+
- `FINA_OLAP_PARTITION_GLOB` — on-store layout with `{tableName}` templating,
|
|
75
|
+
e.g. `{tableName}/region=*/date=*/*.parquet`. Default: `{tableName}*.parquet`
|
|
76
|
+
(flat files directly under the root).
|
|
77
|
+
- `FINA_OLAP_HIVE_PARTITIONING` — `1|0`; when enabled, DuckDB exposes hive
|
|
78
|
+
partition columns (`region`, `date`, …) as normal columns. Default: on for
|
|
79
|
+
`s3`/`gcs`, off for local; may be overridden via `dataSource.hivePartitioning`
|
|
80
|
+
per payload.
|
|
81
|
+
|
|
82
|
+
Remote stores use DuckDB `httpfs` with an S3-compatible secret (GCS via
|
|
83
|
+
`S3_API_KEY`/`S3_API_SECRET` or AWS via `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`
|
|
84
|
+
+ `AWS_ENDPOINT_URL`), configured automatically when store env vars are present.
|
|
85
|
+
`GET /api/health` (and the MCP `status` tool) report the active store config.
|
|
86
|
+
|
|
87
|
+
### Runtime store configuration (MCP tools)
|
|
88
|
+
|
|
89
|
+
The same settings can be switched at runtime via MCP tools, so a UI can point
|
|
90
|
+
the engine at a different store without a restart. Overrides layer on top of
|
|
91
|
+
the env base and reset with `clear=true` (env vars stay the process default):
|
|
92
|
+
|
|
93
|
+
- `store_config` — read the effective config (fields + which are overridden).
|
|
94
|
+
- `store_configure` — set `store`, `parquet_root`, `bucket`, `path`,
|
|
95
|
+
`partition_glob`, `hive_partitioning`; `clear=true` resets all overrides.
|
|
96
|
+
- `store_resolve` — preview how a table resolves (`source`, hive flag,
|
|
97
|
+
partition columns) under the current config.
|
|
98
|
+
|
|
99
|
+
### Upsert (CSV / JSON / Parquet → store)
|
|
100
|
+
|
|
101
|
+
`upsert_store(table_name, key, rows[, file_path][, data_format][, appender][, chunk_size])`
|
|
102
|
+
merges rows (or a CSV/JSON/Parquet file) into the store's Parquet layout. `key`
|
|
103
|
+
is a column or comma-separated columns that identify a stored row: matching keys
|
|
104
|
+
update, unknown keys insert, and the full store is rewritten in place — flat
|
|
105
|
+
files are replaced atomically (temp + rename), hive-partitioned stores are
|
|
106
|
+
rewritten with `COPY ... PARTITION_BY`. The target table is created if absent.
|
|
107
|
+
Return value reports before/after row counts, matched/inserted split, the
|
|
108
|
+
written location, writer used and the resulting schema. Error cases: no key /
|
|
109
|
+
missing key column / unsupported `data_format` / incoming file missing a target
|
|
110
|
+
key column.
|
|
111
|
+
|
|
112
|
+
Appender mode: pass `appender=true` to switch the flat-store write from DuckDB
|
|
113
|
+
`COPY` to an Arrow `ParquetWriter` (pyarrow); each `write_table` call appends one
|
|
114
|
+
row group, and `chunk_size` splits the merged result so it is written chunk by
|
|
115
|
+
chunk (result reports `row_groups`). Appender mode requires a flat local store —
|
|
116
|
+
it is rejected for hive-partitioned globs and object-store targets.
|
|
117
|
+
|
|
118
|
+
### Export (store → CSV / JSON / JSONL / Parquet)
|
|
119
|
+
|
|
120
|
+
`store_export(table_name, out_path[, data_format][, columns][, filters][, limit])`
|
|
121
|
+
dumps a table through the same read path an SSRM query uses (`resolve_source` +
|
|
122
|
+
hive columns), applies the same ag-grid `filterModel` semantics as the live
|
|
123
|
+
query builder (so an export matches what the grid shows), then streams out via
|
|
124
|
+
DuckDB `COPY`:
|
|
125
|
+
|
|
126
|
+
- `csv` → `(FORMAT CSV, HEADER)`
|
|
127
|
+
- `json` → `(FORMAT JSON, ARRAY true)` — a JSON array of objects
|
|
128
|
+
- `jsonl`/`ndjson` → `(FORMAT JSON)` — one object per line
|
|
129
|
+
- `parquet` → a single Parquet file
|
|
130
|
+
|
|
131
|
+
`data_format` overrides extension sniffing on `out_path`; `columns` is a
|
|
132
|
+
comma-separated subset; `filters` is the ag-grid `filterModel` (text / number /
|
|
133
|
+
date / set / combined AND-OR conditions); `limit` caps the rows. Returns the
|
|
134
|
+
written path, format, exported row count and column list.
|
|
135
|
+
|
|
136
|
+
## Deployment
|
|
137
|
+
|
|
138
|
+
Vercel serverless: the function in `api/index.py` serves both `/api/*` (SSRM
|
|
139
|
+
REST, passthrough) and `/mcp` (rewritten to `/api/mcp`, restored to `/mcp` for
|
|
140
|
+
the FastAPI mount). Local: `uv run fina-olap --http` exposes `/api/*` and `/mcp`
|
|
141
|
+
on the same port.
|
|
142
|
+
|
|
143
|
+
## Tests
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
uv run pytest # builder SQL * engine E2E * schema * HTTP * MCP stdio
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
## Repo layout
|
|
150
|
+
|
|
151
|
+
```
|
|
152
|
+
api/ Vercel ASGI adapter
|
|
153
|
+
src/fina_olap/ sdist/wheel (PyPI: fina-olap)
|
|
154
|
+
builder.py SSRM -> DuckDB SQL compiler
|
|
155
|
+
engine.py source resolution + query execution
|
|
156
|
+
schema.py pydantic SSRM request/response model
|
|
157
|
+
fixture.py deterministic sample parquet generator
|
|
158
|
+
gcs.py DuckDB httpfs object-store configuration
|
|
159
|
+
mcp_server.py FastMCP stdio + Streamable HTTP tools
|
|
160
|
+
server.py FastAPI app (REST + /mcp mount)
|
|
161
|
+
vercel.py Vercel path-rewrite ASGI wrapper
|
|
162
|
+
fina-table/ React headless table lib (separate npm package)
|
|
163
|
+
demo/ Next.js sample app using fina-table
|
|
164
|
+
```
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "fina-olap"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "ag-grid Server-Side Row Model OLAP engine over Parquet (S3/GCS/local) powered by DuckDB — with a lean HTTP server, an MCP v2 tool server and a Vercel serverless deployment target"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
keywords = [
|
|
8
|
+
"ag-grid",
|
|
9
|
+
"ssrm",
|
|
10
|
+
"olap",
|
|
11
|
+
"duckdb",
|
|
12
|
+
"parquet",
|
|
13
|
+
"s3",
|
|
14
|
+
"gcs",
|
|
15
|
+
"mcp",
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 4 - Beta",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Topic :: Database :: Front-Ends",
|
|
24
|
+
"Topic :: Scientific/Engineering :: Visualization",
|
|
25
|
+
]
|
|
26
|
+
dependencies = [
|
|
27
|
+
"duckdb>=1.1,<2",
|
|
28
|
+
"mcp>=1.6,<2",
|
|
29
|
+
"fastapi>=0.110,<1",
|
|
30
|
+
"pandas>=2.0,<3",
|
|
31
|
+
"pyarrow>=15,<22",
|
|
32
|
+
"pydantic>=2.8,<3",
|
|
33
|
+
"starlette>=0.40,<1",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.license]
|
|
37
|
+
text = "MIT"
|
|
38
|
+
|
|
39
|
+
[[project.authors]]
|
|
40
|
+
name = "FinA"
|
|
41
|
+
|
|
42
|
+
[project.scripts]
|
|
43
|
+
fina-olap = "fina_olap.cli:main"
|
|
44
|
+
fina-olap-mcp = "fina_olap.mcp_server:main"
|
|
45
|
+
|
|
46
|
+
[build-system]
|
|
47
|
+
requires = ["uv_build>=0.12.1,<0.13.0"]
|
|
48
|
+
build-backend = "uv_build"
|
|
49
|
+
|
|
50
|
+
[tool.uv.build-backend]
|
|
51
|
+
module-name = "fina_olap"
|
|
52
|
+
module-root = "src"
|
|
53
|
+
|
|
54
|
+
[tool.vercel]
|
|
55
|
+
entrypoint = "api.index:app"
|
|
56
|
+
|
|
57
|
+
[tool.pytest.ini_options]
|
|
58
|
+
testpaths = ["tests"]
|
|
59
|
+
addopts = "-q"
|
|
60
|
+
|
|
61
|
+
[tool.ruff]
|
|
62
|
+
line-length = 120
|
|
63
|
+
target-version = "py312"
|
|
64
|
+
extend-exclude = [
|
|
65
|
+
".venv",
|
|
66
|
+
".tmp",
|
|
67
|
+
"data",
|
|
68
|
+
"dist",
|
|
69
|
+
"node_modules",
|
|
70
|
+
"demo",
|
|
71
|
+
]
|
|
72
|
+
src = [
|
|
73
|
+
"src",
|
|
74
|
+
"tests",
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
[tool.ruff.lint]
|
|
78
|
+
select = [
|
|
79
|
+
"E",
|
|
80
|
+
"F",
|
|
81
|
+
"I",
|
|
82
|
+
"UP",
|
|
83
|
+
"B",
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
[tool.mypy]
|
|
87
|
+
python_version = "3.12"
|
|
88
|
+
ignore_missing_imports = true
|
|
89
|
+
|
|
90
|
+
[dependency-groups]
|
|
91
|
+
dev = [
|
|
92
|
+
"asgi-lifespan>=2.1,<3",
|
|
93
|
+
"httpx>=0.27,<1",
|
|
94
|
+
"mypy>=1.11,<2",
|
|
95
|
+
"pytest>=8.0,<9",
|
|
96
|
+
"pytest-cov>=5,<7",
|
|
97
|
+
"ruff>=0.6,<1",
|
|
98
|
+
"uvicorn>=0.30,<1",
|
|
99
|
+
]
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "fina-olap"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "ag-grid Server-Side Row Model OLAP engine over Parquet (S3/GCS/local) powered by DuckDB — with a lean HTTP server, an MCP v2 tool server and a Vercel serverless deployment target"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
license = { text = "MIT" }
|
|
8
|
+
authors = [{ name = "FinA" }]
|
|
9
|
+
keywords = ["ag-grid", "ssrm", "olap", "duckdb", "parquet", "s3", "gcs", "mcp"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 4 - Beta",
|
|
12
|
+
"Intended Audience :: Developers",
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.12",
|
|
15
|
+
"Programming Language :: Python :: 3.13",
|
|
16
|
+
"Topic :: Database :: Front-Ends",
|
|
17
|
+
"Topic :: Scientific/Engineering :: Visualization",
|
|
18
|
+
]
|
|
19
|
+
dependencies = [
|
|
20
|
+
"duckdb>=1.1,<2",
|
|
21
|
+
"mcp>=1.6,<2",
|
|
22
|
+
"fastapi>=0.110,<1",
|
|
23
|
+
"pandas>=2.0,<3",
|
|
24
|
+
"pyarrow>=15,<22",
|
|
25
|
+
"pydantic>=2.8,<3",
|
|
26
|
+
"starlette>=0.40,<1",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
fina-olap = "fina_olap.cli:main"
|
|
31
|
+
fina-olap-mcp = "fina_olap.mcp_server:main"
|
|
32
|
+
|
|
33
|
+
[build-system]
|
|
34
|
+
requires = ["uv_build>=0.12.1,<0.13.0"]
|
|
35
|
+
build-backend = "uv_build"
|
|
36
|
+
|
|
37
|
+
[tool.uv.build-backend]
|
|
38
|
+
module-name = "fina_olap"
|
|
39
|
+
module-root = "src"
|
|
40
|
+
|
|
41
|
+
[dependency-groups]
|
|
42
|
+
dev = [
|
|
43
|
+
"asgi-lifespan>=2.1,<3",
|
|
44
|
+
"httpx>=0.27,<1",
|
|
45
|
+
"mypy>=1.11,<2",
|
|
46
|
+
"pytest>=8.0,<9",
|
|
47
|
+
"pytest-cov>=5,<7",
|
|
48
|
+
"ruff>=0.6,<1",
|
|
49
|
+
"uvicorn>=0.30,<1",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[tool.vercel]
|
|
53
|
+
entrypoint = "api.index:app"
|
|
54
|
+
|
|
55
|
+
[tool.pytest.ini_options]
|
|
56
|
+
testpaths = ["tests"]
|
|
57
|
+
addopts = "-q"
|
|
58
|
+
|
|
59
|
+
[tool.ruff]
|
|
60
|
+
line-length = 120
|
|
61
|
+
target-version = "py312"
|
|
62
|
+
extend-exclude = [".venv", ".tmp", "data", "dist", "node_modules", "demo"]
|
|
63
|
+
src = ["src", "tests"]
|
|
64
|
+
|
|
65
|
+
[tool.ruff.lint]
|
|
66
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
67
|
+
|
|
68
|
+
[tool.mypy]
|
|
69
|
+
python_version = "3.12"
|
|
70
|
+
ignore_missing_imports = true
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""fina-olap: ag-grid SSRM OLAP engine over Parquet (S3/GCS/local), DuckDB-powered."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.2.0"
|
|
4
|
+
|
|
5
|
+
from .builder import BuiltQuery, DuckDBSqlBuilder, default_agg
|
|
6
|
+
from .engine import OlapEngine, resolve_source
|
|
7
|
+
from .export import export_store
|
|
8
|
+
from .schema import LodConfig, SSRMRequest, SSRMResponse, ValueCol
|
|
9
|
+
from .storage import (
|
|
10
|
+
StorageConfig,
|
|
11
|
+
clear_storage_override,
|
|
12
|
+
get_storage_config,
|
|
13
|
+
reload_storage_config,
|
|
14
|
+
set_storage_override,
|
|
15
|
+
storage_overrides,
|
|
16
|
+
storage_status,
|
|
17
|
+
)
|
|
18
|
+
from .upsert import upsert_store
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"__version__",
|
|
22
|
+
"BuiltQuery",
|
|
23
|
+
"DuckDBSqlBuilder",
|
|
24
|
+
"LodConfig",
|
|
25
|
+
"OlapEngine",
|
|
26
|
+
"SSRMRequest",
|
|
27
|
+
"SSRMResponse",
|
|
28
|
+
"StorageConfig",
|
|
29
|
+
"ValueCol",
|
|
30
|
+
"clear_storage_override",
|
|
31
|
+
"default_agg",
|
|
32
|
+
"export_store",
|
|
33
|
+
"get_storage_config",
|
|
34
|
+
"reload_storage_config",
|
|
35
|
+
"resolve_source",
|
|
36
|
+
"set_storage_override",
|
|
37
|
+
"storage_overrides",
|
|
38
|
+
"storage_status",
|
|
39
|
+
"upsert_store",
|
|
40
|
+
]
|