queria 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +57 -1
- package/dist/boot.d.ts +24 -0
- package/dist/boot.js +137 -0
- package/dist/boot.js.map +1 -0
- package/dist/client.d.ts +56 -0
- package/dist/client.js +194 -0
- package/dist/client.js.map +1 -0
- package/dist/elements/define.d.ts +1 -0
- package/dist/elements/define.js +4 -0
- package/dist/elements/define.js.map +1 -0
- package/dist/elements/index.d.ts +40 -0
- package/dist/elements/index.js +144 -0
- package/dist/elements/index.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -0
- package/dist/react/Chart.d.ts +12 -0
- package/dist/react/Chart.js +42 -0
- package/dist/react/Chart.js.map +1 -0
- package/dist/react/DataMap.d.ts +12 -0
- package/dist/react/DataMap.js +44 -0
- package/dist/react/DataMap.js.map +1 -0
- package/dist/react/MapView.d.ts +13 -0
- package/dist/react/MapView.js +31 -0
- package/dist/react/MapView.js.map +1 -0
- package/dist/react/PoweredBy.d.ts +2 -0
- package/dist/react/PoweredBy.js +5 -0
- package/dist/react/PoweredBy.js.map +1 -0
- package/dist/react/index.d.ts +5 -0
- package/dist/react/index.js +6 -0
- package/dist/react/index.js.map +1 -0
- package/dist/react/query.d.ts +20 -0
- package/dist/react/query.js +50 -0
- package/dist/react/query.js.map +1 -0
- package/dist/render/attribution.d.ts +6 -0
- package/dist/render/attribution.js +17 -0
- package/dist/render/attribution.js.map +1 -0
- package/dist/render/chart.d.ts +27 -0
- package/dist/render/chart.js +50 -0
- package/dist/render/chart.js.map +1 -0
- package/dist/render/data-map.d.ts +61 -0
- package/dist/render/data-map.js +200 -0
- package/dist/render/data-map.js.map +1 -0
- package/dist/render/escape.d.ts +2 -0
- package/dist/render/escape.js +4 -0
- package/dist/render/escape.js.map +1 -0
- package/dist/render/index.d.ts +5 -0
- package/dist/render/index.js +6 -0
- package/dist/render/index.js.map +1 -0
- package/dist/render/map.d.ts +7 -0
- package/dist/render/map.js +9 -0
- package/dist/render/map.js.map +1 -0
- package/dist/render/poweredBy.d.ts +2 -0
- package/dist/render/poweredBy.js +5 -0
- package/dist/render/poweredBy.js.map +1 -0
- package/dist/sql.d.ts +10 -0
- package/dist/sql.js +13 -0
- package/dist/sql.js.map +1 -0
- package/package.json +93 -1
- package/src/boot.test.ts +65 -0
- package/src/boot.ts +141 -0
- package/src/client.test.ts +77 -0
- package/src/client.ts +247 -0
- package/src/elements/define.ts +4 -0
- package/src/elements/index.ts +151 -0
- package/src/index.ts +12 -0
- package/src/react/Chart.tsx +49 -0
- package/src/react/DataMap.tsx +52 -0
- package/src/react/MapView.tsx +45 -0
- package/src/react/PoweredBy.tsx +4 -0
- package/src/react/index.ts +5 -0
- package/src/react/query.tsx +64 -0
- package/src/render/attribution.test.ts +22 -0
- package/src/render/attribution.ts +21 -0
- package/src/render/chart.test.ts +23 -0
- package/src/render/chart.ts +80 -0
- package/src/render/data-map.test.ts +40 -0
- package/src/render/data-map.ts +230 -0
- package/src/render/escape.ts +4 -0
- package/src/render/index.ts +12 -0
- package/src/render/map.ts +18 -0
- package/src/render/poweredBy.ts +6 -0
- package/src/sql.test.ts +21 -0
- package/src/sql.ts +19 -0
package/src/boot.ts
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import type { AsyncDuckDB, DuckDBBundles } from "@duckdb/duckdb-wasm";
|
|
2
|
+
import { stripLiterals } from "./sql.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Source put before the DuckDB worker script. It adds a Bearer token to the worker's XHRs, and only to
|
|
6
|
+
* the origins it was given. DuckDB WASM sends no headers of its own (neither CREATE SECRET nor
|
|
7
|
+
* EXTRA_HTTP_HEADERS reach the request in the browser), and keeping the token in a JS variable of the
|
|
8
|
+
* worker keeps it out of reach of SQL. It takes only { queriaAuth: { token, origins } } messages and
|
|
9
|
+
* does not pass them on to DuckDB.
|
|
10
|
+
*/
|
|
11
|
+
export const authPatch = `(() => {
|
|
12
|
+
let token = null;
|
|
13
|
+
let origins = [];
|
|
14
|
+
self.addEventListener("message", (event) => {
|
|
15
|
+
const auth = event.data && typeof event.data === "object" ? event.data.queriaAuth : undefined;
|
|
16
|
+
if (auth === undefined) return;
|
|
17
|
+
event.stopImmediatePropagation();
|
|
18
|
+
token = auth && typeof auth.token === "string" ? auth.token : null;
|
|
19
|
+
origins = auth && Array.isArray(auth.origins) ? auth.origins.map((o) => String(o) + "/") : [];
|
|
20
|
+
});
|
|
21
|
+
const open = XMLHttpRequest.prototype.open;
|
|
22
|
+
XMLHttpRequest.prototype.open = function (method, url, ...rest) {
|
|
23
|
+
open.call(this, method, url, ...rest);
|
|
24
|
+
if (token !== null && origins.some((o) => String(url).startsWith(o))) {
|
|
25
|
+
this.setRequestHeader("Authorization", "Bearer " + token);
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
})();
|
|
29
|
+
`;
|
|
30
|
+
|
|
31
|
+
export const bundlesAt = (base: string): DuckDBBundles => ({
|
|
32
|
+
mvp: { mainModule: `${base}/duckdb-mvp.wasm`, mainWorker: `${base}/duckdb-browser-mvp.worker.js` },
|
|
33
|
+
eh: { mainModule: `${base}/duckdb-eh.wasm`, mainWorker: `${base}/duckdb-browser-eh.worker.js` },
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
const workers = new WeakMap<AsyncDuckDB, Worker>();
|
|
37
|
+
const authOrigins = new WeakMap<AsyncDuckDB, Set<string>>();
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Send the token to these origins from now on, as well as to the origins given before: queries run
|
|
41
|
+
* concurrently, and one that is still reading must keep its origins. A null token sends none.
|
|
42
|
+
*/
|
|
43
|
+
export const setAuth = (db: AsyncDuckDB, token: string | null, origins: readonly string[]): void => {
|
|
44
|
+
const all = token === null ? new Set<string>() : new Set([...(authOrigins.get(db) ?? []), ...origins]);
|
|
45
|
+
authOrigins.set(db, all);
|
|
46
|
+
workers.get(db)?.postMessage({ queriaAuth: { token, origins: [...all] } });
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const instantiate = async (bundles: DuckDBBundles): Promise<AsyncDuckDB> => {
|
|
50
|
+
const duckdb = await import("@duckdb/duckdb-wasm");
|
|
51
|
+
const bundle = await duckdb.selectBundle(bundles);
|
|
52
|
+
if (!bundle.mainWorker) throw new Error("The DuckDB bundle has no worker script");
|
|
53
|
+
// The worker lives on another origin, and a page with a strict-dynamic CSP blocks importScripts,
|
|
54
|
+
// so fetch the script and start it from a blob URL.
|
|
55
|
+
const res = await fetch(bundle.mainWorker);
|
|
56
|
+
if (!res.ok) throw new Error(`${bundle.mainWorker}: ${res.status}`);
|
|
57
|
+
const url = URL.createObjectURL(new Blob([authPatch, await res.text()], { type: "text/javascript" }));
|
|
58
|
+
try {
|
|
59
|
+
// A CSP without blob: in worker-src throws here, so this is inside the try that revokes the URL.
|
|
60
|
+
const worker = new Worker(url);
|
|
61
|
+
const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
|
|
62
|
+
try {
|
|
63
|
+
// Pages without cross-origin isolation cannot start threads.
|
|
64
|
+
await (bundle.pthreadWorker
|
|
65
|
+
? db.instantiate(bundle.mainModule, bundle.pthreadWorker).catch(() => db.instantiate(bundle.mainModule))
|
|
66
|
+
: db.instantiate(bundle.mainModule));
|
|
67
|
+
} catch (e) {
|
|
68
|
+
await db.terminate().catch(() => undefined);
|
|
69
|
+
throw e;
|
|
70
|
+
}
|
|
71
|
+
workers.set(db, worker);
|
|
72
|
+
return db;
|
|
73
|
+
} finally {
|
|
74
|
+
URL.revokeObjectURL(url);
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
/** Start DuckDB from the given WASM URL, and from jsDelivr when that fails. */
|
|
79
|
+
export const boot = async (wasm: string): Promise<AsyncDuckDB> => {
|
|
80
|
+
const db = await instantiate(bundlesAt(wasm)).catch(async (e: unknown) => {
|
|
81
|
+
const { getJsDelivrBundles } = await import("@duckdb/duckdb-wasm");
|
|
82
|
+
return instantiate(getJsDelivrBundles()).catch((e2: unknown) => {
|
|
83
|
+
throw new AggregateError([e, e2], "DuckDB WASM could not be started");
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
try {
|
|
87
|
+
// Read Parquet by range instead of downloading whole files. Arrow returns DECIMAL as an unscaled
|
|
88
|
+
// integer, so let DuckDB convert it to DOUBLE.
|
|
89
|
+
await db.open({ path: ":memory:", filesystem: { forceFullHTTPReads: false }, query: { castDecimalToDouble: true } });
|
|
90
|
+
const conn = await db.connect();
|
|
91
|
+
await conn.query("INSTALL ducklake; LOAD ducklake").finally(() => conn.close());
|
|
92
|
+
} catch (e) {
|
|
93
|
+
await db.terminate().catch(() => undefined);
|
|
94
|
+
throw e;
|
|
95
|
+
}
|
|
96
|
+
return db;
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
// The spatial extension is large, so it is loaded only for SQL that needs it, and on the error
|
|
100
|
+
// DuckDB gives for a missing spatial function.
|
|
101
|
+
|
|
102
|
+
/** Whether the SQL calls an ST_* function outside string literals and comments. */
|
|
103
|
+
export const needsSpatial = (sql: string): boolean => /\bST_\w+\s*\(/i.test(stripLiterals(sql));
|
|
104
|
+
|
|
105
|
+
/** Whether the error comes from the spatial extension not being loaded. */
|
|
106
|
+
export const isSpatialMissing = (message: string): boolean =>
|
|
107
|
+
/exists in the spatial extension/i.test(message) || /\b(function|type) with name (st_\w+|geometry)\b.*does not exist/i.test(message);
|
|
108
|
+
|
|
109
|
+
const spatial = new WeakMap<AsyncDuckDB, Promise<void>>();
|
|
110
|
+
const spatialLoaded = new WeakSet<AsyncDuckDB>();
|
|
111
|
+
|
|
112
|
+
export const loadSpatial = (db: AsyncDuckDB): Promise<void> => {
|
|
113
|
+
const found = spatial.get(db);
|
|
114
|
+
if (found) return found;
|
|
115
|
+
const p = db
|
|
116
|
+
.connect()
|
|
117
|
+
.then((conn) => conn.query("INSTALL spatial; LOAD spatial").finally(() => conn.close()))
|
|
118
|
+
.then(() => {
|
|
119
|
+
spatialLoaded.add(db);
|
|
120
|
+
})
|
|
121
|
+
.catch((e: unknown) => {
|
|
122
|
+
spatial.delete(db);
|
|
123
|
+
throw e;
|
|
124
|
+
});
|
|
125
|
+
spatial.set(db, p);
|
|
126
|
+
return p;
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
/** Run fn, loading spatial first when the SQL needs it, and once more after a missing-spatial error. */
|
|
130
|
+
export const withSpatial = async <T>(db: AsyncDuckDB, sql: string, fn: () => Promise<T>): Promise<T> => {
|
|
131
|
+
if (needsSpatial(sql)) await loadSpatial(db);
|
|
132
|
+
// Once spatial is loaded, running the SQL again gives the same error.
|
|
133
|
+
const loaded = spatialLoaded.has(db);
|
|
134
|
+
try {
|
|
135
|
+
return await fn();
|
|
136
|
+
} catch (e) {
|
|
137
|
+
if (loaded || !isSpatialMissing(String(e))) throw e;
|
|
138
|
+
await loadSpatial(db);
|
|
139
|
+
return fn();
|
|
140
|
+
}
|
|
141
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
2
|
+
import { aliases, connect, dataPath, lookup, parseDatasets } from "./client.js";
|
|
3
|
+
|
|
4
|
+
describe("aliases", () => {
|
|
5
|
+
it("uses the last part of each name as the alias", () => {
|
|
6
|
+
expect([...aliases(["queria/e_stat", "queria/jma"])]).toEqual([
|
|
7
|
+
["e_stat", "queria/e_stat"],
|
|
8
|
+
["jma", "queria/jma"],
|
|
9
|
+
]);
|
|
10
|
+
});
|
|
11
|
+
it("takes explicit aliases", () => {
|
|
12
|
+
expect([...aliases({ a: "alice/e_stat", b: "bob/e_stat" })]).toEqual([
|
|
13
|
+
["a", "alice/e_stat"],
|
|
14
|
+
["b", "bob/e_stat"],
|
|
15
|
+
]);
|
|
16
|
+
});
|
|
17
|
+
it("rejects short names", () => expect(() => aliases(["e_stat"])).toThrow("fully qualified"));
|
|
18
|
+
it("rejects two datasets with one alias", () => expect(() => aliases(["alice/e_stat", "bob/e_stat"])).toThrow("share an alias"));
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
describe("parseDatasets", () => {
|
|
22
|
+
it("reads names and alias=name pairs", () => {
|
|
23
|
+
expect(parseDatasets(" queria/e_stat, a=alice/x ")).toEqual({ e_stat: "queria/e_stat", a: "alice/x" });
|
|
24
|
+
});
|
|
25
|
+
it("rejects two datasets with one alias", () =>
|
|
26
|
+
expect(() => parseDatasets("alice/e_stat bob/e_stat")).toThrow("share an alias"));
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
describe("dataPath", () => {
|
|
30
|
+
it("drops the version and points next to the catalog", () => {
|
|
31
|
+
expect(dataPath("https://data.queria.io/d/k/ducklake.duckdb?v=1")).toBe("https://data.queria.io/d/k/ducklake.duckdb.files/");
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
describe("lookup", () => {
|
|
36
|
+
afterEach(() => vi.unstubAllGlobals());
|
|
37
|
+
it("reads the row by qualified name and ignores unknown fields", async () => {
|
|
38
|
+
const fetch = vi.fn(async () => Response.json({ qualified_name: "queria/e_stat", catalog_url: "u", extra: 1 }));
|
|
39
|
+
vi.stubGlobal("fetch", fetch);
|
|
40
|
+
expect((await lookup("https://api", "queria/e_stat")).catalog_url).toBe("u");
|
|
41
|
+
expect(fetch).toHaveBeenCalledWith("https://api/v1/datasets/queria/e_stat", undefined);
|
|
42
|
+
});
|
|
43
|
+
it("sends the token as Bearer", async () => {
|
|
44
|
+
const fetch = vi.fn(async () => Response.json({ qualified_name: "x/y", catalog_url: "u" }));
|
|
45
|
+
vi.stubGlobal("fetch", fetch);
|
|
46
|
+
await lookup("https://api", "x/y", "qd_t");
|
|
47
|
+
expect(fetch).toHaveBeenCalledWith("https://api/v1/datasets/x/y", { headers: { Authorization: "Bearer qd_t" } });
|
|
48
|
+
});
|
|
49
|
+
it("rejects a response without a catalog URL", async () => {
|
|
50
|
+
vi.stubGlobal("fetch", async () => Response.json({ dataset: { qualified_name: "x/y" } }));
|
|
51
|
+
await expect(lookup("https://api", "x/y")).rejects.toThrow("no catalog_url");
|
|
52
|
+
});
|
|
53
|
+
it("reports a missing dataset", async () => {
|
|
54
|
+
vi.stubGlobal("fetch", async () => new Response(null, { status: 404 }));
|
|
55
|
+
await expect(lookup("https://api", "x/y")).rejects.toThrow("x/y was not found");
|
|
56
|
+
});
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
describe("sources", () => {
|
|
60
|
+
afterEach(() => vi.unstubAllGlobals());
|
|
61
|
+
it("asks a token function for each lookup", async () => {
|
|
62
|
+
const fetch = vi.fn(async (url: string) => Response.json({ qualified_name: url.split("/v1/datasets/")[1], catalog_url: "u" }));
|
|
63
|
+
vi.stubGlobal("fetch", fetch);
|
|
64
|
+
const token = vi.fn(async () => "qd_fresh");
|
|
65
|
+
await connect({ datasets: ["x/a", "x/b"], apiUrl: "https://api", token }).sources();
|
|
66
|
+
expect(token).toHaveBeenCalledTimes(2);
|
|
67
|
+
expect(fetch).toHaveBeenCalledWith("https://api/v1/datasets/x/a", { headers: { Authorization: "Bearer qd_fresh" } });
|
|
68
|
+
});
|
|
69
|
+
it("returns every declared dataset without SQL, and only the referenced ones with SQL", async () => {
|
|
70
|
+
vi.stubGlobal("fetch", async (url: string) =>
|
|
71
|
+
Response.json({ qualified_name: url.split("/v1/datasets/")[1], catalog_url: "u" }),
|
|
72
|
+
);
|
|
73
|
+
const conn = connect({ datasets: ["queria/e_stat", "queria/jma"], apiUrl: "https://api" });
|
|
74
|
+
expect((await conn.sources()).map((d) => d.qualified_name)).toEqual(["queria/e_stat", "queria/jma"]);
|
|
75
|
+
expect((await conn.sources("SELECT * FROM jma.main.t")).map((d) => d.qualified_name)).toEqual(["queria/jma"]);
|
|
76
|
+
});
|
|
77
|
+
});
|
package/src/client.ts
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import type { AsyncDuckDB } from "@duckdb/duckdb-wasm";
|
|
2
|
+
import { boot, setAuth, withSpatial } from "./boot.js";
|
|
3
|
+
import { datasetsIn, lit } from "./sql.js";
|
|
4
|
+
|
|
5
|
+
/** A license of a dataset, as returned by the Queria API. */
|
|
6
|
+
export type License = { id?: string; title?: string; url?: string };
|
|
7
|
+
|
|
8
|
+
/** The part of a dataset row from the Queria API that the package uses. */
|
|
9
|
+
export type Dataset = {
|
|
10
|
+
qualified_name: string;
|
|
11
|
+
title?: string;
|
|
12
|
+
page_url?: string;
|
|
13
|
+
licenses?: License[];
|
|
14
|
+
catalog_url: string;
|
|
15
|
+
versioned_catalog_url?: string;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/** Datasets to read, by fully qualified name. An array uses each name's last part as the SQL alias. */
|
|
19
|
+
export type Declared = readonly string[] | Readonly<Record<string, string>>;
|
|
20
|
+
|
|
21
|
+
export type ConnectOptions = {
|
|
22
|
+
datasets: Declared;
|
|
23
|
+
/** Base URL of the Queria API. */
|
|
24
|
+
apiUrl?: string;
|
|
25
|
+
/** Base URL of the DuckDB WASM bundles. jsDelivr is used when they cannot be loaded. */
|
|
26
|
+
wasmUrl?: string;
|
|
27
|
+
/**
|
|
28
|
+
* A data token (qd_...) for datasets that are not public, or a function that returns one. A function is
|
|
29
|
+
* called before each request, so it can return a fresh token. The token is sent to the API and to the
|
|
30
|
+
* hosts of the declared datasets. DuckDB is shared by the page, so use one token per page.
|
|
31
|
+
*/
|
|
32
|
+
token?: string | (() => string | Promise<string>);
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
export type Connection = {
|
|
36
|
+
/** Run SQL and return rows as plain objects. Datasets the SQL references are attached on first use. */
|
|
37
|
+
sql: <T = Record<string, unknown>>(sql: string) => Promise<T[]>;
|
|
38
|
+
/**
|
|
39
|
+
* The declared datasets the SQL references, for showing their sources and licenses.
|
|
40
|
+
* Without SQL (rows computed elsewhere), every declared dataset.
|
|
41
|
+
*/
|
|
42
|
+
sources: (sql?: string) => Promise<Dataset[]>;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
const API = "https://api.queria.io";
|
|
46
|
+
// Must match the @duckdb/duckdb-wasm version in package.json "dependencies".
|
|
47
|
+
const WASM = "https://assets.queria.io/duckdb-wasm/1.33.1-dev57.0";
|
|
48
|
+
|
|
49
|
+
const ALIAS = /^[a-z_][a-z0-9_]*$/;
|
|
50
|
+
const QUALIFIED = /^[a-z0-9_-]+\/[a-z_][a-z0-9_]*$/i;
|
|
51
|
+
|
|
52
|
+
/** Map each SQL alias to a fully qualified name, rejecting names the package cannot use. */
|
|
53
|
+
export const aliases = (declared: Declared): Map<string, string> => {
|
|
54
|
+
const pairs: [string, string][] = Array.isArray(declared)
|
|
55
|
+
? declared.map((q: string) => [q.split("/")[1] ?? "", q])
|
|
56
|
+
: Object.entries(declared);
|
|
57
|
+
pairs.forEach(([alias, q]) => {
|
|
58
|
+
if (!QUALIFIED.test(q)) throw new Error(`"${q}" is not a fully qualified dataset name such as "queria/e_stat"`);
|
|
59
|
+
if (!ALIAS.test(alias)) throw new Error(`"${alias}" cannot be used as an SQL alias`);
|
|
60
|
+
});
|
|
61
|
+
const map = new Map(pairs);
|
|
62
|
+
if (map.size !== pairs.length) throw new Error("Two datasets share an alias. Declare them as { alias: name }");
|
|
63
|
+
return map;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/** Look up a dataset row. The API returns the catalog URL; the package never builds it. */
|
|
67
|
+
export const lookup = async (api: string, qualified: string, token: string | null = null): Promise<Dataset> => {
|
|
68
|
+
const res = await fetch(`${api}/v1/datasets/${qualified}`, bearer(token));
|
|
69
|
+
if (res.status === 404) throw new Error(`Dataset ${qualified} was not found`);
|
|
70
|
+
if (!res.ok) throw new Error(`${api}/v1/datasets/${qualified}: ${res.status}`);
|
|
71
|
+
const d = (await res.json()) as Partial<Dataset> | null;
|
|
72
|
+
if (typeof d?.catalog_url !== "string" || typeof d.qualified_name !== "string")
|
|
73
|
+
throw new Error(`${api}/v1/datasets/${qualified}: the response has no catalog_url or qualified_name`);
|
|
74
|
+
return d as Dataset;
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const bearer = (token: string | null): RequestInit | undefined =>
|
|
78
|
+
token === null ? undefined : { headers: { Authorization: `Bearer ${token}` } };
|
|
79
|
+
|
|
80
|
+
/** Parquet files sit next to the catalog: .../ducklake.duckdb → .../ducklake.duckdb.files/ */
|
|
81
|
+
export const dataPath = (catalogUrl: string): string => `${catalogUrl.split("?")[0]}.files/`;
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Attach one dataset. The catalog is fetched in a single GET: letting DuckDB read it by range
|
|
85
|
+
* costs hundreds of 16 KiB round trips for large catalogs. Data files are still read by range.
|
|
86
|
+
*/
|
|
87
|
+
const attach = async (db: AsyncDuckDB, alias: string, d: Dataset, token: string | null): Promise<void> => {
|
|
88
|
+
const url = d.versioned_catalog_url ?? d.catalog_url;
|
|
89
|
+
const res = await fetch(url, bearer(token));
|
|
90
|
+
if (!res.ok) throw new Error(`${url}: ${res.status}`);
|
|
91
|
+
// One file per alias: an alias is attached once per DuckDB, while one dataset can be attached
|
|
92
|
+
// under two aliases, and DuckDB refuses to attach one path twice.
|
|
93
|
+
const file = `${alias}.duckdb`;
|
|
94
|
+
await db.registerFileBuffer(file, new Uint8Array(await res.arrayBuffer()));
|
|
95
|
+
const conn = await db.connect();
|
|
96
|
+
try {
|
|
97
|
+
await conn.query(
|
|
98
|
+
`ATTACH ${lit(`ducklake:${file}`)} AS ${alias} (READ_ONLY, DATA_PATH ${lit(dataPath(d.catalog_url))}, OVERRIDE_DATA_PATH true)`,
|
|
99
|
+
);
|
|
100
|
+
} finally {
|
|
101
|
+
await conn.close();
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/** Convert Arrow values to plain JS: lists to arrays, structs to objects, BIGINT and HUGEINT to number. */
|
|
106
|
+
export const plain = (v: unknown): unknown => {
|
|
107
|
+
if (typeof v === "bigint") return Number(v);
|
|
108
|
+
if (v === null || typeof v !== "object") return v;
|
|
109
|
+
if (Array.isArray(v)) return v.map(plain);
|
|
110
|
+
// 128-bit integers come as Arrow big numbers (typed arrays); BLOBs stay Uint8Array.
|
|
111
|
+
if (ArrayBuffer.isView(v)) return Symbol.for("isArrowBigNum") in v ? Number(String(v)) : v;
|
|
112
|
+
if ("toJSON" in v && typeof v.toJSON === "function") return plain(v.toJSON());
|
|
113
|
+
return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, plain(x)]));
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
/** One DuckDB per WASM URL, shared by every connection on the page. Aliases are global within it. */
|
|
117
|
+
type Engine = {
|
|
118
|
+
db: () => Promise<AsyncDuckDB>;
|
|
119
|
+
/** Terminate DuckDB if it was started. Never starts it. */
|
|
120
|
+
stop: () => Promise<void>;
|
|
121
|
+
attached: Map<string, { qualified: string; done: Promise<void> }>;
|
|
122
|
+
};
|
|
123
|
+
const engines = new Map<string, Engine>();
|
|
124
|
+
|
|
125
|
+
const engine = (wasm: string): Engine => {
|
|
126
|
+
const found = engines.get(wasm);
|
|
127
|
+
if (found) return found;
|
|
128
|
+
let db: Promise<AsyncDuckDB> | null = null;
|
|
129
|
+
const e: Engine = {
|
|
130
|
+
db: async () => {
|
|
131
|
+
const p = (db ??= boot(wasm).catch((err: unknown) => {
|
|
132
|
+
if (db === p) {
|
|
133
|
+
db = null;
|
|
134
|
+
e.attached.clear();
|
|
135
|
+
}
|
|
136
|
+
throw err;
|
|
137
|
+
}));
|
|
138
|
+
const d = await p;
|
|
139
|
+
// stop() ran while DuckDB was starting; it terminates this one.
|
|
140
|
+
if (db !== p) throw new Error("DuckDB was stopped");
|
|
141
|
+
return d;
|
|
142
|
+
},
|
|
143
|
+
stop: async () => {
|
|
144
|
+
const started = db;
|
|
145
|
+
db = null;
|
|
146
|
+
e.attached.clear();
|
|
147
|
+
// A failed start was already reported to the query that started it.
|
|
148
|
+
await started?.then((d) => d.terminate(), () => undefined);
|
|
149
|
+
},
|
|
150
|
+
attached: new Map(),
|
|
151
|
+
};
|
|
152
|
+
engines.set(wasm, e);
|
|
153
|
+
return e;
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
/** Stop every DuckDB the package started. A later query starts it again. */
|
|
157
|
+
export const closeAll = async (): Promise<void> => {
|
|
158
|
+
const all = [...engines.values()];
|
|
159
|
+
engines.clear();
|
|
160
|
+
await Promise.all(all.map((e) => e.stop()));
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Connect to Queria datasets. Declare every dataset the SQL reads by its fully qualified name;
|
|
165
|
+
* the SQL refers to it by alias, such as e_stat.census.population. Short names are never resolved.
|
|
166
|
+
*/
|
|
167
|
+
export const connect = ({ datasets, apiUrl = API, wasmUrl = WASM, token }: ConnectOptions): Connection => {
|
|
168
|
+
const declared = aliases(datasets);
|
|
169
|
+
const auth = async (): Promise<string | null> => (token === undefined ? null : typeof token === "string" ? token : await token());
|
|
170
|
+
const rows = new Map<string, Promise<Dataset>>();
|
|
171
|
+
/** The dataset row, looked up once. Pass the token when the caller has already asked for it. */
|
|
172
|
+
const row = (qualified: string, token?: Promise<string | null>): Promise<Dataset> => {
|
|
173
|
+
const p =
|
|
174
|
+
rows.get(qualified) ??
|
|
175
|
+
(token ?? auth())
|
|
176
|
+
.then((t) => lookup(apiUrl, qualified, t))
|
|
177
|
+
.catch((e: unknown) => {
|
|
178
|
+
rows.delete(qualified);
|
|
179
|
+
throw e;
|
|
180
|
+
});
|
|
181
|
+
rows.set(qualified, p);
|
|
182
|
+
return p;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
/** Declared aliases the SQL references. Other three-part names are left to DuckDB (alias.column.field). */
|
|
186
|
+
const used = (sql: string): string[] => datasetsIn(sql).filter((a) => declared.has(a));
|
|
187
|
+
|
|
188
|
+
const ensure = async (e: Engine, db: AsyncDuckDB, alias: string, t: string | null): Promise<void> => {
|
|
189
|
+
const qualified = declared.get(alias) ?? "";
|
|
190
|
+
const prev = e.attached.get(alias);
|
|
191
|
+
if (prev && prev.qualified !== qualified)
|
|
192
|
+
throw new Error(`Alias ${alias} is already used for ${prev.qualified} on this page. Declare ${qualified} with another alias`);
|
|
193
|
+
const done =
|
|
194
|
+
prev?.done ??
|
|
195
|
+
row(qualified, Promise.resolve(t))
|
|
196
|
+
.then((d) => attach(db, alias, d, t))
|
|
197
|
+
.catch((err: unknown) => {
|
|
198
|
+
e.attached.delete(alias);
|
|
199
|
+
throw err;
|
|
200
|
+
});
|
|
201
|
+
e.attached.set(alias, { qualified, done });
|
|
202
|
+
await done;
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
const sql = async <T,>(query: string): Promise<T[]> => {
|
|
206
|
+
const eng = engine(wasmUrl);
|
|
207
|
+
const aliasesUsed = used(query);
|
|
208
|
+
// Look up the datasets while DuckDB starts. The token function is called once per query.
|
|
209
|
+
const token = auth();
|
|
210
|
+
const [db, t, ds] = await Promise.all([eng.db(), token, Promise.all(aliasesUsed.map((a) => row(declared.get(a) ?? "", token)))]);
|
|
211
|
+
// Data files are read by the DuckDB worker, so hand it the token and the hosts it may go to.
|
|
212
|
+
if (t !== null) setAuth(db, t, [...new Set(ds.map((d) => new URL(d.catalog_url).origin))]);
|
|
213
|
+
await Promise.all(aliasesUsed.map((a) => ensure(eng, db, a, t)));
|
|
214
|
+
const run = async (): Promise<T[]> => {
|
|
215
|
+
const conn = await db.connect();
|
|
216
|
+
try {
|
|
217
|
+
return (await conn.query(query)).toArray().map((r) => plain(r) as T);
|
|
218
|
+
} finally {
|
|
219
|
+
await conn.close();
|
|
220
|
+
}
|
|
221
|
+
};
|
|
222
|
+
try {
|
|
223
|
+
return await withSpatial(db, query, run);
|
|
224
|
+
} catch (e) {
|
|
225
|
+
const undeclared = datasetsIn(query).filter((a) => !declared.has(a));
|
|
226
|
+
if (undeclared.length === 0) throw e;
|
|
227
|
+
throw new Error(`${String(e)}\nIf ${undeclared.join(", ")} is a dataset, declare it in connect({ datasets })`, { cause: e });
|
|
228
|
+
}
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
const sources = (query?: string): Promise<Dataset[]> =>
|
|
232
|
+
Promise.all((query === undefined ? [...declared.keys()] : used(query)).map((a) => row(declared.get(a) ?? "")));
|
|
233
|
+
|
|
234
|
+
return { sql, sources };
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
/** Read a list of datasets from an HTML attribute: "queria/e_stat queria/jma" or "a=alice/x b=bob/x". */
|
|
238
|
+
export const parseDatasets = (attr: string): Declared => {
|
|
239
|
+
const pairs = attr
|
|
240
|
+
.split(/[\s,]+/)
|
|
241
|
+
.filter(Boolean)
|
|
242
|
+
.map((w) => (w.includes("=") ? w.split("=", 2) : [w.split("/")[1] ?? "", w]));
|
|
243
|
+
// Object.fromEntries would keep only the last of two datasets with one alias.
|
|
244
|
+
if (new Set(pairs.map(([alias]) => alias)).size !== pairs.length)
|
|
245
|
+
throw new Error("Two datasets share an alias. Declare them as alias=name");
|
|
246
|
+
return Object.fromEntries(pairs);
|
|
247
|
+
};
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { connect, parseDatasets } from "../client.js";
|
|
2
|
+
import { attributionHtml } from "../render/attribution.js";
|
|
3
|
+
import { renderChart } from "../render/chart.js";
|
|
4
|
+
import { renderMap } from "../render/data-map.js";
|
|
5
|
+
import { esc } from "../render/escape.js";
|
|
6
|
+
import { poweredByHtml } from "../render/poweredBy.js";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* <queria-chart datasets="queria/e_stat" sql="..." spec='{"mark": "line", ...}'>
|
|
10
|
+
* Runs the SQL and draws the rows with the Vega-Lite spec, followed by the sources of the datasets.
|
|
11
|
+
* Instead of sql, `rows` takes the rows as JSON, for pages that run the SQL at build time. Declare the datasets the
|
|
12
|
+
* rows came from in `datasets` so their sources are shown.
|
|
13
|
+
* `api-url` replaces the Queria API, for example with a staging host.
|
|
14
|
+
*/
|
|
15
|
+
export class QueriaChart extends HTMLElement {
|
|
16
|
+
static observedAttributes = ["datasets", "sql", "spec", "rows", "api-url"];
|
|
17
|
+
#run = 0;
|
|
18
|
+
#pending = false;
|
|
19
|
+
#finalize: (() => void) | null = null;
|
|
20
|
+
|
|
21
|
+
connectedCallback() {
|
|
22
|
+
this.#schedule();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
attributeChangedCallback() {
|
|
26
|
+
if (this.isConnected) this.#schedule();
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
disconnectedCallback() {
|
|
30
|
+
this.#run++;
|
|
31
|
+
this.#clear();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Draw once per task: an upgrade calls attributeChangedCallback for every attribute, then connectedCallback. */
|
|
35
|
+
#schedule() {
|
|
36
|
+
if (this.#pending) return;
|
|
37
|
+
this.#pending = true;
|
|
38
|
+
queueMicrotask(() => {
|
|
39
|
+
this.#pending = false;
|
|
40
|
+
if (this.isConnected) this.#draw();
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
#clear() {
|
|
45
|
+
this.#finalize?.();
|
|
46
|
+
this.#finalize = null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async #draw() {
|
|
50
|
+
const [datasets, sql, spec, rows] = ["datasets", "sql", "spec", "rows"].map((a) => this.getAttribute(a));
|
|
51
|
+
if (!spec || (!rows && !sql)) return;
|
|
52
|
+
const run = ++this.#run;
|
|
53
|
+
this.#clear();
|
|
54
|
+
this.innerHTML = '<p class="queria-note">Loading…</p>';
|
|
55
|
+
try {
|
|
56
|
+
const conn = connect({ datasets: parseDatasets(datasets ?? ""), apiUrl: this.getAttribute("api-url") ?? undefined });
|
|
57
|
+
// With rows, the page ran the SQL itself; show every declared dataset as a source.
|
|
58
|
+
const [data, sources] = await Promise.all([sql ? conn.sql(sql) : JSON.parse(rows ?? "[]"), conn.sources(sql ?? undefined)]);
|
|
59
|
+
if (run !== this.#run) return;
|
|
60
|
+
this.innerHTML = `<div></div>${attributionHtml(sources)}`;
|
|
61
|
+
const { finalize } = await renderChart(this.firstElementChild as HTMLElement, JSON.parse(spec), data);
|
|
62
|
+
if (run === this.#run) this.#finalize = finalize;
|
|
63
|
+
else finalize();
|
|
64
|
+
} catch (e) {
|
|
65
|
+
if (run === this.#run) this.innerHTML = `<p class="queria-note">${esc(String(e))}</p>`;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* <queria-map datasets="queria/e_stat" sql="..." layer='{"area": "city", "code": "lg_code", "color": "population"}'>
|
|
72
|
+
* Runs the SQL and puts the rows on a map by area code or by lng/lat, followed by the sources of the datasets.
|
|
73
|
+
* `rows`, `api-url` work as on <queria-chart>. `theme` is light or dark; `map-style` takes another MapLibre style URL.
|
|
74
|
+
* Include maplibre-gl's CSS in the page. The map height is the CSS variable --queria-map-height (400px by default).
|
|
75
|
+
*/
|
|
76
|
+
export class QueriaMap extends HTMLElement {
|
|
77
|
+
static observedAttributes = ["datasets", "sql", "layer", "rows", "api-url", "theme", "map-style"];
|
|
78
|
+
#run = 0;
|
|
79
|
+
#pending = false;
|
|
80
|
+
#finalize: (() => void) | null = null;
|
|
81
|
+
|
|
82
|
+
connectedCallback() {
|
|
83
|
+
this.#schedule();
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
attributeChangedCallback() {
|
|
87
|
+
if (this.isConnected) this.#schedule();
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
disconnectedCallback() {
|
|
91
|
+
this.#run++;
|
|
92
|
+
this.#clear();
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
#schedule() {
|
|
96
|
+
if (this.#pending) return;
|
|
97
|
+
this.#pending = true;
|
|
98
|
+
queueMicrotask(() => {
|
|
99
|
+
this.#pending = false;
|
|
100
|
+
if (this.isConnected) this.#draw();
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
#clear() {
|
|
105
|
+
this.#finalize?.();
|
|
106
|
+
this.#finalize = null;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async #draw() {
|
|
110
|
+
const [datasets, sql, layer, rows, theme, style] = ["datasets", "sql", "layer", "rows", "theme", "map-style"].map((a) => this.getAttribute(a));
|
|
111
|
+
if (!layer || (!rows && !sql)) return;
|
|
112
|
+
const run = ++this.#run;
|
|
113
|
+
this.#clear();
|
|
114
|
+
this.innerHTML = '<p class="queria-note">Loading…</p>';
|
|
115
|
+
try {
|
|
116
|
+
const conn = connect({ datasets: parseDatasets(datasets ?? ""), apiUrl: this.getAttribute("api-url") ?? undefined });
|
|
117
|
+
const [data, sources] = await Promise.all([sql ? conn.sql(sql) : JSON.parse(rows ?? "[]"), conn.sources(sql ?? undefined)]);
|
|
118
|
+
if (run !== this.#run) return;
|
|
119
|
+
this.innerHTML = `<div style="height:var(--queria-map-height,400px)"></div>${attributionHtml(sources)}`;
|
|
120
|
+
const { finalize } = await renderMap(this.firstElementChild as HTMLElement, JSON.parse(layer), data, {
|
|
121
|
+
theme: theme === "dark" ? "dark" : "light",
|
|
122
|
+
...(style ? { style } : {}),
|
|
123
|
+
});
|
|
124
|
+
if (run === this.#run) this.#finalize = finalize;
|
|
125
|
+
else finalize();
|
|
126
|
+
} catch (e) {
|
|
127
|
+
if (run === this.#run) this.innerHTML = `<p class="queria-note">${esc(String(e))}</p>`;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** <queria-powered-by> */
|
|
133
|
+
export class QueriaPoweredBy extends HTMLElement {
|
|
134
|
+
connectedCallback() {
|
|
135
|
+
this.innerHTML = poweredByHtml();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const elements = { "queria-chart": QueriaChart, "queria-map": QueriaMap, "queria-powered-by": QueriaPoweredBy };
|
|
140
|
+
|
|
141
|
+
/** Register the custom elements. Safe to call more than once. */
|
|
142
|
+
export const defineElements = (): void =>
|
|
143
|
+
Object.entries(elements).forEach(([name, cls]) => customElements.get(name) ?? customElements.define(name, cls));
|
|
144
|
+
|
|
145
|
+
declare global {
|
|
146
|
+
interface HTMLElementTagNameMap {
|
|
147
|
+
"queria-chart": QueriaChart;
|
|
148
|
+
"queria-map": QueriaMap;
|
|
149
|
+
"queria-powered-by": QueriaPoweredBy;
|
|
150
|
+
}
|
|
151
|
+
}
|