plevinjs 0.2.1 → 0.2.2

package/dist/slim.js

@@ -0,0 +1,639 @@
+/** A database cut down to the fields asked for, rebuilt the way the builder writes it. */
+import { decompress } from "./lzma.js";
+import { CARRIED, COLUMNS, ORDERED, parse, section, TABLES, VOCABULARIES, } from "./selection.js";
+import { collect } from "./stats.js";
+import { write } from "./writer.js";
+export { fileName } from "./selection.js";
+const MAGIC = "PLEVIN\0";
+const FORMAT = 3;
+const HOST_BITS = 64n;
+const UNSET = -1;
+const VERSIONS = [4, 6];
+const SPINE_TABLES = ["place", "network", "abuse"];
+const REACHED = ["network", "operator", "place", "city"];
+const decoder = new TextDecoder();
+const varint = (data, at) => {
+    let value = 0;
+    let shift = 1;
+    for (;;) {
+        const byte = data[at++];
+        value += (byte & 0x7f) * shift;
+        if (byte < 0x80)
+            return [value, at];
+        shift *= 128;
+    }
+};
+const bigVarint = (data, at) => {
+    let value = 0n;
+    let shift = 0n;
+    for (;;) {
+        const byte = data[at++];
+        value |= BigInt(byte & 0x7f) << shift;
+        if (byte < 0x80)
+            return [value, at];
+        shift += 7n;
+    }
+};
+const unsigned = (bytes, at, width) => {
+    let value = 0;
+    for (let step = width - 1; step >= 0; step -= 1)
+        value = value * 256 + bytes[at + step];
+    return value;
+};
+const twos = (value, width, signed) => signed && value >= 2 ** (width * 8 - 1) ? value - 2 ** (width * 8) : value;
+const little = (bytes, at, width, signed) => {
+    if (width <= 4)
+        return twos(unsigned(bytes, at, width), width, signed);
+    const high = twos(unsigned(bytes, at + 4, width - 4), width - 4, signed);
+    return high * 2 ** 32 + unsigned(bytes, at, 4);
+};
+const bisect = (keys, held) => {
+    let low = 0;
+    let high = keys.length;
+    while (low < high) {
+        const middle = (low + high) >> 1;
+        if (keys[middle] <= held)
+            low = middle + 1;
+        else
+            high = middle;
+    }
+    return low;
+};
+const narrowGroup = (raw, at, first, size, out) => {
+    let value = Number(first);
+    out.push(value);
+    for (let step = 1; step < size; step += 1) {
+        const [gap, next] = varint(raw, at);
+        value += gap;
+        out.push(value);
+        at = next;
+    }
+    return at;
+};
+const wideGroup = (raw, at, first, size, out) => {
+    const networks = [first >> HOST_BITS];
+    for (let step = 1; step < size; step += 1) {
+        const [gap, next] = bigVarint(raw, at);
+        networks.push(networks[step - 1] + gap);
+        at = next;
+    }
+    for (const network of networks) {
+        const [host, next] = bigVarint(raw, at);
+        out.push((network << HOST_BITS) | host);
+        at = next;
+    }
+    return at;
+};
+const frontDecoded = (raw, held, group) => {
+    let cursor = 0;
+    for (let skip = Math.ceil(held / group) - 1; skip > 0; skip -= 1) {
+        cursor = varint(raw, cursor)[1];
+    }
+    const names = [];
+    let previous = new Uint8Array(0);
+    for (let at = 0; at < held; at += 1) {
+        if (at % group === 0)
+            previous = new Uint8Array(0);
+        const shared = raw[cursor];
+        const [fresh, next] = varint(raw, cursor + 1);
+        const name = new Uint8Array(shared + fresh);
+        name.set(previous.subarray(0, shared));
+        name.set(raw.subarray(next, next + fresh), shared);
+        names.push(decoder.decode(name));
+        previous = name;
+        cursor = next + fresh;
+    }
+    return names;
+};
+class Source {
+    head;
+    bytes;
+    body;
+    held = new Map();
+    constructor(bytes) {
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        const magic = decoder.decode(bytes.subarray(0, MAGIC.length));
+        if (magic !== MAGIC || bytes[MAGIC.length] !== FORMAT) {
+            throw new Error("not a plevin raw database");
+        }
+        const size = view.getUint32(MAGIC.length + 1, true);
+        const start = MAGIC.length + 5;
+        this.head = JSON.parse(decoder.decode(bytes.subarray(start, start + size)));
+        this.bytes = bytes;
+        this.body = start + size;
+    }
+    has(name) {
+        return name in this.head.sections;
+    }
+    count(name) {
+        return this.head.sections[name].count;
+    }
+    memo(key, make) {
+        if (!this.held.has(key))
+            this.held.set(key, make());
+        return this.held.get(key);
+    }
+    values(name, Type) {
+        return this.memo(`${name}:${Type.name}`, () => this.decodeValues(name, Type));
+    }
+    keys(name, wide) {
+        return this.memo(name, () => this.decodeKeys(name, wide));
+    }
+    pool() {
+        return this.memo("pool", () => this.decodePool());
+    }
+    openSection(name) {
+        const entry = this.head.sections[name];
+        const at = this.body + entry.offset;
+        const view = this.bytes.subarray(at, at + entry.bytes);
+        const words = new DataView(view.buffer, view.byteOffset, view.byteLength);
+        const blocks = words.getUint32(0, true);
+        const width = words.getUint32(4, true);
+        const offsets = Array.from({ length: blocks + 1 }, (_, at) => words.getUint32(8 + at * 4, true));
+        const heads = [];
+        for (let block = 0; block < blocks; block += 1) {
+            let head = 0n;
+            for (let step = 0; step < width; step += 1) {
+                head = (head << 8n) | BigInt(view[8 + 4 * (blocks + 1) + block * width + step]);
+            }
+            heads.push(head);
+        }
+        const data = view.subarray(8 + 4 * (blocks + 1) + width * blocks);
+        const stream = (block) => decompress(data.subarray(offsets[block], offsets[block + 1]), entry.lzma);
+        return { entry, blocks, heads, stream };
+    }
+    decodeValues(name, Type) {
+        const { entry, blocks, stream } = this.openSection(name);
+        const out = new Type(entry.count);
+        const delta = entry.encoding === "delta";
+        const signed = entry.encoding === "signed" || delta;
+        for (let block = 0; block < blocks; block += 1) {
+            const raw = stream(block);
+            const width = raw[0];
+            const held = Math.min(entry.block, entry.count - block * entry.block);
+            let running = 0;
+            for (let at = 0; at < held; at += 1) {
+                const value = little(raw, 1 + at * width, width, signed);
+                running = delta ? running + value : value;
+                out[block * entry.block + at] = running;
+            }
+        }
+        return out;
+    }
+    decodePool() {
+        const { entry, blocks, stream } = this.openSection("strings");
+        const out = [];
+        for (let block = 0; block < blocks; block += 1) {
+            const held = Math.min(entry.block, entry.count - block * entry.block);
+            out.push(...frontDecoded(stream(block), held, entry.group));
+        }
+        return out;
+    }
+    decodeKeys(name, wide) {
+        const { entry, blocks, heads, stream } = this.openSection(name);
+        const readGroup = wide ? wideGroup : narrowGroup;
+        const out = [];
+        for (let block = 0; block < blocks; block += 1) {
+            const raw = stream(block);
+            const [held, start] = varint(raw, 0);
+            const groups = Math.ceil(held / entry.group);
+            const firsts = [heads[block]];
+            let cursor = start;
+            for (let at = 1; at < groups; at += 1) {
+                const [gap, next] = bigVarint(raw, cursor);
+                firsts.push(firsts[at - 1] + gap);
+                cursor = next;
+            }
+            for (let at = 1; at < groups; at += 1)
+                cursor = varint(raw, cursor)[1];
+            for (let group = 0; group < groups; group += 1) {
+                const size = Math.min(entry.group, held - group * entry.group);
+                cursor = readGroup(raw, cursor, firsts[group], size, out);
+            }
+        }
+        return out;
+    }
+}
+const carriedColumns = (make) => Object.fromEntries(CARRIED.map((name) => [name, make(name)]));
+const carries = (selection, name) => SPINE_TABLES.includes(name) ? selection.table(name) : selection.has(`spine.${name}`);
+const touch = (keep, link) => {
+    if (link > 0 && link <= keep.length)
+        keep[link - 1] = 1;
+};
+const tally = (counts, link, weight) => {
+    if (link > 0)
+        counts[link - 1] += weight;
+};
+const linked = (slab, link) => {
+    if (link === 0)
+        return 0;
+    const at = slab.map[link - 1];
+    return at === undefined || at < 0 ? 0 : at + 1;
+};
+const columnsOf = (slab, kind) => slab.columns.flatMap((column, at) => column.kind === kind ? [[at, column.name]] : []);
+const stopsOf = (source, kind, version) => {
+    const name = `${kind}.v${version}`;
+    if (!source.has(name))
+        return null;
+    return source.memo(`${name}:stops`, () => ({
+        keys: source.keys(name, version === 6),
+        columns: carriedColumns((carried) => source.values(`${name}.${carried}`, Int32Array)),
+        whole: source.values(`${name}.whole`, Int32Array).map((row) => row + 1),
+    }));
+};
+const hostsOf = (source, version) => source.memo(`hosts${version}`, () => {
+    const name = `hosts.v${version}`;
+    if (!source.has(name))
+        return { keys: [], links: new Int32Array(0) };
+    const links = source.values(`${name}.abuse`, Int32Array).map((row) => row + 1);
+    return { keys: source.keys(name, version === 6), links };
+});
+const networkedStops = (source, selection, stops, hosts) => {
+    const networked = new Uint8Array(stops.keys.length).fill(1);
+    if (!selection.sparse)
+        return networked;
+    const services = source.values("col.abuse.service", Float64Array);
+    const kept = selection.narrow.get("abuse.service");
+    const serving = (link) => {
+        const service = link > 0 ? services[link - 1] : 0;
+        return service > 0 && (!kept || kept.includes(service));
+    };
+    const links = selection.has("network.abuse") ? stops.columns.abuse : stops.whole;
+    for (let at = 0; at < networked.length; at += 1) {
+        networked[at] = serving(links[at]) ? 1 : 0;
+    }
+    hosts.links.forEach((link, at) => {
+        if (!serving(link))
+            return;
+        const spot = bisect(stops.keys, hosts.keys[at]) - 1;
+        if (spot >= 0)
+            networked[spot] = 1;
+    });
+    return networked;
+};
+const openFamilies = (source, selection) => VERSIONS.flatMap((version) => {
+    const kind = selection.has("place.lat") ? "spine" : "blocks";
+    const stops = stopsOf(source, kind, version);
+    if (!stops)
+        return [];
+    const hosts = hostsOf(source, version);
+    const networked = networkedStops(source, selection, stops, hosts);
+    return [{ version, wide: version === 6, stops, hosts, networked }];
+});
+const columnValues = (source, column) => source.values(section(column), Float64Array);
+const makeSlab = (source, selection, name) => {
+    const columns = COLUMNS.filter((one) => one.table === name && selection.has(one.id));
+    const values = columns.map((column) => columnValues(source, column));
+    const size = values.length ? values[0].length : 0;
+    return {
+        name,
+        columns,
+        values,
+        keep: new Uint8Array(size),
+        map: new Int32Array(size).fill(UNSET),
+        rows: [],
+        uses: [],
+    };
+};
+const makeSlabs = (source, selection) => Object.fromEntries(TABLES.map((name) => [name, makeSlab(source, selection, name)]));
+const followLinks = (slabs, slab) => {
+    for (const [spot, target] of columnsOf(slab, "link")) {
+        for (let row = 0; row < slab.keep.length; row += 1) {
+            if (slab.keep[row])
+                touch(slabs[target].keep, slab.values[spot][row]);
+        }
+    }
+};
+const markReached = (slabs, families) => {
+    touch(slabs.abuse.keep, 1);
+    for (const { stops, networked, hosts } of families) {
+        for (let spot = 0; spot < stops.keys.length; spot += 1) {
+            touch(slabs.place.keep, stops.columns.place[spot]);
+            if (networked[spot])
+                touch(slabs.network.keep, stops.columns.network[spot]);
+            touch(slabs.abuse.keep, stops.columns.abuse[spot]);
+            touch(slabs.abuse.keep, stops.whole[spot]);
+        }
+        for (const link of hosts.links)
+            touch(slabs.abuse.keep, link);
+    }
+    for (const name of REACHED)
+        followLinks(slabs, slabs[name]);
+};
+const slabValue = (selection, column, value, slabs) => {
+    const kept = selection.narrow.get(column.id);
+    const held = kept && !kept.includes(value) ? 0 : value;
+    return column.kind === "link" ? linked(slabs[column.name], held) : held;
+};
+const collapse = (slab, slabs, selection) => {
+    const known = new Map();
+    for (let at = 0; at < slab.keep.length; at += 1) {
+        if (!slab.keep[at])
+            continue;
+        const key = slab.columns.map((column, spot) => slabValue(selection, column, slab.values[spot][at], slabs));
+        const pinned = slab.name === "abuse" && at === 0;
+        if (!pinned && key.every((held) => held === 0))
+            continue;
+        const label = key.join(",");
+        let row = known.get(label);
+        if (row === undefined) {
+            row = slab.rows.length;
+            known.set(label, row);
+            slab.rows.push(key);
+        }
+        slab.map[at] = row;
+    }
+    slab.uses = new Array(slab.rows.length).fill(0);
+};
+const cutters = (selection, slabs, family) => {
+    const { stops, networked } = family;
+    const abuseLinks = selection.has("network.abuse") ? stops.columns.abuse : stops.whole;
+    const cutter = (name) => {
+        const column = stops.columns[name];
+        if (name === "place")
+            return (at) => linked(slabs.place, column[at]);
+        if (name === "network") {
+            return (at) => (networked[at] ? linked(slabs.network, column[at]) : 0);
+        }
+        if (name === "abuse") {
+            return (at) => {
+                const record = linked(slabs.abuse, abuseLinks[at]);
+                return record === 1 ? 0 : record;
+            };
+        }
+        return (at) => column[at];
+    };
+    return CARRIED.map((name) => (carries(selection, name) ? cutter(name) : () => 0));
+};
+const trim = (selection, slabs, family) => {
+    const { version, wide, hosts, stops } = family;
+    const cuts = cutters(selection, slabs, family);
+    const columns = carriedColumns(() => []);
+    const lists = CARRIED.map((name) => columns[name]);
+    const keys = [];
+    const last = new Array(cuts.length).fill(-1);
+    const cut = new Array(cuts.length).fill(0);
+    for (let at = 0; at < stops.keys.length; at += 1) {
+        let same = true;
+        for (let spot = 0; spot < cuts.length; spot += 1) {
+            cut[spot] = cuts[spot](at);
+            if (cut[spot] !== last[spot])
+                same = false;
+        }
+        if (same)
+            continue;
+        keys.push(stops.keys[at]);
+        for (let spot = 0; spot < cuts.length; spot += 1) {
+            lists[spot].push(cut[spot]);
+            last[spot] = cut[spot];
+        }
+    }
+    return { version, wide, hosts, keys, columns };
+};
+const countUses = (slabs, spines) => {
+    for (const { keys, columns, hosts } of spines) {
+        for (const name of SPINE_TABLES) {
+            for (let at = 0; at < keys.length; at += 1) {
+                tally(slabs[name].uses, columns[name][at], 1);
+            }
+        }
+        for (const link of hosts.links)
+            tally(slabs.abuse.uses, linked(slabs.abuse, link), 1);
+    }
+    for (const slab of Object.values(slabs).reverse()) {
+        for (const [spot, target] of columnsOf(slab, "link")) {
+            for (const [at, row] of slab.rows.entries()) {
+                tally(slabs[target].uses, row[spot], Math.max(slab.uses[at], 1));
+            }
+        }
+    }
+};
+const compareRows = (one, two) => {
+    for (let at = 0; at < one.length; at += 1) {
+        if (one[at] !== two[at])
+            return one[at] - two[at];
+    }
+    return 0;
+};
+const sortRows = (slab, order) => {
+    const ordered = ORDERED[slab.name];
+    if (ordered) {
+        const spots = ordered
+            .map((id) => slab.columns.findIndex((column) => column.id === id))
+            .filter((spot) => spot >= 0);
+        const keys = slab.rows.map((row) => [...spots.map((spot) => row[spot]), ...row]);
+        order.sort((one, two) => compareRows(keys[one], keys[two]));
+        return;
+    }
+    if (slab.name === "abuse") {
+        order.sort((one, two) => slab.uses[two] - slab.uses[one] || one - two);
+    }
+};
+const rank = (slab) => {
+    const first = slab.name === "abuse" && slab.rows.length > 0 ? 1 : 0;
+    const order = Array.from({ length: slab.rows.length - first }, (_, at) => at + first);
+    sortRows(slab, order);
+    const ranks = new Array(slab.rows.length).fill(0);
+    order.forEach((at, place) => {
+        ranks[at] = place + first;
+    });
+    return ranks;
+};
+const rankedLink = (order, link) => link === 0 ? 0 : order[link - 1] + 1;
+const relink = (slab, ranks) => {
+    const links = columnsOf(slab, "link");
+    for (const row of slab.rows) {
+        for (const [at, target] of links)
+            row[at] = rankedLink(ranks[target], row[at]);
+    }
+};
+const reorder = (slab, order) => {
+    const moved = new Array(slab.rows.length);
+    slab.rows.forEach((row, at) => {
+        moved[order[at]] = row;
+    });
+    slab.rows = moved;
+};
+const rankSlabs = (slabs) => {
+    const ranks = {};
+    for (const slab of Object.values(slabs)) {
+        relink(slab, ranks);
+        const order = rank(slab);
+        reorder(slab, order);
+        ranks[slab.name] = order;
+    }
+    return ranks;
+};
+const relinkSpine = (spine, ranks) => {
+    for (const name of SPINE_TABLES) {
+        spine.columns[name] = spine.columns[name].map((link) => rankedLink(ranks[name], link));
+    }
+};
+const eachText = (slabs, visit) => {
+    for (const slab of Object.values(slabs)) {
+        for (const [spot] of columnsOf(slab, "text")) {
+            for (const row of slab.rows)
+                visit(row, spot);
+        }
+    }
+};
+const respell = (slabs, pool) => {
+    const used = new Uint8Array(pool.length + 1);
+    eachText(slabs, (row, spot) => {
+        used[row[spot]] = 1;
+    });
+    const renumbered = new Int32Array(pool.length + 1);
+    const names = pool.filter((_, at) => used[at + 1]);
+    let place = 0;
+    for (let id = 1; id < used.length; id += 1) {
+        if (used[id])
+            renumbered[id] = ++place;
+    }
+    eachText(slabs, (row, spot) => {
+        row[spot] = renumbered[row[spot]];
+    });
+    return names;
+};
+const abuseAt = (spine, address, networkAbuse) => {
+    const at = bisect(spine.keys, address) - 1;
+    const link = at < 0 ? 0 : spine.columns.abuse[at];
+    if (link !== 0)
+        return link - 1;
+    return networkAbuse ? null : 0;
+};
+const hostParts = (selection, spine, slabs, order) => {
+    const { hosts } = spine;
+    const networkAbuse = selection.has("network.abuse");
+    const keys = [];
+    const values = [];
+    hosts.links.forEach((link, at) => {
+        const held = linked(slabs.abuse, link);
+        if (held === 0)
+            return;
+        const position = order[held - 1];
+        if (abuseAt(spine, hosts.keys[at], networkAbuse) === position)
+            return;
+        keys.push(hosts.keys[at]);
+        values.push(position);
+    });
+    if (keys.length === 0)
+        return [];
+    const name = `hosts.v${spine.version}`;
+    return [
+        { kind: "index", name, keys, wide: spine.wide },
+        { kind: "values", name: `${name}.abuse`, encoding: "fixed", read: "", values },
+    ];
+};
+const spineColumnParts = (selection, spine) => CARRIED.flatMap((name) => {
+    const values = spine.columns[name];
+    const held = name === "abuse" || values.some((value) => value !== 0);
+    if (!carries(selection, name) || !held)
+        return [];
+    const label = `spine.v${spine.version}.${name}`;
+    return [{ kind: "values", name: label, encoding: "fixed", read: "", values }];
+});
+const spineParts = (selection, spines, slabs, ranks) => {
+    const parts = spines
+        .filter((spine) => spine.keys.length > 0)
+        .flatMap((spine) => [
+        {
+            kind: "index",
+            name: `spine.v${spine.version}`,
+            keys: spine.keys,
+            wide: spine.wide,
+        },
+        ...spineColumnParts(selection, spine),
+    ]);
+    if (!selection.table("abuse"))
+        return parts;
+    const hosts = spines.flatMap((spine) => hostParts(selection, spine, slabs, ranks.abuse));
+    return [...parts, ...hosts];
+};
+const columnParts = (slabs) => COLUMNS.flatMap((column) => {
+    const slab = slabs[column.table];
+    const at = slab.columns.findIndex((one) => one.id === column.id);
+    if (at < 0)
+        return [];
+    const signed = column.kind === "signed" || column.kind === "degrees";
+    const read = column.kind === "text" || column.kind === "degrees" ? column.kind : "";
+    const values = slab.rows.map((row) => row[at]);
+    return [
+        {
+            kind: "values",
+            name: section(column),
+            encoding: signed ? "signed" : "fixed",
+            read,
+            values,
+        },
+    ];
+});
+const vocabulariesFor = (source, selection) => Object.fromEntries(VOCABULARIES.filter(([name, reads]) => name in source.head.vocabularies && reads.some((id) => selection.has(id))).map(([name]) => [name, source.head.vocabularies[name]]));
+const build = (source, selection, progress) => {
+    progress?.("reading");
+    const slabs = makeSlabs(source, selection);
+    const families = openFamilies(source, selection);
+    progress?.("merging");
+    markReached(slabs, families);
+    for (const slab of Object.values(slabs))
+        collapse(slab, slabs, selection);
+    const spines = families.map((family) => trim(selection, slabs, family));
+    countUses(slabs, spines);
+    const pool = respell(slabs, source.pool());
+    const ranks = rankSlabs(slabs);
+    for (const spine of spines)
+        relinkSpine(spine, ranks);
+    const parts = [
+        ...columnParts(slabs),
+        ...spineParts(selection, spines, slabs, ranks),
+        { kind: "strings", pool },
+    ];
+    return {
+        parts,
+        carries: SPINE_TABLES.map((name) => selection.table(name)),
+        fields: selection.fields,
+        books: vocabulariesFor(source, selection),
+    };
+};
+/** One database held open: each selection cut from it reuses what was already decoded. */
+export class Slimmer {
+    source;
+    constructor(bytes) {
+        this.source = new Source(bytes);
+    }
+    get built() {
+        return this.source.head.built;
+    }
+    get fields() {
+        return this.source.head.fields;
+    }
+    plan(terms) {
+        const { fields, vocabularies } = this.source.head;
+        return parse(terms, new Set(fields), vocabularies);
+    }
+    /** The database for `terms` (`place+metro`, `network.asn`), as bytes to save. */
+    slim(terms, progress) {
+        const selection = this.plan(terms);
+        const written = build(this.source, selection, progress);
+        return write(written, this.built, selection.name, progress);
+    }
+    /** The counted facts `estimate` reads instead of this file, a few kilobytes of JSON. */
+    stats() {
+        const { source } = this;
+        const kit = {
+            head: source.head,
+            has: (name) => source.has(name),
+            numbers: (name) => source.values(name, Float64Array),
+            pool: () => source.pool(),
+            spine: (version, coarse) => stopsOf(source, coarse ? "blocks" : "spine", version),
+            hosts: (version) => hostsOf(source, version).links,
+            keys: (version) => ({
+                hosts: hostsOf(source, version).keys,
+                spine: source.keys(`spine.v${version}`, version === 6),
+            }),
+        };
+        return collect(kit);
+    }
+}
+/** A database rebuilt with only what `terms` (`place+metro`, `network.asn`) need. */
+export const slim = (bytes, terms, progress) => new Slimmer(bytes).slim(terms, progress);
+//# sourceMappingURL=slim.js.map