plevinjs 0.2.0 → 0.2.2

package/README.md

@@ -12,7 +12,7 @@ No API, no rate limit, no lookup leaving the machine or the browser tab.
 [![npm](https://img.shields.io/npm/v/plevinjs?color=1868f2)](https://www.npmjs.com/package/plevinjs)
 [![Types](https://img.shields.io/badge/types-included-1868f2)](https://www.npmjs.com/package/plevinjs?activeTab=code)
 [![License](https://img.shields.io/badge/license-Apache--2.0-1868f2)](https://github.com/tn3w/plevin/blob/master/LICENSE)
-[![Fields](https://img.shields.io/badge/fields-101-6f42c1)](#fields)
+[![Fields](https://img.shields.io/badge/fields-108-6f42c1)](#fields)
 [![Warm](https://img.shields.io/badge/warm%20lookups-5M%2Fs-2ea043)](#speed)
 
 </div>
@@ -52,11 +52,11 @@ Open once, reuse for every lookup. Nothing is downloaded or cached for you.
 
 | file                                                  | size    | carries                                  |
 | ----------------------------------------------------- | ------- | ---------------------------------------- |
-| `plevin.plv`                                          | 16.4 MB | every field                              |
-| `plevin.metro-place.plv`                              | 5.1 MB  | city, region, postal, coordinates, metro |
-| `plevin.network.plv`                                  | 7.0 MB  | ASN, operator, routing                   |
-| `plevin.abuse-level-abuse-provider-abuse-service.plv` | 3.8 MB  | abuse level, service and provider        |
-| `plevin.place-country-code.plv`                       | 376 KB  | country code                             |
+| `plevin.plv`                                          | 18.7 MB | every field                              |
+| `plevin.metro-place.plv`                              | 6.3 MB  | city, region, postal, coordinates, metro |
+| `plevin.network.plv`                                  | 7.4 MB  | ASN, operator, routing, registry         |
+| `plevin.abuse-level-abuse-provider-abuse-service.plv` | 4.1 MB  | abuse level, service and provider        |
+| `plevin.place-country-code.plv`                       | 378 KB  | country code                             |
 
 All are served with open CORS from [plevin.tn3w.dev/db](https://plevin.tn3w.dev/db/);
 GitHub release downloads send no CORS header.
@@ -82,8 +82,8 @@ No build step:
 | `https://esm.sh/plevinjs`                      | the modules, imports rewritten    |
 | `https://plevin.tn3w.dev/plevin/plevin.min.js` | the bundle beside the databases   |
 
-- **Pin a version for production:** `cdn.jsdelivr.net/npm/plevinjs@0.2.0`.
-- **Pick the smallest build:** the country build is 376 KB against 16.4 MB.
+- **Pin a version for production:** `cdn.jsdelivr.net/npm/plevinjs@0.2.2`.
+- **Pick the smallest build:** the country build is 378 KB against 18.7 MB.
 - **Cache the file** so it downloads once per visitor:
 
 ```js
@@ -119,7 +119,9 @@ found.place;
   },
   country: {
     code: 'AU', name: 'Australia', official: null, common: null, iso3: 'AUS',
-    numeric: '036', flag: '🇦🇺', european_union: false, driving_side: 'left',
+    numeric: '036', flag: '🇦🇺', currency: 'AUD', currency_name: 'Australian Dollar',
+    calling_code: '+61', languages: ['en'], european_union: false,
+    driving_side: 'left',
   },
   time: {
     timezone: 'Australia/Brisbane', abbreviation: 'AEST',
@@ -131,7 +133,8 @@ found.place;
 found.network;
 {
   asn: 13335, handle: 'CLOUDFLARENET', prefix: 24, cidr: '1.1.1.0/24',
-  start: '1.1.1.0', end: '1.1.1.255', rir: 'apnic', rpki: 'valid', roas: 1,
+  start: '1.1.1.0', end: '1.1.1.255', rir: 'apnic', country: 'AU', since: 2011,
+  rpki: 'valid', roas: 1,
   operator: {
     company: 'Cloudflare, Inc.', brand: 'Cloudflare', domain: 'cloudflare.com',
     website: 'https://www.cloudflare.com', category: 'content', tier: 2,
@@ -146,8 +149,8 @@ found.network;
 db.lookup("185.220.101.1").abuse;
 {
   name: 'Tor', provider: 'Tor', service: 'tor_exit_node', evidence: 'measured',
-  level: 'high', risk: 0.99, network_risk: 0.86, last_seen_days: 1,
-  is_malicious: true, is_anycast: false, is_satellite: false,
+  threat: 'spam', level: 'high', risk: 0.99, network_risk: 0.86, last_seen_days: 1,
+  is_malicious: true, is_anycast: false, is_satellite: false, is_crawler: false,
   is_hosting_provider: true, is_proxy: false, is_public_proxy: false,
   is_residential_proxy: false, is_anonymous_vpn: false, is_tor_exit_node: true,
   is_private_relay: false, is_anonymous: true,
@@ -156,9 +159,10 @@ db.lookup("185.220.101.1").abuse;
 
 JavaScript specifics:
 
-- `country` comes from an ISO 3166 table in the package and `time` from the runtime's
-  `Intl` data, so neither needs an install or a network call. A different zone
-  database can move a daylight-saving boundary.
+- `country` comes from a table in the package, generated from pycountry, Babel and
+  phonenumbers by [`test/countries.py`](test/countries.py), and `time` from the
+  runtime's `Intl` data, so neither needs an install or a network call. A different
+  zone database can move a daylight-saving boundary.
 - `number` is a `bigint` for v6, so `JSON.stringify` needs a replacer.
 
 Address fields (`compressed`, `expanded`, `arpa`, `is_*`, `tunnel`, `embedded_ipv4`,
@@ -252,7 +256,8 @@ of heap with the whole world touched.
 ## LZMA
 
 Blocks are raw LZMA1, which neither `DecompressionStream` nor `zlib` reads, so the
-package ships its own decoder, checked against liblzma on every block.
+package ships its own decoder, checked against liblzma on every block, and, for
+[`slim`](#slim-databases), an encoder that writes the bytes liblzma writes.
 
 ```js
 import { decompress } from "plevinjs/lzma";
@@ -260,6 +265,52 @@ import { decompress } from "plevinjs/lzma";
 decompress(block, [3, 0, 0]);   // tuning: literal context, literal position, match position bits
 ```
 
+## Slim databases
+
+Cut the builder's raw file down to the fields you use, in the browser or on a server.
+It is the builder's selection pipeline in JavaScript, run on `plevin.raw`: the same
+terms, columns, rows and boundaries merged, blocks re-encoded and compressed, so the
+result opens like any other file.
+
+```js
+import { slim } from "plevinjs/slim";
+
+const raw = new Uint8Array(await (await fetch(url)).arrayBuffer());   // plevin.raw
+const small = slim(raw, "place.country.code+network.asn");   // 400 KB, not 18.7 MB
+const db = new Plevin(small);
+```
+
+```bash
+npx plevinjs place+metro   # reads plevin.raw, writes plevin.metro-place.plv
+```
+
+| terms                  | keeps                                                     |
+| ---------------------- | --------------------------------------------------------- |
+| `full`                 | every field the source holds                              |
+| `place+metro`          | every field under `place`, plus `metro`; join with `+`    |
+| `abuse.is_tor_exit_node` | one flag; rows narrowed to Tor exits only               |
+
+- **Raw in:** [`plevin.raw`](https://github.com/tn3w/plevin/releases/latest/download/plevin.raw)
+  (30 MB) is the world the builder cuts from. Slimmed files are not raw: they cannot be
+  cut again and `slim` throws on them.
+- **Same as the builder:** byte for byte. The encoder is a port of liblzma's preset 9
+  (binary-tree matcher, optimal parse, range coder), checked on every block of the
+  release files; the same terms give the same file, `full` included. The build date
+  stays the raw file's.
+- **Only what the source holds:** a term that matches nothing throws.
+- **Size before building:** `estimate(stats, terms)` from `plevinjs/estimate` reads a
+  few hundred KB of counted facts (`new Slimmer(raw).stats()`, or the `stats.json`
+  beside the databases) and answers instantly, without decoding anything. Rates are fitted
+  to 172 builds; on 64 others it landed within 10% for four in five and within
+  about 2× at worst, tiny selections being the loosest. A build is exact.
+- **Without code:** [plevin.tn3w.dev/#slim](https://plevin.tn3w.dev/#slim) picks the
+  fields, shows the size first and saves the file; it runs `slim` in a worker.
+- **Out of the CDN bundle:** `plevinjs` and `plevin.min.js` never import it. Reach it at
+  `plevinjs/slim`, `https://esm.sh/plevinjs/slim` or
+  `https://cdn.jsdelivr.net/npm/plevinjs/dist/slim.min.js`.
+- **Cost:** the full cut needs about 2 GB of heap and under two minutes; a country file
+  under 10 s. No native module is needed.
+
 ## Development
 
 | module       | job                                                 |
@@ -267,6 +318,15 @@ decompress(block, [3, 0, 0]);   // tuning: literal context, literal position, ma
 | `index.ts`   | `Plevin`, `open`, result shaping                    |
 | `reader.ts`  | file format: sections, blocks, groups, bisection    |
 | `lzma.ts`    | resumable LZMA1 decoder                             |
+| `slim.ts`    | `slim`: the raw file's rows, links and boundaries cut to a selection |
+| `selection.ts` | terms, fields, columns, the builder's tables      |
+| `writer.ts`  | blocks, tuning, header: the builder's `file.rs`     |
+| `compress.ts` | LZMA1 encoder, liblzma preset 9 step for step      |
+| `matcher.ts` | its binary-tree match finder                        |
+| `estimate.ts` | `estimate`: a size from counted facts, no decoding |
+| `leaves.ts`  | what decides a boundary, which a selection keeps    |
+| `stats.ts`   | `collect`: the counted facts `estimate` reads       |
+| `cli.ts`     | `npx plevinjs`                                      |
 | `address.ts` | parsing, spelling, special ranges                   |
 | `naming.ts`  | DNS and own address                                 |
 | `extra.ts`   | country and clock, from `countries.ts`/`zones.ts`   |
@@ -281,8 +341,13 @@ npm run build                                   # dist/ and the CDN bundle
 
 node test/compare.ts ../plevin.plv sample.json  # field for field against Python
 node test/blocks.ts ../plevin.plv 100000        # every block against liblzma
+node test/stats.ts ../plevin.raw stats.json     # the facts `estimate` reads
 ```
 
+`npm test` also splits `../plevin.raw` and compares each `plevin.*.plv` beside it, which
+must come from the same builder run (`plevin-builder raw place+metro network ...`), and
+`PLEVIN_RAW` / `PLEVIN_DB` point elsewhere. Files over 8 MB are skipped.
+
 ## License
 
 Apache 2.0, see [LICENSE](https://github.com/tn3w/plevin/blob/master/LICENSE). The

package/dist/cli.d.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+/** Cuts a database down from the shell: npx plevinjs place+metro. */
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,qEAAqE"}

package/dist/cli.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+/** Cuts a database down from the shell: npx plevinjs place+metro. */
+import { readFile, writeFile } from "node:fs/promises";
+import { fileName, slim } from "./slim.js";
+const args = process.argv.slice(2);
+const [input, terms, output] = args[0]?.endsWith(".raw") ? args : ["plevin.raw", ...args];
+if (!terms) {
+    console.error("usage: npx plevinjs [plevin.raw] <terms> [output.plv]");
+    console.error("terms: full, or fields joined by +, e.g. place.country.code+network.asn");
+    process.exit(1);
+}
+const bytes = slim(new Uint8Array(await readFile(input)), terms);
+const target = output ?? fileName(terms);
+await writeFile(target, bytes);
+console.log(`${target}: ${bytes.length} bytes`);
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,qEAAqE;AAErE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE3C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC;AAE1F,IAAI,CAAC,KAAK,EAAE,CAAC;IACX,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;IACvE,OAAO,CAAC,KAAK,CACX,yEAAyE,CAC1E,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACjE,MAAM,MAAM,GAAG,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC;AACzC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,KAAK,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC"}

package/dist/compress.d.ts

@@ -0,0 +1,5 @@
+/** Raw LZMA1 to an end marker, step for step the encoder liblzma runs at preset 9. */
+import { type Tuning } from "./lzma.ts";
+/** One block as a raw stream, the way `lzma.ts` and liblzma read it. */
+export declare const compress: (data: Uint8Array, tuning: Tuning) => Uint8Array;
+//# sourceMappingURL=compress.d.ts.map

package/dist/compress.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compress.d.ts","sourceRoot":"","sources":["../src/compress.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,OAAO,EAiBL,KAAK,MAAM,EACZ,MAAM,WAAW,CAAC;AA6pCnB,wEAAwE;AACxE,eAAO,MAAM,QAAQ,SAAU,UAAU,UAAU,MAAM,KAAG,UAG3D,CAAC"}