querysub 0.2.0

package/src/zipThreaded.ts

@@ -0,0 +1,106 @@
+import { lazy } from "socket-function/src/caching";
+import os from "os";
+import { Worker } from "worker_threads";
+import { isNode } from "typesafecss";
+import { PromiseObj } from "./promise";
+import debugbreak from "debugbreak";
+import { asBuffer } from "./buffers";
+
+
+let getWorkerLimit = lazy(() => {
+    // Half the CPUs should be enough
+    return Math.max(1, Math.floor(os.cpus().length / 2));
+});
+
+interface UnzipWorker {
+    queueDepth: number;
+    unzip(buffer: Buffer): Promise<Buffer>;
+}
+
+let sharedBufferPool: {
+    buffer: SharedArrayBuffer;
+}[] = [];
+function getSharedBuffer(size: number): SharedArrayBuffer {
+    let index = sharedBufferPool.findIndex(buffer => buffer.buffer.byteLength >= size);
+    if (index >= 0) {
+        let bufferObj = sharedBufferPool.splice(index, 1)[0];
+        return bufferObj.buffer;
+    }
+    // Round to power of 2, so it will be reused
+    //size = Math.pow(2, Math.ceil(Math.log2(size)));
+    return new SharedArrayBuffer(size);
+}
+function releaseSharedBuffer(buffer: SharedArrayBuffer) {
+    sharedBufferPool.push({ buffer });
+}
+
+let unzipWorkers: UnzipWorker[] = [];
+async function spawnWorker() {
+    if (!isNode()) throw new Error("Parallel unzipping is support in Node.js at the moment");
+    const worker = new Worker(__dirname + "/zipThreadedWorker.js");
+    let workerObj: UnzipWorker = {
+        queueDepth: 0,
+        unzip
+    };
+    let messageQueue: {
+        onFree: () => void;
+        promise: PromiseObj<Buffer>;
+    }[] = [];
+    worker.on("message", (buffer: Uint8Array | { error: string }) => {
+        if ("error" in buffer) {
+            let { promise, onFree } = messageQueue.shift()!;
+            onFree();
+            workerObj.queueDepth = messageQueue.length;
+            console.log(`Error inside worker: ${buffer.error}`);
+            promise.reject(new Error(buffer.error));
+            return;
+        }
+        let { promise, onFree } = messageQueue.shift()!;
+        onFree();
+        workerObj.queueDepth = messageQueue.length;
+        promise.resolve(asBuffer(buffer));
+    });
+    worker.on("error", (err) => {
+        let { promise, onFree } = messageQueue.shift()!;
+        onFree();
+        workerObj.queueDepth = messageQueue.length;
+        console.log(`Worker error: ${err.message}`);
+        promise.reject(err);
+    });
+    async function unzip(buffer: Buffer): Promise<Buffer> {
+        let promise = new PromiseObj<Buffer>();
+        let sharedBuffer = getSharedBuffer(buffer.length);
+        {
+            let view = Buffer.from(sharedBuffer, 0, buffer.length);
+            view.set(buffer);
+            buffer = view;
+        }
+        messageQueue.push({
+            promise,
+            onFree() {
+                releaseSharedBuffer(sharedBuffer);
+            }
+        });
+        workerObj.queueDepth = messageQueue.length;
+        worker.postMessage(buffer);
+        return await promise.promise;
+    }
+    unzipWorkers.push(workerObj);
+}
+
+export async function unzipThreaded(buffer: Buffer): Promise<Buffer> {
+    let worker = unzipWorkers.find(worker => worker.queueDepth === 0);
+    if (!worker && unzipWorkers.length < getWorkerLimit()) {
+        await spawnWorker();
+        worker = unzipWorkers.find(worker => worker.queueDepth === 0);
+    }
+    if (!worker) {
+        // Use the most free worker
+        let minQueueDepth = Math.min(...unzipWorkers.map(worker => worker.queueDepth));
+        worker = unzipWorkers.find(worker => worker.queueDepth === minQueueDepth);
+    }
+    if (!worker) {
+        throw new Error("Impossible, no workers?");
+    }
+    return worker.unzip(buffer);
+}

package/src/zipThreadedWorker.js

@@ -0,0 +1,19 @@
+const { parentPort } = require("worker_threads");
+const zlib = require("zlib");
+const parentPortConst = parentPort;
+if (!parentPortConst) {
+    throw new Error("This file must be run as a worker thread");
+}
+
+parentPortConst.on("message", (buffer) => {
+    // A synced call is important, as it keeps calls in order
+    try {
+        let unzipped = zlib.unzipSync(buffer);
+        let sharedBuffer = new SharedArrayBuffer(unzipped.length);
+        let sharedArray = new Uint8Array(sharedBuffer);
+        sharedArray.set(unzipped);
+        parentPortConst.postMessage(sharedArray);
+    } catch (err) {
+        parentPortConst.postMessage({ error: err.message });
+    }
+});

package/tsconfig.json

@@ -0,0 +1,27 @@
+{
+    "compilerOptions": {
+        "strict": true,
+        "module": "CommonJS",
+        "esModuleInterop": true,
+        "allowSyntheticDefaultImports": true,
+        "moduleResolution": "node",
+        "target": "es2018",
+        "lib": [
+            "ESNext",
+            "dom",
+            "dom.iterable"
+        ],
+        "jsx": "react",
+        "alwaysStrict": true,
+        "jsxFactory": "qreact.createElement",
+        "jsxFragmentFactory": "qreact.Fragment",
+        "types": [
+            "node",
+        ],
+        "experimentalDecorators": true,
+        "emitDecoratorMetadata": false,
+        "skipLibCheck": true,
+        "inlineSourceMap": true,
+        "inlineSources": true,
+    }
+}

package/yarnSpec.txt

@@ -0,0 +1,56 @@
+- better yarn, `packagedownload`?
+
+ACTUALLY, a lot of this, BUT, have the config format just entirely describe the on disk format, being relative to where each package is located. That way it is obvious how you will change how a specific package resolves just inside of another package, or if one package is using a custom version, etc, etc.
+
+`npm -run packagedownload add package-name`, `npm -run packagedownload install`
+    add
+        --dry-run
+        --force
+        --update
+        --dev
+        --nolink
+    install
+        --dry-run
+        --force
+        --update
+        --nolink
+- DON'T run scripts by default, only running them if requested via `"execute": {}` in the package.json configuration
+    - With a "request-execute" field, which warns if you use a package and don't let it execute (it will still run it though)
+- On install performs a "light" check, only checking the "package.json".version of packages (or package, if a package is specified), not updating anything that looks up to date
+    - Has a --force flag, which will always delete all packages and install a new one (or rather, install all packages, as we will always delete and install the new one when we are installing)
+- Still supports a --production mode, which doesn't install dev dependencies
+- HAS to support "cpu", and "os" restrictions
+    - But, still support "ignore-engines", to skip these
+- ALWAYS flat, warning on conflicts, using package "resolutions" to decide which one to use
+    - Although, maybe we can have "sandbox", which allows certain dependencies to be told to install all of their dependencies as nested? We won't be able to do partial sandboxing, unless the package is a few levels deep, and we pick the package directly above it (which doesn't give us full control, but is probably enough).
+- Call the lock file `resolvedversions.json`, as it isn't for locking, it is for specifying which versions of packages should be used (if compatible with the rough version in package.json)
+- Avoid nesting automatically, by having any install check if the current folder is in a node_modules folder, and if so uses the lock file above that, and installs the dependencies as siblings in that node_modules folder
+    - Allow packages to use `{ "force-sandbox": true }`, to force their dependencies to be installed in a nested node_modules (sandboxing which also cause the child yarn.lock to be checked before the parent yarn.lock)
+- Greatly simplified lock file, given that we generally won't have nested dependencies
+    - The lock file can give more metadata, such as the full chain of dependencies to the root package.json dependency, as well as package sizes and file counts
+    - Marks which dependencies are dev only (or only included because of a dev only dependency)
+- Support for "link" inside of the package json, which instead works like: `{ "local": { "shard": "../shard/" } }`, which uses those paths when they exist
+    - A "nolink" option will disable this
+    - Creates symlinks
+        - Makes symlinks for all nested dependencies as well
+        - Adds metadata in the lock file to indicate the modules are being symlinked?
+- Ability to install only a specific package, aka, `install shard`, which uses the version in the lock file (or package.json)
+    - typenode can start installing on require, only installing the required package
+        - There would have to be a way to turn it off, ex, with `typenodecompat`, which wouldn't install packages. That way you could use typenode without messing up your node_modules folders
+            - This mode could also store the dist folders somewhere else, or even... just not store them, eliminating caching?
+        - We could even use packagedownload via an import, so we don't need to spawn a new processes!
+            - This would allow packagedownload to maintain various caches, making it a lot more efficient when called multiple times.
+        - Add a TODO to support some form of tree-shaking. As in, if we detect out of the module only "x" and "y" are included, then we can strip the file of everything unrelated to "x" and "y" (which is not easy, but... not REALLY hard either). The idea, is not to reduce file size (who cares about that?), but actually to create unused requires, which we can then strip as well!
+            - We would need a flag to know if required modules have side-effects or not (package.json has a flag for this, and we can also support a `module.sideeffects = true` flag)
+            - And, we might as well have a package.json flag to set side-effects on nested packages, probably recursively.
+    - If the package is marked with "execute" it will be recursively installed as well, unless it has "no-recursive" in the package.json
+- Can be added to a package directly, which ADDS A BOOTSTRAPPER, by default, as a .js file, and adds a reference to a new scripts step ("packagedownload.js"?), or some similar step, in package.json. It can then be also called in your runs steps, ex, `node ./packagedownload.js && npm -run typenode ./index.ts`, which then allows `npm -run start` to work after an install (heck, you could even do `git clone git@github.com/user/repo.git ~/repo && cd ~/repo && git -run start`, to easily install it on any system that just has node and git)
+    - Eventually we could add the ability to install node, using the node version from the package.json (using the upper limit, ex `>=11 <15`, would use node 15), which would mean basically any system could use it.
+- Maybe use a global package download cache (which is maintained and kept below a certain percent of available space!)
+- `--update-check` and `--update`, to deal with updating yarn.lock (-check just checks and logs potential updates)
+- ALSO, "sources", which can be used to map packages to servers other than npm! Ex, github, etc. BUT, unlike the other way of doing it, versions are still supported!
+    - `{ "sources": { "source-map-support": "https://github.com/user/source-map-support" } }`
+        - Also supports npm, so you can switch out a package (for your dependencies too), easily
+    - ALSO, could add support for "dependencies2", to prevent npm and yarn from installing the dependencies (as they don't support "sources"), allowing the package to install them at runtime, allowing for better compatibility
+    - In conjunction with optionalDependencies make it possible to add dependencies that don't exist in npm
+    - The lock file will have to mark who set the source, potentially giving multiple setters, with the highest being the one that actual sets the source.