sliftutils 1.7.9 → 1.7.10

package/misc/https/dns.d.ts

@@ -11,8 +11,12 @@ export declare function deleteRecord(type: string, key: string, value: string):
 export declare function setRecord(type: string, key: string, value: string, proxied?: "proxied"): Promise<void>;
 /** Keeps existing records */
 export declare function addRecord(type: string, key: string, value: string, proxied?: "proxied"): Promise<void>;
-/** Provide Cloudflare credentials directly (an API token, or a path to a file containing one), instead of relying on ./cloudflare.json */
+/** Provide Cloudflare credentials directly instead of relying on ./cloudflare.json. Exactly one of
+ *  key (the API token itself) or path (a file to read it from) — TypeScript rejects both/neither. */
 export declare function setCloudflareCredentials(config: {
-    key?: string;
-    path?: string;
+    value: {
+        key: string;
+    } | {
+        path: string;
+    };
 }): void;

package/misc/https/dns.ts

@@ -115,15 +115,10 @@ export async function addRecord(type: string, key: string, value: string, proxie
 
 
 let credsOverride: { key: string } | undefined;
-/** Provide Cloudflare credentials directly (an API token, or a path to a file containing one), instead of relying on ./cloudflare.json */
-export function setCloudflareCredentials(config: { key?: string; path?: string }) {
-    let key = config.key;
-    if (!key && config.path) {
-        key = fs.readFileSync(config.path, "utf8").trim();
-    }
-    if (!key) {
-        throw new Error(`Must provide either key or path in setCloudflareCredentials, received ${JSON.stringify(Object.keys(config))}`);
-    }
+/** Provide Cloudflare credentials directly instead of relying on ./cloudflare.json. Exactly one of
+ *  key (the API token itself) or path (a file to read it from) — TypeScript rejects both/neither. */
+export function setCloudflareCredentials(config: { value: { key: string } | { path: string } }) {
+    let key = "key" in config.value ? config.value.key : fs.readFileSync(config.value.path, "utf8").trim();
     credsOverride = { key };
 }
 

package/misc/https/hostServer.d.ts

@@ -2,9 +2,12 @@ export type HostServerConfig = {
     /** Full domain to host on (e.g. "testsite.example.com"). The HTTPS cert is created for this domain and *.domain, so using a subdomain never touches the root domain (beyond its _acme-challenge TXT record). */
     domain: string;
     port: number;
-    /** Cloudflare API token (or a path to a file containing one). If neither is given, ./cloudflare.json is used. */
-    cloudflareApiToken?: string;
-    cloudflareApiTokenPath?: string;
+    /** Cloudflare API token: either the token string ({ key }) or a path to a file containing it ({ path }). Required — pass { path: "./cloudflare.json" } explicitly for the on-disk file. */
+    cloudflareApiToken: {
+        key: string;
+    } | {
+        path: string;
+    };
     /** Creates an unproxied A record pointing domain at this machine (publicIp, or our detected external IP) */
     setDNSRecord?: boolean;
     publicIp?: string;

package/misc/https/hostServer.ts

@@ -21,9 +21,10 @@ export type HostServerConfig = {
     /** Full domain to host on (e.g. "testsite.example.com"). The HTTPS cert is created for this domain and *.domain, so using a subdomain never touches the root domain (beyond its _acme-challenge TXT record). */
     domain: string;
     port: number;
-    /** Cloudflare API token (or a path to a file containing one). If neither is given, ./cloudflare.json is used. */
-    cloudflareApiToken?: string;
-    cloudflareApiTokenPath?: string;
+
+    // TODO: Eventually we should support running without Cloudflare API tokens. It's annoying though as the user will have to create a self-signed certificate and then they'll have to go through and trust it everywhere, and a lot of the stuff is transparent, and so it'll have to be non-transparent, getting the user to go to the page that owns the domain and trust it from there. It's much better just make a Cloudflare account. You can buy a domain for $15 a year, and then you can use it for GitHub pages to host your own site and do all kinds of things just like any other real site.
+    /** Cloudflare API token: either the token string ({ key }) or a path to a file containing it ({ path }). Required — pass { path: "./cloudflare.json" } explicitly for the on-disk file. */
+    cloudflareApiToken: { key: string } | { path: string };
     /** Creates an unproxied A record pointing domain at this machine (publicIp, or our detected external IP) */
     setDNSRecord?: boolean;
     publicIp?: string;
@@ -33,9 +34,7 @@ export type HostServerConfig = {
 /** Hosts a SocketFunction server on a real domain, with an automatically created and renewed Let's Encrypt HTTPS certificate (cached in the home folder, shared between processes on the machine). Expose your controllers (and any RequireController setup) before calling this. Returns the mounted nodeId. */
 export async function hostServer(config: HostServerConfig): Promise<string> {
     let { domain, port } = config;
-    if (config.cloudflareApiToken || config.cloudflareApiTokenPath) {
-        setCloudflareCredentials({ key: config.cloudflareApiToken, path: config.cloudflareApiTokenPath });
-    }
+    setCloudflareCredentials({ value: config.cloudflareApiToken });
     // The identity CA always lives on the root domain (nodeIds are threadHash.machineHash.root.tld)
     let rootDomain = domain.split(".").slice(-2).join(".");
     await loadIdentityCA(rootDomain);
@@ -56,6 +55,7 @@ export async function hostServer(config: HostServerConfig): Promise<string> {
 
     let nodeId = await SocketFunction.mount({
         public: true,
+        autoForwardPort: true,
         port,
         ...getThreadKeyCert(rootDomain),
         SNICerts: {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.7.9",
+    "version": "1.7.10",
     "main": "index.js",
     "license": "MIT",
     "files": [
@@ -33,6 +33,7 @@
     "bin": {
         "filehoster": "./bin/filehoster.js",
         "autohost": "./bin/autohost.js",
+        "storageserver": "./bin/storageserver.js",
         "build-nodejs": "./builders/nodeJSBuildRun.js",
         "buildnodejs": "./builders/nodeJSBuildRun.js",
         "build-extension": "./builders/extensionBuildRun.js",

package/spec.txt

@@ -1,60 +1,95 @@
 
-Searching
-    1) We want to create something that's database agnostic
-        - So I think we need like multiple phases for reading. And for applying it? We need to be able to know all of our reads ahead of time, which we should be able to roughly know. Or at least we'll have multiple stages. That should be fine. I guess we could do it so it actually understands our database. Like it reads, and then if it's not there, it returns. That might be fine too. 
 
-interface Database<Root> {
-    readData: <T>(fnc: (root: Root) => T) => T | undefined;
-    writeData: <T>(fnc: (root: Root) => T, newValue: T) => void;
-}
-I guess this works. We need multiple stages, so it's gonna have to read in the index, In parallel, and then once it has the full index, then it can do the next step in parallel, etc. 
+IMPORTANT! modified time PLUS size is what's important. Store both!
 
 
-remaining
-    1) The code book creation. We kind of do want to recreate it sometimes. Maybe we can have something that does a full expensive recreation at certain early on points?
-    2) The searching there's a few different stages. Like, we want to search more IVF cells, but then even within the cells, we kind of want to download more of the final embeddings.  because the codes aren't gonna to be perfect. So we have to do both at the same time?
+1) Tests locally, running it in Slift utils and running some code in CYOA to access it. 
+2) Set up a bootstrapper that we can chain with another command that will set up secrets, cloudflare key, etc
+    yarn exposesecrets && yarn storageserver --url https://storage.vidgridweb.com:4446 --folder ~/storagetest
+    - The bootstrapper needs to write to "cloudflare.json" and "backblaze.json"
+        - Using our sync server secrets. Not sure if they're stored in there, but, you know, easy enough...
 
-parts
-    p8g8 format (q8), so we can store 2048 dimensions is a bit over 2KB
-    IVF cell size 32? PQ index reduces by 32X, and if IVF cells reduce by 32X... then 500K 2KB values (1GB) would still just be... 1MB IVF index, so... that's good.
-    0) In IVF, sample 32, then keep sampling more, up to 128 cells
-    1) Pq 64 index (byte values each, break into 64 groups, always... should work well, For face vectors, they're already really optimized. They have so few dimensions, and so we don't compress it by that much. But for text vectors, they're less compressed, so we compress it more. So it should work in all cases)
-    2) Retraining index on a subset
-        - Or... just don't. Freeze after a certain point, and... it might be fine? HMM... Although no, we could allow retraining. It should be cheap enough. 
-    3) Split into transaction logs, so we can read the PQ index efficiently, but still update it
-        UGH... I mean basically you would just be replicating some of what we actually store on disk, but in our database, which is stupid, but I think we'll probably do it anyway. Basically, just a transaction log. We have to read everything every time, but it should help the speed a little bit...
-            - And I think it's fair because it's us saying, hey, we're going to read all this data every time, which the database can't know, so it can't optimize for that, but we can know, and then we can optimize for it
-    4) IVF the pq index to vastly decrease the initial access time
-    5.0) automatically split cells when they get big enough (2X their initial size)
-    5) occasionally regenerate the entire IVF.
-    5.1) Have something to store a subset of the data codes?
-    5.1) Have something that stores some of the data randomly so we can easily access a subset. We won't represent deletions in it. That's fine. This will allow us to rebalance the IVF fairly efficiently. 
-        - And maybe even slowly move stuff over to the new IVF as we access stuff, we see if it's in the new IVF, and if not, we move it?
-        - 10% of the data? And... maybe just the pq index values?
-        - Should make retraining the IVF ten times faster and shouldn't affect the recall by too much, especially if we have more cells
-        - AND, Maybe we can actually sample less the more data we have. Ramping up to 10%. 
-unknown
-    exactly how we're going to update the IVF. 
-    IVF pq index, so we can read part of it
-        mini transaction logs in each cell?
+3) Verify backblaze works, verify basically all the features work. 
 
-    BUT... how do we update IVF
 
 
 
+3) Support sharding in synchronization sources as well, adding new field
+    route: {
+        range: [number, number];
+        level: number;
+    }
+    - Route level is required as if two things have different route ranges, they obviously aren't going to be pulling from each other. So we need another way to stack redundancy, and the route level can do that. If two things are at the same route level, it means they aren't using each other for redundancy. We could automatically detect this, but it's more clear if it's explicit.
 
-1) Converting once is faster, so we should investigate if we can have a pool of memory. If we can reuse it, then the allocations are free. 
+3.1) We will almost always set up copy files on our different shards, but of course, these will only be considered depending on the valid window. 
+3.2) Of course, having our site configuration support sharding as well. So you don't need to fiddle with the shards. You kind of just say there are a certain number of shards. And if you want to change the shards, it will automatically set them up in the future, applying a default switch time of a day that you can also configure yourself, etc. 
 
-1) Fix embedding formats. It's really, really stupid. decodeToLength was just absolutely retarded. 
 
 
 
+5) Shard retry writes - Basically, they give us part of a path, and part of it we can fill in ourselves. We try to keep track of which shards are down and if a shard is down, we will keep changing that random value until we find something that doesn't match that shard Until we hit a certain percentage of shards being down, probably greater than 99%. And then we return the path that we ended up using. 
+    - This solves the problem of not being able to write to the backup sources. As it'll mean you'll be able to read if a shard is down, and you'll also be able to write if a shard is down. So it'll be like everything's fine. 
+    - We only need to use this for certain important things like uploading images. pretty much all of the AI stuff, we can probably use it. 
+    HMMM!!! This causes problems if we want it to be an image or a video. So we should probably have it so by default we try to put nothing there, So it'll be the exact path the user requested. That way, if they try to do the same right multiple times, it'll clobber itself.
+        - Although realistically, I don't think it'll really matter.  because we're not generating the same content most of the time. 
+6) geographical sharding as well. Which tries to change the hash so it's the server that we're closest to
+    - This can massively reduce the write latency, Which is important because we don't want to have to send the data around all the time, we just want to send around the URLs. Well then this requires waiting for the remote to confirm that we can read back. which will be purely limited really on latency if it's close to us. 
 
+7) readerDiskLimit. Only when our first sync source has copyFiles is false and cacheReads is true (and required true). Once we copy over a certain amount, we start deleting the least recently used from our local cache. This allows us to cache large sources without getting overwhelmed. 
 
 
-0) Make sure the AI stops trying to stop when a read didn't work because it doesn't fucking matter. I don't think there's a single place that's doing it where it actually matters. 
-    - I mean, I guess there are a few places. Like when we're rebalancing, that's one place.
-    - I guess some of the returns are valid, but I guess we have to check all of them. 
 
-1) Make sure the AI gets rid of all those four each's. I mean, what the fuck? 
-1)  Ensure the AI efficiently switches from the hard-coded stuff to the regular stuff and correctly regenerates the codebook. 
+
+
+
+
+
+
+
+2) Use our machine configuration to setup a storage service
+
+2.1) cyoa page to setup services + VIEW their storagerouting.JSON
+    WE DO NOT want to configure it via the interface! We want the configuration to always be in code. That way, we can update one place and all of our buckets will update!
+    OH! Make something that will import all of our code just like in a deploy, and then we'll gather all of the bucket configurations that we created using our helper functions so we register them all, and then we'll touch them all to make sure the configuration is updated on all of them!
+    - We should probably make a factory so we can easily create buckets without having to find the full chain every single time, and so we can change the full chain easily in one place. 
+    - We should allow easily chaining them for redundancy
+    - We might even have some UI that shows all of the storage services together, allowing you to easily configure how they chain, etc. 
+    - We should also have it poll for the status of like scanning file stats, etc. 
+
+2.2) Test some of these custom buckets on CYOA (local dev), testing redundancy, speed, etc.
+    - If we run into a lot of issues that we need to debug, we can always host the storage server locally. 
+
+    custom => backblaze
+        - So internal redundancy and then also backblaze at the very end
+2.3) Test using our thing as a front end for back plays with no copy, which should mean there's really no setup. 
+    - Just test reading images or something. 
+
+
+
+
+
+
+
+ArchivesRemote
+    bucketName
+    public
+    fast
+    writeDelay
+
+ArchivesFactory
+    host: https://storage.vidgridweb.com:4444/storagerouting.json
+    redundantHosts:
+        - https://storage2.vidgridweb.com:4445/storagerouting.json
+        - https://f002.backblazeb2.com/file/querysubtest-com-public-immutable/storage/storagerouting.json
+
+HostStorageServerConfig
+    host: https://storage.vidgridweb.com:4444/storagerouting.json
+    redundantHosts:
+        - https://storage.vidgridweb.com:4444/storagerouting.json
+        - https://storage2.vidgridweb.com:4445/storagerouting.json
+        - https://f002.backblazeb2.com/file/querysubtest-com-public-immutable/storage/storagerouting.json
+    folder: os.homedir() + "/storage"
+    cloudflareApiToken: ...
+
+
+https://f002.backblazeb2.com/file/querysubtest-com-public-immutable/image/img_00BloAFyIevA_1024x1536.jpg

package/storage/ArchivesDisk.d.ts

@@ -0,0 +1,65 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { IArchives, ArchiveFileInfo, ArchivesConfig } from "./IArchives";
+export declare class ArchivesDisk implements IArchives {
+    private folder;
+    constructor(folder: string);
+    private filesDir;
+    private uploadsDir;
+    private handles;
+    private largeUploads;
+    private nextLargeUploadId;
+    init: {
+        (): Promise<void>;
+        reset(): void;
+        set(newValue: Promise<void>): void;
+    };
+    getDebugName(): string;
+    getConfig(): Promise<ArchivesConfig>;
+    private filePath;
+    set(key: string, data: Buffer, config?: {
+        lastModified?: number;
+    }): Promise<void>;
+    del(key: string): Promise<void>;
+    get(key: string, config?: {
+        range?: {
+            start: number;
+            end: number;
+        };
+    }): Promise<Buffer | undefined>;
+    get2(key: string, config?: {
+        range?: {
+            start: number;
+            end: number;
+        };
+    }): Promise<{
+        data: Buffer;
+        writeTime: number;
+    } | undefined>;
+    getInfo(key: string): Promise<{
+        writeTime: number;
+        size: number;
+    } | undefined>;
+    find(prefix: string, config?: {
+        shallow?: boolean;
+        type: "files" | "folders";
+    }): Promise<string[]>;
+    findInfo(prefix: string, config?: {
+        shallow?: boolean;
+        type?: "files" | "folders";
+    }): Promise<ArchiveFileInfo[]>;
+    private collectFiles;
+    setLargeFile(config: {
+        path: string;
+        getNextData(): Promise<Buffer | undefined>;
+    }): Promise<void>;
+    startLargeUpload(): Promise<string>;
+    appendLargeUpload(id: string, data: Buffer): Promise<void>;
+    finishLargeUpload(id: string, key: string): Promise<void>;
+    cancelLargeUpload(id: string): Promise<void>;
+    getURL(path: string): Promise<string>;
+}
+export declare function applyFindInfoShape(files: ArchiveFileInfo[], prefix: string, config?: {
+    shallow?: boolean;
+    type?: "files" | "folders";
+}): ArchiveFileInfo[];