querysub 0.700.0 → 0.702.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.700.0",
+  "version": "0.702.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-a-archives/archives2.ts

@@ -41,6 +41,8 @@ function createSourceWindows(
 
 function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     const ONTARIO = `https://99-250-124-91.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
+    // NOTE: This is on an overly large extra disk that's on our 4090 machine. I prefer to not use it because it might slow down diffusion, but it's quite large (2tb, and diffusion will never use more than 1tb), so we could use it if we needed to.
+    const ONTARIO2 = `https://99-250-124-91.querysubtest.com:5235/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const HETZNER = `https://65-109-93-113.querysubtest.com:5234/file/${STORAGE_ACCOUNT}/${bucket}/storage/storagerouting.json`;
     const BACKBLAZE = `https://f002.backblazeb2.com/file/${bucket}/storage/storagerouting.json`;
 
@@ -76,6 +78,12 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
                 // NOTE: Once we add new sources, we need to add something at a new start time in the future so we have time to move the files. 
                 // TODO: EVENTUALLY, Once we have enough data, instead of restriping, we need to bisect the data. That way we can slowly move over the data instead of trying to rewrite all of our data all at once. However, for the moment, moving all the data is probably fine. On the data center, we should be able to move terabytes of data every hour, so we should be able to handle quite a bit, especially if we have a few days to run the restriping. 
                 // IMPORTANT! Every time we re-stripe, we need to increment the route version. Otherwise, we could reuse a folder that we previously used that's no longer in sync. which will cause us to resurrect old data, which is very bad. We don't want to do that...
+                // NOTE: We do full redundancy, so if the ranges are a, b, c, then its:
+                //   s0=a s1=b s2=c
+                //   s0=c s1=a s2=b
+                //   s0=b s1=c s2=a
+                //  Essentially, we just shift it every time. So every server has all the data, But the primary writers are divided between all the servers. 
+                //  (If you follow A, you can see it's diagonal, B is diagonal, they're all diagonal, like stripes).
                 ...stripeSources([ONTARIO, HETZNER], { routeVersion: 2 }),
                 { type: "backblaze", url: BACKBLAZE, name: "backblaze" },
             ]
@@ -83,7 +91,7 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     ]);
 
     return createArchives({
-        version: 28,
+        version: 29,
         sources,
     });
 }

package/src/deployManager/components/MachineDetailPage.tsx

@@ -134,13 +134,21 @@ export class MachineDetailPage extends qreact.Component {
                                 <ShowMore className={css.whiteSpace("pre-wrap")} maxHeight={80}>
                                     {(() => {
                                         if (typeof value === "object") {
-                                            return <UsageBar label={value.label || value.type} value={value.value} max={value.max} {...getUsageThresholds(value.type)} />;
+                                            return <UsageBar label={value.label || value.type} value={value.value} max={value.max} subValue={value.subValue} subLabel={value.subLabel} {...getUsageThresholds(value.type)} />;
                                         }
                                         return value;
                                     })()}
                                 </ShowMore>
                             </div>
                         ))}
+                        {machine.meminfo && (
+                            <div className={css.hbox(10).hsla(0, 0, 0, 0.1).pad2(10, 2)}>
+                                <b className={css.flexShrink0}>meminfo:</b>
+                                <ShowMore className={css.whiteSpace("pre-wrap")} maxHeight={80}>
+                                    {Object.entries(machine.meminfo).map(([key, bytes]) => `${key}: ${formatNumber(bytes)}B`).join("\n")}
+                                </ShowMore>
+                            </div>
+                        )}
                     </div>
                 </div>
 

package/src/deployManager/components/MachinesListPage.tsx

@@ -187,7 +187,7 @@ export class MachinesListPage extends qreact.Component {
                                     <RenderGitRefInfo gitRef={machineInfo.gitRef} />
                                     {Object.values(machineInfo.info).map(value => {
                                         if (typeof value === "string") return undefined;
-                                        return <UsageBar label={value.label || value.type} value={value.value} max={value.max} {...getUsageThresholds(value.type)} />;
+                                        return <UsageBar label={value.label || value.type} value={value.value} max={value.max} subValue={value.subValue} subLabel={value.subLabel} {...getUsageThresholds(value.type)} />;
                                     })}
                                 </div>
                                 <div className={css.vbox(4).flexGrow(1)}>

package/src/deployManager/machineApplyMainCode.ts

@@ -63,6 +63,45 @@ const getMemoryInfo = measureWrap(async function getMemoryInfo(): Promise<{ memo
 });
 
 
+const MEMINFO_LINE_REGEX = /^(\S+):\s+(\d+)(?:\s+(\S+))?\s*$/;
+const MEMINFO_UNIT_MULTIPLIERS: Record<string, number> = {
+    b: 1,
+    kb: 1024,
+    mb: 1024 * 1024,
+    gb: 1024 * 1024 * 1024,
+};
+
+export function parseMeminfo(raw: string): Record<string, number> {
+    let result: Record<string, number> = {};
+    for (let line of raw.split("\n")) {
+        let match = MEMINFO_LINE_REGEX.exec(line);
+        if (!match) continue;
+        let [, key, valueText, unit] = match;
+        let multiplier = unit ? MEMINFO_UNIT_MULTIPLIERS[unit.toLowerCase()] : 1;
+        if (multiplier === undefined) continue;
+        let value = parseInt(valueText);
+        if (!(value >= 0)) continue;
+        result[key] = value * multiplier;
+    }
+    return result;
+}
+
+const getMeminfo = measureWrap(async function getMeminfo(): Promise<Record<string, number> | undefined> {
+    if (os.platform() === "win32") {
+        throw new Error("Windows is not supported for machine resource monitoring");
+    }
+    try {
+        let raw = await fs.promises.readFile("/proc/meminfo", "utf8");
+        let parsed = parseMeminfo(raw);
+        if (Object.keys(parsed).length > 0) {
+            return parsed;
+        }
+    } catch (e: any) {
+        console.warn(`Error reading /proc/meminfo: ${e.stack}`);
+    }
+    return undefined;
+});
+
 const LARGE_DISK_MIN_BYTES = 100 * 1024 * 1024 * 1024;
 const DF_LINE_REGEX = /^(\S+)\s+(\d+)\s+(\d+)\s+\d+\s+\S+\s+(.+?)\s*$/;
 
@@ -93,12 +132,12 @@ const getDiskInfos = measureWrap(async function getDiskInfos(): Promise<DiskInfo
             }
             disks.push({ device, mountPoint, value, max });
         }
+        sort(disks, disk => -disk.max);
         let large = disks.filter(disk => disk.max >= LARGE_DISK_MIN_BYTES);
-        if (large.length > 0) {
-            disks = large;
+        if (large.length === 0) {
+            large = disks.slice(0, 1);
         }
-        sort(disks, disk => -disk.max);
-        return disks;
+        return large;
     } catch (e: any) {
         console.warn(`Error getting disk info: ${e.stack}`);
     }
@@ -117,9 +156,10 @@ const getLiveMachineInfo = measureWrap(async function getLiveMachineInfo() {
     };
 
     // Get system resource information
-    let [memoryInfo, diskInfos] = await Promise.all([
+    let [memoryInfo, diskInfos, meminfo] = await Promise.all([
         getMemoryInfo(),
-        getDiskInfos()
+        getDiskInfos(),
+        getMeminfo(),
     ]);
 
     if (memoryInfo?.memory) {
@@ -127,9 +167,15 @@ const getLiveMachineInfo = measureWrap(async function getLiveMachineInfo() {
             type: "MEMORY",
             value: memoryInfo.memory.value,
             max: memoryInfo.memory.max,
+            subValue: meminfo?.Active,
+            subLabel: "active",
         };
     }
 
+    if (meminfo) {
+        machineInfo.meminfo = meminfo;
+    }
+
     if (memoryInfo?.swap) {
         machineInfo.info.swap = {
             type: "SWAP",

package/src/deployManager/machineSchema.ts

@@ -64,8 +64,12 @@ export type MachineInfo = {
         value: number;
         max: number;
         label?: string;
+        subValue?: number;
+        subLabel?: string;
     }>;
 
+    meminfo?: Record<string, number>;
+
     repoUrl: string;
     gitRef: string;
 

package/src/library-components/InputLabel.tsx

@@ -28,12 +28,15 @@ export type InputLabelProps = Omit<InputProps, "label" | "title"> & {
     tooltip?: string;
 
     fillWidth?: boolean | number;
+    // Height a textarea opens at in edit mode; EDIT_TEXTAREA_MIN_HEIGHT when unset.
+    editMinHeight?: number;
 };
 
 // A checkbox is a small square that reads as belonging to the words beside it, so it sits tight against
 // its label; every other input is a box of its own and wants room between the two.
 const LABEL_GAP = 8;
 const CHECKBOX_LABEL_GAP = 2;
+export const EDIT_TEXTAREA_MIN_HEIGHT = 80;
 
 function roundToDecimals(value: number, decimals: number) {
     return Math.round(value * 10 ** decimals) / 10 ** decimals;
@@ -134,6 +137,10 @@ export class InputLabel extends qreact.Component<InputLabelProps> {
             };
             // Default focus on mount, as editting it is the only reason to click on it.
             props.focusOnMount = props.focusOnMount ?? true;
+            if (props.textarea) {
+                props.fillWidth = props.fillWidth ?? true;
+                style.minHeight = style.minHeight ?? props.editMinHeight ?? EDIT_TEXTAREA_MIN_HEIGHT;
+            }
         }
         let input = <Input
             {...props as any}
@@ -141,8 +148,8 @@ export class InputLabel extends qreact.Component<InputLabelProps> {
             style={style}
         />;
         if (props.edit && !this.state.editting) {
-            input = <span class={css.hbox(2).overflowHidden + " trigger-hover"}>
-                <span class={props.editClass}>
+            input = <span class={css.hbox(2).overflowHidden + (props.textarea && css.fillWidth.alignItems("start") || "") + " trigger-hover"}>
+                <span class={(props.textarea && css.whiteSpace("pre-wrap").fillWidth || "") + " " + props.editClass}>
                     {props.editValue ?? props.value}
                 </span>
                 <span class={css.opacity(0.1).opacity(1, "hover")}>

package/src/library-components/UsageBar.tsx

@@ -3,6 +3,8 @@ import { css } from "typesafecss";
 import { formatNumber } from "socket-function/src/formatting/format";
 
 const FLASH_CLASS_NAME = "UsageBar-flash";
+const SUB_BAR_HEIGHT_PERCENT = 35;
+const SUB_BAR_ALPHA = 0.3;
 
 export const MEMORY_WARNING_THRESHOLD = 0.7;
 export const MEMORY_ERROR_THRESHOLD = 0.85;
@@ -36,11 +38,14 @@ export class UsageBar extends qreact.Component<{
     warningThreshold?: number;
     /** Fraction of max at which the bar fill turns red and flashes. */
     errorThreshold?: number;
+    subValue?: number;
+    subLabel?: string;
 }> {
     render() {
-        let { label, value, max, unit, warningThreshold, errorThreshold } = this.props;
+        let { label, value, max, unit, warningThreshold, errorThreshold, subValue, subLabel } = this.props;
         let suffix = unit || "";
         let fraction = max && value / max || 0;
+        let subFraction = subValue !== undefined && max && subValue / max || 0;
         let isError = errorThreshold !== undefined && fraction >= errorThreshold;
         let isWarning = !isError && warningThreshold !== undefined && fraction >= warningThreshold;
         let fillClass = isError && css.hsl(0, 80, 60) || isWarning && css.hsl(45, 90, 60) || css.hsl(0, 0, 70);
@@ -50,8 +55,12 @@ export class UsageBar extends qreact.Component<{
                 + fillClass
                 + (isError && (" " + FLASH_CLASS_NAME) || "")
             } />
+            {subValue !== undefined && <div className={
+                css.absolute.pos(0, `${100 - SUB_BAR_HEIGHT_PERCENT}%`).size(`${Math.min(subFraction, 1) * 100}%`, `${SUB_BAR_HEIGHT_PERCENT}%`).hsla(0, 0, 0, SUB_BAR_ALPHA)
+            } />}
             <div className={css.relative}>
                 {label} ({formatNumber(value)}{suffix} / {formatNumber(max)}{suffix})
+                {subValue !== undefined && <span className={css.opacity(0.75)}>, {subLabel || "sub"} {formatNumber(subValue)}{suffix}</span>}
             </div>
             {isError && <style>{`
                 @keyframes ${FLASH_CLASS_NAME}-anim {