xindex 1.0.0

package/packages/streamx/src/interval.ts

@@ -0,0 +1,58 @@
+import { Defer } from '@handy/fun/defer';
+import { reader } from './reader';
+import { StreamX } from './index';
+
+export type IInterval = StreamX<number> & {
+  stop: () => void;
+};
+
+export function interval(ms: number, startImmediate = false): IInterval {
+  let point = startImmediate ? null : Date.now();
+
+  let delayDefer = Defer<void>();
+  delayDefer.resolve();
+
+  let delayTimer: any;
+  let stopped = false;
+
+  const intervalStream = reader<number>(async () => {
+    if (stopped) {
+      return reader.DONE;
+    }
+
+    const now = Date.now();
+
+    if (!point) {
+      point = now;
+      return now;
+    }
+
+    const xDelay = now - point;
+
+    if (xDelay < ms) {
+      delayDefer = Defer<void>();
+      const delayTimeout = ms - xDelay;
+      delayTimer = setTimeout(() => delayDefer.resolve(), delayTimeout);
+      await delayDefer.promise;
+
+      if (stopped) {
+        return reader.DONE;
+      }
+    }
+
+    point = Date.now();
+
+    return point;
+  }) as IInterval;
+
+  intervalStream.stop = () => {
+    if (delayTimer) {
+      clearTimeout(delayTimer);
+      delayTimer = undefined;
+    }
+    stopped = true;
+    delayDefer.resolve();
+  };
+
+  return intervalStream;
+}

package/packages/streamx/src/loop.ts

@@ -0,0 +1,8 @@
+import { Promised, StreamX } from './index';
+import { reader } from './reader';
+
+export function loop(condition: () => Promised<boolean>): StreamX<true> {
+  return reader<true>(async () => {
+    return (await condition()) ? true : reader.DONE;
+  });
+}

package/packages/streamx/src/map.ts

@@ -0,0 +1,12 @@
+import { Promised, StreamX, StreamXMapper } from './index';
+
+export function map<Input, Output>(
+  mapper: (input: Input) => Promised<Output>
+): StreamXMapper<Input, Output> {
+  return inputStream =>
+    (async function* mappedStream(): StreamX<Output> {
+      for await (const record of inputStream) {
+        yield await mapper(record);
+      }
+    })();
+}

package/packages/streamx/src/merge.ts

@@ -0,0 +1,89 @@
+import { StreamX } from './index';
+import { IRead, read } from './reader';
+import { IWriter, Writer } from './writer';
+import { syncTick } from '@handy/fun/tick';
+
+export function merge<T1, T2, T3, T4>(
+  stream1: StreamX<T1>,
+  stream2: StreamX<T2>,
+  stream3: StreamX<T3>,
+  stream4: StreamX<T4>
+): StreamX<T1 | T2 | T3 | T4>;
+export function merge<T1, T2, T3>(
+  stream1: StreamX<T1>,
+  stream2: StreamX<T2>,
+  stream3: StreamX<T3>
+): StreamX<T1 | T2 | T3>;
+export function merge<T1, T2>(
+  stream1: StreamX<T1>,
+  stream2: StreamX<T2>
+): StreamX<T1 | T2>;
+export function merge<Type>(...streams: StreamX<any>[]): StreamX<Type> {
+  let outputBuffer: IWriter<Type>;
+  let readOutput: IRead<Type>;
+  let _error: Error | undefined = undefined;
+
+  async function finish() {
+    if (outputBuffer) {
+      await outputBuffer.finish();
+    }
+  }
+
+  return {
+    [Symbol.asyncIterator](): AsyncIterator<Type> {
+      return {
+        async next(): Promise<IteratorResult<Type>> {
+          if (_error) {
+            throw _error;
+          }
+
+          if (!outputBuffer) {
+            outputBuffer = Writer<Type>();
+
+            syncTick(async () => {
+              try {
+                await Promise.all(
+                  streams.map(async stream => {
+                    const readStream = read(stream);
+                    while (true) {
+                      if (_error) {
+                        break;
+                      }
+                      const value = await readStream();
+
+                      if (value === read.DONE) {
+                        break;
+                      } else {
+                        await outputBuffer.write(value);
+                      }
+                    }
+                  })
+                );
+
+                await finish();
+              } catch (error) {
+                _error = error as Error;
+              }
+            });
+          }
+
+          if (!readOutput) {
+            readOutput = read<Type>(outputBuffer.stream);
+          }
+
+          const output = await readOutput();
+
+          if (_error) {
+            throw _error;
+          }
+
+          if (output === read.DONE) {
+            return { done: true, value: undefined };
+          } else {
+            return { done: false, value: output };
+          }
+        },
+      };
+    },
+  };
+}

package/packages/streamx/src/nodeReadable.ts

@@ -0,0 +1,6 @@
+import { of, StreamXOf } from './index';
+import { Readable } from 'stream';
+
+export function nodeReadable<Output>(readable: Readable): StreamXOf<Output> {
+  return of<Output>(readable);
+}

package/packages/streamx/src/nodeTransform.ts

@@ -0,0 +1,9 @@
+import { of, StreamXMapper } from './index';
+import { Readable, ReadableOptions, Transform } from 'stream';
+
+export function nodeTransform<Input, Output>(
+  transform: Transform,
+  options: ReadableOptions = {}
+): StreamXMapper<Input, Output> {
+  return inputStream => of(Readable.from(inputStream, options).pipe(transform));
+}

package/packages/streamx/src/nodeWritable.ts

@@ -0,0 +1,38 @@
+import { StreamX, StreamXMapper } from './index';
+import { Writable } from 'stream';
+import { Defer } from '@handy/fun/defer';
+
+export function nodeWritable<Type>(
+  writable: Writable,
+  encoding: BufferEncoding = 'utf-8'
+): StreamXMapper<Type, Type> {
+  return inputStream =>
+    (async function* _nodeWritable(): StreamX<Type> {
+      const deferEnd = Defer<void>();
+      writable.on('error', error => deferEnd.reject(error));
+
+      try {
+        for await (const value of inputStream) {
+          const defer = Defer<void>();
+          writable.write(value, encoding, error => {
+            if (error) {
+              defer.reject(error);
+            } else {
+              defer.resolve();
+            }
+          });
+
+          await defer.promise;
+          yield value;
+        }
+
+        writable.end(() => {
+          deferEnd.resolve();
+        });
+      } catch (error) {
+        deferEnd.reject(error as Error);
+      }
+
+      await deferEnd.promise;
+    })();
+}

package/packages/streamx/src/objectReader.ts

@@ -0,0 +1,16 @@
+import { Promised, StreamX } from './index';
+import { reader } from './reader';
+
+export function objectReader<T extends object | object[]>(
+  read: () => Promised<T | null | undefined | boolean | number>
+): StreamX<T> {
+  return reader<T>(async () => {
+    const object = await read();
+
+    if (object instanceof Object) {
+      return object;
+    }
+
+    return reader.DONE;
+  });
+}

package/packages/streamx/src/polyfill.ts

@@ -0,0 +1,20 @@
+// polyfill
+if (!Symbol) {
+  // @ts-ignore
+
+  Symbol = (description: string) => description;
+
+  // @ts-ignore
+  Symbol.for = key => key;
+}
+
+if (!(Symbol as any).asyncIterator) {
+  // @ts-ignore
+  (Symbol as any).asyncIterator = Symbol.asyncIterator || Symbol('Symbol.asyncIterator');
+}
+
+if (!Symbol.iterator) {
+  (Symbol as any).iterator = Symbol.iterator || Symbol('Symbol.iterator');
+}
+
+export const StreamXPolyfill = Symbol.for('Polyfill');

package/packages/streamx/src/reader.ts

@@ -0,0 +1,38 @@
+import { Promised, StreamX } from './index';
+
+const DONE = Symbol('DONE');
+
+export type IRead<T> = () => Promised<T | typeof DONE>;
+
+export function reader<T>(read: IRead<T>): StreamX<T> {
+  return {
+    [Symbol.asyncIterator]() {
+      return {
+        async next() {
+          const value = await read();
+          return { done: value === reader.DONE, value };
+        },
+      } as AsyncIterator<T>;
+    },
+  };
+}
+
+export function read<T>(stream: StreamX<T>): IRead<T> {
+  let iterator: AsyncIterator<T>;
+  return async () => {
+    if (!iterator) {
+      iterator = stream[Symbol.asyncIterator]();
+    }
+
+    const { done, value } = await iterator.next();
+
+    if (done) {
+      return reader.DONE;
+    } else {
+      return value;
+    }
+  };
+}
+
+read.DONE = DONE;
+reader.DONE = DONE;

package/packages/streamx/src/reduce.ts

@@ -0,0 +1,15 @@
+import { Promised, StreamX, StreamXMapper } from './index';
+
+export function reduce<Input, Accumulator>(
+  reducer: (accumulator: Accumulator, input: Input) => Promised<Accumulator>,
+  initial: Accumulator
+): StreamXMapper<Input, Accumulator> {
+  return inputStream =>
+    (async function* reduced(): StreamX<Accumulator> {
+      let finalValue: Accumulator = initial;
+      for await (const input of inputStream) {
+        finalValue = await reducer(finalValue, input);
+      }
+      yield finalValue;
+    })();
+}

package/packages/streamx/src/scale.ts

@@ -0,0 +1,93 @@
+import { Promised, StreamXMapper } from './index';
+import { IRead, read } from './reader';
+import { IWriter, Writer } from './writer';
+import { Concurrency, IPublishToConcurrency } from '@handy/fun/concurrency';
+import { syncTick } from '@handy/fun/tick';
+
+export function scale<Input, Output>(
+  max: number,
+  mapper: (input: Input) => Promised<Output>
+): StreamXMapper<Input, Output> {
+  let outputBuffer: IWriter<Output>;
+  let readInput: IRead<Input>;
+  let readOutput: IRead<Output>;
+  let concurrencyControl: IPublishToConcurrency<Input, Output>;
+  let _error: Error | undefined = undefined;
+
+  async function finish() {
+    if (concurrencyControl) {
+      await concurrencyControl.finish();
+    }
+
+    if (outputBuffer) {
+      await outputBuffer.finish();
+    }
+  }
+
+  return inputStream => {
+    return {
+      [Symbol.asyncIterator](): AsyncIterator<Output> {
+        return {
+          async next(): Promise<IteratorResult<Output>> {
+            if (_error) {
+              throw _error;
+            }
+
+            if (!readInput) {
+              readInput = read<Input>(inputStream);
+            }
+
+            if (!outputBuffer) {
+              outputBuffer = Writer<Output>();
+
+              if (!concurrencyControl) {
+                concurrencyControl = Concurrency<Input>(max, async input => {
+                  try {
+                    const output = await mapper(input);
+                    await outputBuffer.write(output);
+                  } catch (error) {
+                    _error = error as Error;
+                    finish();
+                  }
+                });
+              }
+
+              syncTick(async () => {
+                while (true) {
+                  try {
+                    const inputValue = await readInput();
+                    if (inputValue === read.DONE) {
+                      await finish();
+                      break;
+                    } else {
+                      await concurrencyControl(inputValue);
+                    }
+                  } catch (error) {
+                    _error = error as Error;
+                    await finish();
+                  }
+                }
+              });
+            }
+
+            if (!readOutput) {
+              readOutput = read<Output>(outputBuffer.stream);
+            }
+
+            const output = await readOutput();
+
+            if (_error) {
+              throw _error;
+            }
+
+            if (output === read.DONE) {
+              return { done: true, value: undefined };
+            } else {
+              return { done: false, value: output };
+            }
+          },
+        };
+      },
+    };
+  };
+}

package/packages/streamx/src/scaleSync.ts

@@ -0,0 +1,13 @@
+import { pipe, Promised, StreamXMapper } from './index';
+import { batch } from './batch';
+import { map } from './map';
+import { flat } from './flat';
+
+export function scaleSync<Input, Output>(
+  size: number,
+  mapper: (input: Input) => Promised<Output>
+): StreamXMapper<Input, Output> {
+  return pipe(batch<Input>(size))
+    .pipe(map(values => Promise.all(values.map(mapper))))
+    .pipe(flat());
+}

package/packages/streamx/src/sequence.ts

@@ -0,0 +1,7 @@
+import { StreamX } from './index';
+
+export async function* sequence(length: number): StreamX<number> {
+  for (let i = 0; i < length; i++) {
+    yield i;
+  }
+}

package/packages/streamx/src/tap.ts

@@ -0,0 +1,9 @@
+import { map } from './map';
+import { Promised, StreamXMapper } from './index';
+
+export function tap<Input>(fn: (input: Input) => Promised<any>): StreamXMapper<Input, Input> {
+  return map<Input, Input>(async (input): Promise<Input> => {
+    await fn(input);
+    return input;
+  });
+}

package/packages/streamx/src/toArray.ts

@@ -0,0 +1,9 @@
+import { StreamX } from './index';
+
+export async function toArray<T>(input: StreamX<T>): Promise<T[]> {
+  const values: T[] = [];
+  for await (const value of input) {
+    values.push(value);
+  }
+  return values;
+}

package/packages/streamx/src/writer.ts

@@ -0,0 +1,96 @@
+import { Defer, IDefer } from '@handy/fun/defer';
+import { of, StreamX } from './index';
+import { reader } from './reader';
+import { flat } from './flat';
+
+export type IWriter<T> = {
+  stream: StreamX<T>;
+  write: (data: T) => Promise<void>;
+  finish: () => Promise<void>;
+  length(): number;
+};
+
+export function Writer<T>(bufferSize = 1): IWriter<T> {
+  let records: T[] = [];
+
+  let weWaitUntilRecordsConsumed: IDefer<void> | undefined = undefined;
+  let weWaitForRecordsToRead: IDefer<void> | undefined = undefined;
+
+  let finishing = false;
+
+  const read = async (): Promise<T[] | typeof reader.DONE> => {
+    if (!records.length) {
+      if (finishing) {
+        return reader.DONE;
+      }
+
+      if (!weWaitForRecordsToRead) {
+        weWaitForRecordsToRead = Defer<void>();
+      }
+
+      // waiting to read
+      await weWaitForRecordsToRead.promise;
+    }
+
+    const currentRecords = [...records];
+    records = [];
+
+    if (weWaitUntilRecordsConsumed) {
+      weWaitUntilRecordsConsumed.resolve();
+      weWaitUntilRecordsConsumed = undefined;
+    }
+
+    if (finishing) {
+      if (currentRecords.length) {
+        return currentRecords;
+      } else {
+        return reader.DONE;
+      }
+    }
+
+    return currentRecords;
+  };
+
+  return {
+    async write(data: T) {
+      if (finishing) {
+        throw new Error(`Buffer is finishing, impossible to write`);
+      }
+
+      records.push(data);
+
+      if (weWaitForRecordsToRead) {
+        weWaitForRecordsToRead.resolve();
+        weWaitForRecordsToRead = undefined;
+      }
+
+      if (records.length >= bufferSize) {
+        if (!weWaitUntilRecordsConsumed) {
+          weWaitUntilRecordsConsumed = Defer<void>();
+        }
+
+        await weWaitUntilRecordsConsumed.promise;
+      }
+    },
+
+    async finish() {
+      finishing = true;
+
+      if (weWaitForRecordsToRead) {
+        weWaitForRecordsToRead.resolve();
+        weWaitForRecordsToRead = undefined;
+      }
+
+      if (weWaitUntilRecordsConsumed) {
+        await weWaitUntilRecordsConsumed.promise;
+        weWaitUntilRecordsConsumed = undefined;
+      }
+    },
+
+    length(): number {
+      return records.length;
+    },
+
+    stream: of(reader(read)).pipe(flat()),
+  };
+}

package/rnd/hf.ts

@@ -0,0 +1,14 @@
+import { pipeline } from "@huggingface/transformers";
+
+const generator = await pipeline(
+  "text-generation",
+  "HuggingFaceTB/SmolLM2-135M-Instruct"
+);
+
+const messages = [
+  { role: "system", content: "You are a helpful assistant." },
+  { role: "user", content: "Who is Microsoft?" },
+];
+
+const output = await generator(messages, { max_new_tokens: 64 });
+console.log(output[0].generated_text.at(-1).content);

package/rnd/keywords-compromise.ts

@@ -0,0 +1,18 @@
+import nlp from "compromise";
+import { readFile } from "fs/promises";
+
+const filePath = process.argv[2];
+if (!filePath) {
+  console.error("Usage: npx tsx keywords-compromise.ts <file>");
+  process.exit(1);
+}
+
+const text = await readFile(filePath, "utf8");
+const doc = nlp(text);
+
+console.log(`Keywords from: ${filePath}\n`);
+console.log("Topics:", doc.topics().out("array"));
+console.log("\nNouns:", doc.nouns().out("array"));
+console.log("\nVerbs:", doc.verbs().out("array"));
+console.log("\nPeople:", doc.people().out("array"));
+console.log("\nOrganizations:", doc.organizations().out("array"));

package/rnd/keywords-pipeline.ts

@@ -0,0 +1,79 @@
+import nlp from "compromise";
+import {createRequire} from "module";
+import {readFile} from "fs/promises";
+import {pipeline} from "@huggingface/transformers";
+
+const require = createRequire(import.meta.url);
+const keyword_extractor = require("keyword-extractor");
+
+const generator = await pipeline(
+    "text-generation",
+    "HuggingFaceTB/SmolLM2-135M-Instruct"
+);
+
+export async function llm(input: string): Promise<string> {
+    const prompt = "Extract and list the most important keywords from the following text. Return only keywords separated by commas.";
+
+    const messages = [
+        {role: "system", content: prompt},
+        {role: "user", content: input},
+    ];
+
+    const output = await generator(messages, {max_new_tokens: 128});
+    return output[0].generated_text.at(-1).content;
+}
+
+const filePath = process.argv[2];
+if (!filePath) {
+    console.error("Usage: npx tsx keywords-pipeline.ts <file>");
+    process.exit(1);
+}
+
+// Step 1: Read file
+const text = await readFile(filePath, "utf8");
+console.log("=== Step 1: Read file ===");
+console.log(`${text.length} chars\n`);
+
+// Step 2: Compromise — extract nouns, verbs, topics
+const doc = nlp(text);
+const nouns = doc.nouns().out("array") as string[];
+const verbs = doc.verbs().out("array") as string[];
+const topics = doc.topics().out("array") as string[];
+const combined = [...topics, ...nouns, ...verbs].join(" ");
+console.log("=== Step 2: Compromise ===");
+console.log(`${topics.length} topics, ${nouns.length} nouns, ${verbs.length} verbs\n`);
+
+// Step 3: Regex — replace all non-word chars with space
+const cleaned = combined.replace(/\W+/g, " ").trim();
+console.log("=== Step 3: Regex cleanup ===");
+console.log(cleaned.slice(0, 200), "\n");
+
+// Step 4: LLM — pass cleaned text to local model
+const llmResult = await llm(cleaned);
+console.log("=== Step 4: LLM ===");
+console.log(llmResult, "\n");
+
+// Step 5: keyword-extractor
+const keywords: string[] = keyword_extractor.extract(cleaned, {
+    language: "english",
+    remove_digits: false,
+    return_changed_case: true,
+    remove_duplicates: true,
+    return_max_ngrams: 3,
+});
+
+// Score by frequency
+const lower = cleaned.toLowerCase();
+const scored = keywords
+    .filter((kw) => kw.length > 2)
+    .map((kw) => {
+        const re = new RegExp(`\\b${kw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "gi");
+        const count = (lower.match(re) || []).length;
+        return {keyword: kw, count};
+    });
+scored.sort((a, b) => b.count - a.count);
+
+console.log("=== Step 5: Keywords ===");
+for (const {keyword, count} of scored) {
+    console.log(`  ${keyword.padEnd(35)} (${count}x)`);
+}