xindex 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ai/research/2026-04-10-file-watching.md +79 -0
- package/.ai/research/2026-04-10-mcp-output-format.md +129 -0
- package/.ai/task/INDEX.md +12 -0
- package/.ai/task/done/INDEX.md +3 -0
- package/.ai/task/done/task.2026-04-09-local-ai-research-protos.log.md +98 -0
- package/.ai/task/done/task.2026-04-09-local-ai-research-protos.md +102 -0
- package/.ai/task/task.2026-04-10-cluster-config.log.md +19 -0
- package/.ai/task/task.2026-04-10-cluster-config.md +118 -0
- package/.ai/task/task.2026-04-10-dir-indexing.log.md +8 -0
- package/.ai/task/task.2026-04-10-dir-indexing.md +92 -0
- package/.ai/task/task.2026-04-10-line-clustering.log.md +50 -0
- package/.ai/task/task.2026-04-10-line-clustering.md +176 -0
- package/.ai/task/task.2026-04-10-object-store.log.md +7 -0
- package/.ai/task/task.2026-04-10-object-store.md +81 -0
- package/.ai/task/task.2026-04-10-search-config.log.md +46 -0
- package/.ai/task/task.2026-04-10-search-config.md +274 -0
- package/.ai/task/task.2026-04-10-watch-indexing.log.md +32 -0
- package/.ai/task/task.2026-04-10-watch-indexing.md +101 -0
- package/.ai/task/task.2026-04-10-xindex-mcp.log.md +5 -0
- package/.ai/task/task.2026-04-10-xindex-mcp.md +92 -0
- package/.ai/task/task.2026-04-10-xindex-mcp.report.md +113 -0
- package/.claude/settings.local.json +73 -0
- package/.claude/skills/make-hof/SKILL.md +8 -0
- package/.claude/skills/make-hof/playbook.md +38 -0
- package/.cursor/mcp.json +8 -0
- package/.mcp.json +8 -0
- package/.xindex.json +22 -0
- package/CLAUDE.md +54 -0
- package/README.md +206 -0
- package/apps/indexApp.ts +31 -0
- package/apps/mcpApp.ts +119 -0
- package/apps/run.index.ts +19 -0
- package/apps/run.mcp.ts +49 -0
- package/apps/run.reset.ts +10 -0
- package/apps/run.search.ts +21 -0
- package/apps/run.watch.ts +44 -0
- package/apps/searchApp.ts +9 -0
- package/apps/watchApp.ts +53 -0
- package/apps/watchFileEventsApp.ts +39 -0
- package/bin/xindex-index +2 -0
- package/bin/xindex-mcp +2 -0
- package/bin/xindex-reset +2 -0
- package/bin/xindex-search +2 -0
- package/bin/xindex-watch +2 -0
- package/componets/IType.ts +1 -0
- package/componets/appId.ts +3 -0
- package/componets/buildComponents.ts +27 -0
- package/componets/config/loadConfig.ts +43 -0
- package/componets/config/xindexConfig.ts +4 -0
- package/componets/index/contentIndexDriver.ts +39 -0
- package/componets/index/formatSearchResults.ts +18 -0
- package/componets/index/getIndexStats.ts +11 -0
- package/componets/index/handleFileEvent.ts +25 -0
- package/componets/index/indexApi.ts +45 -0
- package/componets/index/vectraIndex.ts +11 -0
- package/componets/index/watcherLock.ts +107 -0
- package/componets/keywords/cleanUpKeywords.ts +38 -0
- package/componets/keywords/extractKeywords.ts +14 -0
- package/componets/keywords/refineKeywords.ts +16 -0
- package/componets/llm/embed.ts +18 -0
- package/componets/llm/queryLLM.ts +20 -0
- package/componets/logger.ts +34 -0
- package/componets/walkFiles.ts +51 -0
- package/componets/watchFiles.ts +106 -0
- package/features/indexContent.ts +16 -0
- package/features/removeContent.ts +9 -0
- package/features/resetIndex.ts +9 -0
- package/features/searchIndex.ts +33 -0
- package/package.json +32 -0
- package/packages/fun/src/IType.ts +5 -0
- package/packages/fun/src/array-finder.ts +55 -0
- package/packages/fun/src/array-index.ts +35 -0
- package/packages/fun/src/array.ts +112 -0
- package/packages/fun/src/assert.ts +5 -0
- package/packages/fun/src/asyncRequest.ts +35 -0
- package/packages/fun/src/callsites.ts +18 -0
- package/packages/fun/src/case-never.ts +9 -0
- package/packages/fun/src/casting.ts +41 -0
- package/packages/fun/src/collect.ts +13 -0
- package/packages/fun/src/concurrency.ts +186 -0
- package/packages/fun/src/container.ts +86 -0
- package/packages/fun/src/counter.ts +45 -0
- package/packages/fun/src/create-map.ts +2 -0
- package/packages/fun/src/dedupe.ts +2 -0
- package/packages/fun/src/defer.ts +55 -0
- package/packages/fun/src/delay.ts +5 -0
- package/packages/fun/src/discriminate.ts +34 -0
- package/packages/fun/src/enum-values.ts +12 -0
- package/packages/fun/src/exponential-backoff.ts +20 -0
- package/packages/fun/src/flatten.ts +11 -0
- package/packages/fun/src/hash.ts +67 -0
- package/packages/fun/src/hash128.ts +6 -0
- package/packages/fun/src/hash256.ts +6 -0
- package/packages/fun/src/hub.ts +53 -0
- package/packages/fun/src/id.ts +10 -0
- package/packages/fun/src/interval.ts +76 -0
- package/packages/fun/src/is-non-nullable.ts +2 -0
- package/packages/fun/src/isIterable.ts +3 -0
- package/packages/fun/src/mailbox.ts +13 -0
- package/packages/fun/src/map-record.ts +19 -0
- package/packages/fun/src/match-collections.ts +57 -0
- package/packages/fun/src/match-left-and-right-arrays.ts +78 -0
- package/packages/fun/src/mem.ts +26 -0
- package/packages/fun/src/memos.ts +28 -0
- package/packages/fun/src/normalizeError.ts +25 -0
- package/packages/fun/src/nothing.ts +3 -0
- package/packages/fun/src/pipe.ts +18 -0
- package/packages/fun/src/prettyJson.ts +3 -0
- package/packages/fun/src/project.ts +8 -0
- package/packages/fun/src/promise.ts +27 -0
- package/packages/fun/src/pubsub.ts +128 -0
- package/packages/fun/src/randomId.ts +14 -0
- package/packages/fun/src/regexp-escape.ts +13 -0
- package/packages/fun/src/retry.ts +15 -0
- package/packages/fun/src/serial.test.ts +107 -0
- package/packages/fun/src/serial.ts +17 -0
- package/packages/fun/src/sleep.ts +3 -0
- package/packages/fun/src/sort-object.ts +46 -0
- package/packages/fun/src/speed-test.ts +56 -0
- package/packages/fun/src/tick.ts +37 -0
- package/packages/fun/src/time-behavior.ts +50 -0
- package/packages/fun/src/time.ts +22 -0
- package/packages/fun/src/timedFallback.ts +37 -0
- package/packages/fun/src/timer.ts +30 -0
- package/packages/fun/src/value.ts +33 -0
- package/packages/fun/src/waitForCounter.ts +15 -0
- package/packages/streamx/src/batch.ts +23 -0
- package/packages/streamx/src/batchTimed.ts +113 -0
- package/packages/streamx/src/buffer.ts +72 -0
- package/packages/streamx/src/concatenate.ts +33 -0
- package/packages/streamx/src/filter.ts +14 -0
- package/packages/streamx/src/flat.ts +19 -0
- package/packages/streamx/src/flatMap.ts +9 -0
- package/packages/streamx/src/from.ts +30 -0
- package/packages/streamx/src/index.ts +49 -0
- package/packages/streamx/src/interval.ts +58 -0
- package/packages/streamx/src/loop.ts +8 -0
- package/packages/streamx/src/map.ts +12 -0
- package/packages/streamx/src/merge.ts +89 -0
- package/packages/streamx/src/nodeReadable.ts +6 -0
- package/packages/streamx/src/nodeTransform.ts +9 -0
- package/packages/streamx/src/nodeWritable.ts +38 -0
- package/packages/streamx/src/objectReader.ts +16 -0
- package/packages/streamx/src/polyfill.ts +20 -0
- package/packages/streamx/src/reader.ts +38 -0
- package/packages/streamx/src/reduce.ts +15 -0
- package/packages/streamx/src/scale.ts +93 -0
- package/packages/streamx/src/scaleSync.ts +13 -0
- package/packages/streamx/src/sequence.ts +7 -0
- package/packages/streamx/src/tap.ts +9 -0
- package/packages/streamx/src/toArray.ts +9 -0
- package/packages/streamx/src/writer.ts +96 -0
- package/rnd/hf.ts +14 -0
- package/rnd/keywords-compromise.ts +18 -0
- package/rnd/keywords-pipeline.ts +79 -0
- package/rnd/keywords.ts +38 -0
- package/rnd/test-vectra-memory.ts +63 -0
- package/rnd/vectra-keywords.ts +95 -0
- package/rnd/vectra.ts +50 -0
- package/tsconfig.json +14 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { IPubSub, PubSub } from './pubsub';
|
|
2
|
+
|
|
3
|
+
export type IMailbox<Inbox, Outbox = never> = Readonly<{
|
|
4
|
+
inbox: IPubSub<Inbox>;
|
|
5
|
+
outbox: IPubSub<Outbox>;
|
|
6
|
+
}>;
|
|
7
|
+
|
|
8
|
+
export function Mailbox<Inbox = never, Outbox = never>(): IMailbox<Inbox, Outbox> {
|
|
9
|
+
return {
|
|
10
|
+
inbox: PubSub<Inbox>(),
|
|
11
|
+
outbox: PubSub<Outbox>(),
|
|
12
|
+
};
|
|
13
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export const mapRecord = <TKey extends keyof TIn, TIn extends Record<any, any>, TOut>(
|
|
2
|
+
record: TIn,
|
|
3
|
+
mapper: (value: TIn[TKey], key: TKey) => TOut
|
|
4
|
+
): Record<TKey, TOut> => {
|
|
5
|
+
const entries = Object.entries(record) as [TKey, TIn[TKey]][];
|
|
6
|
+
const results = entries.map(([key, value]) => [key as TKey, mapper(value, key as TKey)]);
|
|
7
|
+
return Object.fromEntries(results) as Record<TKey, TOut>;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export const mapRecordAsync = async <TKey extends keyof TIn, TIn extends Record<any, any>, TOut>(
|
|
11
|
+
record: TIn,
|
|
12
|
+
mapper: (value: TIn[TKey], key: TKey) => Promise<TOut>
|
|
13
|
+
): Promise<Record<TKey, TOut>> => {
|
|
14
|
+
const entries = Object.entries(record) as [TKey, TIn[TKey]][];
|
|
15
|
+
const results = await Promise.all(
|
|
16
|
+
entries.map<Promise<[TKey, TOut]>>(async ([key, value]) => [key, await mapper(value, key)])
|
|
17
|
+
);
|
|
18
|
+
return Object.fromEntries(results) as Record<TKey, TOut>;
|
|
19
|
+
};
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
export type Identifier<Item> = (previousItem: Item, nextItem: Item) => boolean;
|
|
2
|
+
|
|
3
|
+
export type IsEqual<Item = any> = (previousItem: Item, nextItem: Item) => boolean;
|
|
4
|
+
|
|
5
|
+
export type IMatchCollectionsOutput<Item> = {
|
|
6
|
+
created: Item[];
|
|
7
|
+
matched: {
|
|
8
|
+
previous: Item;
|
|
9
|
+
next: Item;
|
|
10
|
+
}[];
|
|
11
|
+
deleted: Item[];
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export type IMatchCollections<Item = any> = (
|
|
15
|
+
previousCollection: Item[],
|
|
16
|
+
nextCollection: Item[]
|
|
17
|
+
) => IMatchCollectionsOutput<Item>;
|
|
18
|
+
|
|
19
|
+
export function MatchCollections<Item>(identifier: Identifier<Item>): IMatchCollections<Item> {
|
|
20
|
+
return function (prevItems, nextItems) {
|
|
21
|
+
const createdItems: Item[] = nextItems.filter(
|
|
22
|
+
nextItem => !prevItems.find(prevItem => identifier(nextItem, prevItem))
|
|
23
|
+
);
|
|
24
|
+
|
|
25
|
+
const deletedItems: Item[] = prevItems.filter(
|
|
26
|
+
prevItem => !nextItems.find(nextItem => identifier(nextItem, prevItem))
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
const map = nextItems.map(nextItem => {
|
|
30
|
+
const prevItemFound = prevItems.find(prevItem => identifier(prevItem, nextItem));
|
|
31
|
+
|
|
32
|
+
const isMatched = !!prevItemFound;
|
|
33
|
+
|
|
34
|
+
return {
|
|
35
|
+
prevItem: prevItemFound,
|
|
36
|
+
nextItem,
|
|
37
|
+
isMatched,
|
|
38
|
+
};
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
const matchedItems = map
|
|
42
|
+
.filter(({ isMatched }) => isMatched)
|
|
43
|
+
.map(({ prevItem, nextItem }) => ({
|
|
44
|
+
previous: prevItem!,
|
|
45
|
+
next: nextItem!,
|
|
46
|
+
}));
|
|
47
|
+
|
|
48
|
+
return {
|
|
49
|
+
created: createdItems,
|
|
50
|
+
matched: matchedItems,
|
|
51
|
+
deleted: deletedItems,
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export const identifyById = <T extends { id: any }>(item1: T, item2: T) =>
|
|
57
|
+
String(item1.id) === String(item2.id);
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { sortObject } from './sort-object';
|
|
2
|
+
|
|
3
|
+
export type IMatchLeftAndRightArraysResult<Left, Right> = {
|
|
4
|
+
matched: { left: Left; right: Right }[];
|
|
5
|
+
unmatched: { left: Left[]; right: Right[] };
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export function matchLeftAndRightArrays<Left, Right, ID>(
|
|
9
|
+
left: Left[] | ReadonlyArray<Left>,
|
|
10
|
+
right: Right[] | ReadonlyArray<Right>,
|
|
11
|
+
leftToId: (left: Left) => ID,
|
|
12
|
+
rightToId: (left: Right) => ID
|
|
13
|
+
): IMatchLeftAndRightArraysResult<Left, Right> {
|
|
14
|
+
const matched: { left: Left; right: Right }[] = [];
|
|
15
|
+
|
|
16
|
+
const leftSet = new Set<Left>();
|
|
17
|
+
const rightSet = new Set<Right>();
|
|
18
|
+
|
|
19
|
+
if (!left.length) {
|
|
20
|
+
right.forEach(rightItem => {
|
|
21
|
+
rightSet.add(rightItem);
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
if (!right.length) {
|
|
26
|
+
left.forEach(leftItem => {
|
|
27
|
+
leftSet.add(leftItem);
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const serializeId = (id: ID): string => {
|
|
32
|
+
if (id instanceof Object) {
|
|
33
|
+
return JSON.stringify(sortObject(id));
|
|
34
|
+
} else {
|
|
35
|
+
return String(id);
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const leftMap = new Map<string, Left>();
|
|
40
|
+
const rightMap = new Map<string, Right>();
|
|
41
|
+
|
|
42
|
+
left.forEach(leftItem => {
|
|
43
|
+
const id = serializeId(leftToId(leftItem));
|
|
44
|
+
leftMap.set(id, leftItem);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
right.forEach(rightItem => {
|
|
48
|
+
const id = serializeId(rightToId(rightItem));
|
|
49
|
+
rightMap.set(id, rightItem);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// @ts-ignore
|
|
53
|
+
const ids = new Set<string>([...leftMap.keys(), ...rightMap.keys()]);
|
|
54
|
+
|
|
55
|
+
ids.forEach(id => {
|
|
56
|
+
if (leftMap.has(id) && rightMap.has(id)) {
|
|
57
|
+
matched.push({ left: leftMap.get(id)!, right: rightMap.get(id)! });
|
|
58
|
+
} else {
|
|
59
|
+
if (leftMap.has(id)) {
|
|
60
|
+
leftSet.add(leftMap.get(id)!);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (rightMap.has(id)) {
|
|
64
|
+
rightSet.add(rightMap.get(id)!);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
return {
|
|
70
|
+
matched,
|
|
71
|
+
unmatched: {
|
|
72
|
+
// @ts-ignore
|
|
73
|
+
left: [...leftSet],
|
|
74
|
+
// @ts-ignore
|
|
75
|
+
right: [...rightSet],
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export type IMem<T> = (() => T) & {
|
|
2
|
+
reset: () => void;
|
|
3
|
+
};
|
|
4
|
+
|
|
5
|
+
export function mem<T>(fn: () => T): IMem<T> {
|
|
6
|
+
let value: any = undefined;
|
|
7
|
+
let called = false;
|
|
8
|
+
|
|
9
|
+
const iMem = () => {
|
|
10
|
+
if (called) {
|
|
11
|
+
return value;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
value = fn();
|
|
15
|
+
called = true;
|
|
16
|
+
|
|
17
|
+
return value;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
iMem.reset = () => {
|
|
21
|
+
value = undefined;
|
|
22
|
+
called = false;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
return iMem;
|
|
26
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { IMem, mem } from './mem';
|
|
2
|
+
|
|
3
|
+
export type IMemos<Input, Output> = ((value: Input) => Output) & {
|
|
4
|
+
reset: () => void;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
export function memos<Input, Output>(
|
|
8
|
+
toKey: (value: Input) => string | number,
|
|
9
|
+
toValue: (value: Input) => Output
|
|
10
|
+
): IMemos<Input, Output> {
|
|
11
|
+
let cache: { [P in string | number]: IMem<Output> } = {};
|
|
12
|
+
|
|
13
|
+
const iMemos = (value: Input) => {
|
|
14
|
+
const key = toKey(value);
|
|
15
|
+
if (!(key in cache)) {
|
|
16
|
+
cache[key] = mem<Output>(() => toValue(value));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
return cache[key]!();
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
iMemos.reset = () => {
|
|
23
|
+
Object.values(cache).forEach(mem => mem.reset());
|
|
24
|
+
cache = {};
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
return iMemos;
|
|
28
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export function normalizeError<Type extends Error>(error: Type): Type {
|
|
2
|
+
if (!error) {
|
|
3
|
+
return error;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
if(!(error instanceof Object)) {
|
|
7
|
+
return error as Type;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
return {
|
|
11
|
+
...error,
|
|
12
|
+
name: error.name,
|
|
13
|
+
message: error.message,
|
|
14
|
+
stack: error.stack,
|
|
15
|
+
|
|
16
|
+
// @ts-ignore
|
|
17
|
+
type: error.type,
|
|
18
|
+
|
|
19
|
+
// @ts-ignore
|
|
20
|
+
data: error.data,
|
|
21
|
+
|
|
22
|
+
// @ts-ignore
|
|
23
|
+
origin: error.origin,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// TODO @@@@slava specs
|
|
2
|
+
type IMapper = (x: any) => any;
|
|
3
|
+
|
|
4
|
+
export function syncPipe<Input = void, Output = any>(...mappers: IMapper[]) {
|
|
5
|
+
return function (input: Input): Output {
|
|
6
|
+
// @ts-ignore
|
|
7
|
+
return mappers.reduce((current, mapper) => mapper(current), input) as Output;
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function pipe<Input = void, Output = any>(...mappers: IMapper[]) {
|
|
12
|
+
return async function (input: Input): Promise<Output> {
|
|
13
|
+
return (await mappers.reduce(
|
|
14
|
+
async (current, mapper) => mapper(await current),
|
|
15
|
+
(async () => input)()
|
|
16
|
+
)) as Output;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
type IUnoPromise<T> = (() => Promise<T>) & {
|
|
2
|
+
reset: () => Promise<void>;
|
|
3
|
+
};
|
|
4
|
+
|
|
5
|
+
export function UnoPromise<T>(fn: () => Promise<T>): IUnoPromise<T> {
|
|
6
|
+
let promise: undefined | Promise<T> = undefined;
|
|
7
|
+
|
|
8
|
+
const unoPromise = async () => {
|
|
9
|
+
const hadPromise = !!promise;
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
promise = promise || fn();
|
|
13
|
+
return promise as Promise<T>;
|
|
14
|
+
} finally {
|
|
15
|
+
if (!hadPromise) {
|
|
16
|
+
promise = undefined;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
unoPromise.reset = async () => {
|
|
22
|
+
await promise;
|
|
23
|
+
promise = undefined;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
return unoPromise;
|
|
27
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { Defer } from "./defer";
|
|
2
|
+
import { IType, IUseType } from "./IType";
|
|
3
|
+
|
|
4
|
+
export type ISyncPubSub<T> = {
|
|
5
|
+
publish: (message: T) => void;
|
|
6
|
+
subscribe: IPubSubSubscribe<T>;
|
|
7
|
+
unsubscribeAll: () => void;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export type IPubSubSubscribe<T> = (
|
|
11
|
+
subscriber: IPubSubSubscriber<T>,
|
|
12
|
+
) => IPubSubUnsubscribe;
|
|
13
|
+
export type IPubSubPublish<T> = (message: T) => Promise<void>;
|
|
14
|
+
|
|
15
|
+
export type IPubSub<T> = {
|
|
16
|
+
publish: IPubSubPublish<T>;
|
|
17
|
+
subscribe: IPubSubSubscribe<T>;
|
|
18
|
+
unsubscribeAll: () => void;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export type IPubSubSubscriber<T> = (message: T) => any | Promise<any>;
|
|
22
|
+
export type IPubSubUnsubscribe = () => void | any;
|
|
23
|
+
|
|
24
|
+
export function SyncPubSub<T>(): ISyncPubSub<T> {
|
|
25
|
+
let subscribers: IPubSubSubscriber<T>[] = [];
|
|
26
|
+
|
|
27
|
+
function subscribe(subscriber: (value: T) => void) {
|
|
28
|
+
subscribers.push(subscriber);
|
|
29
|
+
return () => unsubscribe(subscriber);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function unsubscribe(subscriber: IPubSubSubscriber<T>) {
|
|
33
|
+
subscribers = subscribers.filter((sub) => sub !== subscriber);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function publish(value: T) {
|
|
37
|
+
subscribers.map((sub) => sub(value));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function unsubscribeAll() {
|
|
41
|
+
subscribers = [];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return {
|
|
45
|
+
publish,
|
|
46
|
+
subscribe,
|
|
47
|
+
unsubscribeAll,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export class PubSubAggregateError extends Error {
|
|
52
|
+
constructor(
|
|
53
|
+
public errors: Error[],
|
|
54
|
+
message?: string,
|
|
55
|
+
) {
|
|
56
|
+
super(message);
|
|
57
|
+
|
|
58
|
+
const error = errors[0] || ({} as any);
|
|
59
|
+
|
|
60
|
+
this.message = message || error.message || "";
|
|
61
|
+
this.stack = error.stack || "";
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function PubSub<T>(): IPubSub<T> {
|
|
66
|
+
let subscribers: IPubSubSubscriber<T>[] = [];
|
|
67
|
+
|
|
68
|
+
function subscribe(subscriber: (value: T) => void) {
|
|
69
|
+
subscribers.push(subscriber);
|
|
70
|
+
return () => unsubscribe(subscriber);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function unsubscribe(subscriber: IPubSubSubscriber<T>) {
|
|
74
|
+
subscribers = subscribers.filter((sub) => sub !== subscriber);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function publish(message: T) {
|
|
78
|
+
await Promise.all(subscribers.map((sub) => sub(message)));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function unsubscribeAll() {
|
|
82
|
+
subscribers = [];
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return {
|
|
86
|
+
publish,
|
|
87
|
+
subscribe,
|
|
88
|
+
unsubscribeAll,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export async function waitForMessage<T>(
|
|
93
|
+
subscribe: IPubSubSubscribe<T> | IPubSub<T>,
|
|
94
|
+
isAcceptable: (message: T) => boolean | Promise<boolean>,
|
|
95
|
+
): Promise<T> {
|
|
96
|
+
const defer = Defer<T>();
|
|
97
|
+
if ("subscribe" in subscribe) {
|
|
98
|
+
subscribe = subscribe.subscribe;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const unsubscribe = subscribe(async (message: T) => {
|
|
102
|
+
try {
|
|
103
|
+
if (await isAcceptable(message)) {
|
|
104
|
+
unsubscribe();
|
|
105
|
+
defer.resolve(message);
|
|
106
|
+
}
|
|
107
|
+
} catch (error) {
|
|
108
|
+
unsubscribe();
|
|
109
|
+
defer.reject(error);
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
return defer.promise;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function ScopedSubscribe<Type extends IType, Scope extends Type["type"]>(
|
|
117
|
+
subscribe: IPubSubSubscribe<Type>,
|
|
118
|
+
scope: ReadonlyArray<Scope>,
|
|
119
|
+
): IPubSubSubscribe<IUseType<Type, Scope>> {
|
|
120
|
+
const set = new Set(scope);
|
|
121
|
+
return (subscriber) => {
|
|
122
|
+
return subscribe((message) => {
|
|
123
|
+
if (set.has(message.type as Scope)) {
|
|
124
|
+
return subscriber(message as IUseType<Type, Scope>);
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
};
|
|
128
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export function RandomId(bits = 128): () => string {
|
|
2
|
+
const radix = 16; // hex
|
|
3
|
+
const size = Math.ceil(bits / 8) * 2;
|
|
4
|
+
|
|
5
|
+
return () => {
|
|
6
|
+
let result = "";
|
|
7
|
+
|
|
8
|
+
while (result.length < size) {
|
|
9
|
+
result += Math.random().toString(radix).slice(2);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
return result.slice(0, size);
|
|
13
|
+
};
|
|
14
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// https://stackoverflow.com/a/9310752
|
|
2
|
+
export const regExpEscape = (value: string) => value.replace(/[.*+\-?^${}()|[\]\\]/g, '\\$&');
|
|
3
|
+
|
|
4
|
+
// copy of https://www.npmjs.com/package/escape-string-regexp
|
|
5
|
+
export function escapeStringRegexp(string: string) {
|
|
6
|
+
if (typeof string !== 'string') {
|
|
7
|
+
throw new TypeError('Expected a string');
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
// Escape characters with special meaning either inside or outside character sets.
|
|
11
|
+
// Use a simple backslash escape when it’s always valid, and a `\xnn` escape when the simpler form would be disallowed by Unicode patterns’ stricter grammar.
|
|
12
|
+
return string.replace(/[|\\{}()[\]^$+*?.]/g, '\\$&').replace(/-/g, '\\x2d');
|
|
13
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export async function retry<Result>(
|
|
2
|
+
fn: (attempt: number) => Result,
|
|
3
|
+
{ attempts }: { attempts: number }
|
|
4
|
+
) {
|
|
5
|
+
let lastError: any;
|
|
6
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
7
|
+
try {
|
|
8
|
+
return await fn(attempt);
|
|
9
|
+
} catch (error) {
|
|
10
|
+
lastError = error;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
throw lastError;
|
|
15
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import { Serial } from "./serial.js";
|
|
4
|
+
|
|
5
|
+
function delay(ms: number): Promise<void> {
|
|
6
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
// Thunk-runner: Serial parameterized to accept `() => Promise<T>` inputs.
|
|
10
|
+
// Lets tests exercise Serial with arbitrary per-call work.
|
|
11
|
+
function ThunkSerial() {
|
|
12
|
+
return Serial<() => Promise<any>, any>((fn) => fn());
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
describe("Serial", () => {
|
|
16
|
+
it("runs tasks sequentially — no overlap when two tasks start together", async () => {
|
|
17
|
+
const serial = ThunkSerial();
|
|
18
|
+
const events: string[] = [];
|
|
19
|
+
|
|
20
|
+
const p1 = serial(async () => {
|
|
21
|
+
events.push("a:start");
|
|
22
|
+
await delay(15);
|
|
23
|
+
events.push("a:end");
|
|
24
|
+
return "a";
|
|
25
|
+
});
|
|
26
|
+
const p2 = serial(async () => {
|
|
27
|
+
events.push("b:start");
|
|
28
|
+
await delay(15);
|
|
29
|
+
events.push("b:end");
|
|
30
|
+
return "b";
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
await Promise.all([p1, p2]);
|
|
34
|
+
|
|
35
|
+
assert.deepEqual(events, ["a:start", "a:end", "b:start", "b:end"]);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it("returns the task's resolved value to the caller", async () => {
|
|
39
|
+
const serial = ThunkSerial();
|
|
40
|
+
const v = await serial(() => Promise.resolve(42));
|
|
41
|
+
assert.equal(v, 42);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it("propagates task rejection to the caller", async () => {
|
|
45
|
+
const serial = ThunkSerial();
|
|
46
|
+
const err = new Error("boom");
|
|
47
|
+
await assert.rejects(serial(() => Promise.reject(err)), (e: unknown) => e === err);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it("does not poison the chain when a task rejects — later tasks still resolve", async () => {
|
|
51
|
+
const serial = ThunkSerial();
|
|
52
|
+
const first = new Error("first");
|
|
53
|
+
await assert.rejects(serial(() => Promise.reject(first)), (e: unknown) => e === first);
|
|
54
|
+
const v = await serial(() => Promise.resolve("after"));
|
|
55
|
+
assert.equal(v, "after");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("flush resolves after all pending tasks settle (success and failure)", async () => {
|
|
59
|
+
const serial = ThunkSerial();
|
|
60
|
+
let rejectedDone = false;
|
|
61
|
+
let okDone = false;
|
|
62
|
+
|
|
63
|
+
void serial(async () => {
|
|
64
|
+
await delay(20);
|
|
65
|
+
throw new Error("planned");
|
|
66
|
+
}).catch(() => { rejectedDone = true; });
|
|
67
|
+
|
|
68
|
+
void serial(async () => {
|
|
69
|
+
await delay(15);
|
|
70
|
+
okDone = true;
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
assert.equal(rejectedDone, false);
|
|
74
|
+
assert.equal(okDone, false);
|
|
75
|
+
|
|
76
|
+
await serial.flush();
|
|
77
|
+
|
|
78
|
+
assert.equal(rejectedDone, true);
|
|
79
|
+
assert.equal(okDone, true);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it("preserves FIFO order across many concurrent enqueues", async () => {
|
|
83
|
+
const serial = ThunkSerial();
|
|
84
|
+
const order: number[] = [];
|
|
85
|
+
const tasks = Array.from({ length: 10 }, (_, i) =>
|
|
86
|
+
serial(async () => {
|
|
87
|
+
order.push(i);
|
|
88
|
+
return i;
|
|
89
|
+
}),
|
|
90
|
+
);
|
|
91
|
+
await Promise.all(tasks);
|
|
92
|
+
assert.deepEqual(order, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it("typed worker: dispatches inputs to the configured worker", async () => {
|
|
96
|
+
type Msg = { kind: "add"; n: number } | { kind: "reset" };
|
|
97
|
+
let sum = 0;
|
|
98
|
+
const run = Serial<Msg, number>(async (m) => {
|
|
99
|
+
if (m.kind === "reset") sum = 0;
|
|
100
|
+
else sum += m.n;
|
|
101
|
+
return sum;
|
|
102
|
+
});
|
|
103
|
+
assert.equal(await run({ kind: "add", n: 2 }), 2);
|
|
104
|
+
assert.equal(await run({ kind: "add", n: 3 }), 5);
|
|
105
|
+
assert.equal(await run({ kind: "reset" }), 0);
|
|
106
|
+
});
|
|
107
|
+
});
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export type ISerial<In, Out> = {
|
|
2
|
+
(input: In): Promise<Out>;
|
|
3
|
+
flush: () => Promise<void>;
|
|
4
|
+
};
|
|
5
|
+
|
|
6
|
+
export function Serial<In, Out>(worker: (input: In) => Promise<Out>): ISerial<In, Out> {
|
|
7
|
+
let tail: Promise<unknown> = Promise.resolve();
|
|
8
|
+
|
|
9
|
+
const publish = ((input: In) => {
|
|
10
|
+
const next = tail.then(() => worker(input));
|
|
11
|
+
tail = next.catch(() => {});
|
|
12
|
+
return next;
|
|
13
|
+
}) as ISerial<In, Out>;
|
|
14
|
+
|
|
15
|
+
publish.flush = () => tail.then(() => {});
|
|
16
|
+
return publish;
|
|
17
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// TODO @@@@slava move to fun lib
|
|
2
|
+
|
|
3
|
+
const jsonStringify = (x: any): string => JSON.stringify(x);
|
|
4
|
+
const stringSorter = (a: string, b: string): number => a.localeCompare(b);
|
|
5
|
+
|
|
6
|
+
export function SortObject({
|
|
7
|
+
sortIterable = true,
|
|
8
|
+
stringify = jsonStringify,
|
|
9
|
+
sorter = stringSorter,
|
|
10
|
+
} = {}) {
|
|
11
|
+
return function sortObject(input: any): any {
|
|
12
|
+
if (Array.isArray(input) || (input instanceof Object && input[Symbol.iterator])) {
|
|
13
|
+
if (sortIterable) {
|
|
14
|
+
return [...input]
|
|
15
|
+
.map(sortObject)
|
|
16
|
+
.map(item => ({ item, value: stringify(item) }))
|
|
17
|
+
.sort((a, b) => sorter(a.value, b.value))
|
|
18
|
+
.map(({ item }) => item);
|
|
19
|
+
} else {
|
|
20
|
+
return [...input].map(sortObject);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
if (input instanceof Object) {
|
|
25
|
+
const result: any = {};
|
|
26
|
+
|
|
27
|
+
Object.keys(input)
|
|
28
|
+
.sort(sorter)
|
|
29
|
+
.forEach(key => (result[key] = sortObject(input[key])));
|
|
30
|
+
|
|
31
|
+
return result;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return input;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export const sortObject = SortObject();
|
|
39
|
+
|
|
40
|
+
export function DeepEqual({ sort = sortObject, stringify = jsonStringify } = {}) {
|
|
41
|
+
return function (a: any, b: any) {
|
|
42
|
+
return stringify(sort(a)) === stringify(sort(b));
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export const deepEqual = DeepEqual();
|