querysub 0.436.0 → 0.438.0

package/src/functional/diff.ts

@@ -1,858 +1,858 @@
-import { binarySearchIndex, compare, keyByArray, list, sort } from "socket-function/src/misc";
-import { fastHash } from "../misc/hash";
-
-export interface LineChange {
-    type: "unchanged" | "added" | "removed";
-    value: string;
-}
-
-// TODO: Replace our base algorithm with something that gives better diffs, at least for small sets of values?
-//  - Ex, https://luppeng.wordpress.com/2020/10/10/when-to-use-each-of-the-git-diff-algorithms/, 
-//      myers, patience, or histogram
-
-//  Algorithm
-//  - Hash values into blocks to make them smaller to deal with
-//      - We want an optimal block size, where we are not too far away from having the cardinality of the
-//          intersection * block size === cardinality of per line intersection (so, most is matched). Of course,
-//          the larger the block size the less matches we will accept.
-//          - And then once we find the optimal block size, divide it by 2, to increase granularity
-//  - The hashes will just be used for comparison
-//  - Remove any blocks with hashes not found in both
-//  - Repeat the algorithm, until the optimal block size is 1
-//  - For N evenly choosen options, match Y random matches in the new array, then iterate (first forward,
-//      then go backwards) matching any ===, and for any different, choosing to both remove existing, or add new one
-//      at a depth of D, picking the series of choices that result in the maximum number of matches.
-//      (D will probably be 6? We can always parameterize it)
-//      - To iterate backwards, just slice, reverse, and the iterate function on that.
-
-//  blockMatch
-//  spreadMatch
-//  iterateMatch
-//  - Skew towards either removal or addition, so we can resolve large changes without having lots of random interspersed
-//      adds and removals, and instead we can batch the differences
-
-
-
-
-export type DiffChange<T> = {
-    type: "unchanged" | "added" | "removed";
-    value: T;
-    // When type is unchanged, this is the value in the new array (which compares equal to value, but
-    //  there might be additional metadata that is of use, such as line numbers)
-    unchangedNextValue?: T;
-};
-interface DiffOptions<T> {
-    base: T[];
-    next: T[];
-    compare: (lhs: T, rhs: T) => number;
-    getHash: (value: T) => number;
-}
-
-
-export function diffArrays<T>(
-    config: {
-        base: T[];
-        next: T[];
-    }
-): DiffChange<T>[] {
-    let { base: baseValues, next: newValues } = config;
-    return blockDiff({
-        base: baseValues,
-        next: newValues,
-        baseRange: { start: 0, end: baseValues.length },
-        nextRange: { start: 0, end: newValues.length },
-        compare: compare,
-        getHash: (x: unknown) => {
-            if (x === undefined) {
-                return -234235479;
-            }
-            if (x === null) {
-                return -31290214;
-            }
-            if (x === false) {
-                return -843348236;
-            }
-            if (x === true) {
-                return -843348236;
-            }
-            if (typeof x === "string") {
-                return fastHash(x);
-            }
-            if (typeof x === "number") {
-                // Not great, but... this will probably be fine? We just want to spread values out a bit more, so their
-                //      bits don't easily collide
-                return ~~(x * 417691);
-            }
-
-            throw new Error(`Unhandled type in getHash, ${typeof x}`);
-        },
-    });
-}
-
-/** Uses rolling hashes to find a diff */
-function blockDiff<T>(
-    config: DiffOptions<T> & {
-        baseRange: { start: number, end: number };
-        nextRange: { start: number, end: number };
-        previous?: {
-            matchFraction: number;
-            maxLines: number;
-        }
-    }
-): DiffChange<T>[] {
-    const { baseRange, nextRange, compare, getHash, base, next, previous } = config;
-    let maxLines = previous?.maxLines ?? Math.max(config.base.length, config.next.length);
-
-    // NOTE: next values won't all have the same block size, as they must be able to be offset,
-    //  which can result in gaps
-    //  - We also allow gaps in base values, as it is better for blocks to match more consistently,
-    //      rather than match all values
-    function getRollingHashAliased(blockSize: number): {
-        base: Block[];
-        next: Block[];
-    } | undefined {
-        if (blockSize < 0) return undefined;
-        let baseHashes = getRollingHashes({ data: base, range: baseRange, getHash, blockSize });
-        if (baseHashes.length === 0) return undefined;
-
-        let nextHashes = getRollingHashes({ data: next, range: nextRange, getHash, blockSize });
-        if (nextHashes.length === 0) return undefined;
-
-        let hashesInNext = new Set(nextHashes.map(x => x.hash));
-
-        // Take non-overlapping blocks, that also exist in next
-        let baseBlocks: Block[] = [];
-        {
-            let baseIndex = 0;
-            while (baseIndex < baseHashes.length) {
-                let baseBlock = baseHashes[baseIndex];
-                if (!hashesInNext.has(baseBlock.hash)) {
-                    baseIndex++;
-                } else {
-                    baseBlocks.push(baseHashes[baseIndex]);
-                    baseIndex += blockSize;
-                }
-            }
-        }
-
-        let hashesInBase = new Set(baseBlocks.map(x => x.hash));
-
-        let matchedNextBlocks = nextHashes.filter(x => hashesInBase.has(x.hash));
-
-        let nextBlocks: Block[] = [];
-
-        // TODO: Replace matchedNextIndexes with a range tree, so we can efficiently filter out overlapping ranges.
-        //  - Although... worse case this is O(N), which this code already is.
-        let matchedNextIndexes = new Set<number>();
-        for (let nextBlock of matchedNextBlocks) {
-            // NOTE: As we match blocks of size blockSize, it means it is impossible for anything
-            //  in the middle of a block to be matched without touching the start/end!
-            if (matchedNextIndexes.has(nextBlock.start)) continue;
-            if (matchedNextIndexes.has(nextBlock.end)) continue;
-
-            nextBlocks.push(nextBlock);
-            // NOTE: This isn't TOO inefficient, as we will only end up matching O(N) indexes anyway, and
-            //  our block overlapping checking will ignore most blocks, and only have to check O(N) blocks.
-            for (let i = nextBlock.start; i < nextBlock.end; i++) {
-                matchedNextIndexes.add(i);
-            }
-        }
-
-        let lastIndex = 0;
-        let lastNextBlock = UnionUndefined(nextBlocks[nextBlocks.length - 1]);
-        if (lastNextBlock) {
-            lastIndex = lastNextBlock.end;
-        }
-
-        return {
-            base: baseBlocks,
-            next: nextBlocks
-        };
-    }
-
-    let maxBlockSize = 2 ** Math.round(Math.log2(Math.min(base.length, next.length)));
-    let perfectBlockSize = maxBlockSize / 4;
-    perfectBlockSize = Math.max(perfectBlockSize, 2);
-
-    // If we match 50% at blockSize=128, but 75% at blockSize=64, we would prefer blockSize=64
-    //  However, if we also match 90% at blockSize=2, blockSize=64 is better, as the values at blockSize=2
-    //  might just be coincidental
-    function estimateBlockSizeQuality(blockSize: number): number {
-        let sizeFactor = (Math.log2(blockSize) / Math.log2(perfectBlockSize)) ** 0.5;
-        sizeFactor = Math.min(sizeFactor, 1);
-        sizeFactor = Math.max(sizeFactor, 0.2);
-
-        let matchFraction = getMatchFraction(blockSize);
-        return matchFraction * sizeFactor;
-    }
-    function getMatchFraction(blockSize: number): number {
-        if (blockSize < 0) return -1;
-        let blockObj = getRollingHashAliased(blockSize);
-        if (!blockObj) return -1;
-
-        let baseMatched = sum(blockObj.base.map(x => x.end - x.start));
-        let nextMatched = sum(blockObj.next.map(x => x.end - x.start));
-
-        return Math.min(baseMatched, nextMatched) / maxLines;
-    }
-
-    // Test all block sizes
-    let pickedBlockSize = maxBlockSize;
-    {
-        let testBlockSize = maxBlockSize;
-        let curQuality = estimateBlockSizeQuality(testBlockSize);
-        testBlockSize = testBlockSize / 2;
-
-        while (testBlockSize > 1) {
-            let nextQuality = estimateBlockSizeQuality(testBlockSize);
-            if (nextQuality > curQuality) {
-                curQuality = nextQuality;
-                pickedBlockSize = testBlockSize;
-            }
-            testBlockSize = testBlockSize / 2;
-        }
-        if (curQuality < 0) {
-            pickedBlockSize = -1;
-        }
-    }
-
-    if (previous) {
-        let matchFraction = getMatchFraction(pickedBlockSize);
-        // If we aren't matching enough, don't pick this. Otherwise we end up basically
-        //  just matching the longest common subsequence.
-        if (matchFraction < previous.matchFraction * 0.7) {
-            return randomDiff(config);
-        }
-    }
-
-    {
-        // Only consider recursive if block size > 1, OR, we are the initial search, which may
-        //  have an ideal block size of 1.
-        let matched = (!previous || pickedBlockSize > 1) && getRollingHashAliased(pickedBlockSize);
-        if (!matched) {
-            return randomDiff(config);
-        }
-        type FullBlock = (Block & { isBase?: boolean });
-        let baseBlocks = matched.base.map(x => ({ ...x, isBase: true })) as FullBlock[];
-        let nextBlocks = matched.next as FullBlock[];
-
-        let blockDiffObj = blockDiff({
-            previous: {
-                matchFraction: getMatchFraction(pickedBlockSize),
-                maxLines,
-            },
-            base: baseBlocks,
-            next: nextBlocks,
-            baseRange: { start: 0, end: baseBlocks.length },
-            nextRange: { start: 0, end: nextBlocks.length },
-            compare: (lhs, rhs) => {
-                let diff = lhs.hash - rhs.hash;
-                if (diff !== 0) return diff;
-                let lhsLength = lhs.end - lhs.start;
-                let rhsLength = rhs.end - rhs.start;
-                diff = lhsLength - rhsLength;
-                if (diff !== 0) return diff;
-                let lhsSource = lhs.isBase ? base : next;
-                let rhsSource = rhs.isBase ? base : next;
-                for (let i = 0; i < lhsLength; i++) {
-                    diff = compare(lhsSource[lhs.start + i], rhsSource[rhs.start + i]);
-                    if (diff !== 0) return diff;
-                }
-                return 0;
-            },
-            getHash: value => value.hash,
-        });
-
-        // Every unchanged block is our base, and then we do randomDiff to find changes between them.
-        //  We don't expect to find many matches, so randomDiff is fine.
-        //  - I believe this is basically the histogram diff, except we find our features with rolling diffs,
-        //      instead of just frequency of lines.
-
-        let fullLineChanges: DiffChange<T>[] = [];
-        let baseAddedEnd = 0;
-        let nextAddedEnd = 0;
-        function emitGap(baseEnd: number, nextEnd: number) {
-            let baseCount = baseEnd - baseAddedEnd;
-            let nextCount = nextEnd - nextAddedEnd;
-            if (baseCount === 0) {
-                for (let i = nextAddedEnd; i < nextEnd; i++) {
-                    fullLineChanges.push({
-                        type: "added",
-                        value: next[i]
-                    });
-                }
-            } else if (nextCount === 0) {
-                for (let i = baseAddedEnd; i < baseEnd; i++) {
-                    fullLineChanges.push({
-                        type: "removed",
-                        value: base[i]
-                    });
-                }
-            } else {
-                let gapDiff = randomDiff({
-                    base: base.slice(baseAddedEnd, baseEnd),
-                    next: next.slice(nextAddedEnd, nextEnd),
-                    compare,
-                    getHash,
-                });
-                for (let value of gapDiff) {
-                    fullLineChanges.push(value);
-                }
-            }
-            baseAddedEnd = baseEnd;
-            nextAddedEnd = nextEnd;
-        }
-
-        // TODO: Maybe extend any matched regions to exact matches that are beside them? Or... just leave it up to randomDiff
-        //  to figure it out, as just extending regions may not be always optimal.
-
-        let matches = blockDiffObj.filter(x => x.type === "unchanged");
-
-        for (let blockObj of matches) {
-            let baseBlock = blockObj.value;
-            if (!baseBlock.isBase) {
-                throw new Error(`Internal error, expected base block in unchanged value`);
-            }
-            let nextBlock = assertDefined(blockObj.unchangedNextValue);
-            if (nextBlock.isBase) {
-                throw new Error(`Internal error, expected next block in unchanged unchangedNextValue`);
-            }
-            emitGap(baseBlock.start, nextBlock.start);
-
-            let arr = baseBlock.isBase ? base : next;
-            for (let i = baseBlock.start; i < baseBlock.end; i++) {
-                fullLineChanges.push({
-                    type: blockObj.type,
-                    value: arr[i],
-                });
-            }
-
-            baseAddedEnd = baseBlock.end;
-            nextAddedEnd = nextBlock.end;
-        }
-        emitGap(base.length, next.length);
-
-        {
-            let resolve = fullLineChanges.filter(x => x.type !== "removed").map(x => x.value);
-            if (resolve.length !== next.length) {
-                throw new Error(`Internal diff error, created invalid diff size, was ${resolve.length}, should be ${next.length}`);
-            }
-            for (let i = 0; i < resolve.length; i++) {
-                if (compare(resolve[i], next[i]) !== 0) {
-                    throw new Error(`Internal diff error, incorrect index at ${i}`);
-                }
-            }
-        }
-
-        return fullLineChanges;
-    }
-}
-
-type Block = {
-    start: number;
-    end: number;
-    hash: number;
-};
-// NOTE: Only returns full blocks, so if data.length < blockSize, doesn't return any hashes.
-function getRollingHashes<T>(config: {
-    range: { start: number; end: number };
-    data: T[];
-    getHash: (value: T) => number;
-    blockSize: number;
-}): Block[] {
-    const { range, data, getHash, blockSize } = config;
-
-    // Get a hash so that the sum of all of our hashes won't exceed Number.MAX_SAFE_INTEGER
-    let safeMod = 2 ** (52 - Math.ceil(Math.log2(blockSize + 2)));
-    function safeHash(value: T) {
-        let hash = getHash(value);
-        hash = hash % safeMod;
-        return hash;
-    }
-    let blocks: Block[] = [];
-    let hash = 0;
-    let inBlock = 0;
-    for (let i = range.start; i < range.end; i++) {
-        hash = hash + safeHash(data[i]);
-        inBlock++;
-        if (inBlock > blockSize) {
-            inBlock--;
-            hash = hash - safeHash(data[i - inBlock]);
-        }
-        if (inBlock === blockSize) {
-            blocks.push({
-                start: i - inBlock + 1,
-                end: i + 1,
-                hash,
-            });
-        }
-    }
-    return blocks;
-}
-
-
-// randomDiff example
-//          - We might have 1214  => 49294, in which case, we want to choose
-//              -
-//                      +4
-//                  -1
-//                  -9
-//                  2   2
-//                  -1  -1
-//                      +9
-//                  4   4
-//              - NOT
-//                  -1
-//                  -2
-//                  -1
-//                  4   4
-//                      +9
-//                      +2
-//                      +9
-//                      +4
-
-/** Uses an underlying diffing algorithm, combined with some randomness for the starting points,
- *      to allow creating a reasonable diff on data that has many changes, but which the order is
- *      relatively the same
- */
-function randomDiff<T>(
-    config: DiffOptions<T> & {
-        // Defaults to some low number, probably 3
-        searchCount?: number;
-    }
-): DiffChange<T>[] {
-    let { searchCount, getHash, base, next } = config;
-    searchCount = (searchCount ?? 3);
-
-    function getFeatures(values: T[]) {
-        let features = new Map<number, number[]>();
-        for (let i = 0; i < values.length; i++) {
-            let value = values[i];
-            let hash = getHash(value);
-            let indexes = features.get(hash);
-            if (!indexes) {
-                indexes = [];
-                features.set(hash, indexes);
-            }
-            indexes.push(i);
-        }
-        return features;
-    }
-
-    let nextFeatures = getFeatures(next);
-
-    let featureEntries = Array.from(getFeatures(base));
-    sort(featureEntries, x => x[1].length);
-    featureEntries = featureEntries.filter(x => nextFeatures.has(x[0]));
-
-    if (featureEntries.length === 0) {
-        let changes: DiffChange<T>[] = [];
-        for (let value of base) {
-            changes.push({ type: "removed", value });
-        }
-        for (let value of next) {
-            changes.push({ type: "added", value });
-        }
-        return changes;
-    }
-
-    searchCount = Math.min(searchCount, featureEntries.length);
-
-    let bestDiff: { diff: DiffChange<T>[]; value: number; } | undefined;
-    function getDiffValue(diff: DiffChange<T>[]): number {
-        return -diff.length;
-    }
-
-    for (let i = 0; i <= searchCount; i++) {
-        let index = Math.round(i / searchCount * featureEntries.length);
-        if (index === featureEntries.length) {
-            index--;
-        }
-        let feature = featureEntries[index];
-        let baseIndex = feature[1][0];
-
-        let nextIndexes = assertDefined(nextFeatures.get(feature[0]));
-        for (let i = 0; i <= searchCount; i++) {
-
-            let index = Math.round(i / searchCount * nextIndexes.length);
-            if (index === nextIndexes.length) {
-                index--;
-            }
-            let nextIndex = nextIndexes[index];
-
-            let diff = getDiffStart({ ...config, baseIndex, nextIndex });
-            let diffValue = getDiffValue(diff);
-            if (!bestDiff || diffValue > bestDiff.value) {
-                bestDiff = { diff, value: diffValue };
-            }
-        }
-    }
-
-    assertDefined2(bestDiff);
-    return bestDiff.diff;
-}
-
-
-function getDiffStart<T>(
-    config: DiffOptions<T> & {
-        baseIndex: number;
-        nextIndex: number;
-    }
-): DiffChange<T>[] {
-    let { base, next, baseIndex, nextIndex } = config;
-
-    let beforeDiff = getDiff({
-        ...config,
-        base: base.slice(0, baseIndex).reverse(),
-        next: next.slice(0, nextIndex).reverse(),
-    }).reverse();
-
-    let afterDiff = getDiff({
-        ...config,
-        base: base.slice(baseIndex),
-        next: next.slice(nextIndex),
-    });
-
-    return [...beforeDiff, ...afterDiff];
-}
-
-function createFindIndex<T>(
-    values: T[],
-    getHash: (value: T) => number,
-    compare: (lhs: T, rhs: T) => number
-): (value: T, searchFirstIndex: number) => number {
-    let nextId = 1;
-    let ids: Map<number, { value: T; id: number }[]> = new Map();
-    let getId = (value: T) => {
-        let hash = getHash(value);
-        let idList = ids.get(hash);
-        if (!idList) {
-            idList = [];
-            ids.set(hash, idList);
-        }
-        // TODO: We could binary sort within this list, but... there shouldn't be that many hash collisions!
-        let idObj = idList.find(x => compare(x.value, value) === 0);
-        if (!idObj) {
-            idObj = { value, id: nextId++ };
-            idList.push(idObj);
-        }
-        return idObj.id;
-    };
-
-    let valuesById = keyByArray(values.map((x, index) => ({ value: x, index })), x => getId(x.value));
-
-    return function (value: T, searchFirstIndex: number): number {
-        const matches = valuesById.get(getId(value));
-        if (!matches) return -1;
-
-        let nextIndex = binarySearchIndex(matches.length, i => matches[i].index - searchFirstIndex);
-        if (nextIndex < 0) nextIndex = ~nextIndex;
-
-        let match = UnionUndefined(matches[nextIndex]);
-        if (!match) return -1;
-        return match.index;
-    };
-}
-
-/** Gets the diff between two arrays, starting at the start of each range and diffing forwards. Just uses
- *      a brute force search to a certain depth, and is really only designed to take an existing match
- *      and extend it, where it is assumed outside the match there won't be any matches.
- */
-function getDiff<T>(
-    config: DiffOptions<T> & {
-        // Defaults to some low number, probably 6
-        searchDepth?: number;
-    }
-): DiffChange<T>[] {
-    let { compare, getHash, base, next, searchDepth } = config;
-    searchDepth = searchDepth ?? 6;
-
-    //iterate matching any ===, and for any different, choosing to both remove existing, or add new one
-    //      at a depth of D, picking the series of choices that result in the maximum number of matches.
-    //      (D will probably be 6? We can always parameterize it)
-    //      - We will have a lookup to go from value => index, which will let us know if
-    //          a value has to be an addition / removal (due to not existing at all in the opposite values),
-    //          and if there is no choice, it won't add to the search depth.
-
-    let baseFindIndex = createFindIndex(config.base, getHash, compare);
-    let nextFindIndex = createFindIndex(config.next, getHash, compare);
-
-    let diff: DiffChange<T>[] = [];
-
-    type DiffOutcome = {
-        removals: number;
-        additions: number;
-        unchanged: number;
-    };
-    function rankOutcome(outcome: DiffOutcome): number {
-        // NOTE: As values with no choice (impossible to match, or is a match immediately),
-        //  results in no branches, choices to remove a lot of values will quickly be resolved to also
-        //  match a lot, and therefore we don't have to worry about a lot of removals never being matched.
-        return outcome.unchanged - outcome.additions - outcome.removals;
-    }
-
-    function createEmptyOutcome(): DiffOutcome {
-        return {
-            removals: 0,
-            additions: 0,
-            unchanged: 0,
-        };
-    }
-    type DiffState = {
-        curIndex: number;
-        nextIndex: number;
-    };
-    let diffState: DiffState = {
-        curIndex: 0,
-        nextIndex: 0,
-    };
-
-    /** Simulate to given depth, returning the best possible outcome */
-    function simulateToDepth(diffState: DiffState, outcome: DiffOutcome, depthLeft: number): DiffOutcome {
-        if (depthLeft <= 0 || diffState.curIndex >= base.length || diffState.nextIndex >= next.length) {
-            return outcome;
-        }
-
-        let { curIndex, nextIndex } = diffState;
-        outcome = { ...outcome };
-
-        let nextMatch = nextFindIndex(base[curIndex], nextIndex);
-        if (nextMatch < 0) {
-            // Can't match it
-            curIndex++;
-            outcome.removals++;
-        } else if (nextMatch === nextIndex) {
-            // Is a perfect match
-            curIndex++;
-            nextIndex++;
-            outcome.unchanged++;
-        } else {
-            let baseMatch = baseFindIndex(next[nextIndex], curIndex);
-            if (baseMatch < 0) {
-                // Can never match the otherside, so it is an insertion
-                outcome.additions++;
-                nextIndex++;
-            } else {
-                // Branch
-                let insertOutcome: DiffOutcome = { ...outcome };
-                let deleteOutcome: DiffOutcome = { ...outcome };
-
-                // 1) Insertion, assume base is matched by nextMatch, which means all the next values
-                //      in between are insertions
-                {
-                    insertOutcome.additions += nextMatch - nextIndex;
-                    insertOutcome = simulateToDepth({ curIndex, nextIndex: nextMatch }, insertOutcome, depthLeft - 1);
-                }
-
-                // 2) Delete base, assume next is a match, and so everything up until baseMatch is a deletion
-                {
-                    deleteOutcome.removals += baseMatch - curIndex;
-                    deleteOutcome = simulateToDepth({ curIndex: baseMatch, nextIndex }, deleteOutcome, depthLeft - 1);
-                }
-
-                if (rankOutcome(insertOutcome) > rankOutcome(deleteOutcome)) {
-                    outcome = insertOutcome;
-                } else {
-                    outcome = deleteOutcome;
-                }
-            }
-        }
-
-        return outcome;
-    }
-
-
-
-    while (diffState.curIndex < base.length && diffState.nextIndex < next.length) {
-        diffArraysComparisonCount++;
-        let nextMatch = nextFindIndex(base[diffState.curIndex], diffState.nextIndex);
-        if (nextMatch < 0) {
-            // We won't ever match it if it doesn't exist in next, so it must be a remove
-            diff.push({ type: "removed", value: base[diffState.curIndex] });
-            diffState.curIndex++;
-        } else if (nextMatch === diffState.nextIndex) {
-            // Is a perfect match
-            diff.push({ type: "unchanged", value: base[diffState.curIndex], unchangedNextValue: next[diffState.nextIndex] });
-            diffState.curIndex++;
-            diffState.nextIndex++;
-        } else {
-            let baseMatch = diffState.nextIndex < next.length ? baseFindIndex(next[diffState.nextIndex], diffState.curIndex) : -1;
-            if (baseMatch < 0) {
-                // Can never match the otherside, so next is an insertion
-                diff.push({ type: "added", value: next[diffState.nextIndex] });
-                diffState.nextIndex++;
-            } else {
-                let insertOutcome = simulateToDepth({ curIndex: diffState.curIndex, nextIndex: nextMatch }, createEmptyOutcome(), searchDepth);
-
-                let deleteOutcome = simulateToDepth({ curIndex: baseMatch, nextIndex: diffState.nextIndex }, createEmptyOutcome(), searchDepth);
-
-                if (rankOutcome(insertOutcome) > rankOutcome(deleteOutcome)) {
-                    // 1) Insertion, assume base is matched by nextMatch, which means all the next values
-                    //      in between are insertions
-                    for (let i = diffState.nextIndex; i < nextMatch; i++) {
-                        diff.push({ type: "added", value: next[i] });
-                    }
-                    diffState.nextIndex = nextMatch;
-                } else {
-                    // 2) Delete base, assume next is a match, and so everything up until baseMatch is a deletion
-                    for (let i = diffState.curIndex; i < baseMatch; i++) {
-                        diff.push({ type: "removed", value: base[i] });
-                    }
-                    diffState.curIndex = baseMatch;
-                }
-            }
-        }
-    }
-
-    // Handle any trailing values
-    while (diffState.curIndex < base.length) {
-        diff.push({ type: "removed", value: base[diffState.curIndex] });
-        diffState.curIndex++;
-    }
-    while (diffState.nextIndex < next.length) {
-        diff.push({ type: "added", value: next[diffState.nextIndex] });
-        diffState.nextIndex++;
-    }
-
-    return diff;
-}
-
-
-
-
-export let diffArraysComparisonCount = 0;
-export function diffArraysOld(
-    config: {
-        baseValues: string[];
-        newValues: string[];
-    }
-): LineChange[] {
-    let { baseValues, newValues } = config;
-
-    let stringLengthCount = new Map<string, number>();
-    for (let value of baseValues) {
-        stringLengthCount.set(value, (stringLengthCount.get(value) || 0) + 1);
-    }
-    for (let value of newValues) {
-        stringLengthCount.set(value, (stringLengthCount.get(value) || 0) + 1);
-    }
-    function getLengthFactor(count: number) {
-        if (count === 1) return 10;
-        if (count === 2) return 5;
-        if (count === 3) return 2;
-        if (count < 10) return 1;
-        return 0.5;
-    }
-    // TODO: Also prefer values that are more in the middle, as this helps our algorithm run more efficiently
-    function getStrLengthFactor(value: string) {
-        return getLengthFactor(stringLengthCount.get(value) || 0);
-    }
-
-    type MatchValue = { baseStart: number; newStart: number; baseEnd: number; newEnd: number; value: number; };
-    function getPairMatchValue(baseIndex: number, newIndex: number): MatchValue | undefined {
-        diffArraysComparisonCount++;
-        if (baseValues[baseIndex] !== newValues[newIndex]) return undefined;
-        while (baseIndex > 1 && newIndex > 1 && baseValues[baseIndex] === newValues[newIndex]) {
-            diffArraysComparisonCount++;
-            baseIndex--;
-            newIndex--;
-        }
-        let baseStart = baseIndex;
-        let newStart = newIndex;
-        let value = 0;
-        while (baseIndex < baseValues.length && newIndex < newValues.length && baseValues[baseIndex] === newValues[newIndex]) {
-            diffArraysComparisonCount++;
-            value += getStrLengthFactor(baseValues[baseIndex]);
-            baseIndex++;
-            newIndex++;
-        }
-        let baseEnd = baseIndex;
-        let newEnd = newIndex;
-        return {
-            baseStart,
-            newStart,
-            baseEnd,
-            newEnd,
-            value
-        };
-    }
-
-    let bestMatch = maxValue(
-        list(baseValues.length).map(baseIndex =>
-            maxValue(
-                list(newValues.length).map(newIndex => getPairMatchValue(baseIndex, newIndex)),
-                x => x?.value ?? 0
-            )
-        ),
-        x => x?.value ?? 0
-    );
-    if (!bestMatch) {
-        // If there are no matches, everything is a change
-        return (
-            ([] as LineChange[])
-                .concat(baseValues.map(value => ({ type: "removed" as const, value })))
-                .concat(newValues.map(value => ({ type: "added" as const, value })))
-        );
-    }
-
-    let changes: LineChange[] = [];
-    // Deal with changes before the match
-    if (bestMatch.baseStart === 0) {
-        if (bestMatch.newStart > 0) {
-            changes.push(...newValues.slice(0, bestMatch.newStart).map(value => ({ type: "added" as const, value })));
-        }
-    } else if (bestMatch.newStart === 0) {
-        if (bestMatch.baseStart > 0) {
-            changes.push(...baseValues.slice(0, bestMatch.baseStart).map(value => ({ type: "removed" as const, value })));
-        }
-    } else {
-        // Values before must be diffed
-        changes.push(...diffArrays({ base: baseValues.slice(0, bestMatch.baseStart), next: newValues.slice(0, bestMatch.newStart) }));
-    }
-
-    // Add the match
-    changes.push(...baseValues.slice(bestMatch.baseStart, bestMatch.baseEnd).map(value => ({ type: "unchanged" as const, value })));
-
-    // Deal with changes after the match
-    if (bestMatch.baseEnd === baseValues.length) {
-        if (bestMatch.newEnd !== newValues.length) {
-            changes.push(...newValues.slice(bestMatch.newEnd).map(value => ({ type: "added" as const, value })));
-        }
-    } else if (bestMatch.newEnd === newValues.length) {
-        if (bestMatch.baseEnd !== baseValues.length) {
-            changes.push(...baseValues.slice(bestMatch.baseEnd).map(value => ({ type: "removed" as const, value })));
-        }
-    } else {
-        changes.push(...diffArrays({ base: baseValues.slice(bestMatch.baseEnd), next: newValues.slice(bestMatch.newEnd) }));
-    }
-
-    return changes;
-}
-
-function sum(arg0: number[]) {
-    return arg0.reduce((a, b) => a + b, 0);
-}
-
-function UnionUndefined<T>(x: T): T | undefined {
-    return x;
-}
-function assertDefined<T>(x: T | undefined): T {
-    if (x === undefined) throw new Error(`Expected value to be defined`);
-    return x;
-}
-
-function assertDefined2<T>(x: T | undefined): asserts x is T {
-    if (x === undefined) throw new Error(`Expected value to be defined`);
-}
-function maxValue<T>(values: T[], getValue: (value: T) => number): T | undefined {
-    let bestValue = -Infinity;
-    let best: T | undefined = undefined;
-    for (let value of values) {
-        let curValue = getValue(value);
-        if (curValue > bestValue) {
-            bestValue = curValue;
-            best = value;
-        }
-    }
-    return best;
+import { binarySearchIndex, compare, keyByArray, list, sort } from "socket-function/src/misc";
+import { fastHash } from "../misc/hash";
+
+export interface LineChange {
+    type: "unchanged" | "added" | "removed";
+    value: string;
+}
+
+// TODO: Replace our base algorithm with something that gives better diffs, at least for small sets of values?
+//  - Ex, https://luppeng.wordpress.com/2020/10/10/when-to-use-each-of-the-git-diff-algorithms/, 
+//      myers, patience, or histogram
+
+//  Algorithm
+//  - Hash values into blocks to make them smaller to deal with
+//      - We want an optimal block size, where we are not too far away from having the cardinality of the
+//          intersection * block size === cardinality of per line intersection (so, most is matched). Of course,
+//          the larger the block size the less matches we will accept.
+//          - And then once we find the optimal block size, divide it by 2, to increase granularity
+//  - The hashes will just be used for comparison
+//  - Remove any blocks with hashes not found in both
+//  - Repeat the algorithm, until the optimal block size is 1
+//  - For N evenly choosen options, match Y random matches in the new array, then iterate (first forward,
+//      then go backwards) matching any ===, and for any different, choosing to both remove existing, or add new one
+//      at a depth of D, picking the series of choices that result in the maximum number of matches.
+//      (D will probably be 6? We can always parameterize it)
+//      - To iterate backwards, just slice, reverse, and the iterate function on that.
+
+//  blockMatch
+//  spreadMatch
+//  iterateMatch
+//  - Skew towards either removal or addition, so we can resolve large changes without having lots of random interspersed
+//      adds and removals, and instead we can batch the differences
+
+
+
+
+export type DiffChange<T> = {
+    type: "unchanged" | "added" | "removed";
+    value: T;
+    // When type is unchanged, this is the value in the new array (which compares equal to value, but
+    //  there might be additional metadata that is of use, such as line numbers)
+    unchangedNextValue?: T;
+};
+interface DiffOptions<T> {
+    base: T[];
+    next: T[];
+    compare: (lhs: T, rhs: T) => number;
+    getHash: (value: T) => number;
+}
+
+
+export function diffArrays<T>(
+    config: {
+        base: T[];
+        next: T[];
+    }
+): DiffChange<T>[] {
+    let { base: baseValues, next: newValues } = config;
+    return blockDiff({
+        base: baseValues,
+        next: newValues,
+        baseRange: { start: 0, end: baseValues.length },
+        nextRange: { start: 0, end: newValues.length },
+        compare: compare,
+        getHash: (x: unknown) => {
+            if (x === undefined) {
+                return -234235479;
+            }
+            if (x === null) {
+                return -31290214;
+            }
+            if (x === false) {
+                return -843348236;
+            }
+            if (x === true) {
+                return -843348236;
+            }
+            if (typeof x === "string") {
+                return fastHash(x);
+            }
+            if (typeof x === "number") {
+                // Not great, but... this will probably be fine? We just want to spread values out a bit more, so their
+                //      bits don't easily collide
+                return ~~(x * 417691);
+            }
+
+            throw new Error(`Unhandled type in getHash, ${typeof x}`);
+        },
+    });
+}
+
+/** Uses rolling hashes to find a diff */
+function blockDiff<T>(
+    config: DiffOptions<T> & {
+        baseRange: { start: number, end: number };
+        nextRange: { start: number, end: number };
+        previous?: {
+            matchFraction: number;
+            maxLines: number;
+        }
+    }
+): DiffChange<T>[] {
+    const { baseRange, nextRange, compare, getHash, base, next, previous } = config;
+    let maxLines = previous?.maxLines ?? Math.max(config.base.length, config.next.length);
+
+    // NOTE: next values won't all have the same block size, as they must be able to be offset,
+    //  which can result in gaps
+    //  - We also allow gaps in base values, as it is better for blocks to match more consistently,
+    //      rather than match all values
+    function getRollingHashAliased(blockSize: number): {
+        base: Block[];
+        next: Block[];
+    } | undefined {
+        if (blockSize < 0) return undefined;
+        let baseHashes = getRollingHashes({ data: base, range: baseRange, getHash, blockSize });
+        if (baseHashes.length === 0) return undefined;
+
+        let nextHashes = getRollingHashes({ data: next, range: nextRange, getHash, blockSize });
+        if (nextHashes.length === 0) return undefined;
+
+        let hashesInNext = new Set(nextHashes.map(x => x.hash));
+
+        // Take non-overlapping blocks, that also exist in next
+        let baseBlocks: Block[] = [];
+        {
+            let baseIndex = 0;
+            while (baseIndex < baseHashes.length) {
+                let baseBlock = baseHashes[baseIndex];
+                if (!hashesInNext.has(baseBlock.hash)) {
+                    baseIndex++;
+                } else {
+                    baseBlocks.push(baseHashes[baseIndex]);
+                    baseIndex += blockSize;
+                }
+            }
+        }
+
+        let hashesInBase = new Set(baseBlocks.map(x => x.hash));
+
+        let matchedNextBlocks = nextHashes.filter(x => hashesInBase.has(x.hash));
+
+        let nextBlocks: Block[] = [];
+
+        // TODO: Replace matchedNextIndexes with a range tree, so we can efficiently filter out overlapping ranges.
+        //  - Although... worse case this is O(N), which this code already is.
+        let matchedNextIndexes = new Set<number>();
+        for (let nextBlock of matchedNextBlocks) {
+            // NOTE: As we match blocks of size blockSize, it means it is impossible for anything
+            //  in the middle of a block to be matched without touching the start/end!
+            if (matchedNextIndexes.has(nextBlock.start)) continue;
+            if (matchedNextIndexes.has(nextBlock.end)) continue;
+
+            nextBlocks.push(nextBlock);
+            // NOTE: This isn't TOO inefficient, as we will only end up matching O(N) indexes anyway, and
+            //  our block overlapping checking will ignore most blocks, and only have to check O(N) blocks.
+            for (let i = nextBlock.start; i < nextBlock.end; i++) {
+                matchedNextIndexes.add(i);
+            }
+        }
+
+        let lastIndex = 0;
+        let lastNextBlock = UnionUndefined(nextBlocks[nextBlocks.length - 1]);
+        if (lastNextBlock) {
+            lastIndex = lastNextBlock.end;
+        }
+
+        return {
+            base: baseBlocks,
+            next: nextBlocks
+        };
+    }
+
+    let maxBlockSize = 2 ** Math.round(Math.log2(Math.min(base.length, next.length)));
+    let perfectBlockSize = maxBlockSize / 4;
+    perfectBlockSize = Math.max(perfectBlockSize, 2);
+
+    // If we match 50% at blockSize=128, but 75% at blockSize=64, we would prefer blockSize=64
+    //  However, if we also match 90% at blockSize=2, blockSize=64 is better, as the values at blockSize=2
+    //  might just be coincidental
+    function estimateBlockSizeQuality(blockSize: number): number {
+        let sizeFactor = (Math.log2(blockSize) / Math.log2(perfectBlockSize)) ** 0.5;
+        sizeFactor = Math.min(sizeFactor, 1);
+        sizeFactor = Math.max(sizeFactor, 0.2);
+
+        let matchFraction = getMatchFraction(blockSize);
+        return matchFraction * sizeFactor;
+    }
+    function getMatchFraction(blockSize: number): number {
+        if (blockSize < 0) return -1;
+        let blockObj = getRollingHashAliased(blockSize);
+        if (!blockObj) return -1;
+
+        let baseMatched = sum(blockObj.base.map(x => x.end - x.start));
+        let nextMatched = sum(blockObj.next.map(x => x.end - x.start));
+
+        return Math.min(baseMatched, nextMatched) / maxLines;
+    }
+
+    // Test all block sizes
+    let pickedBlockSize = maxBlockSize;
+    {
+        let testBlockSize = maxBlockSize;
+        let curQuality = estimateBlockSizeQuality(testBlockSize);
+        testBlockSize = testBlockSize / 2;
+
+        while (testBlockSize > 1) {
+            let nextQuality = estimateBlockSizeQuality(testBlockSize);
+            if (nextQuality > curQuality) {
+                curQuality = nextQuality;
+                pickedBlockSize = testBlockSize;
+            }
+            testBlockSize = testBlockSize / 2;
+        }
+        if (curQuality < 0) {
+            pickedBlockSize = -1;
+        }
+    }
+
+    if (previous) {
+        let matchFraction = getMatchFraction(pickedBlockSize);
+        // If we aren't matching enough, don't pick this. Otherwise we end up basically
+        //  just matching the longest common subsequence.
+        if (matchFraction < previous.matchFraction * 0.7) {
+            return randomDiff(config);
+        }
+    }
+
+    {
+        // Only consider recursive if block size > 1, OR, we are the initial search, which may
+        //  have an ideal block size of 1.
+        let matched = (!previous || pickedBlockSize > 1) && getRollingHashAliased(pickedBlockSize);
+        if (!matched) {
+            return randomDiff(config);
+        }
+        type FullBlock = (Block & { isBase?: boolean });
+        let baseBlocks = matched.base.map(x => ({ ...x, isBase: true })) as FullBlock[];
+        let nextBlocks = matched.next as FullBlock[];
+
+        let blockDiffObj = blockDiff({
+            previous: {
+                matchFraction: getMatchFraction(pickedBlockSize),
+                maxLines,
+            },
+            base: baseBlocks,
+            next: nextBlocks,
+            baseRange: { start: 0, end: baseBlocks.length },
+            nextRange: { start: 0, end: nextBlocks.length },
+            compare: (lhs, rhs) => {
+                let diff = lhs.hash - rhs.hash;
+                if (diff !== 0) return diff;
+                let lhsLength = lhs.end - lhs.start;
+                let rhsLength = rhs.end - rhs.start;
+                diff = lhsLength - rhsLength;
+                if (diff !== 0) return diff;
+                let lhsSource = lhs.isBase ? base : next;
+                let rhsSource = rhs.isBase ? base : next;
+                for (let i = 0; i < lhsLength; i++) {
+                    diff = compare(lhsSource[lhs.start + i], rhsSource[rhs.start + i]);
+                    if (diff !== 0) return diff;
+                }
+                return 0;
+            },
+            getHash: value => value.hash,
+        });
+
+        // Every unchanged block is our base, and then we do randomDiff to find changes between them.
+        //  We don't expect to find many matches, so randomDiff is fine.
+        //  - I believe this is basically the histogram diff, except we find our features with rolling diffs,
+        //      instead of just frequency of lines.
+
+        let fullLineChanges: DiffChange<T>[] = [];
+        let baseAddedEnd = 0;
+        let nextAddedEnd = 0;
+        function emitGap(baseEnd: number, nextEnd: number) {
+            let baseCount = baseEnd - baseAddedEnd;
+            let nextCount = nextEnd - nextAddedEnd;
+            if (baseCount === 0) {
+                for (let i = nextAddedEnd; i < nextEnd; i++) {
+                    fullLineChanges.push({
+                        type: "added",
+                        value: next[i]
+                    });
+                }
+            } else if (nextCount === 0) {
+                for (let i = baseAddedEnd; i < baseEnd; i++) {
+                    fullLineChanges.push({
+                        type: "removed",
+                        value: base[i]
+                    });
+                }
+            } else {
+                let gapDiff = randomDiff({
+                    base: base.slice(baseAddedEnd, baseEnd),
+                    next: next.slice(nextAddedEnd, nextEnd),
+                    compare,
+                    getHash,
+                });
+                for (let value of gapDiff) {
+                    fullLineChanges.push(value);
+                }
+            }
+            baseAddedEnd = baseEnd;
+            nextAddedEnd = nextEnd;
+        }
+
+        // TODO: Maybe extend any matched regions to exact matches that are beside them? Or... just leave it up to randomDiff
+        //  to figure it out, as just extending regions may not be always optimal.
+
+        let matches = blockDiffObj.filter(x => x.type === "unchanged");
+
+        for (let blockObj of matches) {
+            let baseBlock = blockObj.value;
+            if (!baseBlock.isBase) {
+                throw new Error(`Internal error, expected base block in unchanged value`);
+            }
+            let nextBlock = assertDefined(blockObj.unchangedNextValue);
+            if (nextBlock.isBase) {
+                throw new Error(`Internal error, expected next block in unchanged unchangedNextValue`);
+            }
+            emitGap(baseBlock.start, nextBlock.start);
+
+            let arr = baseBlock.isBase ? base : next;
+            for (let i = baseBlock.start; i < baseBlock.end; i++) {
+                fullLineChanges.push({
+                    type: blockObj.type,
+                    value: arr[i],
+                });
+            }
+
+            baseAddedEnd = baseBlock.end;
+            nextAddedEnd = nextBlock.end;
+        }
+        emitGap(base.length, next.length);
+
+        {
+            let resolve = fullLineChanges.filter(x => x.type !== "removed").map(x => x.value);
+            if (resolve.length !== next.length) {
+                throw new Error(`Internal diff error, created invalid diff size, was ${resolve.length}, should be ${next.length}`);
+            }
+            for (let i = 0; i < resolve.length; i++) {
+                if (compare(resolve[i], next[i]) !== 0) {
+                    throw new Error(`Internal diff error, incorrect index at ${i}`);
+                }
+            }
+        }
+
+        return fullLineChanges;
+    }
+}
+
+type Block = {
+    start: number;
+    end: number;
+    hash: number;
+};
+// NOTE: Only returns full blocks, so if data.length < blockSize, doesn't return any hashes.
+function getRollingHashes<T>(config: {
+    range: { start: number; end: number };
+    data: T[];
+    getHash: (value: T) => number;
+    blockSize: number;
+}): Block[] {
+    const { range, data, getHash, blockSize } = config;
+
+    // Get a hash so that the sum of all of our hashes won't exceed Number.MAX_SAFE_INTEGER
+    let safeMod = 2 ** (52 - Math.ceil(Math.log2(blockSize + 2)));
+    function safeHash(value: T) {
+        let hash = getHash(value);
+        hash = hash % safeMod;
+        return hash;
+    }
+    let blocks: Block[] = [];
+    let hash = 0;
+    let inBlock = 0;
+    for (let i = range.start; i < range.end; i++) {
+        hash = hash + safeHash(data[i]);
+        inBlock++;
+        if (inBlock > blockSize) {
+            inBlock--;
+            hash = hash - safeHash(data[i - inBlock]);
+        }
+        if (inBlock === blockSize) {
+            blocks.push({
+                start: i - inBlock + 1,
+                end: i + 1,
+                hash,
+            });
+        }
+    }
+    return blocks;
+}
+
+
+// randomDiff example
+//          - We might have 1214  => 49294, in which case, we want to choose
+//              -
+//                      +4
+//                  -1
+//                  -9
+//                  2   2
+//                  -1  -1
+//                      +9
+//                  4   4
+//              - NOT
+//                  -1
+//                  -2
+//                  -1
+//                  4   4
+//                      +9
+//                      +2
+//                      +9
+//                      +4
+
+/** Uses an underlying diffing algorithm, combined with some randomness for the starting points,
+ *      to allow creating a reasonable diff on data that has many changes, but which the order is
+ *      relatively the same
+ */
+function randomDiff<T>(
+    config: DiffOptions<T> & {
+        // Defaults to some low number, probably 3
+        searchCount?: number;
+    }
+): DiffChange<T>[] {
+    let { searchCount, getHash, base, next } = config;
+    searchCount = (searchCount ?? 3);
+
+    function getFeatures(values: T[]) {
+        let features = new Map<number, number[]>();
+        for (let i = 0; i < values.length; i++) {
+            let value = values[i];
+            let hash = getHash(value);
+            let indexes = features.get(hash);
+            if (!indexes) {
+                indexes = [];
+                features.set(hash, indexes);
+            }
+            indexes.push(i);
+        }
+        return features;
+    }
+
+    let nextFeatures = getFeatures(next);
+
+    let featureEntries = Array.from(getFeatures(base));
+    sort(featureEntries, x => x[1].length);
+    featureEntries = featureEntries.filter(x => nextFeatures.has(x[0]));
+
+    if (featureEntries.length === 0) {
+        let changes: DiffChange<T>[] = [];
+        for (let value of base) {
+            changes.push({ type: "removed", value });
+        }
+        for (let value of next) {
+            changes.push({ type: "added", value });
+        }
+        return changes;
+    }
+
+    searchCount = Math.min(searchCount, featureEntries.length);
+
+    let bestDiff: { diff: DiffChange<T>[]; value: number; } | undefined;
+    function getDiffValue(diff: DiffChange<T>[]): number {
+        return -diff.length;
+    }
+
+    for (let i = 0; i <= searchCount; i++) {
+        let index = Math.round(i / searchCount * featureEntries.length);
+        if (index === featureEntries.length) {
+            index--;
+        }
+        let feature = featureEntries[index];
+        let baseIndex = feature[1][0];
+
+        let nextIndexes = assertDefined(nextFeatures.get(feature[0]));
+        for (let i = 0; i <= searchCount; i++) {
+
+            let index = Math.round(i / searchCount * nextIndexes.length);
+            if (index === nextIndexes.length) {
+                index--;
+            }
+            let nextIndex = nextIndexes[index];
+
+            let diff = getDiffStart({ ...config, baseIndex, nextIndex });
+            let diffValue = getDiffValue(diff);
+            if (!bestDiff || diffValue > bestDiff.value) {
+                bestDiff = { diff, value: diffValue };
+            }
+        }
+    }
+
+    assertDefined2(bestDiff);
+    return bestDiff.diff;
+}
+
+
+function getDiffStart<T>(
+    config: DiffOptions<T> & {
+        baseIndex: number;
+        nextIndex: number;
+    }
+): DiffChange<T>[] {
+    let { base, next, baseIndex, nextIndex } = config;
+
+    let beforeDiff = getDiff({
+        ...config,
+        base: base.slice(0, baseIndex).reverse(),
+        next: next.slice(0, nextIndex).reverse(),
+    }).reverse();
+
+    let afterDiff = getDiff({
+        ...config,
+        base: base.slice(baseIndex),
+        next: next.slice(nextIndex),
+    });
+
+    return [...beforeDiff, ...afterDiff];
+}
+
+function createFindIndex<T>(
+    values: T[],
+    getHash: (value: T) => number,
+    compare: (lhs: T, rhs: T) => number
+): (value: T, searchFirstIndex: number) => number {
+    let nextId = 1;
+    let ids: Map<number, { value: T; id: number }[]> = new Map();
+    let getId = (value: T) => {
+        let hash = getHash(value);
+        let idList = ids.get(hash);
+        if (!idList) {
+            idList = [];
+            ids.set(hash, idList);
+        }
+        // TODO: We could binary sort within this list, but... there shouldn't be that many hash collisions!
+        let idObj = idList.find(x => compare(x.value, value) === 0);
+        if (!idObj) {
+            idObj = { value, id: nextId++ };
+            idList.push(idObj);
+        }
+        return idObj.id;
+    };
+
+    let valuesById = keyByArray(values.map((x, index) => ({ value: x, index })), x => getId(x.value));
+
+    return function (value: T, searchFirstIndex: number): number {
+        const matches = valuesById.get(getId(value));
+        if (!matches) return -1;
+
+        let nextIndex = binarySearchIndex(matches.length, i => matches[i].index - searchFirstIndex);
+        if (nextIndex < 0) nextIndex = ~nextIndex;
+
+        let match = UnionUndefined(matches[nextIndex]);
+        if (!match) return -1;
+        return match.index;
+    };
+}
+
+/** Gets the diff between two arrays, starting at the start of each range and diffing forwards. Just uses
+ *      a brute force search to a certain depth, and is really only designed to take an existing match
+ *      and extend it, where it is assumed outside the match there won't be any matches.
+ */
+function getDiff<T>(
+    config: DiffOptions<T> & {
+        // Defaults to some low number, probably 6
+        searchDepth?: number;
+    }
+): DiffChange<T>[] {
+    let { compare, getHash, base, next, searchDepth } = config;
+    searchDepth = searchDepth ?? 6;
+
+    //iterate matching any ===, and for any different, choosing to both remove existing, or add new one
+    //      at a depth of D, picking the series of choices that result in the maximum number of matches.
+    //      (D will probably be 6? We can always parameterize it)
+    //      - We will have a lookup to go from value => index, which will let us know if
+    //          a value has to be an addition / removal (due to not existing at all in the opposite values),
+    //          and if there is no choice, it won't add to the search depth.
+
+    let baseFindIndex = createFindIndex(config.base, getHash, compare);
+    let nextFindIndex = createFindIndex(config.next, getHash, compare);
+
+    let diff: DiffChange<T>[] = [];
+
+    type DiffOutcome = {
+        removals: number;
+        additions: number;
+        unchanged: number;
+    };
+    function rankOutcome(outcome: DiffOutcome): number {
+        // NOTE: As values with no choice (impossible to match, or is a match immediately),
+        //  results in no branches, choices to remove a lot of values will quickly be resolved to also
+        //  match a lot, and therefore we don't have to worry about a lot of removals never being matched.
+        return outcome.unchanged - outcome.additions - outcome.removals;
+    }
+
+    function createEmptyOutcome(): DiffOutcome {
+        return {
+            removals: 0,
+            additions: 0,
+            unchanged: 0,
+        };
+    }
+    type DiffState = {
+        curIndex: number;
+        nextIndex: number;
+    };
+    let diffState: DiffState = {
+        curIndex: 0,
+        nextIndex: 0,
+    };
+
+    /** Simulate to given depth, returning the best possible outcome */
+    function simulateToDepth(diffState: DiffState, outcome: DiffOutcome, depthLeft: number): DiffOutcome {
+        if (depthLeft <= 0 || diffState.curIndex >= base.length || diffState.nextIndex >= next.length) {
+            return outcome;
+        }
+
+        let { curIndex, nextIndex } = diffState;
+        outcome = { ...outcome };
+
+        let nextMatch = nextFindIndex(base[curIndex], nextIndex);
+        if (nextMatch < 0) {
+            // Can't match it
+            curIndex++;
+            outcome.removals++;
+        } else if (nextMatch === nextIndex) {
+            // Is a perfect match
+            curIndex++;
+            nextIndex++;
+            outcome.unchanged++;
+        } else {
+            let baseMatch = baseFindIndex(next[nextIndex], curIndex);
+            if (baseMatch < 0) {
+                // Can never match the otherside, so it is an insertion
+                outcome.additions++;
+                nextIndex++;
+            } else {
+                // Branch
+                let insertOutcome: DiffOutcome = { ...outcome };
+                let deleteOutcome: DiffOutcome = { ...outcome };
+
+                // 1) Insertion, assume base is matched by nextMatch, which means all the next values
+                //      in between are insertions
+                {
+                    insertOutcome.additions += nextMatch - nextIndex;
+                    insertOutcome = simulateToDepth({ curIndex, nextIndex: nextMatch }, insertOutcome, depthLeft - 1);
+                }
+
+                // 2) Delete base, assume next is a match, and so everything up until baseMatch is a deletion
+                {
+                    deleteOutcome.removals += baseMatch - curIndex;
+                    deleteOutcome = simulateToDepth({ curIndex: baseMatch, nextIndex }, deleteOutcome, depthLeft - 1);
+                }
+
+                if (rankOutcome(insertOutcome) > rankOutcome(deleteOutcome)) {
+                    outcome = insertOutcome;
+                } else {
+                    outcome = deleteOutcome;
+                }
+            }
+        }
+
+        return outcome;
+    }
+
+
+
+    while (diffState.curIndex < base.length && diffState.nextIndex < next.length) {
+        diffArraysComparisonCount++;
+        let nextMatch = nextFindIndex(base[diffState.curIndex], diffState.nextIndex);
+        if (nextMatch < 0) {
+            // We won't ever match it if it doesn't exist in next, so it must be a remove
+            diff.push({ type: "removed", value: base[diffState.curIndex] });
+            diffState.curIndex++;
+        } else if (nextMatch === diffState.nextIndex) {
+            // Is a perfect match
+            diff.push({ type: "unchanged", value: base[diffState.curIndex], unchangedNextValue: next[diffState.nextIndex] });
+            diffState.curIndex++;
+            diffState.nextIndex++;
+        } else {
+            let baseMatch = diffState.nextIndex < next.length ? baseFindIndex(next[diffState.nextIndex], diffState.curIndex) : -1;
+            if (baseMatch < 0) {
+                // Can never match the otherside, so next is an insertion
+                diff.push({ type: "added", value: next[diffState.nextIndex] });
+                diffState.nextIndex++;
+            } else {
+                let insertOutcome = simulateToDepth({ curIndex: diffState.curIndex, nextIndex: nextMatch }, createEmptyOutcome(), searchDepth);
+
+                let deleteOutcome = simulateToDepth({ curIndex: baseMatch, nextIndex: diffState.nextIndex }, createEmptyOutcome(), searchDepth);
+
+                if (rankOutcome(insertOutcome) > rankOutcome(deleteOutcome)) {
+                    // 1) Insertion, assume base is matched by nextMatch, which means all the next values
+                    //      in between are insertions
+                    for (let i = diffState.nextIndex; i < nextMatch; i++) {
+                        diff.push({ type: "added", value: next[i] });
+                    }
+                    diffState.nextIndex = nextMatch;
+                } else {
+                    // 2) Delete base, assume next is a match, and so everything up until baseMatch is a deletion
+                    for (let i = diffState.curIndex; i < baseMatch; i++) {
+                        diff.push({ type: "removed", value: base[i] });
+                    }
+                    diffState.curIndex = baseMatch;
+                }
+            }
+        }
+    }
+
+    // Handle any trailing values
+    while (diffState.curIndex < base.length) {
+        diff.push({ type: "removed", value: base[diffState.curIndex] });
+        diffState.curIndex++;
+    }
+    while (diffState.nextIndex < next.length) {
+        diff.push({ type: "added", value: next[diffState.nextIndex] });
+        diffState.nextIndex++;
+    }
+
+    return diff;
+}
+
+
+
+
+export let diffArraysComparisonCount = 0;
+export function diffArraysOld(
+    config: {
+        baseValues: string[];
+        newValues: string[];
+    }
+): LineChange[] {
+    let { baseValues, newValues } = config;
+
+    let stringLengthCount = new Map<string, number>();
+    for (let value of baseValues) {
+        stringLengthCount.set(value, (stringLengthCount.get(value) || 0) + 1);
+    }
+    for (let value of newValues) {
+        stringLengthCount.set(value, (stringLengthCount.get(value) || 0) + 1);
+    }
+    function getLengthFactor(count: number) {
+        if (count === 1) return 10;
+        if (count === 2) return 5;
+        if (count === 3) return 2;
+        if (count < 10) return 1;
+        return 0.5;
+    }
+    // TODO: Also prefer values that are more in the middle, as this helps our algorithm run more efficiently
+    function getStrLengthFactor(value: string) {
+        return getLengthFactor(stringLengthCount.get(value) || 0);
+    }
+
+    type MatchValue = { baseStart: number; newStart: number; baseEnd: number; newEnd: number; value: number; };
+    function getPairMatchValue(baseIndex: number, newIndex: number): MatchValue | undefined {
+        diffArraysComparisonCount++;
+        if (baseValues[baseIndex] !== newValues[newIndex]) return undefined;
+        while (baseIndex > 1 && newIndex > 1 && baseValues[baseIndex] === newValues[newIndex]) {
+            diffArraysComparisonCount++;
+            baseIndex--;
+            newIndex--;
+        }
+        let baseStart = baseIndex;
+        let newStart = newIndex;
+        let value = 0;
+        while (baseIndex < baseValues.length && newIndex < newValues.length && baseValues[baseIndex] === newValues[newIndex]) {
+            diffArraysComparisonCount++;
+            value += getStrLengthFactor(baseValues[baseIndex]);
+            baseIndex++;
+            newIndex++;
+        }
+        let baseEnd = baseIndex;
+        let newEnd = newIndex;
+        return {
+            baseStart,
+            newStart,
+            baseEnd,
+            newEnd,
+            value
+        };
+    }
+
+    let bestMatch = maxValue(
+        list(baseValues.length).map(baseIndex =>
+            maxValue(
+                list(newValues.length).map(newIndex => getPairMatchValue(baseIndex, newIndex)),
+                x => x?.value ?? 0
+            )
+        ),
+        x => x?.value ?? 0
+    );
+    if (!bestMatch) {
+        // If there are no matches, everything is a change
+        return (
+            ([] as LineChange[])
+                .concat(baseValues.map(value => ({ type: "removed" as const, value })))
+                .concat(newValues.map(value => ({ type: "added" as const, value })))
+        );
+    }
+
+    let changes: LineChange[] = [];
+    // Deal with changes before the match
+    if (bestMatch.baseStart === 0) {
+        if (bestMatch.newStart > 0) {
+            changes.push(...newValues.slice(0, bestMatch.newStart).map(value => ({ type: "added" as const, value })));
+        }
+    } else if (bestMatch.newStart === 0) {
+        if (bestMatch.baseStart > 0) {
+            changes.push(...baseValues.slice(0, bestMatch.baseStart).map(value => ({ type: "removed" as const, value })));
+        }
+    } else {
+        // Values before must be diffed
+        changes.push(...diffArrays({ base: baseValues.slice(0, bestMatch.baseStart), next: newValues.slice(0, bestMatch.newStart) }));
+    }
+
+    // Add the match
+    changes.push(...baseValues.slice(bestMatch.baseStart, bestMatch.baseEnd).map(value => ({ type: "unchanged" as const, value })));
+
+    // Deal with changes after the match
+    if (bestMatch.baseEnd === baseValues.length) {
+        if (bestMatch.newEnd !== newValues.length) {
+            changes.push(...newValues.slice(bestMatch.newEnd).map(value => ({ type: "added" as const, value })));
+        }
+    } else if (bestMatch.newEnd === newValues.length) {
+        if (bestMatch.baseEnd !== baseValues.length) {
+            changes.push(...baseValues.slice(bestMatch.baseEnd).map(value => ({ type: "removed" as const, value })));
+        }
+    } else {
+        changes.push(...diffArrays({ base: baseValues.slice(bestMatch.baseEnd), next: newValues.slice(bestMatch.newEnd) }));
+    }
+
+    return changes;
+}
+
+function sum(arg0: number[]) {
+    return arg0.reduce((a, b) => a + b, 0);
+}
+
+function UnionUndefined<T>(x: T): T | undefined {
+    return x;
+}
+function assertDefined<T>(x: T | undefined): T {
+    if (x === undefined) throw new Error(`Expected value to be defined`);
+    return x;
+}
+
+function assertDefined2<T>(x: T | undefined): asserts x is T {
+    if (x === undefined) throw new Error(`Expected value to be defined`);
+}
+function maxValue<T>(values: T[], getValue: (value: T) => number): T | undefined {
+    let bestValue = -Infinity;
+    let best: T | undefined = undefined;
+    for (let value of values) {
+        let curValue = getValue(value);
+        if (curValue > bestValue) {
+            bestValue = curValue;
+            best = value;
+        }
+    }
+    return best;
 }