typesense-nextra-adapter 0.0.1

package/dist/indexFromHtml.js

@@ -0,0 +1,213 @@
+import crypto_0 from "crypto";
+import * as __rspack_external_cheerio from "cheerio";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class IndexFromHtml {
+    getRecords(html, url, lang) {
+        const $ = __rspack_external_cheerio.load(html);
+        $('[data-pagefind-ignore]').remove();
+        const records = [];
+        const getGlobalText = (selector)=>{
+            const el = $(selector).first();
+            return el.length ? this.cleanText(el) : null;
+        };
+        const globalHierarchy = {
+            lvl0: getGlobalText(this.selectors.lvl0?.selector) || getGlobalText(this.selectors.lvl1?.selector) || getGlobalText('header nav [aria-current=true]') || $('title').text().trim() || 'Documentation'
+        };
+        const selectorString = Object.values(this.selectors).filter((s)=>!s.global).map((s)=>s.selector).join(', ');
+        const nodes = $(selectorString);
+        let previousHierarchy = this.generateEmptyHierarchy();
+        previousHierarchy.lvl0 = globalHierarchy.lvl0 || null;
+        const anchors = this.generateEmptyHierarchy();
+        nodes.each((index, element)=>{
+            const el = $(element);
+            const isNestedMatch = el.parents(selectorString).length > 0;
+            if (isNestedMatch) return;
+            const tagName = element.tagName.toLowerCase();
+            const currentLevel = this.getLevelFromTag(tagName);
+            const hierarchy = {
+                ...previousHierarchy
+            };
+            const currentLevelInt = this.levels.indexOf(currentLevel);
+            if ('content' !== currentLevel) {
+                const text = this.cleanText(el);
+                hierarchy[currentLevel] = text;
+                anchors[currentLevel] = this.getAnchor(el);
+                for(let i = currentLevelInt + 1; i < 7; i++){
+                    const lvlKey = `lvl${i}`;
+                    hierarchy[lvlKey] = null;
+                    anchors[lvlKey] = null;
+                }
+                previousHierarchy = {
+                    ...hierarchy
+                };
+            }
+            let content = null;
+            if ('content' === currentLevel) {
+                content = this.cleanText(el);
+                if (!content) return;
+            }
+            if (currentLevel.startsWith('lvl') && !hierarchy[currentLevel]) return;
+            const resolvedAnchor = this.getClosestAnchor(anchors);
+            let levelWeight;
+            if ('content' !== currentLevel) levelWeight = 100 - 10 * currentLevelInt;
+            else {
+                const isCodeBlock = 'code' === tagName || el.parents('pre').length > 0;
+                levelWeight = isCodeBlock ? 0 : 10;
+            }
+            const weight = {
+                page_rank: 0,
+                level: levelWeight,
+                position: index,
+                position_descending: nodes.length - index
+            };
+            const record = {
+                objectID: '',
+                url_without_anchor: url,
+                url: resolvedAnchor ? `${url}#${resolvedAnchor}` : url,
+                anchor: resolvedAnchor,
+                content: content,
+                hierarchy: hierarchy,
+                hierarchy_radio: this.getHierarchyRadio(hierarchy, currentLevel),
+                type: currentLevel,
+                weight: weight,
+                language: lang
+            };
+            record.objectID = this.getObjectID(record);
+            records.push(record);
+        });
+        return records;
+    }
+    getLevelFromTag(tagName) {
+        if ([
+            'h1',
+            'h2',
+            'h3',
+            'h4',
+            'h5',
+            'h6'
+        ].includes(tagName)) return `lvl${tagName.replace('h', '')}`;
+        return 'content';
+    }
+    generateEmptyHierarchy() {
+        return {
+            lvl0: null,
+            lvl1: null,
+            lvl2: null,
+            lvl3: null,
+            lvl4: null,
+            lvl5: null,
+            lvl6: null
+        };
+    }
+    getHierarchyRadio(hierarchy, currentLevel) {
+        const radio = this.generateEmptyHierarchy();
+        let isFound = false;
+        for(let i = 6; i >= 0; i--){
+            const level = `lvl${i}`;
+            const value = hierarchy[level];
+            if (!isFound && null !== value) {
+                if ('content' === currentLevel || level === currentLevel) {
+                    radio[level] = value;
+                    isFound = true;
+                    continue;
+                }
+            }
+            radio[level] = null;
+        }
+        return radio;
+    }
+    getAnchor(el) {
+        const id = el.attr('id');
+        return id || null;
+    }
+    getClosestAnchor(anchors) {
+        for(let i = 6; i >= 0; i--){
+            const val = anchors[`lvl${i}`];
+            if (val) return val;
+        }
+        return null;
+    }
+    getObjectID(record) {
+        const hierarchyToHash = {};
+        Object.keys(record.hierarchy).forEach((k)=>{
+            const val = record.hierarchy[k];
+            if (val) hierarchyToHash[k] = val;
+        });
+        const payload = {
+            hierarchy: hierarchyToHash,
+            url: record.url_without_anchor,
+            position: record.weight.position
+        };
+        const jsonStr = JSON.stringify(payload, Object.keys(payload).sort());
+        return crypto_0.createHash('sha1').update(jsonStr).digest('hex');
+    }
+    cleanText(el) {
+        const clone = el.clone();
+        clone.find('.rp-header-anchor').remove();
+        clone.find('br').replaceWith(' ');
+        clone.find('li, p, div, tr').append(' ');
+        let text = clone.text();
+        text = text.replace(/\u00A0/g, ' ').replace(/[\u200B-\u200D\uFEFF]/g, '').replace(/\s+/g, ' ').trim();
+        if (text.length > 10000) text = text.slice(0, 10000) + '...';
+        return text;
+    }
+    constructor(options){
+        _define_property(this, "levels", [
+            'lvl0',
+            'lvl1',
+            'lvl2',
+            'lvl3',
+            'lvl4',
+            'lvl5',
+            'lvl6'
+        ]);
+        _define_property(this, "selectors", void 0);
+        const docClass = '[data-pagefind-body]';
+        let contentSelector = `${docClass} p, ${docClass} li, ${docClass} td, ${docClass} th`;
+        if (options?.indexCodeBlocks) contentSelector += `${docClass} pre > code`;
+        this.selectors = {
+            lvl0: {
+                selector: 'header nav [aria-current=true]',
+                global: true
+            },
+            lvl1: {
+                selector: `${docClass} h1`,
+                global: false
+            },
+            lvl2: {
+                selector: `${docClass} h2`,
+                global: false
+            },
+            lvl3: {
+                selector: `${docClass} h3`,
+                global: false
+            },
+            lvl4: {
+                selector: `${docClass} h4`,
+                global: false
+            },
+            lvl5: {
+                selector: `${docClass} h5`,
+                global: false
+            },
+            lvl6: {
+                selector: `${docClass} h6`,
+                global: false
+            },
+            content: {
+                selector: contentSelector,
+                global: false
+            }
+        };
+    }
+}
+export { IndexFromHtml };

package/dist/types.d.ts

@@ -0,0 +1,44 @@
+import type { CollectionFieldSchema } from 'typesense/lib/Typesense/Collection';
+import type { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections';
+export interface Hierarchy {
+    [key: string]: string | null | undefined;
+    lvl0: string | null;
+    lvl1: string | null;
+    lvl2: string | null;
+    lvl3: string | null;
+    lvl4: string | null;
+    lvl5: string | null;
+    lvl6: string | null;
+}
+export interface RecordWeight {
+    page_rank: number;
+    level: number;
+    position: number;
+    position_descending: number;
+}
+export interface DocSearchRecord {
+    objectID: string;
+    url: string;
+    url_without_anchor: string;
+    anchor: string | null;
+    content: string | null;
+    hierarchy: Hierarchy;
+    hierarchy_radio: Hierarchy;
+    type: string;
+    weight: RecordWeight;
+    version?: string;
+    language?: string;
+    [key: string]: any;
+}
+export interface FieldsParams {
+    locale: string;
+    isVersioned: boolean;
+}
+export interface CustomCollectionSettings {
+    token_separators?: CollectionCreateSchema['token_separators'];
+    symbols_to_index?: CollectionCreateSchema['symbols_to_index'];
+    fields?: (params: FieldsParams) => CollectionFieldSchema[];
+    enable_nested_fields?: CollectionCreateSchema['enable_nested_fields'];
+}
+/**  Allows a single global config OR a map of configs keyed by language (e.g. `{ en: {...}, zh: {...} }`) */
+export type CustomCollectionSettingsConfig = CustomCollectionSettings | Record<string, CustomCollectionSettings>;

package/dist/typesenseHelper.d.ts

@@ -0,0 +1,67 @@
+import type { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration';
+import type { DocSearchRecord, CustomCollectionSettings, FieldsParams } from './types.js';
+import type { CollectionFieldSchema } from 'typesense/lib/Typesense/Collection';
+export interface TypesenseHelperOptions {
+    config: ConfigurationOptions;
+    aliasName: string;
+    collectionNameTmp: string;
+    customSettings: CustomCollectionSettings | null;
+    locale: string;
+    isVersioned: boolean;
+    batchSize: number;
+}
+export declare class TypesenseHelper {
+    private typesenseClient;
+    private aliasName;
+    private collectionNameTmp;
+    private collectionLocale;
+    private customSettings;
+    private isVersioned;
+    private typesenseVersion;
+    private recordBatch;
+    private readonly batchSize;
+    constructor(options: TypesenseHelperOptions);
+    init(): Promise<void>;
+    createTmpCollection(): Promise<void>;
+    /**
+     * Adds records to an internal batch queue and logs the file stats instantly.
+     * Uploads to Typesense automatically when the queue hits batchSize.
+     */
+    addRecords(records: DocSearchRecord[], fileName: string, padLength?: number): Promise<number>;
+    /**
+     * Internal method that actually performs the network request to Typesense.
+     * Throws a specific error so the main asyncPool loop knows it's a batch failure, not a file parsing failure.
+     */
+    private uploadBatch;
+    /**
+     * Force uploads any remaining records that did not cleanly hit the batchSize limit.
+     * Must be called before `commitTmpCollection()`.
+     */
+    flushBatch(): Promise<void>;
+    commitTmpCollection(): Promise<void>;
+    static transformRecord(record: DocSearchRecord, isVersioned: boolean): any;
+    private getOldCollectionName;
+    private transferSynonyms;
+    private transferOverrides;
+}
+/**
+ * Returns the default collection fields used when creating a Typesense collection.
+ *
+ * Use this when you need to customize specific fields without redefining the entire schema.
+ * The `version` field is automatically included when `isVersioned` is `true`.
+ *
+ * @param locale - The locale used for text fields (e.g. `'en'`, `'zh'`).
+ * @param isVersioned - When `true`, appends the `version` field to the returned fields.
+ *
+ * @example
+ * // Adding a custom field while keeping defaults:
+ * customLocaleCollectionSettings: {
+ *   en: {
+ *     fields: (params) => [
+ *       ...getDefaultCollectionFields(params),
+ *       { name: 'my_custom_field', type: 'string', locale },
+ *     ],
+ *   },
+ * }
+ */
+export declare function getDefaultCollectionFields({ locale, isVersioned, }: FieldsParams): CollectionFieldSchema[];

package/dist/typesenseHelper.js

@@ -0,0 +1,236 @@
+import { Client } from "typesense";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class TypesenseHelper {
+    async init() {
+        const debugInfo = await this.typesenseClient.debug.retrieve();
+        const version = debugInfo.version;
+        if ('nightly' === version) this.typesenseVersion = 30;
+        else this.typesenseVersion = parseInt(version.split('.')[0], 10);
+    }
+    async createTmpCollection() {
+        if (0 === this.typesenseVersion) await this.init();
+        try {
+            await this.typesenseClient.collections(this.collectionNameTmp).delete();
+        } catch (error) {
+            if (error?.httpStatus !== 404) {
+                console.error('[TypesensePlugin] Error:');
+                throw error;
+            }
+        }
+        const schema = {
+            name: this.collectionNameTmp,
+            fields: getDefaultCollectionFields({
+                locale: this.collectionLocale,
+                isVersioned: this.isVersioned
+            }),
+            default_sorting_field: 'item_priority',
+            token_separators: [
+                '_',
+                '-'
+            ]
+        };
+        if (this.customSettings) {
+            if (this.customSettings.token_separators) schema.token_separators = this.customSettings.token_separators;
+            if (this.customSettings.fields) schema.fields = this.customSettings.fields({
+                locale: this.collectionLocale,
+                isVersioned: this.isVersioned
+            });
+            if (this.customSettings.symbols_to_index) schema.symbols_to_index = this.customSettings.symbols_to_index;
+            if (void 0 !== this.customSettings.enable_nested_fields) schema.enable_nested_fields = this.customSettings.enable_nested_fields;
+        }
+        await this.typesenseClient.collections().create(schema);
+    }
+    async addRecords(records, fileName, padLength = 40) {
+        const transformedRecords = records.map((r)=>TypesenseHelper.transformRecord(r, this.isVersioned));
+        const recordCount = transformedRecords.length;
+        const fillerLength = Math.max(2, padLength - fileName.length);
+        const filler = '\x1b[90m' + '.'.repeat(fillerLength) + '\x1b[0m';
+        const paddedCount = recordCount.toString().padStart(4, ' ');
+        console.log(`  \x1b[32m✔\x1b[0m \x1b[37m${fileName}\x1b[0m ${filler} \x1b[33m${paddedCount}\x1b[0m \x1b[90mrecords\x1b[0m`);
+        this.recordBatch.push(...transformedRecords);
+        if (this.recordBatch.length >= this.batchSize) {
+            const batchToUpload = this.recordBatch.splice(0, this.batchSize);
+            await this.uploadBatch(batchToUpload);
+        }
+        return recordCount;
+    }
+    async uploadBatch(batch) {
+        if (0 === batch.length) return;
+        try {
+            const results = await this.typesenseClient.collections(this.collectionNameTmp).documents().import(batch, {
+                action: 'create'
+            });
+            const failedItems = results.filter((r)=>false === r.success);
+            if (failedItems.length > 0) {
+                console.error('\n\x1b[31m[Batch Error]\x1b[0m Typesense rejected some records in this batch.');
+                console.error(JSON.stringify(failedItems.slice(0, 5), null, 2));
+                throw new Error('TYPESENSE_BATCH_ERROR');
+            }
+        } catch (err) {
+            if ('TYPESENSE_BATCH_ERROR' !== err.message) console.error('\n\x1b[31m[Network Error]\x1b[0m Failed to reach Typesense server during batch upload.');
+            throw new Error('TYPESENSE_BATCH_ERROR');
+        }
+    }
+    async flushBatch() {
+        if (this.recordBatch.length > 0) {
+            const batchToUpload = this.recordBatch.splice(0, this.recordBatch.length);
+            await this.uploadBatch(batchToUpload);
+        }
+    }
+    async commitTmpCollection() {
+        const oldCollectionName = await this.getOldCollectionName();
+        if (oldCollectionName) {
+            await this.transferSynonyms(oldCollectionName);
+            await this.transferOverrides(oldCollectionName);
+        }
+        await this.typesenseClient.aliases().upsert(this.aliasName, {
+            collection_name: this.collectionNameTmp
+        });
+        if (oldCollectionName) await this.typesenseClient.collections(oldCollectionName).delete();
+    }
+    static transformRecord(record, isVersioned) {
+        const transformedRecord = {};
+        const excludeKeys = [
+            'weight',
+            'language',
+            'tags',
+            '.*_tag'
+        ];
+        for(const key in record)if (null !== record[key] && void 0 !== record[key] && !excludeKeys.includes(key)) transformedRecord[key] = record[key];
+        if (record.weight) transformedRecord['item_priority'] = 1000000000 * record.weight.page_rank + 1000 * record.weight.level + record.weight.position_descending;
+        else transformedRecord['item_priority'] = 0;
+        for(let x = 0; x < 7; x++){
+            const lvlKey = `lvl${x}`;
+            if (record.hierarchy && null != record.hierarchy[lvlKey]) transformedRecord[`hierarchy.lvl${x}`] = record.hierarchy[lvlKey];
+            if (record.hierarchy_radio && null != record.hierarchy_radio[lvlKey]) transformedRecord[`hierarchy_radio.lvl${x}`] = record.hierarchy_radio[lvlKey];
+        }
+        if (isVersioned && record.version && 'string' == typeof record.version) transformedRecord['version'] = record.version;
+        else if (!isVersioned) delete transformedRecord['version'];
+        return transformedRecord;
+    }
+    async getOldCollectionName() {
+        try {
+            const alias = await this.typesenseClient.aliases(this.aliasName).retrieve();
+            return alias.collection_name;
+        } catch (error) {
+            if (error?.httpStatus === 404) return null;
+            throw error;
+        }
+    }
+    async transferSynonyms(oldCollectionName) {
+        if (this.typesenseVersion >= 30) {
+            const oldCollection = await this.typesenseClient.collections(oldCollectionName).retrieve();
+            const synonyms = oldCollection.synonym_sets || [];
+            for (const syn of synonyms)await this.typesenseClient.collections(this.collectionNameTmp).synonyms().upsert(syn.id, syn);
+            return;
+        }
+        const result = await this.typesenseClient.collections(oldCollectionName).synonyms().retrieve();
+        const synonyms = result.synonyms || [];
+        for (const synonym of synonyms){
+            const { id, ...synonymKeys } = synonym;
+            await this.typesenseClient.collections(this.collectionNameTmp).synonyms().upsert(synonym.id, synonymKeys);
+        }
+    }
+    async transferOverrides(oldCollectionName) {
+        if (this.typesenseVersion >= 30) {
+            const oldCollection = await this.typesenseClient.collections(oldCollectionName).retrieve();
+            const curations = oldCollection.curation_sets || [];
+            for (const cur of curations)await this.typesenseClient.collections(this.collectionNameTmp).overrides().upsert(cur.id, cur);
+            return;
+        }
+        const result = await this.typesenseClient.collections(oldCollectionName).overrides().retrieve();
+        const overrides = result.overrides || [];
+        for (const override of overrides){
+            const { id, ...overrideKeys } = override;
+            await this.typesenseClient.collections(this.collectionNameTmp).overrides().upsert(override.id, overrideKeys);
+        }
+    }
+    constructor(options){
+        _define_property(this, "typesenseClient", void 0);
+        _define_property(this, "aliasName", void 0);
+        _define_property(this, "collectionNameTmp", void 0);
+        _define_property(this, "collectionLocale", void 0);
+        _define_property(this, "customSettings", void 0);
+        _define_property(this, "isVersioned", void 0);
+        _define_property(this, "typesenseVersion", 0);
+        _define_property(this, "recordBatch", []);
+        _define_property(this, "batchSize", void 0);
+        const clientConfig = {
+            ...options.config
+        };
+        clientConfig.connectionTimeoutSeconds = clientConfig.connectionTimeoutSeconds || 1800;
+        clientConfig.retryIntervalSeconds = clientConfig.retryIntervalSeconds || 1;
+        clientConfig.numRetries = clientConfig.numRetries || 3;
+        clientConfig.healthcheckIntervalSeconds = clientConfig.healthcheckIntervalSeconds || 60;
+        clientConfig.logLevel = clientConfig.logLevel || 'error';
+        this.typesenseClient = new Client(clientConfig);
+        this.aliasName = options.aliasName;
+        this.collectionNameTmp = options.collectionNameTmp;
+        this.collectionLocale = options.locale || process.env.TYPESENSE_COLLECTION_LOCALE || 'en';
+        this.customSettings = options.customSettings;
+        this.isVersioned = options.isVersioned;
+        this.batchSize = options.batchSize;
+    }
+}
+function getDefaultCollectionFields({ locale, isVersioned }) {
+    const baseFields = [
+        {
+            name: 'anchor',
+            type: 'string',
+            optional: true
+        },
+        {
+            name: 'content',
+            type: 'string',
+            locale,
+            optional: true
+        },
+        {
+            name: 'url',
+            type: 'string',
+            facet: true
+        },
+        {
+            name: 'url_without_anchor',
+            type: 'string',
+            facet: true,
+            optional: true
+        },
+        ...Array.from({
+            length: 7
+        }).map((_, i)=>({
+                name: `hierarchy.lvl${i}`,
+                type: 'string',
+                facet: true,
+                locale,
+                optional: true
+            })),
+        {
+            name: 'type',
+            type: 'string',
+            facet: true,
+            optional: true
+        },
+        {
+            name: 'item_priority',
+            type: 'int64'
+        }
+    ];
+    if (isVersioned) baseFields.push({
+        name: 'version',
+        type: 'string',
+        facet: true,
+        optional: true
+    });
+    return baseFields;
+}
+export { TypesenseHelper, getDefaultCollectionFields };

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "typesense-nextra-adapter",
+  "description": "An adapter that adds lightning-fast, typo-tolerant, Typesense-powered search to your Nextra site",
+  "type": "module",
+  "version": "0.0.1",
+  "scripts": {
+    "build": "rslib build",
+    "dev": "rslib build --watch"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./client": {
+      "types": "./dist/client/index.d.ts",
+      "default": "./dist/client/index.js"
+    }
+  },
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "cheerio": "1.2.0",
+    "typesense": "3.0.6",
+    "typesense-docsearch-react": "3.4.1"
+  },
+  "devDependencies": {
+    "@changesets/cli": "2.31.0",
+    "@rsbuild/plugin-react": "2.0.0",
+    "@rslib/core": "0.20.1",
+    "@types/bun": "latest",
+    "@types/react": "19.2.15",
+    "@types/react-dom": "19.2.3",
+    "rsbuild-plugin-publint": "0.3.4"
+  },
+  "peerDependencies": {
+    "typescript": "5"
+  },
+  "keywords": [
+    "typesense",
+    "documentation-search",
+    "search",
+    "nextra-plugin",
+    "nextra-adapter",
+    "nextra-search",
+    "nextra",
+    "plugin"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/typesense/typesense-nextra-adapter.git"
+  },
+  "bugs": "https://github.com/typesense/typesense-nextra-adapter/issues",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public"
+  }
+}