tsense 0.0.14 → 0.0.15

package/README.md

@@ -89,8 +89,8 @@ type("string").configure({
 
 ### Collection Methods
 
-| Method | Description |
-| ------ | ----------- |
+| Method/Property | Description |
+| --------------- | ----------- |
 | `create()` | Creates the collection in Typesense |
 | `drop()` | Deletes the collection |
 | `get(id)` | Retrieves a document by ID |
@@ -100,7 +100,57 @@ type("string").configure({
 | `updateMany(filter, data)` | Updates documents matching filter |
 | `upsert(docs)` | Inserts or updates documents |
 | `search(options)` | Searches the collection |
+| `syncSchema()` | Syncs schema (creates/patches collection) |
+| `syncData(options)` | Syncs data from external source |
+| `fields` | Array of generated field schemas |
 
+### Schema Sync
+
+Automatically sync schema before the first operation:
+
+```typescript
+const Collection = new TSense({
+  // ...
+  autoSyncSchema: true,
+});
+```
+
+Or manually:
+
+```typescript
+await Collection.syncSchema();
+```
+
+### Data Sync
+
+Sync documents from an external source (database, API, etc.):
+
+```typescript
+const Collection = new TSense({
+  // ...
+  dataSync: {
+    getAllIds: async () => {
+      return db.selectFrom("users").select("id").execute().then(rows => rows.map(r => r.id));
+    },
+    getItems: async (ids) => {
+      return db.selectFrom("users").where("id", "in", ids).execute();
+    },
+    chunkSize: 100, // optional, default 500
+  },
+});
+
+// Full sync
+await Collection.syncData();
+
+// Partial sync (specific IDs)
+await Collection.syncData({ ids: ["id1", "id2"] });
+
+// Full sync + remove orphan documents
+await Collection.syncData({ purge: true });
+
+// Override chunk size
+await Collection.syncData({ chunkSize: 50 });
+```
 
 ### Filter Syntax
 

package/dist/index.d.ts

@@ -3,4 +3,4 @@ export { DateTransformer } from "./transformers/date.js";
 export { defaultTransformers } from "./transformers/defaults.js";
 export type { FieldTransformer } from "./transformers/types.js";
 export { TSense } from "./tsense.js";
-export type { ConnectionConfig, DeleteResult, FilterFor, HighlightOptions, SearchListOptions, SearchListResult, SearchOptions, SearchResult, TsenseOptions, UpdateResult, UpsertResult, } from "./types.js";
+export type { ConnectionConfig, DeleteResult, FilterFor, HighlightOptions, SearchListOptions, SearchListResult, SearchOptions, SearchResult, SyncConfig, SyncOptions, SyncResult, TsenseOptions, UpdateResult, UpsertResult, } from "./types.js";

package/dist/tsense.d.ts

@@ -1,6 +1,6 @@
 import type { Type } from "arktype";
 import redaxios from "redaxios";
-import type { DeleteResult, FilterFor, SearchListOptions, SearchListResult, SearchOptions, SearchResult, TsenseOptions, UpdateResult, UpsertResult } from "./types.js";
+import type { DeleteResult, FieldSchema, FilterFor, SearchListOptions, SearchListResult, SearchOptions, SearchResult, SyncOptions, SyncResult, TsenseOptions, UpdateResult, UpsertResult } from "./types.js";
 declare const redaxiosInstance: {
     <T>(urlOrConfig: string | redaxios.Options, config?: redaxios.Options | undefined, _method?: any, data?: any, _undefined?: undefined): Promise<redaxios.Response<T>>;
     request: (<T_1 = any>(config?: redaxios.Options | undefined) => Promise<redaxios.Response<T_1>>) | (<T_2 = any>(url: string, config?: redaxios.Options | undefined) => Promise<redaxios.Response<T_2>>);
@@ -59,11 +59,13 @@ declare const redaxiosInstance: {
 export type AxiosInstance = ReturnType<typeof redaxiosInstance.create>;
 export declare class TSense<T extends Type> {
     private options;
-    private fields;
+    fields: FieldSchema[];
     private axios;
     private synced;
     private fieldTransformers;
+    private dataSyncConfig?;
     infer: T["infer"];
+    getFields(): FieldSchema[];
     constructor(options: TsenseOptions<T>);
     private getBaseType;
     private inferType;
@@ -72,7 +74,7 @@ export declare class TSense<T extends Type> {
     private serializeFilterValue;
     private extractFields;
     private ensureSynced;
-    sync(): Promise<void>;
+    syncSchema(): Promise<void>;
     private buildObjectFilter;
     private buildFilter;
     private buildSort;
@@ -87,5 +89,8 @@ export declare class TSense<T extends Type> {
     searchList(options: SearchListOptions<T["infer"]>): Promise<SearchListResult<T["infer"]>>;
     count(filter?: FilterFor<T["infer"]>): Promise<number>;
     upsert(docs: T["infer"] | T["infer"][]): Promise<UpsertResult[]>;
+    syncData(options?: SyncOptions): Promise<SyncResult>;
+    private purgeOrphans;
+    private exportIds;
 }
 export {};

package/dist/tsense.js

@@ -11,17 +11,28 @@ var _a;
 import redaxios from "redaxios";
 import { TSenseMigrator } from "./migrator.js";
 import { defaultTransformers } from "./transformers/defaults.js";
+function chunkArray(arr, size) {
+    const chunks = [];
+    for (let i = 0; i < arr.length; i += size) {
+        chunks.push(arr.slice(i, i + size));
+    }
+    return chunks;
+}
 const redaxiosInstance = (_a = redaxios.default) !== null && _a !== void 0 ? _a : redaxios;
 const arkToTsense = {
     string: "string",
     number: "float",
     "number.integer": "int64",
+    "number % 1": "int64",
     boolean: "bool",
     "string[]": "string[]",
     "number[]": "float[]",
     "boolean[]": "bool[]",
 };
 export class TSense {
+    getFields() {
+        return this.fields;
+    }
     constructor(options) {
         var _a;
         this.options = options;
@@ -34,6 +45,7 @@ export class TSense {
             headers: { "X-TYPESENSE-API-KEY": options.connection.apiKey },
         });
         this.extractFields((_a = options.transformers) !== null && _a !== void 0 ? _a : defaultTransformers);
+        this.dataSyncConfig = options.dataSync;
     }
     getBaseType(expression, domain) {
         if (domain && domain !== "undefined")
@@ -94,14 +106,15 @@ export class TSense {
         return transformer.serialize(value);
     }
     extractFields(transformers) {
-        var _a;
+        var _a, _b, _c, _d;
         const internal = this.options.schema;
         for (const prop of internal.structure.props) {
-            const meta = prop.value.meta;
+            const innerType = (_b = (_a = prop.value.branches) === null || _a === void 0 ? void 0 : _a[0]) !== null && _b !== void 0 ? _b : prop.value;
+            const meta = ((_c = innerType.meta) !== null && _c !== void 0 ? _c : prop.value.meta);
             const expression = String(prop.value.expression);
             const domain = prop.value.domain;
             const baseType = this.getBaseType(expression, domain);
-            const transformer = transformers.find((t) => t.match(baseType, domain));
+            const transformer = transformers.find((t) => t.match(expression, domain) || t.match(baseType, domain));
             if (transformer) {
                 this.fieldTransformers.set(prop.key, transformer);
                 this.fields.push({
@@ -114,7 +127,7 @@ export class TSense {
                 });
                 continue;
             }
-            const type = (_a = meta === null || meta === void 0 ? void 0 : meta.type) !== null && _a !== void 0 ? _a : this.inferType(baseType);
+            const type = (_d = meta === null || meta === void 0 ? void 0 : meta.type) !== null && _d !== void 0 ? _d : this.inferType(baseType);
             this.fields.push({
                 name: prop.key,
                 type,
@@ -127,13 +140,13 @@ export class TSense {
     }
     ensureSynced(force) {
         return __awaiter(this, void 0, void 0, function* () {
-            if (!force && (this.synced || !this.options.autoSync))
+            if (!force && (this.synced || !this.options.autoSyncSchema))
                 return;
             yield new TSenseMigrator(this.options.name, this.fields, this.options.defaultSortingField, this.axios).sync();
             this.synced = true;
         });
     }
-    sync() {
+    syncSchema() {
         return __awaiter(this, void 0, void 0, function* () {
             yield this.ensureSynced(true);
         });
@@ -224,7 +237,6 @@ export class TSense {
     }
     drop() {
         return __awaiter(this, void 0, void 0, function* () {
-            console.log("dropping");
             yield this.axios({
                 method: "DELETE",
                 url: `/collections/${this.options.name}`,
@@ -309,9 +321,7 @@ export class TSense {
             yield this.ensureSynced();
             const params = {
                 q: (_a = options.query) !== null && _a !== void 0 ? _a : "",
-                query_by: ((_b = options.queryBy) !== null && _b !== void 0 ? _b : [
-                    this.options.defaultSearchField,
-                ]).join(","),
+                query_by: ((_b = options.queryBy) !== null && _b !== void 0 ? _b : [this.options.defaultSearchField]).join(","),
             };
             const sortBy = this.buildSort(options);
             if (sortBy)
@@ -394,9 +404,7 @@ export class TSense {
             const page = options.cursor ? Number(options.cursor) : 1;
             const params = {
                 q: (_b = options.query) !== null && _b !== void 0 ? _b : "",
-                query_by: ((_c = options.queryBy) !== null && _c !== void 0 ? _c : [
-                    this.options.defaultSearchField,
-                ]).join(","),
+                query_by: ((_c = options.queryBy) !== null && _c !== void 0 ? _c : [this.options.defaultSearchField]).join(","),
                 per_page: limit,
                 page,
                 sort_by: `${field}:${options.sort.direction}`,
@@ -456,9 +464,7 @@ export class TSense {
                     this.options.schema.assert(item);
                 }
             }
-            const payload = items
-                .map((item) => JSON.stringify(this.serializeDoc(item)))
-                .join("\n");
+            const payload = items.map((item) => JSON.stringify(this.serializeDoc(item))).join("\n");
             const params = { action: "upsert" };
             if (this.options.batchSize) {
                 params.batch_size = this.options.batchSize;
@@ -476,4 +482,59 @@ export class TSense {
             return [data];
         });
     }
+    syncData(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
+            if (!this.dataSyncConfig) {
+                throw new Error("DATA_SYNC_NOT_CONFIGURED");
+            }
+            const chunkSize = (_b = (_a = options === null || options === void 0 ? void 0 : options.chunkSize) !== null && _a !== void 0 ? _a : this.dataSyncConfig.chunkSize) !== null && _b !== void 0 ? _b : 500;
+            const ids = (_c = options === null || options === void 0 ? void 0 : options.ids) !== null && _c !== void 0 ? _c : (yield this.dataSyncConfig.getAllIds());
+            let upserted = 0;
+            let failed = 0;
+            for (const chunk of chunkArray(ids, chunkSize)) {
+                const items = yield this.dataSyncConfig.getItems(chunk);
+                const results = yield this.upsert(items);
+                for (const r of results) {
+                    if (r.success)
+                        upserted++;
+                    else
+                        failed++;
+                }
+            }
+            let deleted = 0;
+            if (options === null || options === void 0 ? void 0 : options.purge) {
+                deleted = yield this.purgeOrphans(ids, chunkSize);
+            }
+            return { upserted, deleted, failed };
+        });
+    }
+    purgeOrphans(validIds, chunkSize) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const validSet = new Set(validIds);
+            const remoteIds = yield this.exportIds();
+            const orphans = remoteIds.filter((id) => !validSet.has(id));
+            if (!orphans.length)
+                return 0;
+            let deleted = 0;
+            for (const chunk of chunkArray(orphans, chunkSize)) {
+                const result = yield this.deleteMany({ id: chunk });
+                deleted += result.deleted;
+            }
+            return deleted;
+        });
+    }
+    exportIds() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { data } = yield this.axios({
+                method: "GET",
+                url: `/collections/${this.options.name}/documents/export`,
+                params: { include_fields: "id" },
+            });
+            return data
+                .split("\n")
+                .filter((line) => line.length)
+                .map((line) => JSON.parse(line).id);
+        });
+    }
 }

package/dist/types.d.ts

@@ -42,8 +42,9 @@ export type TsenseOptions<T extends Type> = {
     defaultSortingField?: keyof T["infer"];
     batchSize?: number;
     validateOnUpsert?: boolean;
-    autoSync?: boolean;
+    autoSyncSchema?: boolean;
     transformers?: FieldTransformer[];
+    dataSync?: SyncConfig<T["infer"]>;
 };
 type SingleFilter<T> = Partial<{
     [K in keyof T]: BaseIfArray<T[K]> | NonNullable<BaseIfArray<T[K]>>[] | {
@@ -116,4 +117,19 @@ export type SearchListResult<T> = {
     nextCursor: string | null;
     total: number;
 };
+export type SyncConfig<T> = {
+    getAllIds: () => Promise<string[]>;
+    getItems: (ids: string[]) => Promise<T[]>;
+    chunkSize?: number;
+};
+export type SyncOptions = {
+    ids?: string[];
+    purge?: boolean;
+    chunkSize?: number;
+};
+export type SyncResult = {
+    upserted: number;
+    deleted: number;
+    failed: number;
+};
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "tsense",
-	"version": "0.0.14",
+	"version": "0.0.15",
 	"private": false,
 	"description": "Opinionated, fully typed typesense client",
 	"keywords": [
@@ -42,8 +42,8 @@
 		"@changesets/cli": "^2.29.7",
 		"@types/bun": "latest",
 		"arktype": "^2.1.29",
-		"oxfmt": "^0.27.0",
-		"oxlint": "^1.42.0"
+		"oxfmt": "^0.28.0",
+		"oxlint": "^1.43.0"
 	},
 	"peerDependencies": {
 		"arktype": "^2.1.29",