tsense 0.0.13 → 0.0.15

package/README.md

@@ -89,8 +89,8 @@ type("string").configure({
 
 ### Collection Methods
 
-| Method | Description |
-| ------ | ----------- |
+| Method/Property | Description |
+| --------------- | ----------- |
 | `create()` | Creates the collection in Typesense |
 | `drop()` | Deletes the collection |
 | `get(id)` | Retrieves a document by ID |
@@ -100,7 +100,57 @@ type("string").configure({
 | `updateMany(filter, data)` | Updates documents matching filter |
 | `upsert(docs)` | Inserts or updates documents |
 | `search(options)` | Searches the collection |
+| `syncSchema()` | Syncs schema (creates/patches collection) |
+| `syncData(options)` | Syncs data from external source |
+| `fields` | Array of generated field schemas |
 
+### Schema Sync
+
+Automatically sync schema before the first operation:
+
+```typescript
+const Collection = new TSense({
+  // ...
+  autoSyncSchema: true,
+});
+```
+
+Or manually:
+
+```typescript
+await Collection.syncSchema();
+```
+
+### Data Sync
+
+Sync documents from an external source (database, API, etc.):
+
+```typescript
+const Collection = new TSense({
+  // ...
+  dataSync: {
+    getAllIds: async () => {
+      return db.selectFrom("users").select("id").execute().then(rows => rows.map(r => r.id));
+    },
+    getItems: async (ids) => {
+      return db.selectFrom("users").where("id", "in", ids).execute();
+    },
+    chunkSize: 100, // optional, default 500
+  },
+});
+
+// Full sync
+await Collection.syncData();
+
+// Partial sync (specific IDs)
+await Collection.syncData({ ids: ["id1", "id2"] });
+
+// Full sync + remove orphan documents
+await Collection.syncData({ purge: true });
+
+// Override chunk size
+await Collection.syncData({ chunkSize: 50 });
+```
 
 ### Filter Syntax
 

package/dist/index.d.ts

@@ -3,4 +3,4 @@ export { DateTransformer } from "./transformers/date.js";
 export { defaultTransformers } from "./transformers/defaults.js";
 export type { FieldTransformer } from "./transformers/types.js";
 export { TSense } from "./tsense.js";
-export type { ConnectionConfig, DeleteResult, FilterFor, HighlightOptions, SearchListOptions, SearchListResult, SearchOptions, SearchResult, TsenseOptions, UpdateResult, UpsertResult, } from "./types.js";
+export type { ConnectionConfig, DeleteResult, FilterFor, HighlightOptions, SearchListOptions, SearchListResult, SearchOptions, SearchResult, SyncConfig, SyncOptions, SyncResult, TsenseOptions, UpdateResult, UpsertResult, } from "./types.js";

package/dist/tsense.d.ts

@@ -1,6 +1,6 @@
 import type { Type } from "arktype";
 import redaxios from "redaxios";
-import type { DeleteResult, FilterFor, SearchListOptions, SearchListResult, SearchOptions, SearchResult, TsenseOptions, UpdateResult, UpsertResult } from "./types.js";
+import type { DeleteResult, FieldSchema, FilterFor, SearchListOptions, SearchListResult, SearchOptions, SearchResult, SyncOptions, SyncResult, TsenseOptions, UpdateResult, UpsertResult } from "./types.js";
 declare const redaxiosInstance: {
     <T>(urlOrConfig: string | redaxios.Options, config?: redaxios.Options | undefined, _method?: any, data?: any, _undefined?: undefined): Promise<redaxios.Response<T>>;
     request: (<T_1 = any>(config?: redaxios.Options | undefined) => Promise<redaxios.Response<T_1>>) | (<T_2 = any>(url: string, config?: redaxios.Options | undefined) => Promise<redaxios.Response<T_2>>);
@@ -59,19 +59,22 @@ declare const redaxiosInstance: {
 export type AxiosInstance = ReturnType<typeof redaxiosInstance.create>;
 export declare class TSense<T extends Type> {
     private options;
-    private fields;
+    fields: FieldSchema[];
     private axios;
     private synced;
     private fieldTransformers;
+    private dataSyncConfig?;
     infer: T["infer"];
+    getFields(): FieldSchema[];
     constructor(options: TsenseOptions<T>);
+    private getBaseType;
     private inferType;
     private serializeDoc;
     private deserializeDoc;
     private serializeFilterValue;
     private extractFields;
     private ensureSynced;
-    sync(): Promise<void>;
+    syncSchema(): Promise<void>;
     private buildObjectFilter;
     private buildFilter;
     private buildSort;
@@ -84,6 +87,10 @@ export declare class TSense<T extends Type> {
     updateMany(filter: FilterFor<T["infer"]>, data: Partial<T["infer"]>): Promise<UpdateResult>;
     search(options: SearchOptions<T["infer"]>): Promise<SearchResult<T["infer"]>>;
     searchList(options: SearchListOptions<T["infer"]>): Promise<SearchListResult<T["infer"]>>;
+    count(filter?: FilterFor<T["infer"]>): Promise<number>;
     upsert(docs: T["infer"] | T["infer"][]): Promise<UpsertResult[]>;
+    syncData(options?: SyncOptions): Promise<SyncResult>;
+    private purgeOrphans;
+    private exportIds;
 }
 export {};

package/dist/tsense.js

@@ -11,17 +11,28 @@ var _a;
 import redaxios from "redaxios";
 import { TSenseMigrator } from "./migrator.js";
 import { defaultTransformers } from "./transformers/defaults.js";
+function chunkArray(arr, size) {
+    const chunks = [];
+    for (let i = 0; i < arr.length; i += size) {
+        chunks.push(arr.slice(i, i + size));
+    }
+    return chunks;
+}
 const redaxiosInstance = (_a = redaxios.default) !== null && _a !== void 0 ? _a : redaxios;
 const arkToTsense = {
     string: "string",
     number: "float",
     "number.integer": "int64",
+    "number % 1": "int64",
     boolean: "bool",
     "string[]": "string[]",
     "number[]": "float[]",
     "boolean[]": "bool[]",
 };
 export class TSense {
+    getFields() {
+        return this.fields;
+    }
     constructor(options) {
         var _a;
         this.options = options;
@@ -34,6 +45,12 @@ export class TSense {
             headers: { "X-TYPESENSE-API-KEY": options.connection.apiKey },
         });
         this.extractFields((_a = options.transformers) !== null && _a !== void 0 ? _a : defaultTransformers);
+        this.dataSyncConfig = options.dataSync;
+    }
+    getBaseType(expression, domain) {
+        if (domain && domain !== "undefined")
+            return domain;
+        return expression.replace(/ \| undefined$/, "");
     }
     inferType(arkType) {
         const direct = arkToTsense[arkType];
@@ -41,10 +58,10 @@ export class TSense {
             return direct;
         if (arkType.includes("[]"))
             return "object[]";
+        if (arkType.includes("'") || arkType.includes('"'))
+            return "string";
         if (arkType.includes("{") || arkType.includes("|"))
             return "object";
-        if (arkType.startsWith("'") || arkType.includes("'"))
-            return "string";
         return "string";
     }
     serializeDoc(doc) {
@@ -89,13 +106,15 @@ export class TSense {
         return transformer.serialize(value);
     }
     extractFields(transformers) {
-        var _a;
+        var _a, _b, _c, _d;
         const internal = this.options.schema;
         for (const prop of internal.structure.props) {
-            const meta = prop.value.meta;
+            const innerType = (_b = (_a = prop.value.branches) === null || _a === void 0 ? void 0 : _a[0]) !== null && _b !== void 0 ? _b : prop.value;
+            const meta = ((_c = innerType.meta) !== null && _c !== void 0 ? _c : prop.value.meta);
             const expression = String(prop.value.expression);
             const domain = prop.value.domain;
-            const transformer = transformers.find((t) => t.match(expression, domain));
+            const baseType = this.getBaseType(expression, domain);
+            const transformer = transformers.find((t) => t.match(expression, domain) || t.match(baseType, domain));
             if (transformer) {
                 this.fieldTransformers.set(prop.key, transformer);
                 this.fields.push({
@@ -108,7 +127,7 @@ export class TSense {
                 });
                 continue;
             }
-            const type = (_a = meta === null || meta === void 0 ? void 0 : meta.type) !== null && _a !== void 0 ? _a : this.inferType(domain !== null && domain !== void 0 ? domain : expression);
+            const type = (_d = meta === null || meta === void 0 ? void 0 : meta.type) !== null && _d !== void 0 ? _d : this.inferType(baseType);
             this.fields.push({
                 name: prop.key,
                 type,
@@ -121,13 +140,13 @@ export class TSense {
     }
     ensureSynced(force) {
         return __awaiter(this, void 0, void 0, function* () {
-            if (!force && (this.synced || !this.options.autoSync))
+            if (!force && (this.synced || !this.options.autoSyncSchema))
                 return;
             yield new TSenseMigrator(this.options.name, this.fields, this.options.defaultSortingField, this.axios).sync();
             this.synced = true;
         });
     }
-    sync() {
+    syncSchema() {
         return __awaiter(this, void 0, void 0, function* () {
             yield this.ensureSynced(true);
         });
@@ -410,6 +429,30 @@ export class TSense {
             return { data, nextCursor, total: res.found };
         });
     }
+    count(filter) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.ensureSynced();
+            const filterBy = this.buildFilter(filter).join("&&");
+            if (!filterBy) {
+                const { data } = yield this.axios({
+                    method: "GET",
+                    url: `/collections/${this.options.name}`,
+                });
+                return data.num_documents;
+            }
+            const { data } = yield this.axios({
+                method: "GET",
+                url: `/collections/${this.options.name}/documents/search`,
+                params: {
+                    q: "*",
+                    query_by: this.options.defaultSearchField,
+                    per_page: 0,
+                    filter_by: filterBy,
+                },
+            });
+            return data.found;
+        });
+    }
     upsert(docs) {
         return __awaiter(this, void 0, void 0, function* () {
             yield this.ensureSynced();
@@ -439,4 +482,59 @@ export class TSense {
             return [data];
         });
     }
+    syncData(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
+            if (!this.dataSyncConfig) {
+                throw new Error("DATA_SYNC_NOT_CONFIGURED");
+            }
+            const chunkSize = (_b = (_a = options === null || options === void 0 ? void 0 : options.chunkSize) !== null && _a !== void 0 ? _a : this.dataSyncConfig.chunkSize) !== null && _b !== void 0 ? _b : 500;
+            const ids = (_c = options === null || options === void 0 ? void 0 : options.ids) !== null && _c !== void 0 ? _c : (yield this.dataSyncConfig.getAllIds());
+            let upserted = 0;
+            let failed = 0;
+            for (const chunk of chunkArray(ids, chunkSize)) {
+                const items = yield this.dataSyncConfig.getItems(chunk);
+                const results = yield this.upsert(items);
+                for (const r of results) {
+                    if (r.success)
+                        upserted++;
+                    else
+                        failed++;
+                }
+            }
+            let deleted = 0;
+            if (options === null || options === void 0 ? void 0 : options.purge) {
+                deleted = yield this.purgeOrphans(ids, chunkSize);
+            }
+            return { upserted, deleted, failed };
+        });
+    }
+    purgeOrphans(validIds, chunkSize) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const validSet = new Set(validIds);
+            const remoteIds = yield this.exportIds();
+            const orphans = remoteIds.filter((id) => !validSet.has(id));
+            if (!orphans.length)
+                return 0;
+            let deleted = 0;
+            for (const chunk of chunkArray(orphans, chunkSize)) {
+                const result = yield this.deleteMany({ id: chunk });
+                deleted += result.deleted;
+            }
+            return deleted;
+        });
+    }
+    exportIds() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { data } = yield this.axios({
+                method: "GET",
+                url: `/collections/${this.options.name}/documents/export`,
+                params: { include_fields: "id" },
+            });
+            return data
+                .split("\n")
+                .filter((line) => line.length)
+                .map((line) => JSON.parse(line).id);
+        });
+    }
 }

package/dist/types.d.ts

@@ -42,8 +42,9 @@ export type TsenseOptions<T extends Type> = {
     defaultSortingField?: keyof T["infer"];
     batchSize?: number;
     validateOnUpsert?: boolean;
-    autoSync?: boolean;
+    autoSyncSchema?: boolean;
     transformers?: FieldTransformer[];
+    dataSync?: SyncConfig<T["infer"]>;
 };
 type SingleFilter<T> = Partial<{
     [K in keyof T]: BaseIfArray<T[K]> | NonNullable<BaseIfArray<T[K]>>[] | {
@@ -116,4 +117,19 @@ export type SearchListResult<T> = {
     nextCursor: string | null;
     total: number;
 };
+export type SyncConfig<T> = {
+    getAllIds: () => Promise<string[]>;
+    getItems: (ids: string[]) => Promise<T[]>;
+    chunkSize?: number;
+};
+export type SyncOptions = {
+    ids?: string[];
+    purge?: boolean;
+    chunkSize?: number;
+};
+export type SyncResult = {
+    upserted: number;
+    deleted: number;
+    failed: number;
+};
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "tsense",
-	"version": "0.0.13",
+	"version": "0.0.15",
 	"private": false,
 	"description": "Opinionated, fully typed typesense client",
 	"keywords": [
@@ -42,11 +42,11 @@
 		"@changesets/cli": "^2.29.7",
 		"@types/bun": "latest",
 		"arktype": "^2.1.29",
-		"oxfmt": "^0.23.0",
-		"oxlint": "^1.38.0"
+		"oxfmt": "^0.28.0",
+		"oxlint": "^1.43.0"
 	},
 	"peerDependencies": {
 		"arktype": "^2.1.29",
 		"typescript": "^5"
 	}
-}
+}