wn-turso 0.1.0 → 0.2.0

package/README.md

@@ -0,0 +1,385 @@
+# wn-turso
+
+Turso/libsql database backend and data pipeline utilities for WordNet.
+
+## Features
+
+- **Turso Database Integration** - Connect to Turso via HTTP (remote) or embedded replicas (local SQLite with sync)
+- **Kysely Dialect** - Full Kysely support for type-safe SQL queries
+- **Data Pipelines** - Functional, streaming data transfer between databases
+- **Working DB Pattern** - Transform and extend data for runtime use cases
+
+## Installation
+
+```bash
+pnpm add wn-turso
+```
+
+## Usage
+
+### Basic WordNet Queries
+
+```typescript
+import { TursoWordnet } from "wn-turso";
+
+// Remote mode (serverless)
+const wn = new TursoWordnet({
+  url: "libsql://your-db.turso.io",
+  authToken: process.env.TURSO_AUTH_TOKEN,
+  mode: "remote",
+});
+
+await wn.initialize();
+
+const synsets = await wn.synsets({ form: "computer", language: "en" });
+const words = await wn.words({ form: "run", pos: "v" });
+
+await wn.close();
+```
+
+### Embedded Replicas (Offline-capable)
+
+```typescript
+import { TursoWordnet } from "wn-turso";
+
+const wn = new TursoWordnet({
+  url: "file:./local-wordnet.db",
+  syncUrl: "libsql://your-db.turso.io",
+  authToken: process.env.TURSO_AUTH_TOKEN,
+  mode: "embedded",
+});
+
+await wn.initialize();
+
+// Queries run against local SQLite
+const synsets = await wn.synsets({ form: "computer" });
+
+// Sync with remote when needed
+await wn.sync();
+
+await wn.close();
+```
+
+### Direct Database Access
+
+```typescript
+import { TursoDatabase } from "wn-turso";
+
+const db = new TursoDatabase({
+  url: "libsql://your-db.turso.io",
+  authToken: process.env.TURSO_AUTH_TOKEN,
+  mode: "remote",
+});
+
+await db.initialize();
+
+// Get Kysely instance for custom queries
+const kysely = db.getDatabase();
+const results = await kysely
+  .selectFrom("synsets")
+  .where("language", "=", "en")
+  .selectAll()
+  .execute();
+
+// Get query service with all WordNet methods
+const qs = db.getQueryService();
+const lexicons = await qs.getLexicons();
+
+await db.close();
+```
+
+## Pipeline System
+
+The pipeline system enables streaming data transfers between databases with functional transformations.
+
+### Basic Pipeline
+
+```typescript
+import { Pipeline, tursoSource, tursoSink } from "wn-turso/pipeline";
+
+const result = await Pipeline.from(tursoSource(sourceConfig, "synsets"))
+  .filter((row) => row.language === "en")
+  .map((row) => ({ ...row, processed: true }))
+  .to(tursoSink(destConfig, "synsets_backup"));
+
+console.log(`Transferred ${result.inserted} rows in ${result.duration}ms`);
+```
+
+### Working DB Pattern (Source of Truth → Runtime DB)
+
+```typescript
+import { Pipeline, tursoSource, tursoSink } from "wn-turso/pipeline";
+
+// Transfer from canonical DB to working DB with additional columns
+const result = await Pipeline.from(tursoSource(sourceConfig, "synsets"))
+  .filter((row) => row.language === "en")
+  .extend((row) => ({
+    // Add runtime columns
+    cached_count: 0,
+    last_accessed: null,
+    popularity_score: computeScore(row),
+    source_db: "turso-main",
+  }))
+  .to(
+    tursoSink(workingConfig, "synsets_working", {
+      onConflict: "replace",
+    })
+  );
+```
+
+### Checksum Deduplication (Skip Unchanged Rows)
+
+Reduce write operations by skipping rows that haven't changed using checksum-based deduplication.
+
+#### Operator-Level (Manual Control)
+
+```typescript
+import {
+  Pipeline,
+  tursoSource,
+  tursoSink,
+  loadChecksumsFromTable,
+} from "wn-turso/pipeline";
+
+// Load existing checksums from destination
+const db = await getDestinationKysely();
+const existingChecksums = await loadChecksumsFromTable(
+  db,
+  "synsets_working",
+  "id", // Key field
+  "_etl_checksum" // Checksum column
+);
+
+// Pipeline will skip rows with matching checksums
+const result = await Pipeline.from(tursoSource(sourceConfig, "synsets"))
+  .filter((row) => row.language === "en")
+  .deduplicateByChecksum({
+    keyField: "id",
+    checksumFields: ["definition", "pos", "ili"], // Hash these fields only
+    existingChecksums,
+  })
+  .to(tursoSink(destConfig, "synsets_working"));
+
+console.log(`Skipped ${result.skipped_unchanged} unchanged rows`);
+```
+
+#### Sink-Level (Automatic)
+
+```typescript
+import { Pipeline, tursoSource, tursoSink } from "wn-turso/pipeline";
+
+// Sink automatically compares checksums and skips unchanged rows
+const result = await Pipeline.from(tursoSource(sourceConfig, "synsets"))
+  .filter((row) => row.language === "en")
+  .to(
+    tursoSink(destConfig, "synsets_working", {
+      checksumDeduplication: {
+        enabled: true,
+        keyField: "id",
+        fields: ["definition", "pos", "ili"], // Hash business fields only
+        checksumColumn: "_etl_checksum", // Column to store checksum
+        strategy: "skip", // 'skip' = don't write, 'update' = write anyway
+      },
+    })
+  );
+
+console.log(
+  `Processed: ${result.processed}, Skipped: ${result.skipped_unchanged}`
+);
+```
+
+**Benefits:**
+
+- **90%+ write reduction** on incremental syncs (typical)
+- **Selective field hashing** - ignore metadata like timestamps
+- **Production-proven pattern** - used by Airbyte, Fivetran, dbt
+
+### Pipeline Operators
+
+```typescript
+import { Pipeline, arraySource } from 'wn-turso/pipeline';
+
+await Pipeline
+  .from(arraySource(data))
+  // Filtering
+  .filter(row => row.active)
+
+  // Transformation
+  .map(row => ({ ...row, upper: row.name.toUpperCase() }))
+
+  // Extend with new columns
+  .extend(row => ({ score: row.value * 2 }))
+
+  // Transform with null filtering
+  .transform(row => row.valid ? row : null)
+
+  // Side effects (logging, metrics)
+  .tap(row => console.log('Processing:', row.id))
+
+  // Pagination
+  .skip(10)
+  .take(100)
+
+  // Deduplication by key
+  .distinct(row => row.id)
+
+  // Checksum-based deduplication (skip unchanged)
+  .deduplicateByChecksum({ keyField: 'id', existingChecksums, checksumFields: [...] })
+
+  // Batching
+  .batch(50)
+
+  // Terminal operations
+  .toArray();      // Collect to array
+  .count();        // Count rows
+  .to(sink);       // Write to sink
+  .forEach(fn);    // Execute for each
+```
+
+### Low-Level Streaming
+
+```typescript
+import {
+  streamTable,
+  writeBatches,
+  tursoSource,
+  kyselySource,
+} from "wn-turso/pipeline";
+
+// Stream from any Kysely database
+for await (const row of streamTable(kysely, "synsets", { batchSize: 1000 })) {
+  // Process row
+}
+
+// Write in batches
+await writeBatches(kysely, "synsets", asyncIterable, {
+  batchSize: 100,
+  onConflict: "ignore",
+});
+
+// Use with existing Kysely instance
+const source = kyselySource(existingKysely, "words");
+```
+
+## Configuration
+
+### TursoDatabaseConfig
+
+```typescript
+interface TursoDatabaseConfig {
+  // Database URL
+  // Remote: 'libsql://your-db.turso.io'
+  // Embedded: 'file:./local.db'
+  url: string;
+
+  // Auth token (required for remote, optional for embedded without sync)
+  authToken?: string;
+
+  // Connection mode
+  mode: "remote" | "embedded";
+
+  // For embedded: remote URL to sync with
+  syncUrl?: string;
+
+  // Sync interval in ms (0 = manual only)
+  syncInterval?: number;
+
+  // Encryption key for embedded replicas
+  encryptionKey?: string;
+
+  // Read-only mode
+  readonly?: boolean;
+
+  // Enable foreign keys
+  enableForeignKeys?: boolean;
+}
+```
+
+## API Reference
+
+### TursoWordnet
+
+Main class for WordNet queries.
+
+| Method              | Description           |
+| ------------------- | --------------------- |
+| `initialize()`      | Connect to database   |
+| `synsets(options?)` | Query synsets         |
+| `words(options?)`   | Query words           |
+| `senses(options?)`  | Query senses          |
+| `lexicons()`        | Get all lexicons      |
+| `sync()`            | Sync embedded replica |
+| `close()`           | Close connection      |
+
+### TursoDatabase
+
+Low-level database wrapper.
+
+| Method              | Description              |
+| ------------------- | ------------------------ |
+| `initialize()`      | Connect and setup schema |
+| `getDatabase()`     | Get Kysely instance      |
+| `getQueryService()` | Get query service        |
+| `getAdapter()`      | Get underlying adapter   |
+| `sync()`            | Sync embedded replica    |
+| `close()`           | Close connection         |
+
+### Pipeline
+
+Fluent pipeline builder.
+
+| Method                           | Description                     |
+| -------------------------------- | ------------------------------- |
+| `from(source)`                   | Create pipeline from source     |
+| `filter(predicate)`              | Filter rows                     |
+| `map(fn)`                        | Transform rows                  |
+| `extend(fn)`                     | Add properties to rows          |
+| `transform(fn)`                  | Transform with null filtering   |
+| `tap(fn)`                        | Side effect                     |
+| `take(n)`                        | Take first n rows               |
+| `skip(n)`                        | Skip first n rows               |
+| `distinct(keyFn)`                | Deduplicate by key              |
+| `deduplicateByChecksum(options)` | Skip unchanged rows by checksum |
+| `batch(size)`                    | Batch into arrays               |
+| `to(sink)`                       | Write to sink                   |
+| `toArray()`                      | Collect to array                |
+| `count()`                        | Count rows                      |
+
+### Checksum Utilities
+
+Functions for manual checksum management.
+
+| Function                                                        | Description                             |
+| --------------------------------------------------------------- | --------------------------------------- |
+| `computeChecksum(data, fields?)`                                | Compute MD5 hash of row                 |
+| `checksumBatch(rows, fields?)`                                  | Compute checksums for array             |
+| `loadChecksumsFromTable(db, table, keyField, checksumColumn)`   | Load existing checksums from DB         |
+| `compareChecksums(rows, keyField, existingChecksums, fields?)`  | Compare rows against existing checksums |
+| `filterChangedRows(rows, keyField, existingChecksums, fields?)` | Filter to only changed/new rows         |
+
+## CLI
+
+```bash
+# Upload local SQLite to Turso
+wn-turso upload --url libsql://db.turso.io --token $TOKEN --input ./wordnet.db
+
+# Sync embedded replica
+wn-turso sync --url file:./local.db --token $TOKEN
+```
+
+## Testing
+
+```bash
+# Run all tests
+pnpm test
+
+# Run e2e tests only
+pnpm test:e2e
+
+# Watch mode
+pnpm test:watch
+```
+
+## License
+
+MIT

package/dist/adapters.d.ts

@@ -0,0 +1,2 @@
+export * from './src/adapters/index'
+export {}

package/dist/adapters.js

@@ -0,0 +1,5 @@
+import { E as d, R as o } from "./embedded-adapter-aWF095Qo.js";
+export {
+  d as EmbeddedTursoAdapter,
+  o as RemoteTursoAdapter
+};

package/dist/cli/index.d.ts

@@ -1,6 +1 @@
-#!/usr/bin/env node
-/**
- * wn-turso CLI
- */
-export {};
-//# sourceMappingURL=index.d.ts.map
+export {}

package/dist/cli/index.js

@@ -1,37 +1,31 @@
 #!/usr/bin/env node
-/**
- * wn-turso CLI
- */
-import { parseArgs } from 'util';
-async function main() {
-    const { values, positionals } = parseArgs({
-        allowPositionals: true,
-        options: {
-            help: { type: 'boolean', short: 'h' },
-            url: { type: 'string' },
-            token: { type: 'string', short: 't' },
-            input: { type: 'string', short: 'i' },
-            output: { type: 'string', short: 'o' },
-            verbose: { type: 'boolean', short: 'v' },
-        },
-    });
-    const command = positionals[0];
-    switch (command) {
-        case 'upload':
-            console.log('Upload command - not yet implemented');
-            console.log('Will upload a local SQLite database to Turso');
-            break;
-        case 'sync':
-            console.log('Sync command - not yet implemented');
-            console.log('Will sync an embedded replica with Turso cloud');
-            break;
-        case 'help':
-        default:
-            showHelp();
+import { parseArgs as e } from "util";
+async function t() {
+  const { positionals: o } = e({
+    allowPositionals: !0,
+    options: {
+      help: { type: "boolean", short: "h" },
+      url: { type: "string" },
+      token: { type: "string", short: "t" },
+      input: { type: "string", short: "i" },
+      output: { type: "string", short: "o" },
+      verbose: { type: "boolean", short: "v" }
     }
+  });
+  switch (o[0]) {
+    case "upload":
+      console.log("Upload command - not yet implemented"), console.log("Will upload a local SQLite database to Turso");
+      break;
+    case "sync":
+      console.log("Sync command - not yet implemented"), console.log("Will sync an embedded replica with Turso cloud");
+      break;
+    case "help":
+    default:
+      s();
+  }
 }
-function showHelp() {
-    console.log(`
+function s() {
+  console.log(`
 wn-turso - Turso database tools for WordNet
 
 USAGE:
@@ -55,7 +49,6 @@ EXAMPLES:
   wn-turso sync --url file:./local.db --token $TURSO_TOKEN
 `);
 }
-main().catch((error) => {
-    console.error('Error:', error.message);
-    process.exit(1);
+t().catch((o) => {
+  console.error("Error:", o.message), process.exit(1);
 });

package/dist/config.d.ts

@@ -1,77 +1,2 @@
-/**
- * Turso database configuration
- */
-import type { BaseDatabaseConfig } from 'wn-ts-core/shared';
-/**
- * Connection mode for Turso
- * - 'remote': HTTP-only connection to Turso cloud
- * - 'embedded': Local SQLite replica with Turso sync
- */
-export type TursoConnectionMode = 'remote' | 'embedded';
-/**
- * Turso-specific database configuration
- */
-export interface TursoDatabaseConfig extends BaseDatabaseConfig {
-    /**
-     * Turso database URL
-     * - For remote: libsql://db-org.turso.io
-     * - For embedded: file:./local.db
-     */
-    url: string;
-    /**
-     * Auth token for Turso connection
-     * Required for remote connections and embedded sync
-     */
-    authToken?: string;
-    /**
-     * Connection mode
-     * - 'remote': HTTP-only connection to Turso cloud
-     * - 'embedded': Local SQLite with Turso sync
-     */
-    mode: TursoConnectionMode;
-    /**
-     * For embedded replicas: remote Turso URL to sync with
-     */
-    syncUrl?: string;
-    /**
-     * Sync interval in milliseconds (for embedded mode)
-     * Default: 0 (manual sync only)
-     */
-    syncInterval?: number;
-    /**
-     * Encryption key for embedded replicas
-     */
-    encryptionKey?: string;
-    /**
-     * Connection timeout in milliseconds
-     */
-    timeout?: number;
-    /**
-     * Enable foreign key constraints
-     */
-    enableForeignKeys?: boolean;
-}
-/**
- * Sync configuration for embedded replicas
- */
-export interface SyncConfig {
-    /** Enable automatic sync */
-    autoSync: boolean;
-    /** Sync interval in ms */
-    interval: number;
-    /** Sync on startup */
-    syncOnStart: boolean;
-    /** Sync on close */
-    syncOnClose: boolean;
-    /** Maximum retry attempts */
-    maxRetries: number;
-}
-/**
- * Default sync configuration
- */
-export declare const defaultSyncConfig: SyncConfig;
-/**
- * Validate Turso configuration
- */
-export declare function validateTursoConfig(config: TursoDatabaseConfig): void;
-//# sourceMappingURL=config.d.ts.map
+export * from './src/config'
+export {}

package/dist/config.js

@@ -1,37 +1,27 @@
-/**
- * Turso database configuration
- */
-/**
- * Default sync configuration
- */
-export const defaultSyncConfig = {
-    autoSync: false,
-    interval: 0,
-    syncOnStart: true,
-    syncOnClose: true,
-    maxRetries: 3,
+const t = {
+  autoSync: !1,
+  interval: 0,
+  syncOnStart: !0,
+  syncOnClose: !0,
+  maxRetries: 3
 };
-/**
- * Validate Turso configuration
- */
-export function validateTursoConfig(config) {
-    if (!config.url) {
-        throw new Error('Turso config: url is required');
-    }
-    if (config.mode === 'remote') {
-        if (!config.authToken) {
-            throw new Error('Turso config: authToken is required for remote mode');
-        }
-        if (!config.url.startsWith('libsql://') && !config.url.startsWith('https://')) {
-            throw new Error('Turso config: remote URL must start with libsql:// or https://');
-        }
-    }
-    if (config.mode === 'embedded') {
-        if (!config.url.startsWith('file:')) {
-            throw new Error('Turso config: embedded URL must start with file:');
-        }
-        if (config.syncUrl && !config.authToken) {
-            throw new Error('Turso config: authToken is required when syncUrl is provided');
-        }
-    }
+function e(r) {
+  if (!r.url)
+    throw new Error("Turso config: url is required");
+  if (r.mode === "remote") {
+    if (!r.authToken)
+      throw new Error("Turso config: authToken is required for remote mode");
+    if (!r.url.startsWith("libsql://") && !r.url.startsWith("https://"))
+      throw new Error("Turso config: remote URL must start with libsql:// or https://");
+  }
+  if (r.mode === "embedded") {
+    if (!r.url.startsWith("file:"))
+      throw new Error("Turso config: embedded URL must start with file:");
+    if (r.syncUrl && !r.authToken)
+      throw new Error("Turso config: authToken is required when syncUrl is provided");
+  }
 }
+export {
+  t as defaultSyncConfig,
+  e as validateTursoConfig
+};

package/dist/database.d.ts

@@ -0,0 +1,2 @@
+export * from './src/database/index'
+export {}

package/dist/database.js

@@ -0,0 +1,8 @@
+import { d as s, T as e, b as o, a as u, c as T } from "./turso-database-CNWM_LV6.js";
+export {
+  s as TursoConnection,
+  e as TursoDatabase,
+  o as TursoDriver,
+  u as TursoQueryService,
+  T as createTursoDialect
+};