trickle-backend 0.1.64 → 0.1.66

package/Dockerfile

@@ -0,0 +1,23 @@
+FROM node:20-alpine
+
+WORKDIR /app
+
+# Install dependencies
+COPY package.json package-lock.json* ./
+RUN npm ci --production 2>/dev/null || npm install --production
+
+# Copy built files
+COPY dist/ dist/
+
+# Create data directory
+RUN mkdir -p /data
+
+ENV PORT=4888
+ENV TRICKLE_DB_PATH=/data/trickle.db
+
+EXPOSE 4888
+
+HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
+  CMD wget -qO- http://localhost:4888/api/health || exit 1
+
+CMD ["node", "dist/index.js"]

package/dist/db/connection.js

@@ -7,9 +7,9 @@ exports.db = void 0;
 const path_1 = __importDefault(require("path"));
 const fs_1 = __importDefault(require("fs"));
 const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
-const trickleDir = path_1.default.join(process.env.HOME || "~", ".trickle");
+const dbPath = process.env.TRICKLE_DB_PATH || path_1.default.join(process.env.HOME || "~", ".trickle", "trickle.db");
+const trickleDir = path_1.default.dirname(dbPath);
 fs_1.default.mkdirSync(trickleDir, { recursive: true });
-const dbPath = path_1.default.join(trickleDir, "trickle.db");
 const db = new better_sqlite3_1.default(dbPath);
 exports.db = db;
 db.pragma("journal_mode = WAL");

package/dist/index.js

@@ -9,4 +9,35 @@ const cloud_migrations_1 = require("./db/cloud-migrations");
 const PORT = parseInt(process.env.PORT || "4888", 10);
 server_1.app.listen(PORT, () => {
     console.log(`[trickle] Backend listening on http://localhost:${PORT}`);
+    if (process.env.NODE_ENV === "production") {
+        console.log(`[trickle] Production mode enabled`);
+    }
 });
+// ── Data retention — periodic cleanup of expired data ──
+const RETENTION_DAYS = parseInt(process.env.TRICKLE_RETENTION_DAYS || "30", 10);
+const CLEANUP_INTERVAL_MS = 6 * 3600_000; // Every 6 hours
+function runDataRetention() {
+    try {
+        // Delete expired share links
+        const expiredLinks = connection_1.db.prepare("DELETE FROM share_links WHERE expires_at IS NOT NULL AND expires_at < datetime('now')").run();
+        // Delete old push history (keep last 30 days)
+        const oldHistory = connection_1.db.prepare(`DELETE FROM push_history WHERE pushed_at < datetime('now', '-${RETENTION_DAYS} days')`).run();
+        // Delete stale project data (not updated in retention period)
+        const staleData = connection_1.db.prepare(`DELETE FROM project_data WHERE pushed_at < datetime('now', '-${RETENTION_DAYS} days')`).run();
+        const total = (expiredLinks.changes || 0) + (oldHistory.changes || 0) + (staleData.changes || 0);
+        if (total > 0) {
+            console.log(`[trickle] Data retention: cleaned ${total} rows (${RETENTION_DAYS}d retention)`);
+            // Reclaim space
+            try {
+                connection_1.db.pragma("wal_checkpoint(TRUNCATE)");
+            }
+            catch { }
+        }
+    }
+    catch (err) {
+        console.error("[trickle] Data retention error:", err.message);
+    }
+}
+// Run retention on startup and periodically
+setTimeout(runDataRetention, 10_000); // 10s after startup
+setInterval(runDataRetention, CLEANUP_INTERVAL_MS);

package/dist/routes/cloud.js

@@ -70,7 +70,38 @@ router.post("/keys", (req, res) => {
         message: "Save this key — it cannot be retrieved later.",
     });
 });
-// ── POST /api/v1/push — Upload project data ──
+// ── POST /api/v1/ingest — Real-time streaming ingest ──
+// Accepts batched observations and appends to project data files.
+// This enables `trickle run` to stream data to the cloud in real-time.
+router.post("/ingest", requireAuth, (req, res) => {
+    const { project, file, lines } = req.body;
+    if (!project || !file || !lines) {
+        res.status(400).json({ error: "project, file, and lines required" });
+        return;
+    }
+    const projectId = `${req.keyId}:${project}`;
+    // Auto-create project
+    connection_1.db.prepare(`
+    INSERT INTO projects (id, name, owner_key_id, updated_at)
+    VALUES (?, ?, ?, datetime('now'))
+    ON CONFLICT(id) DO UPDATE SET updated_at = datetime('now')
+  `).run(projectId, project, req.keyId);
+    // Append to existing content (or create new)
+    const existing = connection_1.db.prepare("SELECT content FROM project_data WHERE project_id = ? AND filename = ?").get(projectId, file);
+    const newContent = typeof lines === "string" ? lines : lines.join("\n") + "\n";
+    const content = existing ? existing.content + newContent : newContent;
+    const bytes = Buffer.byteLength(content, "utf-8");
+    connection_1.db.prepare(`
+    INSERT INTO project_data (project_id, filename, content, size_bytes, pushed_at)
+    VALUES (?, ?, ?, ?, datetime('now'))
+    ON CONFLICT(project_id, filename) DO UPDATE SET
+      content = excluded.content,
+      size_bytes = excluded.size_bytes,
+      pushed_at = datetime('now')
+  `).run(projectId, file, content, bytes);
+    res.json({ ok: true, file, bytes });
+});
+// ── POST /api/v1/push — Upload project data (full replace) ──
 router.post("/push", requireAuth, (req, res) => {
     const { project, files, timestamp } = req.body;
     if (!project || typeof project !== "string") {

package/dist/server.js

@@ -21,8 +21,56 @@ const search_1 = __importDefault(require("./routes/search"));
 const cloud_1 = __importDefault(require("./routes/cloud"));
 const app = (0, express_1.default)();
 exports.app = app;
-app.use((0, cors_1.default)());
-app.use(express_1.default.json({ limit: "5mb" }));
+// ── Production middleware ──
+// CORS — allow all origins for local dev, restrict in production
+const allowedOrigins = process.env.TRICKLE_CORS_ORIGINS?.split(",") || [];
+app.use((0, cors_1.default)(allowedOrigins.length > 0 ? { origin: allowedOrigins } : {}));
+// Body size limits
+app.use(express_1.default.json({ limit: "10mb" }));
+// Rate limiting — simple in-memory token bucket per IP
+const rateLimits = new Map();
+const RATE_LIMIT_WINDOW_MS = 60_000; // 1 minute
+const RATE_LIMIT_MAX = parseInt(process.env.TRICKLE_RATE_LIMIT || "300", 10); // 300 req/min default
+function rateLimit(req, res, next) {
+    if (process.env.NODE_ENV !== "production" && !process.env.TRICKLE_RATE_LIMIT) {
+        return next(); // Skip rate limiting in dev unless explicitly enabled
+    }
+    const ip = req.ip || req.socket.remoteAddress || "unknown";
+    const now = Date.now();
+    let bucket = rateLimits.get(ip);
+    if (!bucket || now > bucket.resetAt) {
+        bucket = { count: 0, resetAt: now + RATE_LIMIT_WINDOW_MS };
+        rateLimits.set(ip, bucket);
+    }
+    bucket.count++;
+    if (bucket.count > RATE_LIMIT_MAX) {
+        res.status(429).json({ error: "Rate limit exceeded. Try again later." });
+        return;
+    }
+    // Periodic cleanup of old entries
+    if (rateLimits.size > 10000) {
+        for (const [key, val] of rateLimits) {
+            if (now > val.resetAt)
+                rateLimits.delete(key);
+        }
+    }
+    next();
+}
+app.use("/api/v1", rateLimit);
+// Request logging in production
+if (process.env.NODE_ENV === "production") {
+    app.use((req, _res, next) => {
+        const start = Date.now();
+        _res.on("finish", () => {
+            const ms = Date.now() - start;
+            if (ms > 1000 || _res.statusCode >= 400) {
+                console.log(`${req.method} ${req.path} ${_res.statusCode} ${ms}ms`);
+            }
+        });
+        next();
+    });
+}
+// ── Routes ──
 app.use("/api/ingest", ingest_1.default);
 app.use("/api/functions", functions_1.default);
 app.use("/api/types", types_1.default);
@@ -38,5 +86,10 @@ app.use("/api/search", search_1.default);
 app.use("/api/v1", cloud_1.default);
 // Health check
 app.get("/api/health", (_req, res) => {
-    res.json({ ok: true, timestamp: new Date().toISOString() });
+    res.json({ ok: true, timestamp: new Date().toISOString(), version: process.env.npm_package_version || "dev" });
+});
+// Global error handler
+app.use((err, _req, res, _next) => {
+    console.error("[trickle] Unhandled error:", err.message);
+    res.status(500).json({ error: "Internal server error" });
 });

package/fly.toml

@@ -0,0 +1,31 @@
+app = "trickle-cloud"
+primary_region = "lhr"
+
+[build]
+  dockerfile = "Dockerfile"
+
+[env]
+  PORT = "4888"
+  TRICKLE_DB_PATH = "/data/trickle.db"
+  NODE_ENV = "production"
+
+[http_service]
+  internal_port = 4888
+  force_https = true
+  auto_stop_machines = "stop"
+  auto_start_machines = true
+  min_machines_running = 0
+
+  [http_service.concurrency]
+    type = "connections"
+    hard_limit = 100
+    soft_limit = 80
+
+[mounts]
+  source = "trickle_data"
+  destination = "/data"
+
+[[vm]]
+  memory = "512mb"
+  cpu_kind = "shared"
+  cpus = 1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trickle-backend",
-  "version": "0.1.64",
+  "version": "0.1.66",
   "main": "dist/index.js",
   "scripts": {
     "build": "tsc",

package/src/db/connection.ts

@@ -2,12 +2,11 @@ import path from "path";
 import fs from "fs";
 import Database, { Database as DatabaseType } from "better-sqlite3";
 
-const trickleDir = path.join(process.env.HOME || "~", ".trickle");
+const dbPath = process.env.TRICKLE_DB_PATH || path.join(process.env.HOME || "~", ".trickle", "trickle.db");
+const trickleDir = path.dirname(dbPath);
 
 fs.mkdirSync(trickleDir, { recursive: true });
 
-const dbPath = path.join(trickleDir, "trickle.db");
-
 const db: DatabaseType = new Database(dbPath);
 
 db.pragma("journal_mode = WAL");

package/src/index.ts

@@ -10,4 +10,44 @@ const PORT = parseInt(process.env.PORT || "4888", 10);
 
 app.listen(PORT, () => {
   console.log(`[trickle] Backend listening on http://localhost:${PORT}`);
+  if (process.env.NODE_ENV === "production") {
+    console.log(`[trickle] Production mode enabled`);
+  }
 });
+
+// ── Data retention — periodic cleanup of expired data ──
+
+const RETENTION_DAYS = parseInt(process.env.TRICKLE_RETENTION_DAYS || "30", 10);
+const CLEANUP_INTERVAL_MS = 6 * 3600_000; // Every 6 hours
+
+function runDataRetention(): void {
+  try {
+    // Delete expired share links
+    const expiredLinks = db.prepare(
+      "DELETE FROM share_links WHERE expires_at IS NOT NULL AND expires_at < datetime('now')"
+    ).run();
+
+    // Delete old push history (keep last 30 days)
+    const oldHistory = db.prepare(
+      `DELETE FROM push_history WHERE pushed_at < datetime('now', '-${RETENTION_DAYS} days')`
+    ).run();
+
+    // Delete stale project data (not updated in retention period)
+    const staleData = db.prepare(
+      `DELETE FROM project_data WHERE pushed_at < datetime('now', '-${RETENTION_DAYS} days')`
+    ).run();
+
+    const total = (expiredLinks.changes || 0) + (oldHistory.changes || 0) + (staleData.changes || 0);
+    if (total > 0) {
+      console.log(`[trickle] Data retention: cleaned ${total} rows (${RETENTION_DAYS}d retention)`);
+      // Reclaim space
+      try { db.pragma("wal_checkpoint(TRUNCATE)"); } catch {}
+    }
+  } catch (err: any) {
+    console.error("[trickle] Data retention error:", err.message);
+  }
+}
+
+// Run retention on startup and periodically
+setTimeout(runDataRetention, 10_000); // 10s after startup
+setInterval(runDataRetention, CLEANUP_INTERVAL_MS);

package/src/routes/cloud.ts

@@ -94,7 +94,49 @@ router.post("/keys", (req: Request, res: Response) => {
   });
 });
 
-// ── POST /api/v1/push — Upload project data ──
+// ── POST /api/v1/ingest — Real-time streaming ingest ──
+// Accepts batched observations and appends to project data files.
+// This enables `trickle run` to stream data to the cloud in real-time.
+
+router.post("/ingest", requireAuth, (req: AuthedRequest, res: Response) => {
+  const { project, file, lines } = req.body;
+
+  if (!project || !file || !lines) {
+    res.status(400).json({ error: "project, file, and lines required" });
+    return;
+  }
+
+  const projectId = `${req.keyId}:${project}`;
+
+  // Auto-create project
+  db.prepare(`
+    INSERT INTO projects (id, name, owner_key_id, updated_at)
+    VALUES (?, ?, ?, datetime('now'))
+    ON CONFLICT(id) DO UPDATE SET updated_at = datetime('now')
+  `).run(projectId, project, req.keyId);
+
+  // Append to existing content (or create new)
+  const existing = db.prepare(
+    "SELECT content FROM project_data WHERE project_id = ? AND filename = ?"
+  ).get(projectId, file) as any;
+
+  const newContent = typeof lines === "string" ? lines : (lines as string[]).join("\n") + "\n";
+  const content = existing ? existing.content + newContent : newContent;
+  const bytes = Buffer.byteLength(content, "utf-8");
+
+  db.prepare(`
+    INSERT INTO project_data (project_id, filename, content, size_bytes, pushed_at)
+    VALUES (?, ?, ?, ?, datetime('now'))
+    ON CONFLICT(project_id, filename) DO UPDATE SET
+      content = excluded.content,
+      size_bytes = excluded.size_bytes,
+      pushed_at = datetime('now')
+  `).run(projectId, file, content, bytes);
+
+  res.json({ ok: true, file, bytes });
+});
+
+// ── POST /api/v1/push — Upload project data (full replace) ──
 
 router.post("/push", requireAuth, (req: AuthedRequest, res: Response) => {
   const { project, files, timestamp } = req.body;

package/src/server.ts

@@ -1,4 +1,4 @@
-import express from "express";
+import express, { Request, Response, NextFunction } from "express";
 import cors from "cors";
 
 import ingestRouter from "./routes/ingest";
@@ -17,8 +17,67 @@ import cloudRouter from "./routes/cloud";
 
 const app = express();
 
-app.use(cors());
-app.use(express.json({ limit: "5mb" }));
+// ── Production middleware ──
+
+// CORS — allow all origins for local dev, restrict in production
+const allowedOrigins = process.env.TRICKLE_CORS_ORIGINS?.split(",") || [];
+app.use(cors(allowedOrigins.length > 0 ? { origin: allowedOrigins } : {}));
+
+// Body size limits
+app.use(express.json({ limit: "10mb" }));
+
+// Rate limiting — simple in-memory token bucket per IP
+const rateLimits = new Map<string, { count: number; resetAt: number }>();
+const RATE_LIMIT_WINDOW_MS = 60_000; // 1 minute
+const RATE_LIMIT_MAX = parseInt(process.env.TRICKLE_RATE_LIMIT || "300", 10); // 300 req/min default
+
+function rateLimit(req: Request, res: Response, next: NextFunction): void {
+  if (process.env.NODE_ENV !== "production" && !process.env.TRICKLE_RATE_LIMIT) {
+    return next(); // Skip rate limiting in dev unless explicitly enabled
+  }
+
+  const ip = req.ip || req.socket.remoteAddress || "unknown";
+  const now = Date.now();
+  let bucket = rateLimits.get(ip);
+
+  if (!bucket || now > bucket.resetAt) {
+    bucket = { count: 0, resetAt: now + RATE_LIMIT_WINDOW_MS };
+    rateLimits.set(ip, bucket);
+  }
+
+  bucket.count++;
+  if (bucket.count > RATE_LIMIT_MAX) {
+    res.status(429).json({ error: "Rate limit exceeded. Try again later." });
+    return;
+  }
+
+  // Periodic cleanup of old entries
+  if (rateLimits.size > 10000) {
+    for (const [key, val] of rateLimits) {
+      if (now > val.resetAt) rateLimits.delete(key);
+    }
+  }
+
+  next();
+}
+
+app.use("/api/v1", rateLimit);
+
+// Request logging in production
+if (process.env.NODE_ENV === "production") {
+  app.use((req: Request, _res: Response, next: NextFunction) => {
+    const start = Date.now();
+    _res.on("finish", () => {
+      const ms = Date.now() - start;
+      if (ms > 1000 || _res.statusCode >= 400) {
+        console.log(`${req.method} ${req.path} ${_res.statusCode} ${ms}ms`);
+      }
+    });
+    next();
+  });
+}
+
+// ── Routes ──
 
 app.use("/api/ingest", ingestRouter);
 app.use("/api/functions", functionsRouter);
@@ -36,7 +95,13 @@ app.use("/api/v1", cloudRouter);
 
 // Health check
 app.get("/api/health", (_req, res) => {
-  res.json({ ok: true, timestamp: new Date().toISOString() });
+  res.json({ ok: true, timestamp: new Date().toISOString(), version: process.env.npm_package_version || "dev" });
+});
+
+// Global error handler
+app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => {
+  console.error("[trickle] Unhandled error:", err.message);
+  res.status(500).json({ error: "Internal server error" });
 });
 
 export { app };