supastash 0.1.62 → 0.2.0

package/dist/utils/reusedHelpers.js

@@ -0,0 +1,162 @@
+const validOperators = new Set([
+    "eq",
+    "neq",
+    "gt",
+    "lt",
+    "gte",
+    "lte",
+    "is",
+    "in",
+]);
+export const ReusedHelpers = {
+    isValidFilter(filters) {
+        if (!Array.isArray(filters))
+            return false;
+        const validateNode = (node) => {
+            if (!node || typeof node !== "object")
+                return false;
+            // OR group
+            if ("or" in node) {
+                if (!Array.isArray(node.or) || node.or.length === 0) {
+                    return false;
+                }
+                return node.or.every(validateNode);
+            }
+            // Condition
+            const { column, operator, value } = node;
+            if (typeof column !== "string" || column.trim() === "") {
+                return false;
+            }
+            if (!validOperators.has(operator)) {
+                return false;
+            }
+            switch (operator) {
+                case "is":
+                    return (value === null ||
+                        value === true ||
+                        value === false ||
+                        value === "null" ||
+                        value === "true" ||
+                        value === "false");
+                case "in":
+                    return Array.isArray(value) && value.length > 0;
+                default:
+                    return value !== undefined && value !== null;
+            }
+        };
+        return filters.every(validateNode);
+    },
+    applyFilters(q, filters, table) {
+        for (const f of filters) {
+            // OR group
+            if ("or" in f) {
+                if (!Array.isArray(f.or) || f.or.length === 0) {
+                    throw new Error(`Invalid OR filter for ${table}`);
+                }
+                const orString = f.or
+                    .map((sub) => {
+                    if (!ReusedHelpers.isValidFilter([sub])) {
+                        throw new Error(`Invalid OR syncFilter: ${JSON.stringify(sub)} for ${table}`);
+                    }
+                    return `${String(sub.column)}.${sub.operator}.${sub.value}`;
+                })
+                    .join(",");
+                q = q.or(orString);
+                continue;
+            }
+            // Normal AND filter
+            if (!ReusedHelpers.isValidFilter([f])) {
+                throw new Error(`Invalid syncFilter: ${JSON.stringify(f)} for ${table}`);
+            }
+            q = q[f.operator](f.column, f.value);
+        }
+        return q;
+    },
+    buildFilterString(filter) {
+        if (!filter)
+            return undefined;
+        // OR group
+        if ("or" in filter) {
+            if (!Array.isArray(filter.or) || filter.or.length === 0) {
+                return undefined;
+            }
+            const inner = filter.or
+                .map((f) => {
+                if ("and" in f || "or" in f) {
+                    return undefined; // nested groups not supported here
+                }
+                if (f.value === null) {
+                    return `${String(f.column)}.${f.operator}.null`;
+                }
+                if (f.operator === "in" && Array.isArray(f.value)) {
+                    return `${String(f.column)}.in.(${f.value.join(",")})`;
+                }
+                return `${String(f.column)}.${f.operator}.${f.value}`;
+            })
+                .filter(Boolean)
+                .join(",");
+            return inner ? `or=(${inner})` : undefined;
+        }
+        // Condition
+        const { column, operator, value } = filter;
+        if (value === null) {
+            return `${String(column)}=${operator}.null`;
+        }
+        if (operator === "in" && Array.isArray(value)) {
+            return `${String(column)}=in.(${value.join(",")})`;
+        }
+        return `${String(column)}=${operator}.${value}`;
+    },
+    buildFilterForSql(filter) {
+        if (!filter)
+            return undefined;
+        // OR group
+        if ("or" in filter) {
+            const parts = filter.or
+                .map(ReusedHelpers.buildFilterForSql)
+                .filter(Boolean);
+            if (parts.length === 0)
+                return undefined;
+            return `(${parts.join(" OR ")})`;
+        }
+        const { column, operator, value } = filter;
+        switch (operator) {
+            case "eq":
+                return value === null
+                    ? `${String(column)} IS NULL`
+                    : `${String(column)} = ${sqlValue(value)}`;
+            case "neq":
+                return value === null
+                    ? `${String(column)} IS NOT NULL`
+                    : `${String(column)} != ${sqlValue(value)}`;
+            case "gt":
+                return `${String(column)} > ${sqlValue(value)}`;
+            case "lt":
+                return `${String(column)} < ${sqlValue(value)}`;
+            case "gte":
+                return `${String(column)} >= ${sqlValue(value)}`;
+            case "lte":
+                return `${String(column)} <= ${sqlValue(value)}`;
+            case "in":
+                if (!Array.isArray(value))
+                    return undefined;
+                return `${String(column)} IN (${value.map(sqlValue).join(", ")})`;
+            case "is":
+                return `${String(column)} IS ${sqlValue(value)}`;
+            default:
+                throw new Error(`Unsupported operator: ${operator}`);
+        }
+    },
+};
+function sqlValue(val) {
+    if (Array.isArray(val)) {
+        return val.map(sqlValue).join(", ");
+    }
+    if (typeof val === "boolean")
+        return val ? "1" : "0";
+    if (val === null)
+        return "NULL";
+    if (typeof val === "number")
+        return val.toString();
+    return `'${val.replace(/'/g, "''")}'`; // Escape single quotes
+}

package/dist/utils/schema/createSyncStatus.d.ts

@@ -5,9 +5,11 @@
  * @deprecated Use createSyncStatusTable instead
  */
 export declare function createDeletedStatusTable(): Promise<void>;
-export declare const SYNC_STATUS_TABLES_SQL = "\n  CREATE TABLE IF NOT EXISTS supastash_sync_marks (\n  table_name       TEXT NOT NULL,\n  filter_key       TEXT NOT NULL,         \n  filter_json      TEXT NULL,             \n  last_created_at  TEXT NULL,             \n  last_synced_at   TEXT NULL,   \n  last_synced_at_pk     TEXT NULL,          \n  last_deleted_at  TEXT NULL,             \n  updated_at       TEXT NOT NULL DEFAULT (datetime('now')),\n  PRIMARY KEY (table_name, filter_key)\n);";
+export declare const SYNC_STATUS_TABLES_SQL = "\n  CREATE TABLE IF NOT EXISTS supastash_sync_marks (\n  table_name       TEXT NOT NULL,\n  filter_key       TEXT NOT NULL,         \n  filter_json      TEXT NULL,             \n  last_synced_at   TEXT NULL,   \n  last_synced_at_pk     TEXT NULL,          \n  last_deleted_at  TEXT NULL,             \n  updated_at       TEXT NOT NULL DEFAULT (datetime('now')),\n  PRIMARY KEY (table_name, filter_key)\n);";
+export declare const SERVER_SYNC_STATUS_TABLES_SQL = "\n  CREATE TABLE IF NOT EXISTS supastash_server_sync_marks (\n    table_name            TEXT NOT NULL,\n    filter_key            TEXT NOT NULL,\n    filter_json           TEXT NULL,\n\n    last_deleted_at       TEXT NULL,\n\n    last_synced_at        TEXT NULL,        -- represents arrived_at\n    last_synced_at_pk     TEXT NULL,        -- composite cursor safety\n\n    updated_at            TEXT NOT NULL DEFAULT (datetime('now')),\n\n    PRIMARY KEY (table_name, filter_key)\n  );\n";
 export declare const ADD_PK_TO_SYNC_MARKS_SQL = "\n  ALTER TABLE supastash_sync_marks ADD COLUMN last_synced_at_pk TEXT NULL;\n";
 export declare const INDEX_SYNC_MARKS_SQL = "\n  CREATE INDEX IF NOT EXISTS idx_supastash_marks_updated\n    ON supastash_sync_marks(updated_at);\n";
+export declare const INDEX_SERVER_SYNC_MARKS_SQL = "\n  CREATE INDEX IF NOT EXISTS idx_supastash_server_marks_updated\n    ON supastash_server_sync_marks(updated_at);\n";
 /**
  * Creates the supastash_sync_marks table if it doesn't exist
  *

package/dist/utils/schema/createSyncStatus.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createSyncStatus.d.ts","sourceRoot":"","sources":["../../../src/utils/schema/createSyncStatus.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,wBAAsB,wBAAwB,kBAE7C;AAED,eAAO,MAAM,sBAAsB,scAWhC,CAAC;AAEJ,eAAO,MAAM,wBAAwB,mFAEpC,CAAC;AAEF,eAAO,MAAM,oBAAoB,2GAGhC,CAAC;AAIF;;;;GAIG;AACH,wBAAsB,qBAAqB,kBAY1C"}
+{"version":3,"file":"createSyncStatus.d.ts","sourceRoot":"","sources":["../../../src/utils/schema/createSyncStatus.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,wBAAsB,wBAAwB,kBAE7C;AAED,eAAO,MAAM,sBAAsB,0ZAUhC,CAAC;AAEJ,eAAO,MAAM,6BAA6B,6eAezC,CAAC;AAEF,eAAO,MAAM,wBAAwB,mFAEpC,CAAC;AAEF,eAAO,MAAM,oBAAoB,2GAGhC,CAAC;AAEF,eAAO,MAAM,2BAA2B,yHAGvC,CAAC;AAIF;;;;GAIG;AACH,wBAAsB,qBAAqB,kBAkB1C"}

package/dist/utils/schema/createSyncStatus.js

@@ -1,3 +1,4 @@
+import { getSupastashConfig } from "../../core/config";
 import { getSupastashDb } from "../../db/dbInitializer";
 /**
  * Creates the supastash_deleted_status table if it doesn't exist
@@ -13,13 +14,28 @@ export const SYNC_STATUS_TABLES_SQL = `
   table_name       TEXT NOT NULL,
   filter_key       TEXT NOT NULL,         
   filter_json      TEXT NULL,             
-  last_created_at  TEXT NULL,             
   last_synced_at   TEXT NULL,   
   last_synced_at_pk     TEXT NULL,          
   last_deleted_at  TEXT NULL,             
   updated_at       TEXT NOT NULL DEFAULT (datetime('now')),
   PRIMARY KEY (table_name, filter_key)
 );`;
+export const SERVER_SYNC_STATUS_TABLES_SQL = `
+  CREATE TABLE IF NOT EXISTS supastash_server_sync_marks (
+    table_name            TEXT NOT NULL,
+    filter_key            TEXT NOT NULL,
+    filter_json           TEXT NULL,
+
+    last_deleted_at       TEXT NULL,
+
+    last_synced_at        TEXT NULL,        -- represents arrived_at
+    last_synced_at_pk     TEXT NULL,        -- composite cursor safety
+
+    updated_at            TEXT NOT NULL DEFAULT (datetime('now')),
+
+    PRIMARY KEY (table_name, filter_key)
+  );
+`;
 export const ADD_PK_TO_SYNC_MARKS_SQL = `
   ALTER TABLE supastash_sync_marks ADD COLUMN last_synced_at_pk TEXT NULL;
 `;
@@ -27,6 +43,10 @@ export const INDEX_SYNC_MARKS_SQL = `
   CREATE INDEX IF NOT EXISTS idx_supastash_marks_updated
     ON supastash_sync_marks(updated_at);
 `;
+export const INDEX_SERVER_SYNC_MARKS_SQL = `
+  CREATE INDEX IF NOT EXISTS idx_supastash_server_marks_updated
+    ON supastash_server_sync_marks(updated_at);
+`;
 let addedPk = false;
 /**
  * Creates the supastash_sync_marks table if it doesn't exist
@@ -35,7 +55,15 @@ let addedPk = false;
  */
 export async function createSyncStatusTable() {
     const db = await getSupastashDb();
-    await db.execAsync(SYNC_STATUS_TABLES_SQL);
+    const cfg = getSupastashConfig();
+    if (cfg.replicationMode === "server-side") {
+        await db.execAsync(SERVER_SYNC_STATUS_TABLES_SQL);
+        await db.execAsync(INDEX_SERVER_SYNC_MARKS_SQL);
+    }
+    else {
+        await db.execAsync(SYNC_STATUS_TABLES_SQL);
+        await db.execAsync(INDEX_SYNC_MARKS_SQL);
+    }
     try {
         if (addedPk)
             return;
@@ -45,5 +73,4 @@ export async function createSyncStatusTable() {
     catch (error) {
         // Ignore error if column already exists
     }
-    await db.execAsync(INDEX_SYNC_MARKS_SQL);
 }

package/dist/utils/sync/pullFromRemote/fetchOlder.d.ts

@@ -0,0 +1,44 @@
+import { PayloadData } from "../../../types/query.types";
+import { SupastashFilter } from "../../../types/realtimeData.types";
+/**
+ * Fetches a backward page of records from the server relative to the current
+ * earliest locally stored row.
+ *
+ * This function performs cursor-based backward pagination by:
+ * - Determining the earliest known local record for the table
+ * - Validating the provided boundary timestamp
+ * - Fetching older records from the server up to the specified limit
+ * - Optionally persisting the results into the local database
+ *
+ * Designed for infinite scroll and historical lookback scenarios.
+ *
+ * @returns An object containing:
+ * - `data`: The fetched records
+ * - `hasMore`: Whether additional older records may exist
+ */
+export declare function fetchOlderPage({ boundaryTs, table, filters, limit, shouldStoreToLocalDb, }: {
+    /**
+     * The timestamp to cap lookback at.
+     */
+    boundaryTs: string;
+    /**
+     * The table to fetch data from.
+     */
+    table: string;
+    /**
+     * The filters to apply to the data.
+     */
+    filters?: SupastashFilter[];
+    /**
+     * The maximum number of records to fetch.
+     */
+    limit: number;
+    /**
+     * The data to store to the database.
+     */
+    shouldStoreToLocalDb?: boolean;
+}): Promise<{
+    hasMore: boolean;
+    data: PayloadData[];
+}>;
+//# sourceMappingURL=fetchOlder.d.ts.map

package/dist/utils/sync/pullFromRemote/fetchOlder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchOlder.d.ts","sourceRoot":"","sources":["../../../../src/utils/sync/pullFromRemote/fetchOlder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AAIpE;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,cAAc,CAAC,EACnC,UAAU,EACV,KAAK,EACL,OAAO,EACP,KAAK,EACL,oBAA2B,GAC5B,EAAE;IACD;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;IAE5B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,WAAW,EAAE,CAAA;CAAE,CAAC,CAwCrD"}

package/dist/utils/sync/pullFromRemote/fetchOlder.js

@@ -0,0 +1,55 @@
+import { getSupastashConfig } from "../../../core/config";
+import { logWarn } from "../../logs";
+import { FetchOlderHelpers } from "./fetchOlderHelpers";
+/**
+ * Fetches a backward page of records from the server relative to the current
+ * earliest locally stored row.
+ *
+ * This function performs cursor-based backward pagination by:
+ * - Determining the earliest known local record for the table
+ * - Validating the provided boundary timestamp
+ * - Fetching older records from the server up to the specified limit
+ * - Optionally persisting the results into the local database
+ *
+ * Designed for infinite scroll and historical lookback scenarios.
+ *
+ * @returns An object containing:
+ * - `data`: The fetched records
+ * - `hasMore`: Whether additional older records may exist
+ */
+export async function fetchOlderPage({ boundaryTs, table, filters, limit, shouldStoreToLocalDb = true, }) {
+    try {
+        const isGhost = getSupastashConfig().supastashMode === "ghost";
+        if (isGhost) {
+            return { hasMore: false, data: [] };
+        }
+        const lookbackDays = await FetchOlderHelpers.getLookbackDays({
+            table,
+            filters,
+        });
+        if (boundaryTs) {
+            FetchOlderHelpers.validateBoundaryTs({
+                boundaryTs,
+                earliestDate: lookbackDays.createdAt,
+            });
+        }
+        const data = await FetchOlderHelpers.fetchData({
+            table,
+            filters,
+            limit,
+            boundaryTs,
+            earliestDate: lookbackDays.createdAt,
+            earliestId: lookbackDays.id,
+        });
+        if (shouldStoreToLocalDb) {
+            await FetchOlderHelpers.storeToDb({ table, data });
+        }
+        if (data.length > limit) {
+            logWarn(`[Supastash] Pagination overflow detected for table ${table}`);
+        }
+        return { hasMore: data.length === limit, data };
+    }
+    catch (error) {
+        return FetchOlderHelpers.handleError(error);
+    }
+}

package/dist/utils/sync/pullFromRemote/fetchOlderHelpers.d.ts

@@ -0,0 +1,33 @@
+import { PayloadData } from "../../../types/query.types";
+import { SupastashFilter } from "../../../types/realtimeData.types";
+import { SupastashError } from "../../errorHandler";
+export declare const FetchOlderHelpers: {
+    validateBoundaryTs({ boundaryTs, earliestDate, }: {
+        boundaryTs: string;
+        earliestDate: string;
+    }): void;
+    getLookbackDays({ table, filters, }: {
+        table: string;
+        filters?: SupastashFilter[];
+    }): Promise<{
+        createdAt: any;
+        id: any;
+    }>;
+    fetchData({ table, filters, limit, earliestDate, boundaryTs, earliestId, }: {
+        table: string;
+        filters?: SupastashFilter[];
+        limit: number;
+        boundaryTs?: string;
+        earliestDate: string;
+        earliestId: string;
+    }): Promise<any[]>;
+    handleError(error: SupastashError): {
+        hasMore: boolean;
+        data: PayloadData[];
+    };
+    storeToDb({ table, data }: {
+        table: string;
+        data: PayloadData[];
+    }): Promise<void>;
+};
+//# sourceMappingURL=fetchOlderHelpers.d.ts.map

package/dist/utils/sync/pullFromRemote/fetchOlderHelpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchOlderHelpers.d.ts","sourceRoot":"","sources":["../../../../src/utils/sync/pullFromRemote/fetchOlderHelpers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AAEpE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAQpD,eAAO,MAAM,iBAAiB;sDAIzB;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB;yCAmBE;QACD,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;KAC7B;;;;gFA8BE;QACD,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;QAC5B,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;KACpB;uBAqDkB,cAAc,GAAG;QAClC,OAAO,EAAE,OAAO,CAAC;QACjB,IAAI,EAAE,WAAW,EAAE,CAAC;KACrB;+BAagC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,WAAW,EAAE,CAAA;KAAE;CAcxE,CAAC"}

package/dist/utils/sync/pullFromRemote/fetchOlderHelpers.js

@@ -0,0 +1,110 @@
+import { getSupastashConfig } from "../../../core/config";
+import { getSupastashDb } from "../../../db/dbInitializer";
+import { ReusedHelpers } from "../../../utils/reusedHelpers";
+import { SupastashError } from "../../errorHandler";
+import { logWarn } from "../../logs";
+import { supabaseClientErr } from "../../supabaseClientErr";
+import { upsertData } from "./updateLocalDb";
+const DEFAULT_DATE = "2000-01-01T00:00:00Z";
+const DEFAULT_ID = "00000000-0000-0000-0000-000000000000";
+export const FetchOlderHelpers = {
+    validateBoundaryTs({ boundaryTs, earliestDate, }) {
+        const boundaryTsDate = new Date(boundaryTs);
+        if (isNaN(boundaryTsDate.getTime())) {
+            logWarn(`Invalid boundary timestamp: ${boundaryTs}`);
+            throw new SupastashError(`Invalid boundary timestamp: ${boundaryTs}`, "INVALID_BOUNDARY_TS");
+        }
+        if (boundaryTsDate > new Date(earliestDate)) {
+            throw new SupastashError(`Boundary timestamp is more recent than oldest data: ${boundaryTs} > ${earliestDate}`, "BOUNDARY_TS_MORE_RECENT_THAN_OLDEST_DATA");
+        }
+    },
+    async getLookbackDays({ table, filters, }) {
+        // If no cursor, build the SQL filter
+        let sqlFilter = "";
+        if (filters) {
+            for (const filter of filters) {
+                sqlFilter += ReusedHelpers.buildFilterForSql(filter);
+            }
+        }
+        // Fetch the earliest date and id
+        const db = await getSupastashDb();
+        const result = await db.getFirstAsync(`
+        SELECT id, created_at FROM ${table} 
+        ${sqlFilter}
+        ORDER BY created_at ASC, id ASC
+        LIMIT 1
+        `);
+        if (!result)
+            return { createdAt: DEFAULT_DATE, id: DEFAULT_ID };
+        return { createdAt: result.created_at, id: result.id };
+    },
+    async fetchData({ table, filters, limit, earliestDate, boundaryTs, earliestId, }) {
+        const supabase = getSupastashConfig().supabaseClient;
+        if (!supabase)
+            throw new Error(`No supabase client found: ${supabaseClientErr}`);
+        let cursorId = earliestId;
+        let cursorDate = earliestDate;
+        const results = [];
+        let remainingLimit = limit;
+        while (true) {
+            const pageSize = Math.min(1000, remainingLimit);
+            // Build the query
+            let q = supabase
+                .from(table)
+                .select("*")
+                .limit(pageSize)
+                .order("created_at", { ascending: false })
+                .order("id", { ascending: false })
+                .is("deleted_at", null);
+            q = q
+                .or(`created_at.lt.${cursorDate},and(created_at.eq.${cursorDate},id.lt.${cursorId})`)
+                .gte("created_at", boundaryTs ?? DEFAULT_DATE);
+            if (filters) {
+                q = ReusedHelpers.applyFilters(q, filters, table);
+            }
+            const { data, error } = await q;
+            if (error)
+                throw error;
+            if (!data || data.length === 0)
+                break;
+            results.push(...data);
+            const last = data[data.length - 1];
+            cursorId = last.id;
+            cursorDate = last.created_at;
+            if (data.length < pageSize) {
+                break;
+            }
+            if (remainingLimit <= pageSize) {
+                break;
+            }
+            remainingLimit -= pageSize;
+        }
+        return results;
+    },
+    handleError(error) {
+        switch (error.code) {
+            case "INVALID_BOUNDARY_TS":
+                logWarn(error.message);
+                return { hasMore: false, data: [] };
+            case "BOUNDARY_TS_MORE_RECENT_THAN_OLDEST_DATA":
+                logWarn(error.message);
+                return { hasMore: false, data: [] };
+            default:
+                throw error;
+        }
+    },
+    async storeToDb({ table, data }) {
+        if (data.length === 0)
+            return;
+        const batchSize = 500;
+        const db = await getSupastashDb();
+        await db.withTransaction(async (tx) => {
+            for (let i = 0; i < data.length; i++) {
+                await upsertData({ tx, table, record: data[i] });
+                if ((i + 1) % batchSize === 0) {
+                    await new Promise((res) => setTimeout(res, 0));
+                }
+            }
+        });
+    },
+};

package/dist/utils/sync/pullFromRemote/helpers.d.ts

@@ -1,12 +1,12 @@
 import { PayloadData } from "../../../types/query.types";
-import { RealtimeFilter } from "../../../types/realtimeData.types";
+import { SupastashFilter } from "../../../types/realtimeData.types";
 import { ReceivedDataCompleted } from "../../../types/syncEngine.types";
 export declare function pageThrough(base: {
-    tsCol: "created_at" | "updated_at" | "deleted_at";
+    tsCol: "arrived_at" | "updated_at";
     since: string;
     table: string;
     select?: string;
-    filters?: RealtimeFilter[];
+    filters?: SupastashFilter[];
     includeDeleted?: boolean;
     batchId: string;
     previousPk?: string | null;
@@ -17,21 +17,24 @@ export declare function returnMaxDate({ row, prevMax, col, }: {
         value: string;
         pk: string | null;
     } | null;
-    col: "created_at" | "updated_at" | "deleted_at";
+    col: "arrived_at" | "updated_at" | "deleted_at";
 }): {
     value: string;
     pk: string | null;
 } | null;
-export declare function getMaxDate(rows: PayloadData[], col: "created_at" | "updated_at" | "deleted_at"): string | null;
+export declare function getMaxDate(rows: PayloadData[], col: "arrived_at" | "updated_at"): string | null;
 export declare function logNoUpdates(table: string): void;
 export declare function getReceivedDataCompleted({ batchId, col, }: {
     batchId: string;
-    col: "created_at" | "updated_at" | "deleted_at";
+    col: "arrived_at" | "updated_at";
 }): ReceivedDataCompleted;
 export declare function setReceivedDataCompleted({ batchId, col, completed, }: {
     batchId: string;
-    col: "created_at" | "updated_at" | "deleted_at";
+    col: "arrived_at" | "updated_at";
     completed: ReceivedDataCompleted;
 }): void;
 export declare function deleteReceivedDataCompleted(batchId: string): void;
+export declare function getMaxSyncLookBack({ table, }: {
+    table: string;
+}): string | undefined;
 //# sourceMappingURL=helpers.d.ts.map

package/dist/utils/sync/pullFromRemote/helpers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/utils/sync/pullFromRemote/helpers.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,iCAAiC,CAAC;AAuBxE,wBAAsB,WAAW,CAAC,IAAI,EAAE;IACtC,KAAK,EAAE,YAAY,GAAG,YAAY,GAAG,YAAY,CAAC;IAClD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,kBA2EA;AAED,wBAAgB,aAAa,CAAC,EAC5B,GAAG,EACH,OAAO,EACP,GAAG,GACJ,EAAE;IACD,GAAG,EAAE,WAAW,CAAC;IACjB,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IACrD,GAAG,EAAE,YAAY,GAAG,YAAY,GAAG,YAAY,CAAC;CACjD,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CAoB9C;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,WAAW,EAAE,EACnB,GAAG,EAAE,YAAY,GAAG,YAAY,GAAG,YAAY,GAC9C,MAAM,GAAG,IAAI,CAUf;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,QAczC;AAED,wBAAgB,wBAAwB,CAAC,EACvC,OAAO,EACP,GAAG,GACJ,EAAE;IACD,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,YAAY,GAAG,YAAY,GAAG,YAAY,CAAC;CACjD,GAAG,qBAAqB,CAUxB;AAED,wBAAgB,wBAAwB,CAAC,EACvC,OAAO,EACP,GAAG,EACH,SAAS,GACV,EAAE;IACD,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,YAAY,GAAG,YAAY,GAAG,YAAY,CAAC;IAChD,SAAS,EAAE,qBAAqB,CAAC;CAClC,QAKA;AAED,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,MAAM,QAE1D"}
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/utils/sync/pullFromRemote/helpers.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,iCAAiC,CAAC;AAaxE,wBAAsB,WAAW,CAAC,IAAI,EAAE;IACtC,KAAK,EAAE,YAAY,GAAG,YAAY,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;IAC5B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,kBAgFA;AAED,wBAAgB,aAAa,CAAC,EAC5B,GAAG,EACH,OAAO,EACP,GAAG,GACJ,EAAE;IACD,GAAG,EAAE,WAAW,CAAC;IACjB,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IACrD,GAAG,EAAE,YAAY,GAAG,YAAY,GAAG,YAAY,CAAC;CACjD,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CAoB9C;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,WAAW,EAAE,EACnB,GAAG,EAAE,YAAY,GAAG,YAAY,GAC/B,MAAM,GAAG,IAAI,CAUf;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,QAczC;AAED,wBAAgB,wBAAwB,CAAC,EACvC,OAAO,EACP,GAAG,GACJ,EAAE;IACD,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,YAAY,GAAG,YAAY,CAAC;CAClC,GAAG,qBAAqB,CAUxB;AAED,wBAAgB,wBAAwB,CAAC,EACvC,OAAO,EACP,GAAG,EACH,SAAS,GACV,EAAE;IACD,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,YAAY,GAAG,YAAY,CAAC;IACjC,SAAS,EAAE,qBAAqB,CAAC;CAClC,QAKA;AAED,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,MAAM,QAE1D;AAED,wBAAgB,kBAAkB,CAAC,EACjC,KAAK,GACN,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,MAAM,GAAG,SAAS,CAcrB"}

package/dist/utils/sync/pullFromRemote/helpers.js

@@ -1,8 +1,8 @@
 import { getSupastashConfig } from "../../../core/config";
 import { RECEIVED_DATA_COMPLETED_MAP, RECEIVED_DATA_THRESHOLD, } from "../../../store/syncStatus";
 import log from "../../logs";
+import { ReusedHelpers } from "../../reusedHelpers";
 import { supabaseClientErr } from "../../supabaseClientErr";
-import isValidFilter from "./validateFilters";
 const RANDOM_OLD_DATE = "2000-01-01T00:00:00Z";
 const PAGE_SIZE = RECEIVED_DATA_THRESHOLD;
 const MAX_PAGE_SIZE = 2000;
@@ -10,15 +10,6 @@ const timesPulled = new Map();
 const lastPulled = new Map();
 const DEFAULT_MAX_PULL_ATTEMPTS = 150;
 const DEFAULT_PK = "00000000-0000-0000-0000-000000000000";
-function applyFilters(q, filters, table) {
-    for (const f of filters) {
-        if (!isValidFilter([f])) {
-            throw new Error(`Invalid syncFilter: ${JSON.stringify(f)} for ${table}`);
-        }
-        q = q[f.operator](f.column, f.value);
-    }
-    return q;
-}
 export async function pageThrough(base) {
     const supabase = getSupastashConfig().supabaseClient;
     if (!supabase)
@@ -34,7 +25,11 @@ export async function pageThrough(base) {
     let cursorId = base.previousPk ?? lastWork.lastId;
     let lastDataSize = 0;
     const { table, filters = [], select = "*" } = base;
+    const maxSyncLookBack = getMaxSyncLookBack({ table });
     while (true) {
+        const ts = maxSyncLookBack && Date.parse(cursorTs) < Date.parse(maxSyncLookBack)
+            ? maxSyncLookBack
+            : cursorTs;
         let q = supabase
             .from(table)
             .select(select)
@@ -42,13 +37,13 @@ export async function pageThrough(base) {
             .order("id", { ascending: true })
             .limit(PAGE_SIZE);
         if (cursorId) {
-            q = q.or(`${base.tsCol}.gt.${cursorTs},and(${base.tsCol}.eq.${cursorTs},id.gt.${cursorId})`);
+            q = q.or(`${base.tsCol}.gt.${ts},and(${base.tsCol}.eq.${ts},id.gt.${cursorId})`);
         }
         else {
-            q = q.gte(base.tsCol, cursorTs);
+            q = q.gte(base.tsCol, ts);
         }
-        if (filters) {
-            q = applyFilters(q, filters, table);
+        if (filters && filters.length > 0) {
+            q = ReusedHelpers.applyFilters(q, filters, table);
         }
         const { data, error } = await q;
         if (error)
@@ -150,3 +145,14 @@ export function setReceivedDataCompleted({ batchId, col, completed, }) {
 export function deleteReceivedDataCompleted(batchId) {
     delete RECEIVED_DATA_COMPLETED_MAP[batchId];
 }
+export function getMaxSyncLookBack({ table, }) {
+    const cfg = getSupastashConfig();
+    if (cfg.fullSyncTables?.includes(table)) {
+        return undefined;
+    }
+    const perTable = cfg.perTableSyncLookbackDays?.[table];
+    const days = perTable !== undefined ? perTable : cfg.maxSyncLookbackDays;
+    if (days === undefined)
+        return undefined;
+    return new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString();
+}

package/dist/utils/sync/pullFromRemote/pullData.d.ts

@@ -1,5 +1,5 @@
 import { PayloadData } from "../../../types/query.types";
-import { RealtimeFilter } from "../../../types/realtimeData.types";
+import { SupastashFilter } from "../../../types/realtimeData.types";
 /**
  * Pulls data from the remote database for a given table
  * @param table - The table to pull data from
@@ -7,13 +7,12 @@ import { RealtimeFilter } from "../../../types/realtimeData.types";
  */
 export declare function pullData({ table, filters, batchId, }: {
     table: string;
-    filters?: RealtimeFilter[];
+    filters?: SupastashFilter[];
     batchId: string;
 }): Promise<{
     data: PayloadData[];
     deletedIds: string[];
     timestamps: {
-        createdMax: string | null;
         updatedMax: string | null;
         deletedMax: string | null;
         updatedMaxPk: string | null;

package/dist/utils/sync/pullFromRemote/pullData.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pullData.d.ts","sourceRoot":"","sources":["../../../../src/utils/sync/pullFromRemote/pullData.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAQnE;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,EAC7B,KAAK,EACL,OAAO,EACP,OAAO,GACR,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,WAAW,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;CACH,GAAG,IAAI,CAAC,CAqFR"}
+{"version":3,"file":"pullData.d.ts","sourceRoot":"","sources":["../../../../src/utils/sync/pullFromRemote/pullData.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AAQpE;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,EAC7B,KAAK,EACL,OAAO,EACP,OAAO,GACR,EAAE;IACD,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG,OAAO,CAAC;IACV,IAAI,EAAE,WAAW,EAAE,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;CACH,GAAG,IAAI,CAAC,CAmFR"}