teleton 0.3.0 → 0.5.1

package/dist/scraper-KXRBQMVQ.js

@@ -1,282 +0,0 @@
-import {
-  addPriceHistory,
-  getCollectionId,
-  getScraperStats,
-  initScraperDb,
-  upsertCollection,
-  upsertModel
-} from "./chunk-DUW5VBAZ.js";
-import {
-  MARKETAPP_BASE_URL
-} from "./chunk-DAMFGHXV.js";
-import {
-  SCRAPER_PARALLEL_WORKERS
-} from "./chunk-UYF4TT44.js";
-import "./chunk-EYWNOHMJ.js";
-import {
-  BROWSER_NAVIGATION_TIMEOUT_MS,
-  SCRAPER_COLLECTION_NAV_MS,
-  SCRAPER_COLLECTION_SCROLL_MS,
-  SCRAPER_FILTER_CLICK_MS,
-  SCRAPER_FILTER_OPEN_MS,
-  SCRAPER_MAX_SCROLL_ITERATIONS,
-  SCRAPER_MODEL_CLICK_MS,
-  SCRAPER_MODEL_OPEN_MS,
-  SCRAPER_PAGE_LOAD_MS,
-  SCRAPER_PRE_SCROLL_MS,
-  SCRAPER_SCROLL_INCREMENT_PX,
-  SCRAPER_SCROLL_PADDING_PX,
-  SCRAPER_SCROLL_STEP_MS
-} from "./chunk-LCMHAUNK.js";
-import "./chunk-QGM4M3NI.js";
-
-// src/market/scraper.ts
-import { chromium } from "playwright";
-async function scrapeAllModels(page, collection, db) {
-  try {
-    const url = `${MARKETAPP_BASE_URL}/collection/${collection.address}/?tab=nfts`;
-    await page.goto(url, { waitUntil: "domcontentloaded", timeout: BROWSER_NAVIGATION_TIMEOUT_MS });
-    await page.waitForTimeout(SCRAPER_PAGE_LOAD_MS);
-    try {
-      await page.click('button:has-text("Filters")', { timeout: SCRAPER_FILTER_CLICK_MS });
-      await page.waitForTimeout(SCRAPER_FILTER_OPEN_MS);
-    } catch (e) {
-      return 0;
-    }
-    try {
-      await page.click("text=Model", { timeout: SCRAPER_MODEL_CLICK_MS });
-      await page.waitForTimeout(SCRAPER_MODEL_OPEN_MS);
-    } catch (e) {
-      return 0;
-    }
-    const allModels = /* @__PURE__ */ new Map();
-    const wrapperHeight = await page.evaluate(() => {
-      const wrappers = document.querySelectorAll(".virtual-scroll-wrapper");
-      const wrapper = wrappers[1];
-      return wrapper ? wrapper.scrollHeight : 0;
-    });
-    for (let scrollPos = 0; scrollPos <= wrapperHeight + SCRAPER_SCROLL_PADDING_PX; scrollPos += SCRAPER_SCROLL_INCREMENT_PX) {
-      const text = await page.evaluate((pos) => {
-        const wrappers = document.querySelectorAll(".virtual-scroll-wrapper");
-        const wrapper = wrappers[1];
-        if (wrapper) {
-          wrapper.scrollTop = pos;
-          return wrapper.innerText;
-        }
-        return "";
-      }, scrollPos);
-      if (text) {
-        const lines = text.split("\n").map((l) => l.trim()).filter((l) => l);
-        let currentModel = null;
-        for (const line of lines) {
-          if (line.length > 1 && line.length < 50 && !line.match(/^[\d,.]+$/) && !line.startsWith("Floor:") && !line.includes("%")) {
-            currentModel = { name: line, floor: null, count: null, pct: null };
-          }
-          if (line.startsWith("Floor:") && currentModel) {
-            const match = line.match(/Floor:\s*([\d,.]+)/);
-            if (match) currentModel.floor = parseFloat(match[1].replace(/,/g, ""));
-          }
-          if (currentModel && line.match(/^\d+$/) && !currentModel.count) {
-            currentModel.count = parseInt(line);
-          }
-          if (line.includes("%") && currentModel) {
-            currentModel.pct = line;
-            if (currentModel.name && currentModel.floor) {
-              allModels.set(currentModel.name, { ...currentModel });
-            }
-            currentModel = null;
-          }
-        }
-      }
-      await page.waitForTimeout(SCRAPER_SCROLL_STEP_MS);
-    }
-    const models = [...allModels.values()];
-    const collectionId = getCollectionId(db, collection.address);
-    if (!collectionId) return 0;
-    for (const model of models) {
-      const modelId = upsertModel(db, collectionId, model);
-      if (model.floor) {
-        addPriceHistory(db, collectionId, modelId, model.floor);
-      }
-    }
-    return models.length;
-  } catch (error) {
-    return -1;
-  }
-}
-async function createWorker(browser, db) {
-  const context = await browser.newContext({
-    userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
-    viewport: { width: 1920, height: 1080 }
-  });
-  const page = await context.newPage();
-  return {
-    page,
-    async scrape(collection) {
-      return await scrapeAllModels(page, collection, db);
-    },
-    async close() {
-      await context.close();
-    }
-  };
-}
-async function getCollections(page, db) {
-  await page.goto(`${MARKETAPP_BASE_URL}/?tab=gifts&sort_by=floor_desc`, {
-    waitUntil: "domcontentloaded",
-    timeout: SCRAPER_COLLECTION_NAV_MS
-  });
-  await page.waitForTimeout(SCRAPER_PRE_SCROLL_MS);
-  for (let i = 0; i < SCRAPER_MAX_SCROLL_ITERATIONS; i++) {
-    await page.evaluate(() => window.scrollBy(0, 2e3));
-    await page.waitForTimeout(SCRAPER_COLLECTION_SCROLL_MS);
-  }
-  const collections = await page.evaluate(() => {
-    const results = [];
-    const text = document.body.innerText;
-    const lines = text.split("\n").map((l) => l.trim()).filter((l) => l);
-    for (let i = 0; i < lines.length; i++) {
-      if (lines[i] === "1% fee" && i > 0) {
-        const name = lines[i - 1];
-        if (name.length < 3 || name.length > 40 || name === "Name") continue;
-        let floorTON = null;
-        let floorUSD = null;
-        let volume7d = null;
-        let skipNext = 0;
-        for (let j = i + 1; j < Math.min(i + 12, lines.length); j++) {
-          const val = lines[j];
-          if (floorTON === null && val.match(/^[\d,.]+$/)) {
-            floorTON = parseFloat(val.replace(/,/g, ""));
-            continue;
-          }
-          if (floorTON !== null && floorUSD === null && val.startsWith("~$")) {
-            floorUSD = parseFloat(val.replace("~$", "").replace(/,/g, ""));
-            skipNext = 2;
-            continue;
-          }
-          if (skipNext > 0 && (val.match(/^[\d,.]+$/) || val.startsWith("~$"))) {
-            skipNext--;
-            continue;
-          }
-          if (floorUSD !== null && volume7d === null && skipNext === 0) {
-            const volMatch = val.match(/^([\d,.]+)(K|M)?$/);
-            if (volMatch) {
-              let vol = parseFloat(volMatch[1].replace(/,/g, ""));
-              if (volMatch[2] === "K") vol *= 1e3;
-              if (volMatch[2] === "M") vol *= 1e6;
-              volume7d = vol;
-              break;
-            }
-          }
-          if (val === "1% fee") break;
-        }
-        if (name && floorTON) {
-          results.push({ name, floorTON, floorUSD, volume7d, address: null });
-        }
-      }
-    }
-    const links = document.querySelectorAll('a[href*="/collection/"]');
-    const addressMap = /* @__PURE__ */ new Map();
-    links.forEach((link) => {
-      const href = link.getAttribute("href");
-      if (!href) return;
-      const match = href.match(/\/collection\/([^/?]+)/);
-      if (match) {
-        const text2 = link.textContent?.trim().split("\n")[0];
-        if (text2 && text2.length > 2) addressMap.set(text2, match[1]);
-      }
-    });
-    return results.map((r) => ({
-      ...r,
-      address: addressMap.get(r.name) || [...addressMap.entries()].find(
-        ([k]) => k.toLowerCase().includes(r.name.toLowerCase().slice(0, 10))
-      )?.[1] || null
-    })).filter((r) => r.address);
-  });
-  for (const col of collections) {
-    const collectionId = upsertCollection(db, col);
-    addPriceHistory(db, collectionId, null, col.floorTON, col.floorUSD);
-  }
-  return collections;
-}
-async function runScraper(options) {
-  const workers = options.workers || SCRAPER_PARALLEL_WORKERS;
-  const limit = options.limit || 0;
-  console.log("=".repeat(60));
-  console.log(`SCRAPING ALL MODELS (${workers} workers)`);
-  console.log("=".repeat(60));
-  const db = initScraperDb();
-  const startTime = Date.now();
-  let browser = null;
-  try {
-    browser = await chromium.launch({ headless: true });
-    console.log("\n1. Collections...");
-    const mainCtx = await browser.newContext({
-      userAgent: "Mozilla/5.0",
-      viewport: { width: 1920, height: 1080 }
-    });
-    const mainPage = await mainCtx.newPage();
-    const collections = await getCollections(mainPage, db);
-    await mainCtx.close();
-    console.log(`   \u2713 ${collections.length} collections`);
-    console.log(`
-2. Workers (${workers})...`);
-    const workerPool = await Promise.all(
-      Array(workers).fill(null).map(() => createWorker(browser, db))
-    );
-    const toProcess = limit > 0 ? collections.slice(0, limit) : collections;
-    console.log(`
-3. Scraping ${toProcess.length} collections (all models)...
-`);
-    let completed = 0;
-    let totalModels = 0;
-    const queue = [...toProcess];
-    async function processNext(worker) {
-      while (queue.length > 0) {
-        const col = queue.shift();
-        if (!col) break;
-        const count = await worker.scrape(col);
-        completed++;
-        const status = count > 0 ? `\u2713 ${count.toString().padStart(2)}` : count === 0 ? "- 0 " : "\u2717   ";
-        if (count > 0) totalModels += count;
-        const elapsed2 = ((Date.now() - startTime) / 1e3).toFixed(1);
-        console.log(
-          `   [${completed.toString().padStart(3)}/${toProcess.length}] ${col.name.padEnd(
-            22
-          )} ${status} (${elapsed2}s)`
-        );
-      }
-    }
-    await Promise.all(workerPool.map((w) => processNext(w)));
-    await Promise.all(workerPool.map((w) => w.close()));
-    const elapsed = ((Date.now() - startTime) / 1e3).toFixed(1);
-    const stats = getScraperStats(db);
-    console.log("\n" + "=".repeat(60));
-    console.log(`DONE in ${elapsed}s`);
-    console.log("=".repeat(60));
-    console.log(`Collections: ${stats.collections}`);
-    console.log(`Models: ${stats.models}`);
-    console.log(`History entries: ${stats.historyEntries}`);
-    return {
-      success: true,
-      collections: stats.collections,
-      models: stats.models,
-      duration: parseInt(elapsed)
-    };
-  } catch (error) {
-    return {
-      success: false,
-      collections: 0,
-      models: 0,
-      duration: Math.round((Date.now() - startTime) / 1e3),
-      error: error instanceof Error ? error.message : String(error)
-    };
-  } finally {
-    if (browser) {
-      await browser.close();
-    }
-    db.close();
-  }
-}
-export {
-  runScraper
-};

package/dist/timeouts-ZAK6NELA.js

@@ -1,63 +0,0 @@
-import {
-  BATCH_TRIGGER_DELAY_MS,
-  BROWSER_NAVIGATION_TIMEOUT_MS,
-  DEFAULT_FETCH_TIMEOUT_MS,
-  GRAMJS_RETRY_DELAY_MS,
-  MESSAGE_HANDLER_LOCK_TIMEOUT_MS,
-  ONBOARDING_PROMPT_TIMEOUT_MS,
-  RETRY_BLOCKCHAIN_BASE_DELAY_MS,
-  RETRY_BLOCKCHAIN_MAX_DELAY_MS,
-  RETRY_BLOCKCHAIN_TIMEOUT_MS,
-  RETRY_DEFAULT_BASE_DELAY_MS,
-  RETRY_DEFAULT_MAX_ATTEMPTS,
-  RETRY_DEFAULT_MAX_DELAY_MS,
-  RETRY_DEFAULT_TIMEOUT_MS,
-  SCRAPER_COLLECTION_NAV_MS,
-  SCRAPER_COLLECTION_SCROLL_MS,
-  SCRAPER_FILTER_CLICK_MS,
-  SCRAPER_FILTER_OPEN_MS,
-  SCRAPER_MAX_SCROLL_ITERATIONS,
-  SCRAPER_MODEL_CLICK_MS,
-  SCRAPER_MODEL_OPEN_MS,
-  SCRAPER_PAGE_LOAD_MS,
-  SCRAPER_PRE_SCROLL_MS,
-  SCRAPER_SCROLL_INCREMENT_PX,
-  SCRAPER_SCROLL_PADDING_PX,
-  SCRAPER_SCROLL_STEP_MS,
-  SCRAPER_WINDOW_SCROLL_PX,
-  SHUTDOWN_TIMEOUT_MS,
-  TOOL_EXECUTION_TIMEOUT_MS,
-  TTS_TIMEOUT_MS
-} from "./chunk-LCMHAUNK.js";
-import "./chunk-QGM4M3NI.js";
-export {
-  BATCH_TRIGGER_DELAY_MS,
-  BROWSER_NAVIGATION_TIMEOUT_MS,
-  DEFAULT_FETCH_TIMEOUT_MS,
-  GRAMJS_RETRY_DELAY_MS,
-  MESSAGE_HANDLER_LOCK_TIMEOUT_MS,
-  ONBOARDING_PROMPT_TIMEOUT_MS,
-  RETRY_BLOCKCHAIN_BASE_DELAY_MS,
-  RETRY_BLOCKCHAIN_MAX_DELAY_MS,
-  RETRY_BLOCKCHAIN_TIMEOUT_MS,
-  RETRY_DEFAULT_BASE_DELAY_MS,
-  RETRY_DEFAULT_MAX_ATTEMPTS,
-  RETRY_DEFAULT_MAX_DELAY_MS,
-  RETRY_DEFAULT_TIMEOUT_MS,
-  SCRAPER_COLLECTION_NAV_MS,
-  SCRAPER_COLLECTION_SCROLL_MS,
-  SCRAPER_FILTER_CLICK_MS,
-  SCRAPER_FILTER_OPEN_MS,
-  SCRAPER_MAX_SCROLL_ITERATIONS,
-  SCRAPER_MODEL_CLICK_MS,
-  SCRAPER_MODEL_OPEN_MS,
-  SCRAPER_PAGE_LOAD_MS,
-  SCRAPER_PRE_SCROLL_MS,
-  SCRAPER_SCROLL_INCREMENT_PX,
-  SCRAPER_SCROLL_PADDING_PX,
-  SCRAPER_SCROLL_STEP_MS,
-  SCRAPER_WINDOW_SCROLL_PX,
-  SHUTDOWN_TIMEOUT_MS,
-  TOOL_EXECUTION_TIMEOUT_MS,
-  TTS_TIMEOUT_MS
-};