teleton 0.4.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +88 -13
  2. package/dist/BigInteger-DQ33LTTE.js +5 -0
  3. package/dist/chunk-4DU3C27M.js +30 -0
  4. package/dist/chunk-5WWR4CU3.js +124 -0
  5. package/dist/{chunk-E2NXSWOS.js → chunk-NUGDTPE4.js} +24 -64
  6. package/dist/{chunk-OA5L7GM6.js → chunk-O4R7V5Y2.js} +37 -5
  7. package/dist/chunk-QUAPFI2N.js +42 -0
  8. package/dist/chunk-TSKJCWQQ.js +1263 -0
  9. package/dist/{chunk-B2PRMXOH.js → chunk-WL2Q3VRD.js} +0 -2
  10. package/dist/{chunk-QU4ZOR35.js → chunk-WOXBZOQX.js} +3179 -3368
  11. package/dist/{chunk-7UPH62J2.js → chunk-WUTMT6DW.js} +293 -261
  12. package/dist/{chunk-OQGNS2FV.js → chunk-YBA6IBGT.js} +20 -5
  13. package/dist/cli/index.js +41 -172
  14. package/dist/{endpoint-FT2B2RZ2.js → endpoint-FLYNEZ2F.js} +1 -1
  15. package/dist/{get-my-gifts-AFKBG4YQ.js → get-my-gifts-KVULMBJ3.js} +1 -1
  16. package/dist/index.js +12 -12
  17. package/dist/{memory-SYTQ5P7P.js → memory-Y5J7CXAR.js} +4 -5
  18. package/dist/{migrate-ITXMRRSZ.js → migrate-UEQCDWL2.js} +4 -5
  19. package/dist/server-BQY7CM2N.js +1120 -0
  20. package/dist/{task-dependency-resolver-GY6PEBIS.js → task-dependency-resolver-TRPILAHM.js} +2 -2
  21. package/dist/{task-executor-4QKTZZ3P.js → task-executor-N7XNVK5N.js} +1 -1
  22. package/dist/{tasks-M3QDPTGY.js → tasks-QSCWSMPS.js} +1 -1
  23. package/dist/{transcript-DF2Y6CFY.js → transcript-7V4UNID4.js} +1 -1
  24. package/dist/web/assets/index-CDMbujHf.css +1 -0
  25. package/dist/web/assets/index-DDX8oQ2z.js +67 -0
  26. package/dist/web/index.html +16 -0
  27. package/dist/web/logo_dark.png +0 -0
  28. package/package.json +23 -6
  29. package/dist/chunk-67QC5FBN.js +0 -60
  30. package/dist/chunk-A64NPEFL.js +0 -74
  31. package/dist/chunk-DUW5VBAZ.js +0 -133
  32. package/dist/chunk-QBGUCUOW.js +0 -16
  33. package/dist/scraper-SH7GS7TO.js +0 -282
package/dist/web/index.html ADDED
@@ -0,0 +1,16 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
+ <link rel="preconnect" href="https://fonts.googleapis.com" />
7
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
8
+ <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&display=swap" rel="stylesheet" />
9
+ <title>Teleton</title>
10
+ <script type="module" crossorigin src="/assets/index-DDX8oQ2z.js"></script>
11
+ <link rel="stylesheet" crossorigin href="/assets/index-CDMbujHf.css">
12
+ </head>
13
+ <body>
14
+ <div id="root"></div>
15
+ </body>
16
+ </html>
package/dist/web/logo_dark.png ADDED
Binary file
package/package.json CHANGED
@@ -1,6 +1,9 @@
1
1
  {
2
2
  "name": "teleton",
3
- "version": "0.4.0",
3
+ "version": "0.5.2",
4
+ "workspaces": [
5
+ "packages/*"
6
+ ],
4
7
  "description": "Personal AI Agent for Telegram",
5
8
  "author": "ZKProof (https://t.me/zkproof)",
6
9
  "license": "MIT",
@@ -33,16 +36,24 @@
33
36
  "src/templates/"
34
37
  ],
35
38
  "scripts": {
36
- "build": "tsup",
39
+ "build": "npm run build:sdk && npm run build:backend && npm run build:web",
40
+ "build:sdk": "npm run build -w @teleton-agent/sdk",
41
+ "build:backend": "tsup --config tsup.config.ts",
42
+ "build:web": "cd web && npm run build",
43
+ "prestart": "npm run build",
37
44
  "start": "node dist/cli/index.js start",
38
45
  "dev": "tsx watch src/index.ts",
39
46
  "dev:cli": "tsx src/cli/index.ts",
47
+ "dev:web": "cd web && npm run dev",
40
48
  "setup": "node dist/cli/index.js setup",
41
49
  "doctor": "node dist/cli/index.js doctor",
42
50
  "lint": "eslint src --ext .ts",
43
51
  "lint:fix": "eslint src --ext .ts --fix",
44
52
  "format": "prettier --write \"src/**/*.ts\"",
45
53
  "format:check": "prettier --check \"src/**/*.ts\"",
54
+ "test": "vitest run",
55
+ "test:watch": "vitest",
56
+ "test:coverage": "vitest run --coverage",
46
57
  "typecheck": "tsc --noEmit",
47
58
  "prepublishOnly": "npm run build",
48
59
  "prepare": "husky"
@@ -50,21 +61,21 @@
50
61
  "dependencies": {
51
62
  "@clack/prompts": "^0.7.0",
52
63
  "@dedust/sdk": "^0.8.7",
53
- "@evaafi/sdk": "^0.9.5",
64
+ "@hono/node-server": "^1.19.9",
54
65
  "@huggingface/transformers": "^3.8.1",
55
66
  "@mariozechner/pi-ai": "^0.50.9",
56
67
  "@orbs-network/ton-access": "^2.3.3",
57
68
  "@sinclair/typebox": "^0.34.48",
58
- "@storm-trade/sdk": "^1.0.0-rc.4",
59
69
  "@ton/core": "^0.63.0",
60
70
  "@ton/crypto": "^3.3.0",
61
71
  "@ton/ton": "^16.1.0",
62
72
  "better-sqlite3": "^11.7.0",
73
+ "chokidar": "^5.0.0",
63
74
  "commander": "^12.0.0",
64
75
  "crypto-js": "^4.2.0",
65
76
  "grammy": "^1.39.3",
77
+ "hono": "^4.11.9",
66
78
  "js-tiktoken": "^1.0.21",
67
- "playwright": "^1.58.1",
68
79
  "sqlite-vec": "^0.1.7-alpha.2",
69
80
  "telegram": "^2.26.22",
70
81
  "yaml": "^2.7.0",
@@ -77,13 +88,15 @@
77
88
  "@types/node": "^22.0.0",
78
89
  "@typescript-eslint/eslint-plugin": "^8.54.0",
79
90
  "@typescript-eslint/parser": "^8.54.0",
91
+ "@vitest/coverage-v8": "^4.0.18",
80
92
  "eslint": "^9.39.2",
81
93
  "husky": "^9.1.7",
82
94
  "lint-staged": "^16.2.7",
83
95
  "prettier": "^3.8.1",
84
96
  "tsup": "^8.5.1",
85
97
  "tsx": "^4.19.0",
86
- "typescript": "^5.7.0"
98
+ "typescript": "^5.7.0",
99
+ "vitest": "^4.0.18"
87
100
  },
88
101
  "optionalDependencies": {
89
102
  "edge-tts": "^1.0.1"
@@ -95,6 +108,10 @@
95
108
  "src/**/*.ts": [
96
109
  "eslint --fix",
97
110
  "prettier --write"
111
+ ],
112
+ "packages/sdk/src/**/*.ts": [
113
+ "eslint --fix",
114
+ "prettier --write"
98
115
  ]
99
116
  }
100
117
  }
package/dist/chunk-67QC5FBN.js DELETED
@@ -1,60 +0,0 @@
1
- // src/constants/timeouts.ts
2
- var TTS_TIMEOUT_MS = 3e4;
3
- var BROWSER_NAVIGATION_TIMEOUT_MS = 3e4;
4
- var MESSAGE_HANDLER_LOCK_TIMEOUT_MS = 12e4;
5
- var ONBOARDING_PROMPT_TIMEOUT_MS = 12e4;
6
- var BATCH_TRIGGER_DELAY_MS = 500;
7
- var DEFAULT_FETCH_TIMEOUT_MS = 15e3;
8
- var SCRAPER_PAGE_LOAD_MS = 2500;
9
- var SCRAPER_FILTER_CLICK_MS = 3e3;
10
- var SCRAPER_MODEL_CLICK_MS = 2e3;
11
- var SCRAPER_FILTER_OPEN_MS = 600;
12
- var SCRAPER_MODEL_OPEN_MS = 800;
13
- var SCRAPER_SCROLL_STEP_MS = 80;
14
- var SCRAPER_PRE_SCROLL_MS = 4e3;
15
- var SCRAPER_COLLECTION_SCROLL_MS = 200;
16
- var SCRAPER_SCROLL_INCREMENT_PX = 250;
17
- var SCRAPER_SCROLL_PADDING_PX = 500;
18
- var SCRAPER_MAX_SCROLL_ITERATIONS = 15;
19
- var SCRAPER_COLLECTION_NAV_MS = 6e4;
20
- var RETRY_DEFAULT_MAX_ATTEMPTS = 3;
21
- var RETRY_DEFAULT_BASE_DELAY_MS = 1e3;
22
- var RETRY_DEFAULT_MAX_DELAY_MS = 1e4;
23
- var RETRY_DEFAULT_TIMEOUT_MS = 15e3;
24
- var RETRY_BLOCKCHAIN_BASE_DELAY_MS = 2e3;
25
- var RETRY_BLOCKCHAIN_MAX_DELAY_MS = 15e3;
26
- var RETRY_BLOCKCHAIN_TIMEOUT_MS = 3e4;
27
- var GRAMJS_RETRY_DELAY_MS = 1e3;
28
- var TOOL_EXECUTION_TIMEOUT_MS = 9e4;
29
- var SHUTDOWN_TIMEOUT_MS = 1e4;
30
-
31
- export {
32
- TTS_TIMEOUT_MS,
33
- BROWSER_NAVIGATION_TIMEOUT_MS,
34
- MESSAGE_HANDLER_LOCK_TIMEOUT_MS,
35
- ONBOARDING_PROMPT_TIMEOUT_MS,
36
- BATCH_TRIGGER_DELAY_MS,
37
- DEFAULT_FETCH_TIMEOUT_MS,
38
- SCRAPER_PAGE_LOAD_MS,
39
- SCRAPER_FILTER_CLICK_MS,
40
- SCRAPER_MODEL_CLICK_MS,
41
- SCRAPER_FILTER_OPEN_MS,
42
- SCRAPER_MODEL_OPEN_MS,
43
- SCRAPER_SCROLL_STEP_MS,
44
- SCRAPER_PRE_SCROLL_MS,
45
- SCRAPER_COLLECTION_SCROLL_MS,
46
- SCRAPER_SCROLL_INCREMENT_PX,
47
- SCRAPER_SCROLL_PADDING_PX,
48
- SCRAPER_MAX_SCROLL_ITERATIONS,
49
- SCRAPER_COLLECTION_NAV_MS,
50
- RETRY_DEFAULT_MAX_ATTEMPTS,
51
- RETRY_DEFAULT_BASE_DELAY_MS,
52
- RETRY_DEFAULT_MAX_DELAY_MS,
53
- RETRY_DEFAULT_TIMEOUT_MS,
54
- RETRY_BLOCKCHAIN_BASE_DELAY_MS,
55
- RETRY_BLOCKCHAIN_MAX_DELAY_MS,
56
- RETRY_BLOCKCHAIN_TIMEOUT_MS,
57
- GRAMJS_RETRY_DELAY_MS,
58
- TOOL_EXECUTION_TIMEOUT_MS,
59
- SHUTDOWN_TIMEOUT_MS
60
- };
package/dist/chunk-A64NPEFL.js DELETED
@@ -1,74 +0,0 @@
1
- import {
2
- DEFAULT_FETCH_TIMEOUT_MS
3
- } from "./chunk-67QC5FBN.js";
4
-
5
- // src/utils/fetch.ts
6
- var DEFAULT_TIMEOUT_MS = DEFAULT_FETCH_TIMEOUT_MS;
7
- function fetchWithTimeout(url, init) {
8
- const { timeoutMs = DEFAULT_TIMEOUT_MS, ...fetchInit } = init ?? {};
9
- if (fetchInit.signal) {
10
- return fetch(url, fetchInit);
11
- }
12
- return fetch(url, {
13
- ...fetchInit,
14
- signal: AbortSignal.timeout(timeoutMs)
15
- });
16
- }
17
-
18
- // src/constants/api-endpoints.ts
19
- var TONAPI_BASE_URL = "https://tonapi.io/v2";
20
- var _tonapiKey;
21
- function setTonapiKey(key) {
22
- _tonapiKey = key;
23
- }
24
- function tonapiHeaders() {
25
- const headers = { Accept: "application/json" };
26
- if (_tonapiKey) {
27
- headers["Authorization"] = `Bearer ${_tonapiKey}`;
28
- }
29
- return headers;
30
- }
31
- var TONAPI_MAX_RPS = 5;
32
- var _tonapiTimestamps = [];
33
- async function waitForTonapiSlot() {
34
- const clean = () => {
35
- const cutoff = Date.now() - 1e3;
36
- while (_tonapiTimestamps.length > 0 && _tonapiTimestamps[0] <= cutoff) {
37
- _tonapiTimestamps.shift();
38
- }
39
- };
40
- clean();
41
- if (_tonapiTimestamps.length >= TONAPI_MAX_RPS) {
42
- const waitMs = _tonapiTimestamps[0] + 1e3 - Date.now() + 50;
43
- if (waitMs > 0) await new Promise((r) => setTimeout(r, waitMs));
44
- clean();
45
- }
46
- _tonapiTimestamps.push(Date.now());
47
- }
48
- async function tonapiFetch(path, init) {
49
- await waitForTonapiSlot();
50
- return fetchWithTimeout(`${TONAPI_BASE_URL}${path}`, {
51
- ...init,
52
- headers: { ...tonapiHeaders(), ...init?.headers }
53
- });
54
- }
55
- var STONFI_API_BASE_URL = "https://api.ston.fi/v1";
56
- var GECKOTERMINAL_API_URL = "https://api.geckoterminal.com/api/v2";
57
- var COINGECKO_API_URL = "https://api.coingecko.com/api/v3";
58
- var MARKETAPP_BASE_URL = "https://marketapp.ws";
59
- var OPENAI_TTS_URL = "https://api.openai.com/v1/audio/speech";
60
- var ELEVENLABS_TTS_URL = "https://api.elevenlabs.io/v1/text-to-speech";
61
- var VOYAGE_API_URL = "https://api.voyageai.com/v1";
62
-
63
- export {
64
- fetchWithTimeout,
65
- setTonapiKey,
66
- tonapiFetch,
67
- STONFI_API_BASE_URL,
68
- GECKOTERMINAL_API_URL,
69
- COINGECKO_API_URL,
70
- MARKETAPP_BASE_URL,
71
- OPENAI_TTS_URL,
72
- ELEVENLABS_TTS_URL,
73
- VOYAGE_API_URL
74
- };
package/dist/chunk-DUW5VBAZ.js DELETED
@@ -1,133 +0,0 @@
1
- import {
2
- TELETON_ROOT
3
- } from "./chunk-EYWNOHMJ.js";
4
-
5
- // src/market/scraper-db.ts
6
- import Database from "better-sqlite3";
7
- import { join } from "path";
8
- var DB_PATH = join(TELETON_ROOT, "gifts.db");
9
- function initScraperDb() {
10
- const db = new Database(DB_PATH);
11
- db.pragma("journal_mode = WAL");
12
- db.exec(`
13
- -- Gift collections (Plush Pepes, Heart Lockets, etc.)
14
- CREATE TABLE IF NOT EXISTS gift_collections (
15
- id INTEGER PRIMARY KEY AUTOINCREMENT,
16
- address TEXT UNIQUE NOT NULL,
17
- name TEXT NOT NULL,
18
- floor_ton REAL,
19
- floor_usd REAL,
20
- volume_7d REAL,
21
- listed_count INTEGER,
22
- owners INTEGER,
23
- supply INTEGER,
24
- created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
25
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
26
- );
27
-
28
- -- Models per collection (Cozy Galaxy, Milano, etc.)
29
- CREATE TABLE IF NOT EXISTS gift_models (
30
- id INTEGER PRIMARY KEY AUTOINCREMENT,
31
- collection_id INTEGER NOT NULL,
32
- name TEXT NOT NULL,
33
- floor_ton REAL,
34
- rarity_percent REAL,
35
- count INTEGER,
36
- created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
37
- updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
38
- FOREIGN KEY (collection_id) REFERENCES gift_collections(id),
39
- UNIQUE(collection_id, name)
40
- );
41
-
42
- -- Price history (for trends)
43
- CREATE TABLE IF NOT EXISTS price_history (
44
- id INTEGER PRIMARY KEY AUTOINCREMENT,
45
- collection_id INTEGER,
46
- model_id INTEGER,
47
- floor_ton REAL NOT NULL,
48
- floor_usd REAL,
49
- timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
50
- FOREIGN KEY (collection_id) REFERENCES gift_collections(id),
51
- FOREIGN KEY (model_id) REFERENCES gift_models(id)
52
- );
53
-
54
- -- Indexes for frequent queries
55
- CREATE INDEX IF NOT EXISTS idx_price_history_collection ON price_history(collection_id, timestamp);
56
- CREATE INDEX IF NOT EXISTS idx_price_history_model ON price_history(model_id, timestamp);
57
- CREATE INDEX IF NOT EXISTS idx_models_collection ON gift_models(collection_id);
58
- `);
59
- return db;
60
- }
61
- function upsertCollection(db, collection) {
62
- const stmt = db.prepare(`
63
- INSERT INTO gift_collections (address, name, floor_ton, floor_usd, volume_7d, updated_at)
64
- VALUES (@address, @name, @floor_ton, @floor_usd, @volume_7d, CURRENT_TIMESTAMP)
65
- ON CONFLICT(address) DO UPDATE SET
66
- name = @name,
67
- floor_ton = @floor_ton,
68
- floor_usd = @floor_usd,
69
- volume_7d = @volume_7d,
70
- updated_at = CURRENT_TIMESTAMP
71
- RETURNING id
72
- `);
73
- const result = stmt.get({
74
- address: collection.address,
75
- name: collection.name,
76
- floor_ton: collection.floorTON || null,
77
- floor_usd: collection.floorUSD || null,
78
- volume_7d: collection.volume7d || null
79
- });
80
- return result.id;
81
- }
82
- function getCollectionId(db, address) {
83
- const row = db.prepare(`SELECT id FROM gift_collections WHERE address = ?`).get(address);
84
- return row?.id ?? null;
85
- }
86
- function upsertModel(db, collectionId, model) {
87
- const stmt = db.prepare(`
88
- INSERT INTO gift_models (collection_id, name, floor_ton, rarity_percent, count, updated_at)
89
- VALUES (@collection_id, @name, @floor_ton, @rarity_percent, @count, CURRENT_TIMESTAMP)
90
- ON CONFLICT(collection_id, name) DO UPDATE SET
91
- floor_ton = @floor_ton,
92
- rarity_percent = @rarity_percent,
93
- count = @count,
94
- updated_at = CURRENT_TIMESTAMP
95
- RETURNING id
96
- `);
97
- const result = stmt.get({
98
- collection_id: collectionId,
99
- name: model.name,
100
- floor_ton: model.floor || null,
101
- rarity_percent: model.pct ? parseFloat(model.pct) : null,
102
- count: model.count || null
103
- });
104
- return result.id;
105
- }
106
- function addPriceHistory(db, collectionId, modelId, floorTon, floorUsd = null) {
107
- const stmt = db.prepare(`
108
- INSERT INTO price_history (collection_id, model_id, floor_ton, floor_usd)
109
- VALUES (?, ?, ?, ?)
110
- `);
111
- stmt.run(collectionId, modelId, floorTon, floorUsd);
112
- }
113
- function getScraperStats(db) {
114
- const collections = db.prepare("SELECT COUNT(*) as count FROM gift_collections").get();
115
- const models = db.prepare("SELECT COUNT(*) as count FROM gift_models").get();
116
- const history = db.prepare("SELECT COUNT(*) as count FROM price_history").get();
117
- const lastUpdate = db.prepare("SELECT MAX(updated_at) as last FROM gift_collections").get();
118
- return {
119
- collections: collections.count,
120
- models: models.count,
121
- historyEntries: history.count,
122
- lastUpdate: lastUpdate.last
123
- };
124
- }
125
-
126
- export {
127
- initScraperDb,
128
- upsertCollection,
129
- getCollectionId,
130
- upsertModel,
131
- addPriceHistory,
132
- getScraperStats
133
- };
package/dist/chunk-QBGUCUOW.js DELETED
@@ -1,16 +0,0 @@
1
- // src/ton/endpoint.ts
2
- import { getHttpEndpoint } from "@orbs-network/ton-access";
3
- var ENDPOINT_CACHE_TTL_MS = 6e4;
4
- var _cache = null;
5
- async function getCachedHttpEndpoint() {
6
- if (_cache && Date.now() - _cache.ts < ENDPOINT_CACHE_TTL_MS) {
7
- return _cache.url;
8
- }
9
- const url = await getHttpEndpoint({ network: "mainnet" });
10
- _cache = { url, ts: Date.now() };
11
- return url;
12
- }
13
-
14
- export {
15
- getCachedHttpEndpoint
16
- };
package/dist/scraper-SH7GS7TO.js DELETED
@@ -1,282 +0,0 @@
1
- import {
2
- addPriceHistory,
3
- getCollectionId,
4
- getScraperStats,
5
- initScraperDb,
6
- upsertCollection,
7
- upsertModel
8
- } from "./chunk-DUW5VBAZ.js";
9
- import {
10
- MARKETAPP_BASE_URL
11
- } from "./chunk-A64NPEFL.js";
12
- import {
13
- BROWSER_NAVIGATION_TIMEOUT_MS,
14
- SCRAPER_COLLECTION_NAV_MS,
15
- SCRAPER_COLLECTION_SCROLL_MS,
16
- SCRAPER_FILTER_CLICK_MS,
17
- SCRAPER_FILTER_OPEN_MS,
18
- SCRAPER_MAX_SCROLL_ITERATIONS,
19
- SCRAPER_MODEL_CLICK_MS,
20
- SCRAPER_MODEL_OPEN_MS,
21
- SCRAPER_PAGE_LOAD_MS,
22
- SCRAPER_PRE_SCROLL_MS,
23
- SCRAPER_SCROLL_INCREMENT_PX,
24
- SCRAPER_SCROLL_PADDING_PX,
25
- SCRAPER_SCROLL_STEP_MS
26
- } from "./chunk-67QC5FBN.js";
27
- import {
28
- SCRAPER_PARALLEL_WORKERS
29
- } from "./chunk-OA5L7GM6.js";
30
- import "./chunk-EYWNOHMJ.js";
31
- import "./chunk-QGM4M3NI.js";
32
-
33
- // src/market/scraper.ts
34
- import { chromium } from "playwright";
35
- async function scrapeAllModels(page, collection, db) {
36
- try {
37
- const url = `${MARKETAPP_BASE_URL}/collection/${collection.address}/?tab=nfts`;
38
- await page.goto(url, { waitUntil: "domcontentloaded", timeout: BROWSER_NAVIGATION_TIMEOUT_MS });
39
- await page.waitForTimeout(SCRAPER_PAGE_LOAD_MS);
40
- try {
41
- await page.click('button:has-text("Filters")', { timeout: SCRAPER_FILTER_CLICK_MS });
42
- await page.waitForTimeout(SCRAPER_FILTER_OPEN_MS);
43
- } catch (e) {
44
- return 0;
45
- }
46
- try {
47
- await page.click("text=Model", { timeout: SCRAPER_MODEL_CLICK_MS });
48
- await page.waitForTimeout(SCRAPER_MODEL_OPEN_MS);
49
- } catch (e) {
50
- return 0;
51
- }
52
- const allModels = /* @__PURE__ */ new Map();
53
- const wrapperHeight = await page.evaluate(() => {
54
- const wrappers = document.querySelectorAll(".virtual-scroll-wrapper");
55
- const wrapper = wrappers[1];
56
- return wrapper ? wrapper.scrollHeight : 0;
57
- });
58
- for (let scrollPos = 0; scrollPos <= wrapperHeight + SCRAPER_SCROLL_PADDING_PX; scrollPos += SCRAPER_SCROLL_INCREMENT_PX) {
59
- const text = await page.evaluate((pos) => {
60
- const wrappers = document.querySelectorAll(".virtual-scroll-wrapper");
61
- const wrapper = wrappers[1];
62
- if (wrapper) {
63
- wrapper.scrollTop = pos;
64
- return wrapper.innerText;
65
- }
66
- return "";
67
- }, scrollPos);
68
- if (text) {
69
- const lines = text.split("\n").map((l) => l.trim()).filter((l) => l);
70
- let currentModel = null;
71
- for (const line of lines) {
72
- if (line.length > 1 && line.length < 50 && !line.match(/^[\d,.]+$/) && !line.startsWith("Floor:") && !line.includes("%")) {
73
- currentModel = { name: line, floor: null, count: null, pct: null };
74
- }
75
- if (line.startsWith("Floor:") && currentModel) {
76
- const match = line.match(/Floor:\s*([\d,.]+)/);
77
- if (match) currentModel.floor = parseFloat(match[1].replace(/,/g, ""));
78
- }
79
- if (currentModel && line.match(/^\d+$/) && !currentModel.count) {
80
- currentModel.count = parseInt(line);
81
- }
82
- if (line.includes("%") && currentModel) {
83
- currentModel.pct = line;
84
- if (currentModel.name && currentModel.floor) {
85
- allModels.set(currentModel.name, { ...currentModel });
86
- }
87
- currentModel = null;
88
- }
89
- }
90
- }
91
- await page.waitForTimeout(SCRAPER_SCROLL_STEP_MS);
92
- }
93
- const models = [...allModels.values()];
94
- const collectionId = getCollectionId(db, collection.address);
95
- if (!collectionId) return 0;
96
- for (const model of models) {
97
- const modelId = upsertModel(db, collectionId, model);
98
- if (model.floor) {
99
- addPriceHistory(db, collectionId, modelId, model.floor);
100
- }
101
- }
102
- return models.length;
103
- } catch (error) {
104
- return -1;
105
- }
106
- }
107
- async function createWorker(browser, db) {
108
- const context = await browser.newContext({
109
- userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
110
- viewport: { width: 1920, height: 1080 }
111
- });
112
- const page = await context.newPage();
113
- return {
114
- page,
115
- async scrape(collection) {
116
- return await scrapeAllModels(page, collection, db);
117
- },
118
- async close() {
119
- await context.close();
120
- }
121
- };
122
- }
123
- async function getCollections(page, db) {
124
- await page.goto(`${MARKETAPP_BASE_URL}/?tab=gifts&sort_by=floor_desc`, {
125
- waitUntil: "domcontentloaded",
126
- timeout: SCRAPER_COLLECTION_NAV_MS
127
- });
128
- await page.waitForTimeout(SCRAPER_PRE_SCROLL_MS);
129
- for (let i = 0; i < SCRAPER_MAX_SCROLL_ITERATIONS; i++) {
130
- await page.evaluate(() => window.scrollBy(0, 2e3));
131
- await page.waitForTimeout(SCRAPER_COLLECTION_SCROLL_MS);
132
- }
133
- const collections = await page.evaluate(() => {
134
- const results = [];
135
- const text = document.body.innerText;
136
- const lines = text.split("\n").map((l) => l.trim()).filter((l) => l);
137
- for (let i = 0; i < lines.length; i++) {
138
- if (lines[i] === "1% fee" && i > 0) {
139
- const name = lines[i - 1];
140
- if (name.length < 3 || name.length > 40 || name === "Name") continue;
141
- let floorTON = null;
142
- let floorUSD = null;
143
- let volume7d = null;
144
- let skipNext = 0;
145
- for (let j = i + 1; j < Math.min(i + 12, lines.length); j++) {
146
- const val = lines[j];
147
- if (floorTON === null && val.match(/^[\d,.]+$/)) {
148
- floorTON = parseFloat(val.replace(/,/g, ""));
149
- continue;
150
- }
151
- if (floorTON !== null && floorUSD === null && val.startsWith("~$")) {
152
- floorUSD = parseFloat(val.replace("~$", "").replace(/,/g, ""));
153
- skipNext = 2;
154
- continue;
155
- }
156
- if (skipNext > 0 && (val.match(/^[\d,.]+$/) || val.startsWith("~$"))) {
157
- skipNext--;
158
- continue;
159
- }
160
- if (floorUSD !== null && volume7d === null && skipNext === 0) {
161
- const volMatch = val.match(/^([\d,.]+)(K|M)?$/);
162
- if (volMatch) {
163
- let vol = parseFloat(volMatch[1].replace(/,/g, ""));
164
- if (volMatch[2] === "K") vol *= 1e3;
165
- if (volMatch[2] === "M") vol *= 1e6;
166
- volume7d = vol;
167
- break;
168
- }
169
- }
170
- if (val === "1% fee") break;
171
- }
172
- if (name && floorTON) {
173
- results.push({ name, floorTON, floorUSD, volume7d, address: null });
174
- }
175
- }
176
- }
177
- const links = document.querySelectorAll('a[href*="/collection/"]');
178
- const addressMap = /* @__PURE__ */ new Map();
179
- links.forEach((link) => {
180
- const href = link.getAttribute("href");
181
- if (!href) return;
182
- const match = href.match(/\/collection\/([^/?]+)/);
183
- if (match) {
184
- const text2 = link.textContent?.trim().split("\n")[0];
185
- if (text2 && text2.length > 2) addressMap.set(text2, match[1]);
186
- }
187
- });
188
- return results.map((r) => ({
189
- ...r,
190
- address: addressMap.get(r.name) || [...addressMap.entries()].find(
191
- ([k]) => k.toLowerCase().includes(r.name.toLowerCase().slice(0, 10))
192
- )?.[1] || null
193
- })).filter((r) => r.address);
194
- });
195
- for (const col of collections) {
196
- const collectionId = upsertCollection(db, col);
197
- addPriceHistory(db, collectionId, null, col.floorTON, col.floorUSD);
198
- }
199
- return collections;
200
- }
201
- async function runScraper(options) {
202
- const workers = options.workers || SCRAPER_PARALLEL_WORKERS;
203
- const limit = options.limit || 0;
204
- console.log("=".repeat(60));
205
- console.log(`SCRAPING ALL MODELS (${workers} workers)`);
206
- console.log("=".repeat(60));
207
- const db = initScraperDb();
208
- const startTime = Date.now();
209
- let browser = null;
210
- try {
211
- browser = await chromium.launch({ headless: true });
212
- console.log("\n1. Collections...");
213
- const mainCtx = await browser.newContext({
214
- userAgent: "Mozilla/5.0",
215
- viewport: { width: 1920, height: 1080 }
216
- });
217
- const mainPage = await mainCtx.newPage();
218
- const collections = await getCollections(mainPage, db);
219
- await mainCtx.close();
220
- console.log(` \u2713 ${collections.length} collections`);
221
- console.log(`
222
- 2. Workers (${workers})...`);
223
- const workerPool = await Promise.all(
224
- Array(workers).fill(null).map(() => createWorker(browser, db))
225
- );
226
- const toProcess = limit > 0 ? collections.slice(0, limit) : collections;
227
- console.log(`
228
- 3. Scraping ${toProcess.length} collections (all models)...
229
- `);
230
- let completed = 0;
231
- let totalModels = 0;
232
- const queue = [...toProcess];
233
- async function processNext(worker) {
234
- while (queue.length > 0) {
235
- const col = queue.shift();
236
- if (!col) break;
237
- const count = await worker.scrape(col);
238
- completed++;
239
- const status = count > 0 ? `\u2713 ${count.toString().padStart(2)}` : count === 0 ? "- 0 " : "\u2717 ";
240
- if (count > 0) totalModels += count;
241
- const elapsed2 = ((Date.now() - startTime) / 1e3).toFixed(1);
242
- console.log(
243
- ` [${completed.toString().padStart(3)}/${toProcess.length}] ${col.name.padEnd(
244
- 22
245
- )} ${status} (${elapsed2}s)`
246
- );
247
- }
248
- }
249
- await Promise.all(workerPool.map((w) => processNext(w)));
250
- await Promise.all(workerPool.map((w) => w.close()));
251
- const elapsed = ((Date.now() - startTime) / 1e3).toFixed(1);
252
- const stats = getScraperStats(db);
253
- console.log("\n" + "=".repeat(60));
254
- console.log(`DONE in ${elapsed}s`);
255
- console.log("=".repeat(60));
256
- console.log(`Collections: ${stats.collections}`);
257
- console.log(`Models: ${stats.models}`);
258
- console.log(`History entries: ${stats.historyEntries}`);
259
- return {
260
- success: true,
261
- collections: stats.collections,
262
- models: stats.models,
263
- duration: parseInt(elapsed)
264
- };
265
- } catch (error) {
266
- return {
267
- success: false,
268
- collections: 0,
269
- models: 0,
270
- duration: Math.round((Date.now() - startTime) / 1e3),
271
- error: error instanceof Error ? error.message : String(error)
272
- };
273
- } finally {
274
- if (browser) {
275
- await browser.close();
276
- }
277
- db.close();
278
- }
279
- }
280
- export {
281
- runScraper
282
- };