teleton 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import {
|
|
2
|
+
TELETON_ROOT
|
|
3
|
+
} from "./chunk-EYWNOHMJ.js";
|
|
4
|
+
|
|
5
|
+
// src/market/scraper-db.ts
|
|
6
|
+
import Database from "better-sqlite3";
|
|
7
|
+
import { join } from "path";
|
|
8
|
+
var DB_PATH = join(TELETON_ROOT, "gifts.db");
|
|
9
|
+
function initScraperDb() {
|
|
10
|
+
const db = new Database(DB_PATH);
|
|
11
|
+
db.pragma("journal_mode = WAL");
|
|
12
|
+
db.exec(`
|
|
13
|
+
-- Gift collections (Plush Pepes, Heart Lockets, etc.)
|
|
14
|
+
CREATE TABLE IF NOT EXISTS gift_collections (
|
|
15
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
16
|
+
address TEXT UNIQUE NOT NULL,
|
|
17
|
+
name TEXT NOT NULL,
|
|
18
|
+
floor_ton REAL,
|
|
19
|
+
floor_usd REAL,
|
|
20
|
+
volume_7d REAL,
|
|
21
|
+
listed_count INTEGER,
|
|
22
|
+
owners INTEGER,
|
|
23
|
+
supply INTEGER,
|
|
24
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
25
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
-- Models per collection (Cozy Galaxy, Milano, etc.)
|
|
29
|
+
CREATE TABLE IF NOT EXISTS gift_models (
|
|
30
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
31
|
+
collection_id INTEGER NOT NULL,
|
|
32
|
+
name TEXT NOT NULL,
|
|
33
|
+
floor_ton REAL,
|
|
34
|
+
rarity_percent REAL,
|
|
35
|
+
count INTEGER,
|
|
36
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
37
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
38
|
+
FOREIGN KEY (collection_id) REFERENCES gift_collections(id),
|
|
39
|
+
UNIQUE(collection_id, name)
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
-- Price history (for trends)
|
|
43
|
+
CREATE TABLE IF NOT EXISTS price_history (
|
|
44
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
45
|
+
collection_id INTEGER,
|
|
46
|
+
model_id INTEGER,
|
|
47
|
+
floor_ton REAL NOT NULL,
|
|
48
|
+
floor_usd REAL,
|
|
49
|
+
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
50
|
+
FOREIGN KEY (collection_id) REFERENCES gift_collections(id),
|
|
51
|
+
FOREIGN KEY (model_id) REFERENCES gift_models(id)
|
|
52
|
+
);
|
|
53
|
+
|
|
54
|
+
-- Indexes for frequent queries
|
|
55
|
+
CREATE INDEX IF NOT EXISTS idx_price_history_collection ON price_history(collection_id, timestamp);
|
|
56
|
+
CREATE INDEX IF NOT EXISTS idx_price_history_model ON price_history(model_id, timestamp);
|
|
57
|
+
CREATE INDEX IF NOT EXISTS idx_models_collection ON gift_models(collection_id);
|
|
58
|
+
`);
|
|
59
|
+
return db;
|
|
60
|
+
}
|
|
61
|
+
function upsertCollection(db, collection) {
|
|
62
|
+
const stmt = db.prepare(`
|
|
63
|
+
INSERT INTO gift_collections (address, name, floor_ton, floor_usd, volume_7d, updated_at)
|
|
64
|
+
VALUES (@address, @name, @floor_ton, @floor_usd, @volume_7d, CURRENT_TIMESTAMP)
|
|
65
|
+
ON CONFLICT(address) DO UPDATE SET
|
|
66
|
+
name = @name,
|
|
67
|
+
floor_ton = @floor_ton,
|
|
68
|
+
floor_usd = @floor_usd,
|
|
69
|
+
volume_7d = @volume_7d,
|
|
70
|
+
updated_at = CURRENT_TIMESTAMP
|
|
71
|
+
RETURNING id
|
|
72
|
+
`);
|
|
73
|
+
const result = stmt.get({
|
|
74
|
+
address: collection.address,
|
|
75
|
+
name: collection.name,
|
|
76
|
+
floor_ton: collection.floorTON || null,
|
|
77
|
+
floor_usd: collection.floorUSD || null,
|
|
78
|
+
volume_7d: collection.volume7d || null
|
|
79
|
+
});
|
|
80
|
+
return result.id;
|
|
81
|
+
}
|
|
82
|
+
function getCollectionId(db, address) {
|
|
83
|
+
const row = db.prepare(`SELECT id FROM gift_collections WHERE address = ?`).get(address);
|
|
84
|
+
return row?.id ?? null;
|
|
85
|
+
}
|
|
86
|
+
function upsertModel(db, collectionId, model) {
|
|
87
|
+
const stmt = db.prepare(`
|
|
88
|
+
INSERT INTO gift_models (collection_id, name, floor_ton, rarity_percent, count, updated_at)
|
|
89
|
+
VALUES (@collection_id, @name, @floor_ton, @rarity_percent, @count, CURRENT_TIMESTAMP)
|
|
90
|
+
ON CONFLICT(collection_id, name) DO UPDATE SET
|
|
91
|
+
floor_ton = @floor_ton,
|
|
92
|
+
rarity_percent = @rarity_percent,
|
|
93
|
+
count = @count,
|
|
94
|
+
updated_at = CURRENT_TIMESTAMP
|
|
95
|
+
RETURNING id
|
|
96
|
+
`);
|
|
97
|
+
const result = stmt.get({
|
|
98
|
+
collection_id: collectionId,
|
|
99
|
+
name: model.name,
|
|
100
|
+
floor_ton: model.floor || null,
|
|
101
|
+
rarity_percent: model.pct ? parseFloat(model.pct) : null,
|
|
102
|
+
count: model.count || null
|
|
103
|
+
});
|
|
104
|
+
return result.id;
|
|
105
|
+
}
|
|
106
|
+
function addPriceHistory(db, collectionId, modelId, floorTon, floorUsd = null) {
|
|
107
|
+
const stmt = db.prepare(`
|
|
108
|
+
INSERT INTO price_history (collection_id, model_id, floor_ton, floor_usd)
|
|
109
|
+
VALUES (?, ?, ?, ?)
|
|
110
|
+
`);
|
|
111
|
+
stmt.run(collectionId, modelId, floorTon, floorUsd);
|
|
112
|
+
}
|
|
113
|
+
function getScraperStats(db) {
|
|
114
|
+
const collections = db.prepare("SELECT COUNT(*) as count FROM gift_collections").get();
|
|
115
|
+
const models = db.prepare("SELECT COUNT(*) as count FROM gift_models").get();
|
|
116
|
+
const history = db.prepare("SELECT COUNT(*) as count FROM price_history").get();
|
|
117
|
+
const lastUpdate = db.prepare("SELECT MAX(updated_at) as last FROM gift_collections").get();
|
|
118
|
+
return {
|
|
119
|
+
collections: collections.count,
|
|
120
|
+
models: models.count,
|
|
121
|
+
historyEntries: history.count,
|
|
122
|
+
lastUpdate: lastUpdate.last
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export {
|
|
127
|
+
initScraperDb,
|
|
128
|
+
upsertCollection,
|
|
129
|
+
getCollectionId,
|
|
130
|
+
upsertModel,
|
|
131
|
+
addPriceHistory,
|
|
132
|
+
getScraperStats
|
|
133
|
+
};
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import {
|
|
2
|
+
initScraperDb
|
|
3
|
+
} from "./chunk-DUW5VBAZ.js";
|
|
1
4
|
import {
|
|
2
5
|
appendToTranscript,
|
|
3
6
|
archiveTranscript,
|
|
@@ -23556,11 +23559,6 @@ var module_default2 = casinoModule;
|
|
|
23556
23559
|
import Database from "better-sqlite3";
|
|
23557
23560
|
import { join as join15 } from "path";
|
|
23558
23561
|
var DB_PATH3 = join15(TELETON_ROOT, "gifts.db");
|
|
23559
|
-
function getDb2() {
|
|
23560
|
-
const db3 = new Database(DB_PATH3);
|
|
23561
|
-
db3.pragma("journal_mode = WAL");
|
|
23562
|
-
return db3;
|
|
23563
|
-
}
|
|
23564
23562
|
function getStats(db3) {
|
|
23565
23563
|
const collections = db3.prepare("SELECT COUNT(*) as count FROM gift_collections").get();
|
|
23566
23564
|
const models = db3.prepare("SELECT COUNT(*) as count FROM gift_models").get();
|
|
@@ -23588,7 +23586,7 @@ var MarketScraperService = class {
|
|
|
23588
23586
|
this.isScrapingInProgress = true;
|
|
23589
23587
|
console.log("\u{1F504} Starting full market scrape...");
|
|
23590
23588
|
try {
|
|
23591
|
-
const { runScraper } = await import("./scraper-
|
|
23589
|
+
const { runScraper } = await import("./scraper-DSAYK6QJ.js");
|
|
23592
23590
|
const result = await runScraper({
|
|
23593
23591
|
workers: 4,
|
|
23594
23592
|
limit: 0
|
|
@@ -23657,7 +23655,7 @@ var MarketPriceService = class {
|
|
|
23657
23655
|
return;
|
|
23658
23656
|
}
|
|
23659
23657
|
try {
|
|
23660
|
-
this.db =
|
|
23658
|
+
this.db = initScraperDb();
|
|
23661
23659
|
const stats = getStats(this.db);
|
|
23662
23660
|
let dataAgeMs = Infinity;
|
|
23663
23661
|
let timeUntilNextRefresh = this.fullRefreshIntervalMs;
|
package/dist/cli/index.js
CHANGED
package/dist/index.js
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
import {
|
|
2
|
+
addPriceHistory,
|
|
3
|
+
getCollectionId,
|
|
4
|
+
getScraperStats,
|
|
5
|
+
initScraperDb,
|
|
6
|
+
upsertCollection,
|
|
7
|
+
upsertModel
|
|
8
|
+
} from "./chunk-DUW5VBAZ.js";
|
|
1
9
|
import {
|
|
2
10
|
MARKETAPP_BASE_URL
|
|
3
11
|
} from "./chunk-WMIN6AGX.js";
|
|
@@ -19,136 +27,11 @@ import {
|
|
|
19
27
|
SCRAPER_SCROLL_PADDING_PX,
|
|
20
28
|
SCRAPER_SCROLL_STEP_MS
|
|
21
29
|
} from "./chunk-LJXYESJJ.js";
|
|
22
|
-
import
|
|
23
|
-
TELETON_ROOT
|
|
24
|
-
} from "./chunk-EYWNOHMJ.js";
|
|
30
|
+
import "./chunk-EYWNOHMJ.js";
|
|
25
31
|
import "./chunk-QGM4M3NI.js";
|
|
26
32
|
|
|
27
33
|
// src/market/scraper.ts
|
|
28
34
|
import { chromium } from "playwright";
|
|
29
|
-
|
|
30
|
-
// src/market/scraper-db.ts
|
|
31
|
-
import Database from "better-sqlite3";
|
|
32
|
-
import { join } from "path";
|
|
33
|
-
var DB_PATH = join(TELETON_ROOT, "gifts.db");
|
|
34
|
-
function initScraperDb() {
|
|
35
|
-
const db = new Database(DB_PATH);
|
|
36
|
-
db.pragma("journal_mode = WAL");
|
|
37
|
-
db.exec(`
|
|
38
|
-
-- Gift collections (Plush Pepes, Heart Lockets, etc.)
|
|
39
|
-
CREATE TABLE IF NOT EXISTS gift_collections (
|
|
40
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
41
|
-
address TEXT UNIQUE NOT NULL,
|
|
42
|
-
name TEXT NOT NULL,
|
|
43
|
-
floor_ton REAL,
|
|
44
|
-
floor_usd REAL,
|
|
45
|
-
volume_7d REAL,
|
|
46
|
-
listed_count INTEGER,
|
|
47
|
-
owners INTEGER,
|
|
48
|
-
supply INTEGER,
|
|
49
|
-
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
50
|
-
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
51
|
-
);
|
|
52
|
-
|
|
53
|
-
-- Models per collection (Cozy Galaxy, Milano, etc.)
|
|
54
|
-
CREATE TABLE IF NOT EXISTS gift_models (
|
|
55
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
56
|
-
collection_id INTEGER NOT NULL,
|
|
57
|
-
name TEXT NOT NULL,
|
|
58
|
-
floor_ton REAL,
|
|
59
|
-
rarity_percent REAL,
|
|
60
|
-
count INTEGER,
|
|
61
|
-
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
62
|
-
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
63
|
-
FOREIGN KEY (collection_id) REFERENCES gift_collections(id),
|
|
64
|
-
UNIQUE(collection_id, name)
|
|
65
|
-
);
|
|
66
|
-
|
|
67
|
-
-- Price history (for trends)
|
|
68
|
-
CREATE TABLE IF NOT EXISTS price_history (
|
|
69
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
70
|
-
collection_id INTEGER,
|
|
71
|
-
model_id INTEGER,
|
|
72
|
-
floor_ton REAL NOT NULL,
|
|
73
|
-
floor_usd REAL,
|
|
74
|
-
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
75
|
-
FOREIGN KEY (collection_id) REFERENCES gift_collections(id),
|
|
76
|
-
FOREIGN KEY (model_id) REFERENCES gift_models(id)
|
|
77
|
-
);
|
|
78
|
-
|
|
79
|
-
-- Indexes for frequent queries
|
|
80
|
-
CREATE INDEX IF NOT EXISTS idx_price_history_collection ON price_history(collection_id, timestamp);
|
|
81
|
-
CREATE INDEX IF NOT EXISTS idx_price_history_model ON price_history(model_id, timestamp);
|
|
82
|
-
CREATE INDEX IF NOT EXISTS idx_models_collection ON gift_models(collection_id);
|
|
83
|
-
`);
|
|
84
|
-
return db;
|
|
85
|
-
}
|
|
86
|
-
function upsertCollection(db, collection) {
|
|
87
|
-
const stmt = db.prepare(`
|
|
88
|
-
INSERT INTO gift_collections (address, name, floor_ton, floor_usd, volume_7d, updated_at)
|
|
89
|
-
VALUES (@address, @name, @floor_ton, @floor_usd, @volume_7d, CURRENT_TIMESTAMP)
|
|
90
|
-
ON CONFLICT(address) DO UPDATE SET
|
|
91
|
-
name = @name,
|
|
92
|
-
floor_ton = @floor_ton,
|
|
93
|
-
floor_usd = @floor_usd,
|
|
94
|
-
volume_7d = @volume_7d,
|
|
95
|
-
updated_at = CURRENT_TIMESTAMP
|
|
96
|
-
RETURNING id
|
|
97
|
-
`);
|
|
98
|
-
const result = stmt.get({
|
|
99
|
-
address: collection.address,
|
|
100
|
-
name: collection.name,
|
|
101
|
-
floor_ton: collection.floorTON || null,
|
|
102
|
-
floor_usd: collection.floorUSD || null,
|
|
103
|
-
volume_7d: collection.volume7d || null
|
|
104
|
-
});
|
|
105
|
-
return result.id;
|
|
106
|
-
}
|
|
107
|
-
function getCollectionId(db, address) {
|
|
108
|
-
const row = db.prepare(`SELECT id FROM gift_collections WHERE address = ?`).get(address);
|
|
109
|
-
return row?.id ?? null;
|
|
110
|
-
}
|
|
111
|
-
function upsertModel(db, collectionId, model) {
|
|
112
|
-
const stmt = db.prepare(`
|
|
113
|
-
INSERT INTO gift_models (collection_id, name, floor_ton, rarity_percent, count, updated_at)
|
|
114
|
-
VALUES (@collection_id, @name, @floor_ton, @rarity_percent, @count, CURRENT_TIMESTAMP)
|
|
115
|
-
ON CONFLICT(collection_id, name) DO UPDATE SET
|
|
116
|
-
floor_ton = @floor_ton,
|
|
117
|
-
rarity_percent = @rarity_percent,
|
|
118
|
-
count = @count,
|
|
119
|
-
updated_at = CURRENT_TIMESTAMP
|
|
120
|
-
RETURNING id
|
|
121
|
-
`);
|
|
122
|
-
const result = stmt.get({
|
|
123
|
-
collection_id: collectionId,
|
|
124
|
-
name: model.name,
|
|
125
|
-
floor_ton: model.floor || null,
|
|
126
|
-
rarity_percent: model.pct ? parseFloat(model.pct) : null,
|
|
127
|
-
count: model.count || null
|
|
128
|
-
});
|
|
129
|
-
return result.id;
|
|
130
|
-
}
|
|
131
|
-
function addPriceHistory(db, collectionId, modelId, floorTon, floorUsd = null) {
|
|
132
|
-
const stmt = db.prepare(`
|
|
133
|
-
INSERT INTO price_history (collection_id, model_id, floor_ton, floor_usd)
|
|
134
|
-
VALUES (?, ?, ?, ?)
|
|
135
|
-
`);
|
|
136
|
-
stmt.run(collectionId, modelId, floorTon, floorUsd);
|
|
137
|
-
}
|
|
138
|
-
function getScraperStats(db) {
|
|
139
|
-
const collections = db.prepare("SELECT COUNT(*) as count FROM gift_collections").get();
|
|
140
|
-
const models = db.prepare("SELECT COUNT(*) as count FROM gift_models").get();
|
|
141
|
-
const history = db.prepare("SELECT COUNT(*) as count FROM price_history").get();
|
|
142
|
-
const lastUpdate = db.prepare("SELECT MAX(updated_at) as last FROM gift_collections").get();
|
|
143
|
-
return {
|
|
144
|
-
collections: collections.count,
|
|
145
|
-
models: models.count,
|
|
146
|
-
historyEntries: history.count,
|
|
147
|
-
lastUpdate: lastUpdate.last
|
|
148
|
-
};
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
// src/market/scraper.ts
|
|
152
35
|
async function scrapeAllModels(page, collection, db) {
|
|
153
36
|
try {
|
|
154
37
|
const url = `${MARKETAPP_BASE_URL}/collection/${collection.address}/?tab=nfts`;
|