vlr-stats-scraper 1.4.1 → 2.0.0

package/main.mjs

@@ -1,91 +1,166 @@
 #!/usr/bin/env node
-import promptSync from 'prompt-sync';
-import { scrapeWebsite } from './scripts/player-link-scraper.mjs';
-import scrapePlayer from './scripts/player-info-scraper.mjs';
+import promptSync from 'prompt-sync'
+import { scrapeWebsite } from './scripts/player-link-scraper.mjs'
+import scrapePlayer from './scripts/player-info-scraper.mjs'
 import {
   savePlayer,
   saveEvent,
-  addPlayerToEvent,
   getEventByName,
+  getEventId,
+  getPlayersByEvent,
   emptyEvent,
   clearEvent,
   clearDatabase,
-} from './scripts/db.mjs';
+  searchPlayers,
+} from './scripts/db.mjs'
 
-const args = process.argv.slice(2);
+const args = process.argv.slice(2)
+
+function printPlayerStats(player) {
+  // Players saved from the database belong to an event; ones scraped live don't.
+  const event = player.event_name ? ` (${player.event_name}${player.event_year ? `, ${player.event_year}` : ''})` : ''
+  console.log(`\n${player.name}${event}`)
+  console.log(`  Team:        ${player.team ?? 'Free agent'}`)
+  console.log(`  Past teams:  ${player.past_teams?.length ? player.past_teams.join(', ') : 'None'}`)
+  console.log(`  Nationality: ${player.nationality || 'Unknown'}`)
+  console.log(`  Region:      ${player.region ?? 'Unknown'}`)
+  console.log(`  Winnings:    ${player.winnings ?? 'Unknown'}`)
+  // Older saves stored agents as raw <img> tags, so pull the name out of the alt text.
+  const agents = (player.most_played_agents ?? []).map(agent => agent?.match(/alt="([^"]*)"/)?.[1] ?? agent)
+  console.log(`  Top agents:  ${agents.join(', ') || 'None'}`)
+  console.log(`  Avg K:D:     ${Number(player.avg_kda).toFixed(2)}`)
+  if (player.stats?.length) {
+    console.table(player.stats)
+  }
+}
+
+// Manually adds one player to an event that's already saved.
+if (args[0] === '--player') {
+  const [, query, eventName, eventYear, regionArg] = args
+  if (!query?.startsWith('http') || !eventName || !eventYear) {
+    console.error('Usage: valstats --player <vlr.gg player link> <event-name> <year> [region]')
+    process.exit(1)
+  }
+
+  // Look the event up before scraping so a typo doesn't waste a scrape.
+  const eventId = getEventId(eventName, parseInt(eventYear, 10))
+  if (!eventId) {
+    console.error(`No event named "${eventName}" (${eventYear}) in players.db`)
+    process.exit(1)
+  }
+
+  // Without a region argument, use the region the event's other players were saved with.
+  const region = regionArg ?? getPlayersByEvent(eventName)[0]?.region ?? null
+
+  const url = query.includes('timespan=') ? query : query.replace(/\/?$/, '/?timespan=all')
+  const player = await scrapePlayer(url, region)
+  if (!player) {
+    console.error(`Couldn't scrape a player from ${query}`)
+    process.exit(1)
+  }
+  savePlayer(eventId, player)
+  console.log(`Saved ${player.name} to players.db under "${eventName}" (${eventYear})`)
+  printPlayerStats(player)
+  process.exit(0)
+}
+
+if (args[0] === '--search') {
+  const query = args[1]
+  if (!query) {
+    console.error('Usage: valstats --search <name | vlr.gg player link>')
+    process.exit(1)
+  }
+
+  // A vlr.gg link is scraped live anything else is searched for in players.db.
+  if (query.startsWith('http')) {
+    const url = query.includes('timespan=') ? query : query.replace(/\/?$/, '/?timespan=all')
+    const player = await scrapePlayer(url, null)
+    if (!player) {
+      console.error(`Couldn't scrape a player from ${query}`)
+      process.exit(1)
+    }
+    printPlayerStats(player)
+    process.exit(0)
+  }
+
+  const players = searchPlayers(query)
+  if (!players.length) {
+    console.error(`No players matching "${query}" in players.db (pass a vlr.gg player link to scrape them live)`)
+    process.exit(1)
+  }
+  console.log(`${players.length} player${players.length === 1 ? '' : 's'} matching "${query}":`)
+  players.forEach(printPlayerStats)
+  process.exit(0)
+}
 
 if (args[0] === '--clear') {
-  const prompt = promptSync();
-  const answer = prompt('This deletes every event and player in players.db. Type "yes" to continue: ');
+  const prompt = promptSync()
+  const answer = prompt('This deletes every event and player in players.db. Type "yes" to continue: ')
   if (answer?.trim().toLowerCase() === 'yes') {
-    clearDatabase();
-    console.log('Cleared players.db');
+    clearDatabase()
+    console.log('Cleared players.db')
   } else {
-    console.log('Cancelled');
+    console.log('Cancelled')
   }
-  process.exit(0);
+  process.exit(0)
 }
 
 if (args[0] === '--clear-event') {
-  const name = args[1];
+  const name = args[1]
   if (!name) {
-    console.error('Usage: valstats --clear-event <event-name>');
-    process.exit(1);
+    console.error('Usage: valstats --clear-event <event-name>')
+    process.exit(1)
   }
   if (clearEvent(name)) {
-    console.log(`Removed "${name}" and its players from players.db`);
+    console.log(`Removed "${name}" and its players from players.db`)
   } else {
-    console.error(`No event named "${name}" in players.db`);
-    process.exit(1);
+    console.error(`No event named "${name}" in players.db`)
+    process.exit(1)
   }
-  process.exit(0);
+  process.exit(0)
 }
 
-const [link, year, eventName] = args;
+const [link, year, eventName] = args
 
 if (!link || !year || !eventName) {
-  console.error('Usage: valstats <link> <year> <event-name>');
-  console.error('       valstats --clear-event <event-name>');
-  console.error('       valstats --clear');
-  process.exit(1);
+  console.error('Usage: valstats <link> <year> <event-name>')
+  console.error('       valstats --player <vlr.gg player link> <event-name> <year> [region]')
+  console.error('       valstats --search <name | vlr.gg player link>')
+  console.error('       valstats --clear-event <event-name>')
+  console.error('       valstats --clear')
+  process.exit(1)
 }
 
-const isStage = eventName.toLowerCase().includes('stage');
+// Scrape first so a failed scrape doesn't wipe an event that's already saved.
+const scraped = await scrapeWebsite(link)
 
-let region = null;
-if (isStage) {
-  const prompt = promptSync();
-  region = prompt('Enter the region: ');
+if (!scraped) {
+  console.error(`Couldn't get the player list from ${link}`)
+  process.exit(1)
 }
 
+const { playerLinks, region } = scraped
+console.log(`Found ${playerLinks.length} players, region: ${region ?? 'unknown'}`)
+
 // Re-adding an event + year that's already saved replaces its old players.
-const existing = getEventByName(eventName);
+const existing = getEventByName(eventName)
 if (existing && existing.year === parseInt(year, 10)) {
-  emptyEvent(eventName);
-  console.log(`"${eventName}" (${year}) is already in players.db, replacing it`);
+  emptyEvent(eventName)
+  console.log(`"${eventName}" (${year}) is already in players.db, replacing it`)
 }
 
 const eventId = saveEvent({
   name: eventName,
   url: link,
-  region,
   year: parseInt(year, 10),
   stage: null,
-});
-
-const links = await scrapeWebsite(link);
-
-if (!links) {
-  console.error(`Couldn't get the player list from ${link}`);
-  process.exit(1);
-}
+})
 
-for (const playerLink of links) {
-  const player = await scrapePlayer(playerLink, region);
+for (const playerLink of playerLinks) {
+  const player = await scrapePlayer(playerLink, region)
   if (player) {
-    savePlayer(player);
-    addPlayerToEvent(eventId, player.name);
-    console.log(`Saved ${player.name} to players.db under "${eventName}"`);
-    console.log(player);
+    savePlayer(eventId, player)
+    console.log(`Saved ${player.name} to players.db under "${eventName}"`)
+    console.log(player)
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vlr-stats-scraper",
-  "version": "1.4.1",
+  "version": "2.0.0",
   "description": "Scrapes player stats from vlr.gg and stores them in a local SQLite database, grouped by tournament or stage.",
   "homepage": "https://github.com/OverYander/VLR-Webscraper#readme",
   "bugs": {

package/scripts/db.mjs

@@ -1,34 +1,16 @@
-import Database from 'better-sqlite3';
-import path from 'path';
+import Database from 'better-sqlite3'
+import path from 'path'
 
-const db = new Database(path.join(process.cwd(), 'players.db'));
+const db = new Database(path.join(process.cwd(), 'players.db'))
+db.pragma('foreign_keys = ON')
 
-db.exec(`
-  CREATE TABLE IF NOT EXISTS players (
-    name TEXT PRIMARY KEY,
-    team TEXT,
-    past_teams TEXT,
-    winnings TEXT,
-    most_played_agents TEXT,
-    avg_kda REAL,
-    nationality TEXT,
-    region TEXT,
-    stats TEXT
-  )
-`);
-
-const existingPlayerColumns = db.prepare('PRAGMA table_info(players)').all().map(col => col.name);
-if (!existingPlayerColumns.includes('stats')) {
-  db.exec('ALTER TABLE players ADD COLUMN stats TEXT');
-}
-
-const existingTableNames = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(t => t.name);
+const existingTableNames = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(t => t.name)
 if (existingTableNames.includes('tournaments') && !existingTableNames.includes('events')) {
-  db.exec('ALTER TABLE tournaments RENAME TO events');
+  db.exec('ALTER TABLE tournaments RENAME TO events')
 }
 if (existingTableNames.includes('tournament_players') && !existingTableNames.includes('event_players')) {
-  db.exec('ALTER TABLE tournament_players RENAME TO event_players');
-  db.exec('ALTER TABLE event_players RENAME COLUMN tournament_id TO event_id');
+  db.exec('ALTER TABLE tournament_players RENAME TO event_players')
+  db.exec('ALTER TABLE event_players RENAME COLUMN tournament_id TO event_id')
 }
 
 db.exec(`
@@ -36,32 +18,72 @@ db.exec(`
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     name TEXT UNIQUE NOT NULL,
     url TEXT,
-    region TEXT,
     year INTEGER,
     stage INTEGER
   )
-`);
+`)
 
-const existingEventColumns = db.prepare('PRAGMA table_info(events)').all().map(col => col.name);
+const existingEventColumns = db.prepare('PRAGMA table_info(events)').all().map(col => col.name)
 if (!existingEventColumns.includes('year')) {
-  db.exec('ALTER TABLE events ADD COLUMN year INTEGER');
+  db.exec('ALTER TABLE events ADD COLUMN year INTEGER')
 }
 if (!existingEventColumns.includes('stage')) {
-  db.exec('ALTER TABLE events ADD COLUMN stage INTEGER');
+  db.exec('ALTER TABLE events ADD COLUMN stage INTEGER')
+}
+// Region is stored on each player, not on the event.
+if (existingEventColumns.includes('region')) {
+  db.exec('ALTER TABLE events DROP COLUMN region')
 }
 
-db.exec(`
-  CREATE TABLE IF NOT EXISTS event_players (
-    event_id INTEGER NOT NULL REFERENCES events(id),
-    player_name TEXT NOT NULL REFERENCES players(name),
-    PRIMARY KEY (event_id, player_name)
+// Each row is a player as they were in one event, so the same player can have a row per event
+// with different stats.
+const createPlayersTableSql = (tableName) => `
+  CREATE TABLE IF NOT EXISTS ${tableName} (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    event_id INTEGER NOT NULL REFERENCES events(id) ON DELETE CASCADE,
+    name TEXT NOT NULL,
+    team TEXT,
+    past_teams TEXT,
+    winnings TEXT,
+    most_played_agents TEXT,
+    avg_kda REAL,
+    nationality TEXT,
+    region TEXT,
+    stats TEXT,
+    UNIQUE (event_id, name)
   )
-`);
+`
+
+// Older databases kept one shared row per player and linked them to events through event_players.
+// Copy each of those links into its own per-event player row.
+const existingPlayerColumns = db.prepare('PRAGMA table_info(players)').all().map(col => col.name)
+if (existingPlayerColumns.length && !existingPlayerColumns.includes('event_id')) {
+  db.transaction(() => {
+    if (!existingPlayerColumns.includes('stats')) {
+      db.exec('ALTER TABLE players ADD COLUMN stats TEXT')
+    }
+    db.exec(createPlayersTableSql('players_new'))
+    db.exec(`
+      INSERT INTO players_new (event_id, name, team, past_teams, winnings, most_played_agents, avg_kda, nationality, region, stats)
+      SELECT event_players.event_id, players.name, players.team, players.past_teams, players.winnings,
+        players.most_played_agents, players.avg_kda, players.nationality, players.region, players.stats
+      FROM event_players
+      JOIN players ON players.name = event_players.player_name
+      JOIN events ON events.id = event_players.event_id
+    `)
+    db.exec('DROP TABLE event_players')
+    db.exec('DROP TABLE players')
+    db.exec('ALTER TABLE players_new RENAME TO players')
+  })()
+}
+
+db.exec(createPlayersTableSql('players'))
+db.exec('DROP TABLE IF EXISTS event_players')
 
 const upsertPlayerStmt = db.prepare(`
-  INSERT INTO players (name, team, past_teams, winnings, most_played_agents, avg_kda, nationality, region, stats)
-  VALUES (@name, @team, @past_teams, @winnings, @most_played_agents, @avg_kda, @nationality, @region, @stats)
-  ON CONFLICT(name) DO UPDATE SET
+  INSERT INTO players (event_id, name, team, past_teams, winnings, most_played_agents, avg_kda, nationality, region, stats)
+  VALUES (@event_id, @name, @team, @past_teams, @winnings, @most_played_agents, @avg_kda, @nationality, @region, @stats)
+  ON CONFLICT(event_id, name) DO UPDATE SET
     team = excluded.team,
     past_teams = excluded.past_teams,
     winnings = excluded.winnings,
@@ -70,136 +92,142 @@ const upsertPlayerStmt = db.prepare(`
     nationality = excluded.nationality,
     region = excluded.region,
     stats = excluded.stats
-`);
+`)
 
 const upsertEventStmt = db.prepare(`
-  INSERT INTO events (name, url, region, year, stage)
-  VALUES (@name, @url, @region, @year, @stage)
+  INSERT INTO events (name, url, year, stage)
+  VALUES (@name, @url, @year, @stage)
   ON CONFLICT(name) DO UPDATE SET
     url = excluded.url,
-    region = excluded.region,
     year = excluded.year,
     stage = excluded.stage
-`);
+`)
 
-const linkPlayerToEventStmt = db.prepare(`
-  INSERT OR IGNORE INTO event_players (event_id, player_name)
-  VALUES (@eventId, @playerName)
-`);
+// Every player query also pulls in the event's name and year, newest event first.
+const selectPlayersSql = `
+  SELECT players.*, events.name AS event_name, events.year AS event_year FROM players
+  JOIN events ON events.id = players.event_id
+`
+const newestFirst = 'ORDER BY events.year DESC, players.id DESC'
 
 function rowToPlayer(row) {
-  if (!row) return null;
+  if (!row) return null
   return {
     ...row,
     past_teams: JSON.parse(row.past_teams),
     most_played_agents: JSON.parse(row.most_played_agents),
     stats: JSON.parse(row.stats),
-  };
+  }
 }
 
-function savePlayer(player) {
+// Saves a player's stats for one event, replacing any earlier save of them in that same event.
+// Returns the player row's id.
+function savePlayer(eventId, player) {
   upsertPlayerStmt.run({
     ...player,
+    event_id: eventId,
     past_teams: JSON.stringify(player.past_teams),
     most_played_agents: JSON.stringify(player.most_played_agents),
     stats: JSON.stringify(player.stats),
-  });
+  })
+  return db.prepare('SELECT id FROM players WHERE event_id = ? AND name = ?').get(eventId, player.name).id
 }
 
+function getPlayerById(id) {
+  return rowToPlayer(db.prepare(`${selectPlayersSql} WHERE players.id = ?`).get(id))
+}
+
+// The player as they were in their most recent event.
 function getPlayer(name) {
-  return rowToPlayer(db.prepare('SELECT * FROM players WHERE name = ?').get(name));
+  return rowToPlayer(db.prepare(`${selectPlayersSql} WHERE players.name = ? ${newestFirst} LIMIT 1`).get(name))
+}
+
+// Every saved version of a player, one per event, newest first.
+function getPlayerHistory(name) {
+  return db.prepare(`${selectPlayersSql} WHERE players.name = ? ${newestFirst}`).all(name).map(rowToPlayer)
+}
+
+// Case-insensitive prefix match, e.g. "nat" finds "nats" but not "gnat". Returns one row per event.
+function searchPlayers(query) {
+  return db.prepare(`${selectPlayersSql} WHERE players.name LIKE ? ESCAPE '\\' ORDER BY players.name, events.year DESC, players.id DESC`)
+    .all(`${query.replace(/[\\%_]/g, '\\$&')}%`)
+    .map(rowToPlayer)
 }
 
 function getAllPlayers() {
-  return db.prepare('SELECT * FROM players').all().map(rowToPlayer);
+  return db.prepare(selectPlayersSql).all().map(rowToPlayer)
 }
 
 function getRandomPlayer() {
-  return rowToPlayer(db.prepare('SELECT * FROM players ORDER BY RANDOM() LIMIT 1').get());
+  return rowToPlayer(db.prepare(`${selectPlayersSql} ORDER BY RANDOM() LIMIT 1`).get())
 }
 
+
 // Creates the event (tournament, stage, etc.) if it doesn't exist yet, and returns its id either way.
-function saveEvent({ name, url, region, year, stage }) {
+function saveEvent({ name, url, year, stage }) {
   upsertEventStmt.run({
     name,
     url: url ?? null,
-    region: region ?? null,
     year: year ?? null,
     stage: stage ?? null,
-  });
-  return db.prepare('SELECT id FROM events WHERE name = ?').get(name).id;
-}
-
-function addPlayerToEvent(eventId, playerName) {
-  linkPlayerToEventStmt.run({ eventId, playerName });
+  })
+  return db.prepare('SELECT id FROM events WHERE name = ?').get(name).id
 }
 
 function getEvents() {
-  return db.prepare('SELECT * FROM events').all();
+  return db.prepare('SELECT * FROM events').all()
 }
 
 function getEventByName(name) {
-  return db.prepare('SELECT * FROM events WHERE name = ?').get(name);
+  return db.prepare('SELECT * FROM events WHERE name = ?').get(name)
+}
+
+function getEventId(name, year) {
+  return db.prepare('SELECT id FROM events WHERE name = ? AND year = ?').get(name, year)?.id
 }
 
 function getPlayersByEvent(eventName) {
-  return db.prepare(`
-    SELECT players.* FROM players
-    JOIN event_players ON event_players.player_name = players.name
-    JOIN events ON events.id = event_players.event_id
-    WHERE events.name = ?
-  `).all(eventName).map(rowToPlayer);
+  return db.prepare(`${selectPlayersSql} WHERE events.name = ?`).all(eventName).map(rowToPlayer)
 }
 
 function getRandomPlayerFromEvent(eventName) {
-  return rowToPlayer(db.prepare(`
-    SELECT players.* FROM players
-    JOIN event_players ON event_players.player_name = players.name
-    JOIN events ON events.id = event_players.event_id
-    WHERE events.name = ?
-    ORDER BY RANDOM() LIMIT 1
-  `).get(eventName));
+  return rowToPlayer(db.prepare(`${selectPlayersSql} WHERE events.name = ? ORDER BY RANDOM() LIMIT 1`).get(eventName))
 }
 
-// Deletes players that no longer belong to any event.
-function deleteOrphanedPlayers() {
-  db.prepare('DELETE FROM players WHERE name NOT IN (SELECT player_name FROM event_players)').run();
+// Removes every player from an event but keeps the event itself, so it can be re-scraped.
+function emptyEvent(eventName) {
+  const event = getEventByName(eventName)
+  if (!event) return false
+  db.prepare('DELETE FROM players WHERE event_id = ?').run(event.id)
+  return true
 }
 
-// Removes every player from an event but keeps the event itself, so it can be re-scraped.
-// Players that are still in other events are kept.
-const emptyEvent = db.transaction((eventName) => {
-  const event = getEventByName(eventName);
-  if (!event) return false;
-  db.prepare('DELETE FROM event_players WHERE event_id = ?').run(event.id);
-  deleteOrphanedPlayers();
-  return true;
-});
-
-// Removes an event and its players. Players that are still in other events are kept.
-const clearEvent = db.transaction((eventName) => {
-  if (!emptyEvent(eventName)) return false;
-  db.prepare('DELETE FROM events WHERE name = ?').run(eventName);
-  return true;
-});
+// Removes an event and its players.
+function clearEvent(eventName) {
+  // The event's players are deleted along with it (ON DELETE CASCADE).
+  return db.prepare('DELETE FROM events WHERE name = ?').run(eventName).changes > 0
+}
 
 // Wipes every event and player from the database.
 const clearDatabase = db.transaction(() => {
-  db.exec('DELETE FROM event_players; DELETE FROM events; DELETE FROM players;');
-});
+  db.exec('DELETE FROM players; DELETE FROM events;')
+})
 
 export {
   savePlayer,
+  getPlayerById,
   getPlayer,
+  getPlayerHistory,
+  searchPlayers,
   getAllPlayers,
   getRandomPlayer,
   saveEvent,
-  addPlayerToEvent,
   getEvents,
   getEventByName,
+  getEventId,
   getPlayersByEvent,
   getRandomPlayerFromEvent,
   emptyEvent,
   clearEvent,
   clearDatabase,
-};
+}