vlr-stats-scraper 2.0.0 → 3.1.0

package/scripts/migrate.mjs

@@ -0,0 +1,181 @@
+// Brings any players.db from an older version up to the current schema.
+// Runs automatically when the database is opened for writing. Read-only connections never migrate.
+import { normalizeAgentRow, careerKd, cleanAgentName, toInteger, vlrIdFromUrl, AGENT_STAT_COLUMNS } from './stats.mjs'
+
+const SCHEMA_VERSION = 3
+
+const SCHEMA_SQL = `
+  CREATE TABLE IF NOT EXISTS events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    year INTEGER,
+    url TEXT,
+    vlr_event_id INTEGER,
+    stage INTEGER,
+    UNIQUE (name, year)
+  );
+
+  -- One row per player per event: who they were (team, region, ...) when that event was scraped.
+  CREATE TABLE IF NOT EXISTS players (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    event_id INTEGER NOT NULL REFERENCES events(id) ON DELETE CASCADE,
+    vlr_id INTEGER,
+    name TEXT NOT NULL COLLATE NOCASE,
+    url TEXT,
+    team TEXT,
+    past_teams TEXT NOT NULL DEFAULT '[]',
+    winnings INTEGER,
+    most_played_agents TEXT NOT NULL DEFAULT '[]',
+    avg_kda REAL,
+    nationality TEXT,
+    region TEXT,
+    scraped_at TEXT,
+    UNIQUE (event_id, vlr_id)
+  );
+  CREATE INDEX IF NOT EXISTS players_name ON players(name);
+  CREATE INDEX IF NOT EXISTS players_vlr_id ON players(vlr_id);
+
+  -- Career stats on the player's top agents, as they were at players.scraped_at.
+  CREATE TABLE IF NOT EXISTS agent_stats (
+    player_id INTEGER NOT NULL REFERENCES players(id) ON DELETE CASCADE,
+    position INTEGER NOT NULL,
+    agent TEXT NOT NULL,
+    maps INTEGER, use_pct REAL, rounds INTEGER, rating REAL, acs REAL, kd REAL, kast REAL,
+    adr REAL, kpr REAL, apr REAL, fkpr REAL, fdpr REAL, fk_fd REAL,
+    kills INTEGER, deaths INTEGER, assists INTEGER, first_kills INTEGER, first_deaths INTEGER,
+    PRIMARY KEY (player_id, agent)
+  );
+  CREATE INDEX IF NOT EXISTS agent_stats_agent ON agent_stats(agent);
+
+  -- The player's stats in this event only, from the event's stats page.
+  CREATE TABLE IF NOT EXISTS event_stats (
+    player_id INTEGER PRIMARY KEY REFERENCES players(id) ON DELETE CASCADE,
+    team TEXT,
+    agents TEXT NOT NULL DEFAULT '[]',
+    maps INTEGER, rounds INTEGER, rating REAL, acs REAL, kd REAL, kast REAL, adr REAL,
+    kpr REAL, apr REAL, fk_fd REAL, fk_pct REAL, fd_pct REAL, hs_pct REAL,
+    clutch_pct REAL, clutches_won INTEGER, clutches_played INTEGER, kills_max INTEGER,
+    kills INTEGER, deaths INTEGER, assists INTEGER, first_kills INTEGER, first_deaths INTEGER
+  );
+`
+
+const tableNames = db => db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(t => t.name)
+const columnNames = (db, table) => db.prepare(`PRAGMA table_info(${table})`).all().map(col => col.name)
+
+// 1.x: tournaments/tournament_players, then events/event_players with one shared row per player.
+// Copies each event link into its own player row, as 2.x did.
+function migrateFromV1(db) {
+  let tables = tableNames(db)
+  if (tables.includes('tournaments') && !tables.includes('events')) {
+    db.exec('ALTER TABLE tournaments RENAME TO events')
+  }
+  if (tables.includes('tournament_players') && !tables.includes('event_players')) {
+    db.exec('ALTER TABLE tournament_players RENAME TO event_players')
+    db.exec('ALTER TABLE event_players RENAME COLUMN tournament_id TO event_id')
+  }
+  tables = tableNames(db)
+  if (!tables.includes('players') || columnNames(db, 'players').includes('event_id')) return
+
+  // Players that weren't linked to any event have nowhere to go, so they're dropped.
+  if (!tables.includes('event_players') || !tables.includes('events')) {
+    db.exec('DROP TABLE IF EXISTS event_players; DROP TABLE players;')
+    return
+  }
+  const stats = columnNames(db, 'players').includes('stats') ? 'players.stats' : 'NULL'
+  db.exec(`
+    CREATE TABLE players_v2 AS
+    SELECT event_players.event_id AS event_id, players.name, players.team, players.past_teams,
+      players.winnings, players.most_played_agents, players.avg_kda, players.nationality,
+      players.region, ${stats} AS stats
+    FROM event_players
+    JOIN players ON players.name = event_players.player_name
+    JOIN events ON events.id = event_players.event_id
+  `)
+  db.exec('DROP TABLE event_players')
+  db.exec('DROP TABLE players')
+  db.exec('ALTER TABLE players_v2 RENAME TO players')
+}
+
+// 2.x -> 3: events unique by (name, year), vlr ids, numeric winnings, stats split into tables.
+function migrateFromV2(db) {
+  const oldEvents = tableNames(db).includes('events') ? db.prepare('SELECT * FROM events').all() : []
+  const oldPlayers = tableNames(db).includes('players') ? db.prepare('SELECT * FROM players').all() : []
+
+  db.exec('DROP TABLE IF EXISTS players; DROP TABLE IF EXISTS events;')
+  db.exec(SCHEMA_SQL)
+
+  const insertEvent = db.prepare(`
+    INSERT INTO events (id, name, year, url, vlr_event_id, stage)
+    VALUES (@id, @name, @year, @url, @vlr_event_id, @stage)
+  `)
+  for (const event of oldEvents) {
+    insertEvent.run({
+      id: event.id,
+      name: event.name,
+      year: event.year ?? null,
+      url: event.url ?? null,
+      vlr_event_id: vlrIdFromUrl(event.url, 'event'),
+      stage: event.stage ?? null,
+    })
+  }
+
+  const insertPlayer = db.prepare(`
+    INSERT INTO players (id, event_id, vlr_id, name, url, team, past_teams, winnings,
+      most_played_agents, avg_kda, nationality, region, scraped_at)
+    VALUES (@id, @event_id, NULL, @name, NULL, @team, @past_teams, @winnings,
+      @most_played_agents, @avg_kda, @nationality, @region, NULL)
+  `)
+  const insertAgent = db.prepare(`
+    INSERT OR IGNORE INTO agent_stats (player_id, position, agent, ${AGENT_STAT_COLUMNS.join(', ')})
+    VALUES (@player_id, @position, @agent, ${AGENT_STAT_COLUMNS.map(c => `@${c}`).join(', ')})
+  `)
+  const parse = (json, fallback) => { try { return JSON.parse(json) ?? fallback } catch { return fallback } }
+
+  oldPlayers.forEach((player, index) => {
+    if (!player.event_id) return
+    const id = player.id ?? index + 1
+    const agentRows = parse(player.stats, []).map(normalizeAgentRow).filter(row => row.agent)
+    const agents = parse(player.most_played_agents, []).map(cleanAgentName)
+    insertPlayer.run({
+      id,
+      event_id: player.event_id,
+      name: player.name,
+      team: player.team ?? null,
+      past_teams: JSON.stringify(parse(player.past_teams, [])),
+      winnings: toInteger(player.winnings),
+      most_played_agents: JSON.stringify(agents.length ? agents : agentRows.map(row => row.agent)),
+      // 2.x stored a plain average of each agent's K:D; recompute it weighted when we can.
+      avg_kda: (agentRows.length ? careerKd(agentRows) : null) ?? player.avg_kda ?? null,
+      nationality: player.nationality ?? null,
+      region: player.region ?? null,
+    })
+    agentRows.forEach((row, position) => insertAgent.run({ ...row, player_id: id, position }))
+  })
+}
+
+function migrate(db) {
+  const version = db.pragma('user_version', { simple: true })
+  if (version >= SCHEMA_VERSION) return
+
+  const hasOldData = tableNames(db).some(name => ['players', 'events', 'tournaments'].includes(name))
+
+  // Foreign keys have to be off while tables are rebuilt, and that can't change inside a transaction.
+  db.pragma('foreign_keys = OFF')
+  try {
+    db.transaction(() => {
+      if (hasOldData) {
+        migrateFromV1(db)
+        migrateFromV2(db)
+      } else {
+        db.exec(SCHEMA_SQL)
+      }
+      const problems = db.pragma('foreign_key_check')
+      if (problems.length) throw new Error(`Migration left ${problems.length} broken references`)
+      db.pragma(`user_version = ${SCHEMA_VERSION}`)
+    })()
+  } finally {
+    db.pragma('foreign_keys = ON')
+  }
+}
+
+export { migrate, SCHEMA_VERSION }

package/scripts/player-info-scraper.mjs

@@ -1,95 +1,59 @@
 import * as cheerio from 'cheerio'
-import { Hyperbrowser } from "@hyperbrowser/sdk"
-import { config } from "dotenv"
-
-config()
-
-let client;
-function getClient() {
-  if (!client) {
-    client = new Hyperbrowser({
-      apiKey: process.env.HYPERBROWSER_API_KEY,
-    })
-  }
-  return client;
-}
+import { fetchHtml } from './client.mjs'
+import { normalizeAgentRow, careerKd, toInteger, vlrIdFromUrl } from './stats.mjs'
 
+// Scrapes a player's profile page. The stats are whatever timespan the URL asks for
+// (the CLI uses ?timespan=all), so they're a career snapshot taken at `scraped_at`.
 async function scrapePlayer(url, region) {
-  // console.log('scrapePlayer ran')
   try {
-    const scrapeData = await getClient().scrape.startAndWait({
-      url: url,
-      scrapeOptions: {
-        formats: ["html"]
-      }
-    })
-
-    const $ = cheerio.load(scrapeData.data.html)
+    const $ = cheerio.load(await fetchHtml(url))
     const agentsPlayed = findAgents($)
 
     return {
+      vlr_id: vlrIdFromUrl(url, 'player'),
       name: getPlayerName(url),
+      url: url.split('?')[0],
       team: getCurTeam($),
       past_teams: getPastTeam($),
       winnings: getWinnings($),
-      most_played_agents: agentsPlayed.map(data => data.Agent),
-      avg_kda: agentsPlayed.reduce((acc, data) => acc + (parseFloat(data['K:D'])) / agentsPlayed.length, 0).toFixed(2),
+      most_played_agents: agentsPlayed.map(row => row.agent),
+      avg_kda: careerKd(agentsPlayed),
       nationality: getNationality($),
-      region: region,
-      stats: agentsPlayed
+      region: region ?? null,
+      stats: agentsPlayed,
+      scraped_at: new Date().toISOString(),
     }
   }
   catch (error) {
-    console.error("Error scraping website:", error);
+    console.error('Error scraping website:', error)
   }
 }
 
-// testing
-/*
-node scripts/player-info-scraper.mjs
-scrapePlayer("https://www.vlr.gg/player/939/mwzera/?timespan=all", "AMER")
-*/
-
-
 function getPlayerName(link) {
-  let name = new URL(link).pathname.split('/').filter(ele => ele !== '').pop()
-  return name
+  return new URL(link).pathname.split('/').filter(ele => ele !== '').pop()
 }
 
+// The player's top 3 agents, one normalized row each (see normalizeAgentRow in stats.mjs).
 function findAgents($) {
-  // console.log('findAgents ran')
-  const header = $('.mod-agent-rows').first().children().first()
-
-  const statCategories = header.text().split('\n').map(ele => ele.trim()).filter(ele => ele !== '')
-  statCategories.unshift('Agent')
-
-  const agentStats = header.next().children()
-  const agents = agentStats.map((i, el) => {
-    el = $(el)
-    // console.log('agent found:', el.children().first().find('img').attr('alt'))?.trim()
-    return {
-      agent: el.children().first().find('img').attr('alt')?.trim(),
-      stats: el.text().split('\t').map(ele => ele.trim()).filter(ele => ele !== '')
-    }
-  }).get()
-
-  const agentData = agents.map(agent => {
-    return statCategories.reduce((acc, category, index) => {
-      if(index == 0) {
-        acc[category] = agent.agent
-      }
-      else {
-        acc[category] = agent.stats[index-1]
-      }
-      return acc
-    }, {})
-  })
-
-  return agentData.length<3 ? agentData : agentData.slice(0,3)
+  const table = $('.mod-agent-rows').first()
+  const headers = table.find('thead th').map((i, el) => $(el).text().trim()).get()
+  headers[0] = 'Agent'
+
+  return table.find('tbody tr').map((i, row) => {
+    const cells = $(row).children('td')
+    const raw = {}
+    cells.each((index, cell) => {
+      const header = headers[index]
+      if (!header) return
+      raw[header] = index === 0
+        ? $(cell).find('img').attr('alt')?.trim()
+        : $(cell).text().trim()
+    })
+    return normalizeAgentRow(raw)
+  }).get().slice(0, 3)
 }
 
 function getCurTeam($) {
-  // console.log('getCurTeam ran')
   const currentTeam = $('h2:contains("Current Teams")').first().next().text().trim()
   const currentTeamName = currentTeam.split('\t').filter(ele => ele.trim() !== '')
   // Free agents have no current team
@@ -97,31 +61,23 @@ function getCurTeam($) {
 }
 
 function getPastTeam($) {
-    // console.log('getPastTeam ran')
-    const pastTeams = $('h2:contains("Past Teams")').first().next().text().trim()
-    const pastTeamNames = pastTeams.split('\t').filter(ele => ele.trim() !== ''
-                          && !ele.includes('joined')
-                          && !ele.includes(' – ')
-                          && !ele.includes('left'))
-                          .map(ele => ele.trim())
-    return pastTeamNames
+  const pastTeams = $('h2:contains("Past Teams")').first().next().text().trim()
+  return pastTeams.split('\t').filter(ele => ele.trim() !== ''
+    && !ele.includes('joined')
+    && !ele.includes(' – ')
+    && !ele.includes('left'))
+    .map(ele => ele.trim())
 }
 
+// Total event winnings in dollars, e.g. 123456. null if the player has none listed.
 function getWinnings($) {
-    // console.log('getWinnings ran')
-    const events = $('h2:contains("Event Placements")').first().next()
-    const winnings = events.children().first().text().trim()
-    // // console.log(winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1])
-    return winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1]
-  
+  const events = $('h2:contains("Event Placements")').first().next()
+  const winnings = events.children().first().text().trim()
+  return toInteger(winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1])
 }
 
 function getNationality($) {
-  // console.log('getNationality ran')
-  const header = $('.flag').first()
-  // console.log('flags found:', $('.flag').length);
-  // console.log(header.parent().text().trim())
-  return header.parent().text().trim()
+  return $('.flag').first().parent().text().trim()
 }
 
-export default scrapePlayer
+export default scrapePlayer

package/scripts/player-link-scraper.mjs

@@ -1,69 +1,41 @@
-import * as cheerio from 'cheerio'
-import { Hyperbrowser } from "@hyperbrowser/sdk"
-import { config } from "dotenv";
-
-config();
-
-let client;
-function getClient() {
-  if (!client) {
-    client = new Hyperbrowser({
-      apiKey: process.env.HYPERBROWSER_API_KEY,
-    })
-  }
-  return client;
-}
-
-async function scrapeWebsite(url) {
-  try {
-    const scrapeData = await getClient().scrape.startAndWait({
-      url: url,
-      scrapeOptions: {
-        formats: ["html"],
-        includeTags: [".event-header-main-title", ".event-team-players-item"]
-      },
-    })
-    const data = scrapeData.data.html
-    const $ = cheerio.load(data)
-
-    /*
-<h1 class="event-header-main-title">
-				VCT 2027: America Kickoff - North America Open Qualifier  
-			</h1>
-    */
-    const playerLinks = $('.event-team-players-item')
-    .map((i, el) => $(el).attr('href'))
-    .get()
-    .filter(href => href.includes('/player/'))
-    .map(href => href + '/?timespan=all')
-
-    let region = $('.event-header-main-title').first().text().trim()
-    console.log(region)
-
-    if (region.includes('America')) {
-      region = 'AMER'
-    } 
-    else if (region.includes('EMEA')) {
-      region = 'EMEA'
-    } 
-    else if (region.includes('Pacific')) {
-      region = 'PAC'
-    } 
-    else if (region.includes('China')) {
-      region = 'CN'
-    }
-    else {
-      region = null
-    }
-  
-    return { playerLinks, region }
-  } 
-  catch (error) {
-    console.error("Error scraping website:", error)
-  }
-}
-
-export { scrapeWebsite }
-/*
-node teamscraper.mjs
-*/
+import * as cheerio from 'cheerio'
+import { fetchHtml } from './client.mjs'
+import { vlrIdFromUrl } from './stats.mjs'
+
+// Scrapes an event page for its title, region and every player profile linked on it.
+async function scrapeWebsite(url) {
+  try {
+    const html = await fetchHtml(url, {
+      includeTags: ['.event-header-main-title', '.event-team-players-item'],
+    })
+    const $ = cheerio.load(html)
+
+    const playerLinks = [...new Set($('.event-team-players-item')
+      .map((i, el) => $(el).attr('href'))
+      .get()
+      .filter(href => href.includes('/player/'))
+      .map(href => new URL(href, 'https://www.vlr.gg').href.replace(/\/?$/, '/?timespan=all')))]
+
+    const title = $('.event-header-main-title').first().text().trim()
+
+    return {
+      playerLinks,
+      region: regionFromTitle(title),
+      title,
+      vlrEventId: vlrIdFromUrl(url, 'event'),
+    }
+  }
+  catch (error) {
+    console.error('Error scraping website:', error)
+  }
+}
+
+function regionFromTitle(title) {
+  if (title.includes('America')) return 'AMER'
+  if (title.includes('EMEA')) return 'EMEA'
+  if (title.includes('Pacific')) return 'PAC'
+  if (title.includes('China')) return 'CN'
+  return null
+}
+
+export { scrapeWebsite, regionFromTitle }

package/scripts/stats.mjs

@@ -0,0 +1,111 @@
+// Helpers that turn vlr.gg's display strings into numbers we can sort and filter on.
+
+// "1.25" -> 1.25, "77%" -> 77, "$123,456" -> 123456, "" / "-" / undefined -> null
+function toNumber(value) {
+  if (value === null || value === undefined) return null
+  if (typeof value === 'number') return Number.isFinite(value) ? value : null
+  const cleaned = String(value).replace(/[$,%\s]/g, '')
+  if (cleaned === '' || cleaned === '-') return null
+  const number = Number(cleaned)
+  return Number.isFinite(number) ? number : null
+}
+
+function toInteger(value) {
+  const number = toNumber(value)
+  return number === null ? null : Math.round(number)
+}
+
+// "(207) 45%" -> { maps: 207, use_pct: 45 }
+function parseUse(value) {
+  const match = String(value ?? '').match(/\((\d+)\)\s*([\d.]+)%?/)
+  return match ? { maps: Number(match[1]), use_pct: Number(match[2]) } : { maps: null, use_pct: null }
+}
+
+// Older saves stored agents as raw <img> tags, so pull the name out of the alt text.
+function cleanAgentName(agent) {
+  if (!agent) return agent ?? null
+  return String(agent).match(/alt="([^"]*)"/)?.[1] ?? String(agent)
+}
+
+// Column names of the per-agent stats table, keyed by every header vlr.gg has used for them.
+const AGENT_HEADER_TO_COLUMN = {
+  'RND': 'rounds',
+  'R': 'rating',
+  'RATING': 'rating',
+  'R2.0': 'rating',
+  'ACS': 'acs',
+  'K:D': 'kd',
+  'KAST': 'kast',
+  'ADR': 'adr',
+  'KPR': 'kpr',
+  'APR': 'apr',
+  'FKPR': 'fkpr',
+  'FDPR': 'fdpr',
+  'FK:FD': 'fk_fd',
+  'K': 'kills',
+  'D': 'deaths',
+  'A': 'assists',
+  'FK': 'first_kills',
+  'FD': 'first_deaths',
+}
+
+const AGENT_STAT_COLUMNS = [
+  'maps', 'use_pct', 'rounds', 'rating', 'acs', 'kd', 'kast', 'adr', 'kpr', 'apr',
+  'fkpr', 'fdpr', 'fk_fd', 'kills', 'deaths', 'assists', 'first_kills', 'first_deaths',
+]
+const INTEGER_COLUMNS = new Set(['maps', 'rounds', 'kills', 'deaths', 'assists', 'first_kills', 'first_deaths'])
+
+// Turns one agent row keyed by vlr.gg's headers ({ Agent: 'jett', Use: '(207) 45%', 'K:D': '1.32', ... })
+// into numeric columns ({ agent: 'jett', maps: 207, use_pct: 45, kd: 1.32, ... }).
+function normalizeAgentRow(raw) {
+  const row = Object.fromEntries(AGENT_STAT_COLUMNS.map(column => [column, null]))
+  row.agent = null
+  for (const [header, value] of Object.entries(raw ?? {})) {
+    const key = header.trim().toUpperCase()
+    if (key === 'AGENT') {
+      row.agent = cleanAgentName(value)?.toLowerCase() ?? null
+    } else if (key === 'USE') {
+      Object.assign(row, parseUse(value))
+    } else if (AGENT_HEADER_TO_COLUMN[key]) {
+      const column = AGENT_HEADER_TO_COLUMN[key]
+      row[column] = INTEGER_COLUMNS.has(column) ? toInteger(value) : toNumber(value)
+    } else if (AGENT_STAT_COLUMNS.includes(header)) {
+      // Already normalized (e.g. re-saving a player read back from the database).
+      row[header] = value ?? null
+    }
+  }
+  return row
+}
+
+// K:D across several agents, weighted by how much each was played: total kills / total deaths.
+// Falls back to a rounds-weighted average of each agent's K:D when kill/death totals are missing.
+function careerKd(agentRows) {
+  const rows = agentRows ?? []
+  const kills = rows.reduce((sum, row) => sum + (row.kills ?? 0), 0)
+  const deaths = rows.reduce((sum, row) => sum + (row.deaths ?? 0), 0)
+  if (deaths > 0 && rows.every(row => row.kills !== null && row.deaths !== null)) {
+    return Number((kills / deaths).toFixed(2))
+  }
+  const weighted = rows.filter(row => row.kd !== null)
+  if (!weighted.length) return null
+  const weightOf = row => row.rounds ?? 1
+  const totalWeight = weighted.reduce((sum, row) => sum + weightOf(row), 0)
+  return Number((weighted.reduce((sum, row) => sum + row.kd * weightOf(row), 0) / totalWeight).toFixed(2))
+}
+
+// https://www.vlr.gg/player/9/tenz/?timespan=all -> 9
+function vlrIdFromUrl(url, kind) {
+  const match = String(url ?? '').match(new RegExp(`/${kind}/(\\d+)`))
+  return match ? Number(match[1]) : null
+}
+
+export {
+  toNumber,
+  toInteger,
+  parseUse,
+  cleanAgentName,
+  normalizeAgentRow,
+  careerKd,
+  vlrIdFromUrl,
+  AGENT_STAT_COLUMNS,
+}