vlr-stats-scraper 2.0.0 → 3.1.0

package/index.mjs

@@ -1,20 +1,27 @@
-export { default as scrapePlayer } from './scripts/player-info-scraper.mjs';
-export { scrapeWebsite } from './scripts/player-link-scraper.mjs';
-export {
-  savePlayer,
-  getPlayerById,
-  getPlayer,
-  getPlayerHistory,
-  searchPlayers,
-  getAllPlayers,
-  getRandomPlayer,
-  saveEvent,
-  getEvents,
-  getEventByName,
-  getEventId,
-  getPlayersByEvent,
-  getRandomPlayerFromEvent,
-  emptyEvent,
-  clearEvent,
-  clearDatabase,
-} from './scripts/db.mjs';
+export { default as scrapePlayer } from './scripts/player-info-scraper.mjs';
+export { scrapeWebsite } from './scripts/player-link-scraper.mjs';
+export { scrapeEventStats, eventStatsUrl } from './scripts/event-stats-scraper.mjs';
+export {
+  openDatabase,
+  closeDatabase,
+  savePlayer,
+  getPlayerById,
+  getPlayer,
+  getPlayerHistory,
+  searchPlayers,
+  getAllPlayers,
+  getRandomPlayer,
+  getPlayersByTeam,
+  getEventLeaderboard,
+  getAgentLeaderboard,
+  saveEvent,
+  getEvents,
+  getEvent,
+  getEventByName,
+  getEventId,
+  getPlayersByEvent,
+  getRandomPlayerFromEvent,
+  emptyEvent,
+  clearEvent,
+  clearDatabase,
+} from './scripts/db.mjs';

package/main.mjs

@@ -1,166 +1,194 @@
-#!/usr/bin/env node
-import promptSync from 'prompt-sync'
-import { scrapeWebsite } from './scripts/player-link-scraper.mjs'
-import scrapePlayer from './scripts/player-info-scraper.mjs'
-import {
-  savePlayer,
-  saveEvent,
-  getEventByName,
-  getEventId,
-  getPlayersByEvent,
-  emptyEvent,
-  clearEvent,
-  clearDatabase,
-  searchPlayers,
-} from './scripts/db.mjs'
-
-const args = process.argv.slice(2)
-
-function printPlayerStats(player) {
-  // Players saved from the database belong to an event; ones scraped live don't.
-  const event = player.event_name ? ` (${player.event_name}${player.event_year ? `, ${player.event_year}` : ''})` : ''
-  console.log(`\n${player.name}${event}`)
-  console.log(`  Team:        ${player.team ?? 'Free agent'}`)
-  console.log(`  Past teams:  ${player.past_teams?.length ? player.past_teams.join(', ') : 'None'}`)
-  console.log(`  Nationality: ${player.nationality || 'Unknown'}`)
-  console.log(`  Region:      ${player.region ?? 'Unknown'}`)
-  console.log(`  Winnings:    ${player.winnings ?? 'Unknown'}`)
-  // Older saves stored agents as raw <img> tags, so pull the name out of the alt text.
-  const agents = (player.most_played_agents ?? []).map(agent => agent?.match(/alt="([^"]*)"/)?.[1] ?? agent)
-  console.log(`  Top agents:  ${agents.join(', ') || 'None'}`)
-  console.log(`  Avg K:D:     ${Number(player.avg_kda).toFixed(2)}`)
-  if (player.stats?.length) {
-    console.table(player.stats)
-  }
-}
-
-// Manually adds one player to an event that's already saved.
-if (args[0] === '--player') {
-  const [, query, eventName, eventYear, regionArg] = args
-  if (!query?.startsWith('http') || !eventName || !eventYear) {
-    console.error('Usage: valstats --player <vlr.gg player link> <event-name> <year> [region]')
-    process.exit(1)
-  }
-
-  // Look the event up before scraping so a typo doesn't waste a scrape.
-  const eventId = getEventId(eventName, parseInt(eventYear, 10))
-  if (!eventId) {
-    console.error(`No event named "${eventName}" (${eventYear}) in players.db`)
-    process.exit(1)
-  }
-
-  // Without a region argument, use the region the event's other players were saved with.
-  const region = regionArg ?? getPlayersByEvent(eventName)[0]?.region ?? null
-
-  const url = query.includes('timespan=') ? query : query.replace(/\/?$/, '/?timespan=all')
-  const player = await scrapePlayer(url, region)
-  if (!player) {
-    console.error(`Couldn't scrape a player from ${query}`)
-    process.exit(1)
-  }
-  savePlayer(eventId, player)
-  console.log(`Saved ${player.name} to players.db under "${eventName}" (${eventYear})`)
-  printPlayerStats(player)
-  process.exit(0)
-}
-
-if (args[0] === '--search') {
-  const query = args[1]
-  if (!query) {
-    console.error('Usage: valstats --search <name | vlr.gg player link>')
-    process.exit(1)
-  }
-
-  // A vlr.gg link is scraped live anything else is searched for in players.db.
-  if (query.startsWith('http')) {
-    const url = query.includes('timespan=') ? query : query.replace(/\/?$/, '/?timespan=all')
-    const player = await scrapePlayer(url, null)
-    if (!player) {
-      console.error(`Couldn't scrape a player from ${query}`)
-      process.exit(1)
-    }
-    printPlayerStats(player)
-    process.exit(0)
-  }
-
-  const players = searchPlayers(query)
-  if (!players.length) {
-    console.error(`No players matching "${query}" in players.db (pass a vlr.gg player link to scrape them live)`)
-    process.exit(1)
-  }
-  console.log(`${players.length} player${players.length === 1 ? '' : 's'} matching "${query}":`)
-  players.forEach(printPlayerStats)
-  process.exit(0)
-}
-
-if (args[0] === '--clear') {
-  const prompt = promptSync()
-  const answer = prompt('This deletes every event and player in players.db. Type "yes" to continue: ')
-  if (answer?.trim().toLowerCase() === 'yes') {
-    clearDatabase()
-    console.log('Cleared players.db')
-  } else {
-    console.log('Cancelled')
-  }
-  process.exit(0)
-}
-
-if (args[0] === '--clear-event') {
-  const name = args[1]
-  if (!name) {
-    console.error('Usage: valstats --clear-event <event-name>')
-    process.exit(1)
-  }
-  if (clearEvent(name)) {
-    console.log(`Removed "${name}" and its players from players.db`)
-  } else {
-    console.error(`No event named "${name}" in players.db`)
-    process.exit(1)
-  }
-  process.exit(0)
-}
-
-const [link, year, eventName] = args
-
-if (!link || !year || !eventName) {
-  console.error('Usage: valstats <link> <year> <event-name>')
-  console.error('       valstats --player <vlr.gg player link> <event-name> <year> [region]')
-  console.error('       valstats --search <name | vlr.gg player link>')
-  console.error('       valstats --clear-event <event-name>')
-  console.error('       valstats --clear')
-  process.exit(1)
-}
-
-// Scrape first so a failed scrape doesn't wipe an event that's already saved.
-const scraped = await scrapeWebsite(link)
-
-if (!scraped) {
-  console.error(`Couldn't get the player list from ${link}`)
-  process.exit(1)
-}
-
-const { playerLinks, region } = scraped
-console.log(`Found ${playerLinks.length} players, region: ${region ?? 'unknown'}`)
-
-// Re-adding an event + year that's already saved replaces its old players.
-const existing = getEventByName(eventName)
-if (existing && existing.year === parseInt(year, 10)) {
-  emptyEvent(eventName)
-  console.log(`"${eventName}" (${year}) is already in players.db, replacing it`)
-}
-
-const eventId = saveEvent({
-  name: eventName,
-  url: link,
-  year: parseInt(year, 10),
-  stage: null,
-})
-
-for (const playerLink of playerLinks) {
-  const player = await scrapePlayer(playerLink, region)
-  if (player) {
-    savePlayer(eventId, player)
-    console.log(`Saved ${player.name} to players.db under "${eventName}"`)
-    console.log(player)
-  }
-}
+#!/usr/bin/env node
+import promptSync from 'prompt-sync'
+import { scrapeWebsite } from './scripts/player-link-scraper.mjs'
+import { scrapeEventStats } from './scripts/event-stats-scraper.mjs'
+import scrapePlayer from './scripts/player-info-scraper.mjs'
+import {
+  savePlayer,
+  saveEvent,
+  getEvent,
+  getPlayersByEvent,
+  emptyEvent,
+  clearEvent,
+  clearDatabase,
+  searchPlayers,
+} from './scripts/db.mjs'
+
+const args = process.argv.slice(2)
+
+const USAGE = [
+  'Usage: valstats <link> <year> <event-name>',
+  '       valstats --player <vlr.gg player link> <event-name> <year> [region]',
+  '       valstats --search <name | vlr.gg player link>',
+  '       valstats --clear-event <event-name> [year]',
+  '       valstats --clear',
+].join('\n')
+
+const fmt = (value, digits = 2) => (value === null || value === undefined ? '-' : Number(value).toFixed(digits))
+
+function printPlayerStats(player) {
+  // Players saved from the database belong to an event; ones scraped live don't.
+  const event = player.event_name ? ` (${player.event_name}${player.event_year ? `, ${player.event_year}` : ''})` : ''
+  console.log(`\n${player.name}${event}`)
+  console.log(`  Team:        ${player.team ?? 'Free agent'}`)
+  console.log(`  Past teams:  ${player.past_teams?.length ? player.past_teams.join(', ') : 'None'}`)
+  console.log(`  Nationality: ${player.nationality || 'Unknown'}`)
+  console.log(`  Region:      ${player.region ?? 'Unknown'}`)
+  console.log(`  Winnings:    ${player.winnings === null || player.winnings === undefined ? 'Unknown' : `$${player.winnings.toLocaleString('en-US')}`}`)
+  console.log(`  Top agents:  ${player.most_played_agents?.join(', ') || 'None'}`)
+  console.log(`  Career K:D:  ${fmt(player.avg_kda)} (top agents)`)
+  if (player.scraped_at) console.log(`  Scraped:     ${player.scraped_at.slice(0, 10)}`)
+
+  const es = player.event_stats
+  if (es) {
+    const agents = es.agents.map(a => `${a.agent}${a.use_pct !== null ? ` ${a.use_pct}%` : ''}`).join(', ')
+    console.log(`  This event:  ${es.maps ?? '-'} maps, ${es.rounds ?? '-'} rounds on ${agents || 'unknown agents'}`)
+    console.log(`               Rating ${fmt(es.rating)} | ACS ${fmt(es.acs, 0)} | K:D ${fmt(es.kd)} | ADR ${fmt(es.adr, 1)} | KAST ${fmt(es.kast, 0)}% | HS ${fmt(es.hs_pct, 0)}%`)
+  } else if (player.event_name) {
+    console.log('  This event:  no stats saved')
+  }
+
+  if (player.stats?.length) {
+    console.log('  Career stats on top agents:')
+    console.table(player.stats.map(({ agent, maps, rounds, rating, acs, kd, kast, adr, kpr, apr, kills, deaths }) =>
+      ({ agent, maps, rounds, rating, acs, kd, kast, adr, kpr, apr, kills, deaths })))
+  }
+}
+
+const withTimespan = url => (url.includes('timespan=') ? url : url.replace(/\/?$/, '/?timespan=all'))
+
+// Manually adds one player to an event that's already saved.
+if (args[0] === '--player') {
+  const [, query, eventName, eventYear, regionArg] = args
+  if (!query?.startsWith('http') || !eventName || !eventYear) {
+    console.error('Usage: valstats --player <vlr.gg player link> <event-name> <year> [region]')
+    process.exit(1)
+  }
+
+  // Look the event up before scraping so a typo doesn't waste a scrape.
+  const event = getEvent(eventName, parseInt(eventYear, 10))
+  if (!event) {
+    console.error(`No event named "${eventName}" (${eventYear}) in players.db`)
+    process.exit(1)
+  }
+
+  // Without a region argument, use the region the event's other players were saved with.
+  const region = regionArg ?? getPlayersByEvent(event.name, event.year)[0]?.region ?? null
+
+  const player = await scrapePlayer(withTimespan(query), region)
+  if (!player) {
+    console.error(`Couldn't scrape a player from ${query}`)
+    process.exit(1)
+  }
+  if (event.url) {
+    const eventStats = await scrapeEventStats(event.url)
+    player.event_stats = eventStats?.get(player.vlr_id) ?? null
+  }
+  savePlayer(event.id, player)
+  console.log(`Saved ${player.name} to players.db under "${eventName}" (${eventYear})`)
+  printPlayerStats({ ...player, event_name: event.name, event_year: event.year })
+  process.exit(0)
+}
+
+if (args[0] === '--search') {
+  const query = args[1]
+  if (!query) {
+    console.error('Usage: valstats --search <name | vlr.gg player link>')
+    process.exit(1)
+  }
+
+  // A vlr.gg link is scraped live; anything else is searched for in players.db.
+  if (query.startsWith('http')) {
+    const player = await scrapePlayer(withTimespan(query), null)
+    if (!player) {
+      console.error(`Couldn't scrape a player from ${query}`)
+      process.exit(1)
+    }
+    printPlayerStats(player)
+    process.exit(0)
+  }
+
+  const players = searchPlayers(query)
+  if (!players.length) {
+    console.error(`No players matching "${query}" in players.db (pass a vlr.gg player link to scrape them live)`)
+    process.exit(1)
+  }
+  console.log(`${players.length} player${players.length === 1 ? '' : 's'} matching "${query}":`)
+  players.forEach(printPlayerStats)
+  process.exit(0)
+}
+
+if (args[0] === '--clear') {
+  const prompt = promptSync()
+  const answer = prompt('This deletes every event and player in players.db. Type "yes" to continue: ')
+  if (answer?.trim().toLowerCase() === 'yes') {
+    clearDatabase()
+    console.log('Cleared players.db')
+  } else {
+    console.log('Cancelled')
+  }
+  process.exit(0)
+}
+
+if (args[0] === '--clear-event') {
+  const [, name, year] = args
+  if (!name) {
+    console.error('Usage: valstats --clear-event <event-name> [year]')
+    process.exit(1)
+  }
+  try {
+    if (clearEvent(name, year ? parseInt(year, 10) : undefined)) {
+      console.log(`Removed "${name}"${year ? ` (${year})` : ''} and its players from players.db`)
+    } else {
+      console.error(`No event named "${name}"${year ? ` (${year})` : ''} in players.db`)
+      process.exit(1)
+    }
+  } catch (error) {
+    console.error(error.message)
+    process.exit(1)
+  }
+  process.exit(0)
+}
+
+const [link, yearArg, eventName] = args
+const year = parseInt(yearArg, 10)
+
+if (!link || !yearArg || !eventName || Number.isNaN(year)) {
+  console.error(USAGE)
+  process.exit(1)
+}
+
+// Scrape first so a failed scrape doesn't wipe an event that's already saved.
+const scraped = await scrapeWebsite(link)
+
+if (!scraped) {
+  console.error(`Couldn't get the player list from ${link}`)
+  process.exit(1)
+}
+
+const { playerLinks, region } = scraped
+console.log(`Found ${playerLinks.length} players, region: ${region ?? 'unknown'}`)
+
+// One scrape of the event's stats page gives every player's stats in this event only.
+const eventStats = await scrapeEventStats(link)
+if (!eventStats) {
+  console.warn('Couldn\'t get this event\'s stats page; players will be saved without event stats')
+}
+
+// Re-adding an event + year that's already saved replaces its old players.
+if (getEvent(eventName, year)) {
+  emptyEvent(eventName, year)
+  console.log(`"${eventName}" (${year}) is already in players.db, replacing it`)
+}
+
+const eventId = saveEvent({ name: eventName, url: link, year, stage: null })
+
+for (const playerLink of playerLinks) {
+  const player = await scrapePlayer(playerLink, region)
+  if (player) {
+    player.event_stats = eventStats?.get(player.vlr_id) ?? null
+    savePlayer(eventId, player)
+    console.log(`Saved ${player.name} to players.db under "${eventName}" (${year})`)
+    printPlayerStats({ ...player, event_name: eventName, event_year: year })
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vlr-stats-scraper",
-  "version": "2.0.0",
+  "version": "3.1.0",
   "description": "Scrapes player stats from vlr.gg and stores them in a local SQLite database, grouped by tournament or stage.",
   "homepage": "https://github.com/OverYander/VLR-Webscraper#readme",
   "bugs": {

package/scripts/client.mjs

@@ -0,0 +1,27 @@
+import { Hyperbrowser } from '@hyperbrowser/sdk'
+import { config } from 'dotenv'
+
+config()
+
+let client
+function getClient() {
+  if (!client) {
+    client = new Hyperbrowser({
+      apiKey: process.env.HYPERBROWSER_API_KEY,
+    })
+  }
+  return client
+}
+
+// Fetches a page's HTML through Hyperbrowser. Throws if the scrape fails.
+async function fetchHtml(url, scrapeOptions = {}) {
+  const scrapeData = await getClient().scrape.startAndWait({
+    url,
+    scrapeOptions: { formats: ['html'], ...scrapeOptions },
+  })
+  const html = scrapeData?.data?.html
+  if (!html) throw new Error(`No HTML returned for ${url}`)
+  return html
+}
+
+export { fetchHtml }