vlr-stats-scraper 1.5.0 → 3.1.0

package/main.mjs

@@ -1,12 +1,13 @@
 #!/usr/bin/env node
 import promptSync from 'prompt-sync'
 import { scrapeWebsite } from './scripts/player-link-scraper.mjs'
+import { scrapeEventStats } from './scripts/event-stats-scraper.mjs'
 import scrapePlayer from './scripts/player-info-scraper.mjs'
 import {
   savePlayer,
   saveEvent,
-  addPlayerToEvent,
-  getEventByName,
+  getEvent,
+  getPlayersByEvent,
   emptyEvent,
   clearEvent,
   clearDatabase,
@@ -15,33 +16,90 @@ import {
 
 const args = process.argv.slice(2)
 
+const USAGE = [
+  'Usage: valstats <link> <year> <event-name>',
+  '       valstats --player <vlr.gg player link> <event-name> <year> [region]',
+  '       valstats --search <name | vlr.gg player link>',
+  '       valstats --clear-event <event-name> [year]',
+  '       valstats --clear',
+].join('\n')
+
+const fmt = (value, digits = 2) => (value === null || value === undefined ? '-' : Number(value).toFixed(digits))
+
 function printPlayerStats(player) {
-  console.log(`\n${player.name}`)
+  // Players saved from the database belong to an event; ones scraped live don't.
+  const event = player.event_name ? ` (${player.event_name}${player.event_year ? `, ${player.event_year}` : ''})` : ''
+  console.log(`\n${player.name}${event}`)
   console.log(`  Team:        ${player.team ?? 'Free agent'}`)
   console.log(`  Past teams:  ${player.past_teams?.length ? player.past_teams.join(', ') : 'None'}`)
   console.log(`  Nationality: ${player.nationality || 'Unknown'}`)
   console.log(`  Region:      ${player.region ?? 'Unknown'}`)
-  console.log(`  Winnings:    ${player.winnings ?? 'Unknown'}`)
-  // Older saves stored agents as raw <img> tags, so pull the name out of the alt text.
-  const agents = (player.most_played_agents ?? []).map(agent => agent?.match(/alt="([^"]*)"/)?.[1] ?? agent)
-  console.log(`  Top agents:  ${agents.join(', ') || 'None'}`)
-  console.log(`  Avg K:D:     ${Number(player.avg_kda).toFixed(2)}`)
+  console.log(`  Winnings:    ${player.winnings === null || player.winnings === undefined ? 'Unknown' : `$${player.winnings.toLocaleString('en-US')}`}`)
+  console.log(`  Top agents:  ${player.most_played_agents?.join(', ') || 'None'}`)
+  console.log(`  Career K:D:  ${fmt(player.avg_kda)} (top agents)`)
+  if (player.scraped_at) console.log(`  Scraped:     ${player.scraped_at.slice(0, 10)}`)
+
+  const es = player.event_stats
+  if (es) {
+    const agents = es.agents.map(a => `${a.agent}${a.use_pct !== null ? ` ${a.use_pct}%` : ''}`).join(', ')
+    console.log(`  This event:  ${es.maps ?? '-'} maps, ${es.rounds ?? '-'} rounds on ${agents || 'unknown agents'}`)
+    console.log(`               Rating ${fmt(es.rating)} | ACS ${fmt(es.acs, 0)} | K:D ${fmt(es.kd)} | ADR ${fmt(es.adr, 1)} | KAST ${fmt(es.kast, 0)}% | HS ${fmt(es.hs_pct, 0)}%`)
+  } else if (player.event_name) {
+    console.log('  This event:  no stats saved')
+  }
+
   if (player.stats?.length) {
-    console.table(player.stats)
+    console.log('  Career stats on top agents:')
+    console.table(player.stats.map(({ agent, maps, rounds, rating, acs, kd, kast, adr, kpr, apr, kills, deaths }) =>
+      ({ agent, maps, rounds, rating, acs, kd, kast, adr, kpr, apr, kills, deaths })))
   }
 }
 
+const withTimespan = url => (url.includes('timespan=') ? url : url.replace(/\/?$/, '/?timespan=all'))
+
+// Manually adds one player to an event that's already saved.
 if (args[0] === '--player') {
+  const [, query, eventName, eventYear, regionArg] = args
+  if (!query?.startsWith('http') || !eventName || !eventYear) {
+    console.error('Usage: valstats --player <vlr.gg player link> <event-name> <year> [region]')
+    process.exit(1)
+  }
+
+  // Look the event up before scraping so a typo doesn't waste a scrape.
+  const event = getEvent(eventName, parseInt(eventYear, 10))
+  if (!event) {
+    console.error(`No event named "${eventName}" (${eventYear}) in players.db`)
+    process.exit(1)
+  }
+
+  // Without a region argument, use the region the event's other players were saved with.
+  const region = regionArg ?? getPlayersByEvent(event.name, event.year)[0]?.region ?? null
+
+  const player = await scrapePlayer(withTimespan(query), region)
+  if (!player) {
+    console.error(`Couldn't scrape a player from ${query}`)
+    process.exit(1)
+  }
+  if (event.url) {
+    const eventStats = await scrapeEventStats(event.url)
+    player.event_stats = eventStats?.get(player.vlr_id) ?? null
+  }
+  savePlayer(event.id, player)
+  console.log(`Saved ${player.name} to players.db under "${eventName}" (${eventYear})`)
+  printPlayerStats({ ...player, event_name: event.name, event_year: event.year })
+  process.exit(0)
+}
+
+if (args[0] === '--search') {
   const query = args[1]
   if (!query) {
-    console.error('Usage: valstats --player <name | vlr.gg player link>')
+    console.error('Usage: valstats --search <name | vlr.gg player link>')
     process.exit(1)
   }
 
-  // A vlr.gg link is scraped live anything else is searched for in players.db.
+  // A vlr.gg link is scraped live; anything else is searched for in players.db.
   if (query.startsWith('http')) {
-    const url = query.includes('timespan=') ? query : query.replace(/\/?$/, '/?timespan=all')
-    const player = await scrapePlayer(url, null)
+    const player = await scrapePlayer(withTimespan(query), null)
     if (!player) {
       console.error(`Couldn't scrape a player from ${query}`)
       process.exit(1)
@@ -73,66 +131,64 @@ if (args[0] === '--clear') {
 }
 
 if (args[0] === '--clear-event') {
-  const name = args[1]
+  const [, name, year] = args
   if (!name) {
-    console.error('Usage: valstats --clear-event <event-name>')
+    console.error('Usage: valstats --clear-event <event-name> [year]')
     process.exit(1)
   }
-  if (clearEvent(name)) {
-    console.log(`Removed "${name}" and its players from players.db`)
-  } else {
-    console.error(`No event named "${name}" in players.db`)
+  try {
+    if (clearEvent(name, year ? parseInt(year, 10) : undefined)) {
+      console.log(`Removed "${name}"${year ? ` (${year})` : ''} and its players from players.db`)
+    } else {
+      console.error(`No event named "${name}"${year ? ` (${year})` : ''} in players.db`)
+      process.exit(1)
+    }
+  } catch (error) {
+    console.error(error.message)
     process.exit(1)
   }
   process.exit(0)
 }
 
-const [link, year, eventName] = args
+const [link, yearArg, eventName] = args
+const year = parseInt(yearArg, 10)
 
-if (!link || !year || !eventName) {
-  console.error('Usage: valstats <link> <year> <event-name>')
-  console.error('       valstats --player <name | vlr.gg player link>')
-  console.error('       valstats --clear-event <event-name>')
-  console.error('       valstats --clear')
+if (!link || !yearArg || !eventName || Number.isNaN(year)) {
+  console.error(USAGE)
   process.exit(1)
 }
 
-const isStage = eventName.toLowerCase().includes('stage')
+// Scrape first so a failed scrape doesn't wipe an event that's already saved.
+const scraped = await scrapeWebsite(link)
 
-let region = null
-if (isStage) {
-  const prompt = promptSync()
-  region = prompt('Enter the region: ')
+if (!scraped) {
+  console.error(`Couldn't get the player list from ${link}`)
+  process.exit(1)
+}
+
+const { playerLinks, region } = scraped
+console.log(`Found ${playerLinks.length} players, region: ${region ?? 'unknown'}`)
+
+// One scrape of the event's stats page gives every player's stats in this event only.
+const eventStats = await scrapeEventStats(link)
+if (!eventStats) {
+  console.warn('Couldn\'t get this event\'s stats page; players will be saved without event stats')
 }
 
 // Re-adding an event + year that's already saved replaces its old players.
-const existing = getEventByName(eventName)
-if (existing && existing.year === parseInt(year, 10)) {
-  emptyEvent(eventName)
+if (getEvent(eventName, year)) {
+  emptyEvent(eventName, year)
   console.log(`"${eventName}" (${year}) is already in players.db, replacing it`)
 }
 
-const eventId = saveEvent({
-  name: eventName,
-  url: link,
-  region,
-  year: parseInt(year, 10),
-  stage: null,
-})
-
-const links = await scrapeWebsite(link)
-
-if (!links) {
-  console.error(`Couldn't get the player list from ${link}`)
-  process.exit(1)
-}
+const eventId = saveEvent({ name: eventName, url: link, year, stage: null })
 
-for (const playerLink of links) {
+for (const playerLink of playerLinks) {
   const player = await scrapePlayer(playerLink, region)
   if (player) {
-    savePlayer(player)
-    addPlayerToEvent(eventId, player.name)
-    console.log(`Saved ${player.name} to players.db under "${eventName}"`)
-    console.log(player)
+    player.event_stats = eventStats?.get(player.vlr_id) ?? null
+    savePlayer(eventId, player)
+    console.log(`Saved ${player.name} to players.db under "${eventName}" (${year})`)
+    printPlayerStats({ ...player, event_name: eventName, event_year: year })
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vlr-stats-scraper",
-  "version": "1.5.0",
+  "version": "3.1.0",
   "description": "Scrapes player stats from vlr.gg and stores them in a local SQLite database, grouped by tournament or stage.",
   "homepage": "https://github.com/OverYander/VLR-Webscraper#readme",
   "bugs": {

package/scripts/client.mjs

@@ -0,0 +1,27 @@
+import { Hyperbrowser } from '@hyperbrowser/sdk'
+import { config } from 'dotenv'
+
+config()
+
+let client
+function getClient() {
+  if (!client) {
+    client = new Hyperbrowser({
+      apiKey: process.env.HYPERBROWSER_API_KEY,
+    })
+  }
+  return client
+}
+
+// Fetches a page's HTML through Hyperbrowser. Throws if the scrape fails.
+async function fetchHtml(url, scrapeOptions = {}) {
+  const scrapeData = await getClient().scrape.startAndWait({
+    url,
+    scrapeOptions: { formats: ['html'], ...scrapeOptions },
+  })
+  const html = scrapeData?.data?.html
+  if (!html) throw new Error(`No HTML returned for ${url}`)
+  return html
+}
+
+export { fetchHtml }