vlr-stats-scraper 0.0.0-stage → 1.0.0

package/LICENSE

@@ -0,0 +1,15 @@
+ISC License
+
+Copyright (c) 2026, OverYander
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

package/README.md

@@ -1,3 +1,74 @@
-# Temporary Holding Version
+# vlr-stats-scraper
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Scrapes player stats from [vlr.gg](https://www.vlr.gg) and stores them in a local SQLite
+database, grouped by tournament or stage, so you can pull a backlog of players for a
+specific event or game mode later.
+
+## Setup
+
+Requires a [Hyperbrowser](https://hyperbrowser.ai) API key for scraping.
+
+```bash
+npm install vlr-stats-scraper
+```
+
+Create a `.env` file in the directory you run it from:
+
+```
+HYPERBROWSER_API_KEY=your_key_here
+```
+
+## CLI usage
+
+```bash
+npx valstats <link> <year> <event-name>
+```
+
+```bash
+npx valstats https://www.vlr.gg/event/2977/vct-2026-americas-stage-2/group-stage 2026 "Stage 2"
+```
+
+- `link` — the vlr.gg event/group-stage page to scrape.
+- `year` — the year the event took place.
+- `event-name` — a label for this tournament/stage (used to group players together later).
+
+Each player on that event page gets scraped and saved to `players.db` in your current
+directory, linked to that event name.
+
+## Library usage
+
+```js
+import {
+  scrapePlayer,
+  scrapeWebsite,
+  savePlayer,
+  saveEvent,
+  addPlayerToEvent,
+  getPlayersByEvent,
+  getRandomPlayerFromEvent,
+} from 'vlr-stats-scraper';
+
+const links = await scrapeWebsite('https://www.vlr.gg/event/.../group-stage');
+const player = await scrapePlayer(links[0], 'AMER');
+savePlayer(player);
+```
+
+### Exports
+
+- `scrapePlayer(url, region)` — scrapes one player's profile page.
+- `scrapeWebsite(url)` — scrapes an event page for player profile links.
+- `savePlayer(player)` / `getPlayer(name)` / `getAllPlayers()` / `getRandomPlayer()`
+- `saveEvent({ name, url, region, year, stage })` — creates/updates a tournament or stage,
+  returns its id.
+- `addPlayerToEvent(eventId, playerName)`
+- `getEvents()` / `getEventByName(name)`
+- `getPlayersByEvent(eventName)` / `getRandomPlayerFromEvent(eventName)`
+
+## Data storage
+
+Player and event data is stored in a `players.db` SQLite file created in whichever
+directory you run the tool from.
+
+## License
+
+ISC

package/index.mjs

@@ -0,0 +1,14 @@
+export { default as scrapePlayer } from './scripts/player-info-scraper.mjs';
+export { scrapeWebsite } from './scripts/player-link-scraper.mjs';
+export {
+  savePlayer,
+  getPlayer,
+  getAllPlayers,
+  getRandomPlayer,
+  saveEvent,
+  addPlayerToEvent,
+  getEvents,
+  getEventByName,
+  getPlayersByEvent,
+  getRandomPlayerFromEvent,
+} from './scripts/db.mjs';

package/main.mjs

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+import promptSync from 'prompt-sync';
+import { scrapeWebsite } from './scripts/player-link-scraper.mjs';
+import scrapePlayer from './scripts/player-info-scraper.mjs';
+import { savePlayer, saveEvent, addPlayerToEvent } from './scripts/db.mjs';
+
+const [link, year, eventName] = process.argv.slice(2);
+
+if (!link || !year || !eventName) {
+  console.error('Usage: valstats <link> <year> <event-name>');
+  process.exit(1);
+}
+
+const isStage = eventName.toLowerCase().includes('stage');
+
+let region = null;
+if (isStage) {
+  const prompt = promptSync();
+  region = prompt('Enter the region: ');
+}
+
+const eventId = saveEvent({
+  name: eventName,
+  url: link,
+  region,
+  year: parseInt(year, 10),
+  stage: null,
+});
+
+const links = await scrapeWebsite(link);
+
+for (const playerLink of links) {
+  const player = await scrapePlayer(playerLink, region);
+  if (player) {
+    savePlayer(player);
+    addPlayerToEvent(eventId, player.name);
+    console.log(`Saved ${player.name} to players.db under "${eventName}"`);
+    console.log(player);
+  }
+}

package/package.json

@@ -1,6 +1,49 @@
 {
   "name": "vlr-stats-scraper",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.0.0",
+  "description": "Scrapes player stats from vlr.gg and stores them in a local SQLite database, grouped by tournament or stage.",
+  "homepage": "https://github.com/OverYander/VLR-Webscraper#readme",
+  "bugs": {
+    "url": "https://github.com/OverYander/VLR-Webscraper/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/OverYander/VLR-Webscraper.git"
+  },
+  "license": "ISC",
+  "author": "OverYander <gavinchen8706@gmail.com>",
+  "type": "module",
+  "main": "./index.mjs",
+  "exports": "./index.mjs",
+  "bin": {
+    "valstats": "./main.mjs"
+  },
+  "files": [
+    "index.mjs",
+    "main.mjs",
+    "scripts/",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "vlr",
+    "valorant",
+    "esports",
+    "scraper",
+    "sqlite",
+    "stats"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "@hyperbrowser/sdk": "^0.92.3",
+    "better-sqlite3": "^13.0.3",
+    "cheerio": "^1.2.0",
+    "dotenv": "^17.4.2",
+    "prompt-sync": "^4.2.0"
+  }
+}

package/scripts/db.mjs

@@ -0,0 +1,175 @@
+import Database from 'better-sqlite3';
+import path from 'path';
+
+const db = new Database(path.join(process.cwd(), 'players.db'));
+
+db.exec(`
+  CREATE TABLE IF NOT EXISTS players (
+    name TEXT PRIMARY KEY,
+    team TEXT,
+    past_teams TEXT,
+    winnings TEXT,
+    most_played_agents TEXT,
+    avg_kda REAL,
+    nationality TEXT,
+    region TEXT,
+    stats TEXT
+  )
+`);
+
+const existingPlayerColumns = db.prepare('PRAGMA table_info(players)').all().map(col => col.name);
+if (!existingPlayerColumns.includes('stats')) {
+  db.exec('ALTER TABLE players ADD COLUMN stats TEXT');
+}
+
+const existingTableNames = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(t => t.name);
+if (existingTableNames.includes('tournaments') && !existingTableNames.includes('events')) {
+  db.exec('ALTER TABLE tournaments RENAME TO events');
+}
+if (existingTableNames.includes('tournament_players') && !existingTableNames.includes('event_players')) {
+  db.exec('ALTER TABLE tournament_players RENAME TO event_players');
+  db.exec('ALTER TABLE event_players RENAME COLUMN tournament_id TO event_id');
+}
+
+db.exec(`
+  CREATE TABLE IF NOT EXISTS events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT UNIQUE NOT NULL,
+    url TEXT,
+    region TEXT,
+    year INTEGER,
+    stage INTEGER
+  )
+`);
+
+const existingEventColumns = db.prepare('PRAGMA table_info(events)').all().map(col => col.name);
+if (!existingEventColumns.includes('year')) {
+  db.exec('ALTER TABLE events ADD COLUMN year INTEGER');
+}
+if (!existingEventColumns.includes('stage')) {
+  db.exec('ALTER TABLE events ADD COLUMN stage INTEGER');
+}
+
+db.exec(`
+  CREATE TABLE IF NOT EXISTS event_players (
+    event_id INTEGER NOT NULL REFERENCES events(id),
+    player_name TEXT NOT NULL REFERENCES players(name),
+    PRIMARY KEY (event_id, player_name)
+  )
+`);
+
+const upsertPlayerStmt = db.prepare(`
+  INSERT INTO players (name, team, past_teams, winnings, most_played_agents, avg_kda, nationality, region, stats)
+  VALUES (@name, @team, @past_teams, @winnings, @most_played_agents, @avg_kda, @nationality, @region, @stats)
+  ON CONFLICT(name) DO UPDATE SET
+    team = excluded.team,
+    past_teams = excluded.past_teams,
+    winnings = excluded.winnings,
+    most_played_agents = excluded.most_played_agents,
+    avg_kda = excluded.avg_kda,
+    nationality = excluded.nationality,
+    region = excluded.region,
+    stats = excluded.stats
+`);
+
+const upsertEventStmt = db.prepare(`
+  INSERT INTO events (name, url, region, year, stage)
+  VALUES (@name, @url, @region, @year, @stage)
+  ON CONFLICT(name) DO UPDATE SET
+    url = excluded.url,
+    region = excluded.region,
+    year = excluded.year,
+    stage = excluded.stage
+`);
+
+const linkPlayerToEventStmt = db.prepare(`
+  INSERT OR IGNORE INTO event_players (event_id, player_name)
+  VALUES (@eventId, @playerName)
+`);
+
+function rowToPlayer(row) {
+  if (!row) return null;
+  return {
+    ...row,
+    past_teams: JSON.parse(row.past_teams),
+    most_played_agents: JSON.parse(row.most_played_agents),
+    stats: JSON.parse(row.stats),
+  };
+}
+
+function savePlayer(player) {
+  upsertPlayerStmt.run({
+    ...player,
+    past_teams: JSON.stringify(player.past_teams),
+    most_played_agents: JSON.stringify(player.most_played_agents),
+    stats: JSON.stringify(player.stats),
+  });
+}
+
+function getPlayer(name) {
+  return rowToPlayer(db.prepare('SELECT * FROM players WHERE name = ?').get(name));
+}
+
+function getAllPlayers() {
+  return db.prepare('SELECT * FROM players').all().map(rowToPlayer);
+}
+
+function getRandomPlayer() {
+  return rowToPlayer(db.prepare('SELECT * FROM players ORDER BY RANDOM() LIMIT 1').get());
+}
+
+// Creates the event (tournament, stage, etc.) if it doesn't exist yet, and returns its id either way.
+function saveEvent({ name, url, region, year, stage }) {
+  upsertEventStmt.run({
+    name,
+    url: url ?? null,
+    region: region ?? null,
+    year: year ?? null,
+    stage: stage ?? null,
+  });
+  return db.prepare('SELECT id FROM events WHERE name = ?').get(name).id;
+}
+
+function addPlayerToEvent(eventId, playerName) {
+  linkPlayerToEventStmt.run({ eventId, playerName });
+}
+
+function getEvents() {
+  return db.prepare('SELECT * FROM events').all();
+}
+
+function getEventByName(name) {
+  return db.prepare('SELECT * FROM events WHERE name = ?').get(name);
+}
+
+function getPlayersByEvent(eventName) {
+  return db.prepare(`
+    SELECT players.* FROM players
+    JOIN event_players ON event_players.player_name = players.name
+    JOIN events ON events.id = event_players.event_id
+    WHERE events.name = ?
+  `).all(eventName).map(rowToPlayer);
+}
+
+function getRandomPlayerFromEvent(eventName) {
+  return rowToPlayer(db.prepare(`
+    SELECT players.* FROM players
+    JOIN event_players ON event_players.player_name = players.name
+    JOIN events ON events.id = event_players.event_id
+    WHERE events.name = ?
+    ORDER BY RANDOM() LIMIT 1
+  `).get(eventName));
+}
+
+export {
+  savePlayer,
+  getPlayer,
+  getAllPlayers,
+  getRandomPlayer,
+  saveEvent,
+  addPlayerToEvent,
+  getEvents,
+  getEventByName,
+  getPlayersByEvent,
+  getRandomPlayerFromEvent,
+};

package/scripts/player-info-scraper.mjs

@@ -0,0 +1,116 @@
+import * as cheerio from 'cheerio';
+import { Hyperbrowser } from "@hyperbrowser/sdk"
+import { config } from "dotenv";
+
+config();
+
+const client = new Hyperbrowser({
+  apiKey: process.env.HYPERBROWSER_API_KEY,
+})
+
+async function scrapePlayer(url, region) {
+  console.log('scrapePlayer ran')
+  try {
+    const scrapeData = await client.scrape.startAndWait({
+      url: url,
+      scrapeOptions: {
+        formats: ["html"]
+      }
+    })
+
+    const $ = cheerio.load(scrapeData.data.html);
+
+    const agentsPlayed = findAgents($)
+    return {
+      name: getPlayerName(url),
+      team: getCurTeam($),
+      past_teams: getPastTeam($),
+      winnings: getWinnings($),
+      most_played_agents: agentsPlayed.map(data => data.Agent),
+      avg_kda: agentsPlayed.reduce((acc, data) => acc + (parseFloat(data['K:D'])) / agentsPlayed.length, 0),
+      nationality: getNationality($),
+      region: region,
+      stats: agentsPlayed
+    }
+  }
+  catch (error) {
+    console.error("Error scraping website:", error);
+  }
+}
+
+export default scrapePlayer;
+
+/*
+node scripts/player-info-scraper.mjs
+*/
+
+function getPlayerName(link) {
+  console.log('getPlayerName ran')
+  return link.substring(link.lastIndexOf('/') + 1)
+}
+
+function findAgents($) {
+  console.log('findAgents ran')
+  const header = $('.mod-agent-rows').first().children().first()
+
+  const statCategories = header.text().split('\n').map(ele => ele.trim()).filter(ele => ele !== '')
+  statCategories.unshift('Agent')
+
+  const agentStats = header.next().children()
+  const agents = agentStats.map((i, el) => {
+    el = $(el)
+    return {
+      agent: el.children().first().html().trim(),
+      stats: el.text().split('\t').map(ele => ele.trim()).filter(ele => ele !== '')
+    }
+  }).get()
+
+  const agentData = agents.map(agent => {
+    return statCategories.reduce((acc, category, index) => {
+      if(index == 0) {
+        acc[category] = agent.agent
+      }
+      else {
+        acc[category] = agent.stats[index-1]
+      }
+      return acc
+    }, {})
+  })
+
+  return agentData.length<3 ? agentData : agentData.slice(0,3)
+}
+
+function getCurTeam($) {
+  console.log('getCurTeam ran')
+  const currentTeam = $('h2:contains("Current Teams")').first().next().text().trim()
+  const currentTeamName = currentTeam.split('\t').filter(ele => ele.trim() !== '')
+  return currentTeamName[0].trim()
+}
+
+function getPastTeam($) {
+    console.log('getPastTeam ran')
+    const pastTeams = $('h2:contains("Past Teams")').first().next().text().trim()
+    const pastTeamNames = pastTeams.split('\t').filter(ele => ele.trim() !== ''
+                          && !ele.includes('joined')
+                          && !ele.includes(' – ')
+                          && !ele.includes('left'))
+                          .map(ele => ele.trim())
+    return pastTeamNames
+}
+
+function getWinnings($) {
+    console.log('getWinnings ran')
+    const events = $('h2:contains("Event Placements")').first().next()
+    const winnings = events.children().first().text().trim()
+    // console.log(winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1])
+    return winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1]
+  
+}
+
+function getNationality($) {
+  console.log('getNationality ran')
+  const header = $('.flag').first()
+  console.log('flags found:', $('.flag').length);
+  console.log(header.parent().text().trim())
+  return header.parent().text().trim()
+}

package/scripts/player-link-scraper.mjs

@@ -0,0 +1,36 @@
+import { Hyperbrowser } from "@hyperbrowser/sdk"
+import { config } from "dotenv";
+
+config();
+
+const client = new Hyperbrowser({
+  apiKey: process.env.HYPERBROWSER_API_KEY,
+})
+
+async function scrapeWebsite(url) {
+  try {
+    const scrapeData = await client.scrape.startAndWait({
+      url: url,
+      scrapeOptions: {
+        formats: ["html"],
+        includeTags: [".event-team-players-item"]
+      },
+    })
+
+    const players = scrapeData.data.html.split('\n').filter(line => line.includes('event-team-players-item'))
+    const playerLinks = players.map(player => {
+      const match = player.match(/href="(https:\/\/www\.vlr\.gg\/player\/[^"]*)"/);
+      return match ? match[1] : null;
+    }).filter(link => link !== null);
+    
+    return playerLinks;
+  } 
+  catch (error) {
+    console.error("Error scraping website:", error);
+  }
+}
+
+export { scrapeWebsite };
+/*
+node teamscraper.mjs
+*/