vlr-stats-scraper 1.0.0 → 1.1.0

package/README.md

@@ -1,74 +1,88 @@
-# vlr-stats-scraper
-
-Scrapes player stats from [vlr.gg](https://www.vlr.gg) and stores them in a local SQLite
-database, grouped by tournament or stage, so you can pull a backlog of players for a
-specific event or game mode later.
-
-## Setup
-
-Requires a [Hyperbrowser](https://hyperbrowser.ai) API key for scraping.
-
-```bash
-npm install vlr-stats-scraper
-```
-
-Create a `.env` file in the directory you run it from:
-
-```
-HYPERBROWSER_API_KEY=your_key_here
-```
-
-## CLI usage
-
-```bash
-npx valstats <link> <year> <event-name>
-```
-
-```bash
-npx valstats https://www.vlr.gg/event/2977/vct-2026-americas-stage-2/group-stage 2026 "Stage 2"
-```
-
-- `link` — the vlr.gg event/group-stage page to scrape.
-- `year` — the year the event took place.
-- `event-name` — a label for this tournament/stage (used to group players together later).
-
-Each player on that event page gets scraped and saved to `players.db` in your current
-directory, linked to that event name.
-
-## Library usage
-
-```js
-import {
-  scrapePlayer,
-  scrapeWebsite,
-  savePlayer,
-  saveEvent,
-  addPlayerToEvent,
-  getPlayersByEvent,
-  getRandomPlayerFromEvent,
-} from 'vlr-stats-scraper';
-
-const links = await scrapeWebsite('https://www.vlr.gg/event/.../group-stage');
-const player = await scrapePlayer(links[0], 'AMER');
-savePlayer(player);
-```
-
-### Exports
-
-- `scrapePlayer(url, region)` — scrapes one player's profile page.
-- `scrapeWebsite(url)` — scrapes an event page for player profile links.
-- `savePlayer(player)` / `getPlayer(name)` / `getAllPlayers()` / `getRandomPlayer()`
-- `saveEvent({ name, url, region, year, stage })` — creates/updates a tournament or stage,
-  returns its id.
-- `addPlayerToEvent(eventId, playerName)`
-- `getEvents()` / `getEventByName(name)`
-- `getPlayersByEvent(eventName)` / `getRandomPlayerFromEvent(eventName)`
-
-## Data storage
-
-Player and event data is stored in a `players.db` SQLite file created in whichever
-directory you run the tool from.
-
-## License
-
-ISC
+# vlr-stats-scraper
+
+Scrapes player stats from [vlr.gg](https://www.vlr.gg) and stores them in a local SQLite
+database, grouped by tournament or stage, so you can pull a backlog of players.
+
+## Setup
+
+Requires a [Hyperbrowser](https://hyperbrowser.ai) API key for scraping.
+
+```bash
+npm install vlr-stats-scraper
+```
+
+Create a `.env` file in the directory you run it from:
+
+```
+HYPERBROWSER_API_KEY=your_key_here
+```
+
+## CLI usage
+
+```bash
+npx valstats <link> <year> <event-name>
+```
+
+```bash
+npx valstats https://www.vlr.gg/event/2977/vct-2026-americas-stage-2/group-stage 2026 "Stage 2"
+```
+
+- `link` — the vlr.gg event/group-stage page to scrape.
+- `year` — the year the event took place.
+- `event-name` — a label for this tournament/stage (used to group players together later).
+
+Each player on that event page gets scraped and saved to `players.db` in your current
+directory, linked to that event name.
+
+Running it again with an event name and year that are already saved replaces that event's
+players with the freshly scraped ones.
+
+### Clearing data
+
+```bash
+npx valstats --clear-event "Stage 2"   # remove one event and its players
+npx valstats --clear                   # wipe the whole database (asks for confirmation)
+```
+
+Players that also belong to another event are kept when a single event is cleared.
+
+## Library usage
+
+```js
+import {
+  scrapePlayer,
+  scrapeWebsite,
+  savePlayer,
+  saveEvent,
+  addPlayerToEvent,
+  getPlayersByEvent,
+  getRandomPlayerFromEvent,
+} from 'vlr-stats-scraper';
+
+const links = await scrapeWebsite('https://www.vlr.gg/event/.../group-stage');
+const player = await scrapePlayer(links[0], 'AMER');
+savePlayer(player);
+```
+
+### Exports
+
+- `scrapePlayer(url, region)` — scrapes one player's profile page.
+- `scrapeWebsite(url)` — scrapes an event page for player profile links.
+- `savePlayer(player)` / `getPlayer(name)` / `getAllPlayers()` / `getRandomPlayer()`
+- `saveEvent({ name, url, region, year, stage })` — creates/updates a tournament or stage,
+  returns its id.
+- `addPlayerToEvent(eventId, playerName)`
+- `getEvents()` / `getEventByName(name)`
+- `getPlayersByEvent(eventName)` / `getRandomPlayerFromEvent(eventName)`
+- `emptyEvent(eventName)` — removes an event's players but keeps the event.
+- `clearEvent(eventName)` — removes an event and its players.
+- `clearDatabase()` — deletes every event and player.
+
+## Data storage
+
+Player and event data is stored in a `players.db` SQLite file created in whichever
+directory you run the tool from.
+
+## License
+
+ISC

package/index.mjs

@@ -11,4 +11,7 @@ export {
   getEventByName,
   getPlayersByEvent,
   getRandomPlayerFromEvent,
+  emptyEvent,
+  clearEvent,
+  clearDatabase,
 } from './scripts/db.mjs';

package/main.mjs

@@ -2,12 +2,51 @@
 import promptSync from 'prompt-sync';
 import { scrapeWebsite } from './scripts/player-link-scraper.mjs';
 import scrapePlayer from './scripts/player-info-scraper.mjs';
-import { savePlayer, saveEvent, addPlayerToEvent } from './scripts/db.mjs';
+import {
+  savePlayer,
+  saveEvent,
+  addPlayerToEvent,
+  getEventByName,
+  emptyEvent,
+  clearEvent,
+  clearDatabase,
+} from './scripts/db.mjs';
 
-const [link, year, eventName] = process.argv.slice(2);
+const args = process.argv.slice(2);
+
+if (args[0] === '--clear') {
+  const prompt = promptSync();
+  const answer = prompt('This deletes every event and player in players.db. Type "yes" to continue: ');
+  if (answer?.trim().toLowerCase() === 'yes') {
+    clearDatabase();
+    console.log('Cleared players.db');
+  } else {
+    console.log('Cancelled');
+  }
+  process.exit(0);
+}
+
+if (args[0] === '--clear-event') {
+  const name = args[1];
+  if (!name) {
+    console.error('Usage: valstats --clear-event <event-name>');
+    process.exit(1);
+  }
+  if (clearEvent(name)) {
+    console.log(`Removed "${name}" and its players from players.db`);
+  } else {
+    console.error(`No event named "${name}" in players.db`);
+    process.exit(1);
+  }
+  process.exit(0);
+}
+
+const [link, year, eventName] = args;
 
 if (!link || !year || !eventName) {
   console.error('Usage: valstats <link> <year> <event-name>');
+  console.error('       valstats --clear-event <event-name>');
+  console.error('       valstats --clear');
   process.exit(1);
 }
 
@@ -19,6 +58,13 @@ if (isStage) {
   region = prompt('Enter the region: ');
 }
 
+// Re-adding an event + year that's already saved replaces its old players.
+const existing = getEventByName(eventName);
+if (existing && existing.year === parseInt(year, 10)) {
+  emptyEvent(eventName);
+  console.log(`"${eventName}" (${year}) is already in players.db, replacing it`);
+}
+
 const eventId = saveEvent({
   name: eventName,
   url: link,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vlr-stats-scraper",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Scrapes player stats from vlr.gg and stores them in a local SQLite database, grouped by tournament or stage.",
   "homepage": "https://github.com/OverYander/VLR-Webscraper#readme",
   "bugs": {
@@ -16,7 +16,7 @@
   "main": "./index.mjs",
   "exports": "./index.mjs",
   "bin": {
-    "valstats": "./main.mjs"
+    "valstats": "main.mjs"
   },
   "files": [
     "index.mjs",

package/scripts/db.mjs

@@ -161,6 +161,33 @@ function getRandomPlayerFromEvent(eventName) {
   `).get(eventName));
 }
 
+// Deletes players that no longer belong to any event.
+function deleteOrphanedPlayers() {
+  db.prepare('DELETE FROM players WHERE name NOT IN (SELECT player_name FROM event_players)').run();
+}
+
+// Removes every player from an event but keeps the event itself, so it can be re-scraped.
+// Players that are still in other events are kept.
+const emptyEvent = db.transaction((eventName) => {
+  const event = getEventByName(eventName);
+  if (!event) return false;
+  db.prepare('DELETE FROM event_players WHERE event_id = ?').run(event.id);
+  deleteOrphanedPlayers();
+  return true;
+});
+
+// Removes an event and its players. Players that are still in other events are kept.
+const clearEvent = db.transaction((eventName) => {
+  if (!emptyEvent(eventName)) return false;
+  db.prepare('DELETE FROM events WHERE name = ?').run(eventName);
+  return true;
+});
+
+// Wipes every event and player from the database.
+const clearDatabase = db.transaction(() => {
+  db.exec('DELETE FROM event_players; DELETE FROM events; DELETE FROM players;');
+});
+
 export {
   savePlayer,
   getPlayer,
@@ -172,4 +199,7 @@ export {
   getEventByName,
   getPlayersByEvent,
   getRandomPlayerFromEvent,
+  emptyEvent,
+  clearEvent,
+  clearDatabase,
 };

package/scripts/player-info-scraper.mjs

@@ -4,14 +4,20 @@ import { config } from "dotenv";
 
 config();
 
-const client = new Hyperbrowser({
-  apiKey: process.env.HYPERBROWSER_API_KEY,
-})
+let client;
+function getClient() {
+  if (!client) {
+    client = new Hyperbrowser({
+      apiKey: process.env.HYPERBROWSER_API_KEY,
+    })
+  }
+  return client;
+}
 
 async function scrapePlayer(url, region) {
   console.log('scrapePlayer ran')
   try {
-    const scrapeData = await client.scrape.startAndWait({
+    const scrapeData = await getClient().scrape.startAndWait({
       url: url,
       scrapeOptions: {
         formats: ["html"]

package/scripts/player-link-scraper.mjs

@@ -3,13 +3,19 @@ import { config } from "dotenv";
 
 config();
 
-const client = new Hyperbrowser({
-  apiKey: process.env.HYPERBROWSER_API_KEY,
-})
+let client;
+function getClient() {
+  if (!client) {
+    client = new Hyperbrowser({
+      apiKey: process.env.HYPERBROWSER_API_KEY,
+    })
+  }
+  return client;
+}
 
 async function scrapeWebsite(url) {
   try {
-    const scrapeData = await client.scrape.startAndWait({
+    const scrapeData = await getClient().scrape.startAndWait({
       url: url,
       scrapeOptions: {
         formats: ["html"],
@@ -19,14 +25,14 @@ async function scrapeWebsite(url) {
 
     const players = scrapeData.data.html.split('\n').filter(line => line.includes('event-team-players-item'))
     const playerLinks = players.map(player => {
-      const match = player.match(/href="(https:\/\/www\.vlr\.gg\/player\/[^"]*)"/);
-      return match ? match[1] : null;
+      const match = player.match(/href="(https:\/\/www\.vlr\.gg\/player\/[^"]*)"/)
+      return match ? match[1] + "/?timespan=all" : null;
     }).filter(link => link !== null);
     
-    return playerLinks;
+    return playerLinks
   } 
   catch (error) {
-    console.error("Error scraping website:", error);
+    console.error("Error scraping website:", error)
   }
 }