vlr-stats-scraper 1.4.1 → 2.0.0

package/scripts/player-info-scraper.mjs

@@ -1,128 +1,127 @@
-import * as cheerio from 'cheerio'
-import { Hyperbrowser } from "@hyperbrowser/sdk"
-import { config } from "dotenv"
-
-config()
-
-let client;
-function getClient() {
-  if (!client) {
-    client = new Hyperbrowser({
-      apiKey: process.env.HYPERBROWSER_API_KEY,
-    })
-  }
-  return client;
-}
-
-async function scrapePlayer(url, region) {
-  // console.log('scrapePlayer ran')
-  try {
-    const scrapeData = await getClient().scrape.startAndWait({
-      url: url,
-      scrapeOptions: {
-        formats: ["html"]
-      }
-    })
-
-    const $ = cheerio.load(scrapeData.data.html);
-    const agentsPlayed = findAgents($)
-
-    return {
-      name: getPlayerName(url),
-      team: getCurTeam($),
-      past_teams: getPastTeam($),
-      winnings: getWinnings($),
-      most_played_agents: agentsPlayed.map(data => data.Agent),
-      avg_kda: agentsPlayed.reduce((acc, data) => acc + (parseFloat(data['K:D'])) / agentsPlayed.length, 0).toFixed(2),
-      nationality: getNationality($),
-      region: region,
-      stats: agentsPlayed
-    }
-  }
-  catch (error) {
-    console.error("Error scraping website:", error);
-  }
-}
-
-// testing
-/*
-node scripts/player-info-scraper.mjs
-*/
-
-scrapePlayer("https://www.vlr.gg/player/939/mwzera/?timespan=all", "AMER")
-
-
-function getPlayerName(link) {
-  let name = new URL(link).pathname.split('/').filter(ele => ele !== '').pop()
-  return name
-}
-
-function findAgents($) {
-  // console.log('findAgents ran')
-  const header = $('.mod-agent-rows').first().children().first()
-
-  const statCategories = header.text().split('\n').map(ele => ele.trim()).filter(ele => ele !== '')
-  statCategories.unshift('Agent')
-
-  const agentStats = header.next().children()
-  const agents = agentStats.map((i, el) => {
-    el = $(el)
-    // console.log('agent found:', el.children().first().find('img').attr('alt'))?.trim()
-    return {
-      agent: el.children().first().find('img').attr('alt')?.trim(),
-      stats: el.text().split('\t').map(ele => ele.trim()).filter(ele => ele !== '')
-    }
-  }).get()
-
-  const agentData = agents.map(agent => {
-    return statCategories.reduce((acc, category, index) => {
-      if(index == 0) {
-        acc[category] = agent.agent
-      }
-      else {
-        acc[category] = agent.stats[index-1]
-      }
-      return acc
-    }, {})
-  })
-
-  return agentData.length<3 ? agentData : agentData.slice(0,3)
-}
-
-function getCurTeam($) {
-  // console.log('getCurTeam ran')
-  const currentTeam = $('h2:contains("Current Teams")').first().next().text().trim()
-  const currentTeamName = currentTeam.split('\t').filter(ele => ele.trim() !== '')
-  // Free agents have no current team
-  return currentTeamName.length ? currentTeamName[0].trim() : null
-}
-
-function getPastTeam($) {
-    // console.log('getPastTeam ran')
-    const pastTeams = $('h2:contains("Past Teams")').first().next().text().trim()
-    const pastTeamNames = pastTeams.split('\t').filter(ele => ele.trim() !== ''
-                          && !ele.includes('joined')
-                          && !ele.includes(' – ')
-                          && !ele.includes('left'))
-                          .map(ele => ele.trim())
-    return pastTeamNames
-}
-
-function getWinnings($) {
-    // console.log('getWinnings ran')
-    const events = $('h2:contains("Event Placements")').first().next()
-    const winnings = events.children().first().text().trim()
-    // // console.log(winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1])
-    return winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1]
-  
-}
-
-function getNationality($) {
-  // console.log('getNationality ran')
-  const header = $('.flag').first()
-  // console.log('flags found:', $('.flag').length);
-  // console.log(header.parent().text().trim())
-  return header.parent().text().trim()
-}
-
+import * as cheerio from 'cheerio'
+import { Hyperbrowser } from "@hyperbrowser/sdk"
+import { config } from "dotenv"
+
+config()
+
+let client;
+function getClient() {
+  if (!client) {
+    client = new Hyperbrowser({
+      apiKey: process.env.HYPERBROWSER_API_KEY,
+    })
+  }
+  return client;
+}
+
+async function scrapePlayer(url, region) {
+  // console.log('scrapePlayer ran')
+  try {
+    const scrapeData = await getClient().scrape.startAndWait({
+      url: url,
+      scrapeOptions: {
+        formats: ["html"]
+      }
+    })
+
+    const $ = cheerio.load(scrapeData.data.html)
+    const agentsPlayed = findAgents($)
+
+    return {
+      name: getPlayerName(url),
+      team: getCurTeam($),
+      past_teams: getPastTeam($),
+      winnings: getWinnings($),
+      most_played_agents: agentsPlayed.map(data => data.Agent),
+      avg_kda: agentsPlayed.reduce((acc, data) => acc + (parseFloat(data['K:D'])) / agentsPlayed.length, 0).toFixed(2),
+      nationality: getNationality($),
+      region: region,
+      stats: agentsPlayed
+    }
+  }
+  catch (error) {
+    console.error("Error scraping website:", error);
+  }
+}
+
+// testing
+/*
+node scripts/player-info-scraper.mjs
+scrapePlayer("https://www.vlr.gg/player/939/mwzera/?timespan=all", "AMER")
+*/
+
+
+function getPlayerName(link) {
+  let name = new URL(link).pathname.split('/').filter(ele => ele !== '').pop()
+  return name
+}
+
+function findAgents($) {
+  // console.log('findAgents ran')
+  const header = $('.mod-agent-rows').first().children().first()
+
+  const statCategories = header.text().split('\n').map(ele => ele.trim()).filter(ele => ele !== '')
+  statCategories.unshift('Agent')
+
+  const agentStats = header.next().children()
+  const agents = agentStats.map((i, el) => {
+    el = $(el)
+    // console.log('agent found:', el.children().first().find('img').attr('alt'))?.trim()
+    return {
+      agent: el.children().first().find('img').attr('alt')?.trim(),
+      stats: el.text().split('\t').map(ele => ele.trim()).filter(ele => ele !== '')
+    }
+  }).get()
+
+  const agentData = agents.map(agent => {
+    return statCategories.reduce((acc, category, index) => {
+      if(index == 0) {
+        acc[category] = agent.agent
+      }
+      else {
+        acc[category] = agent.stats[index-1]
+      }
+      return acc
+    }, {})
+  })
+
+  return agentData.length<3 ? agentData : agentData.slice(0,3)
+}
+
+function getCurTeam($) {
+  // console.log('getCurTeam ran')
+  const currentTeam = $('h2:contains("Current Teams")').first().next().text().trim()
+  const currentTeamName = currentTeam.split('\t').filter(ele => ele.trim() !== '')
+  // Free agents have no current team
+  return currentTeamName.length ? currentTeamName[0].trim() : null
+}
+
+function getPastTeam($) {
+    // console.log('getPastTeam ran')
+    const pastTeams = $('h2:contains("Past Teams")').first().next().text().trim()
+    const pastTeamNames = pastTeams.split('\t').filter(ele => ele.trim() !== ''
+                          && !ele.includes('joined')
+                          && !ele.includes(' – ')
+                          && !ele.includes('left'))
+                          .map(ele => ele.trim())
+    return pastTeamNames
+}
+
+function getWinnings($) {
+    // console.log('getWinnings ran')
+    const events = $('h2:contains("Event Placements")').first().next()
+    const winnings = events.children().first().text().trim()
+    // // console.log(winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1])
+    return winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1]
+  
+}
+
+function getNationality($) {
+  // console.log('getNationality ran')
+  const header = $('.flag').first()
+  // console.log('flags found:', $('.flag').length);
+  // console.log(header.parent().text().trim())
+  return header.parent().text().trim()
+}
+
 export default scrapePlayer

package/scripts/player-link-scraper.mjs

@@ -1,3 +1,4 @@
+import * as cheerio from 'cheerio'
 import { Hyperbrowser } from "@hyperbrowser/sdk"
 import { config } from "dotenv";
 
@@ -19,24 +20,50 @@ async function scrapeWebsite(url) {
       url: url,
       scrapeOptions: {
         formats: ["html"],
-        includeTags: [".event-team-players-item"]
+        includeTags: [".event-header-main-title", ".event-team-players-item"]
       },
     })
+    const data = scrapeData.data.html
+    const $ = cheerio.load(data)
 
-    const players = scrapeData.data.html.split('\n').filter(line => line.includes('event-team-players-item'))
-    const playerLinks = players.map(player => {
-      const match = player.match(/href="(https:\/\/www\.vlr\.gg\/player\/[^"]*)"/)
-      return match ? match[1] + "/?timespan=all" : null;
-    }).filter(link => link !== null);
-    
-    return playerLinks
+    /*
+<h1 class="event-header-main-title">
+				VCT 2027: America Kickoff - North America Open Qualifier  
+			</h1>
+    */
+    const playerLinks = $('.event-team-players-item')
+    .map((i, el) => $(el).attr('href'))
+    .get()
+    .filter(href => href.includes('/player/'))
+    .map(href => href + '/?timespan=all')
+
+    let region = $('.event-header-main-title').first().text().trim()
+    console.log(region)
+
+    if (region.includes('America')) {
+      region = 'AMER'
+    } 
+    else if (region.includes('EMEA')) {
+      region = 'EMEA'
+    } 
+    else if (region.includes('Pacific')) {
+      region = 'PAC'
+    } 
+    else if (region.includes('China')) {
+      region = 'CN'
+    }
+    else {
+      region = null
+    }
+  
+    return { playerLinks, region }
   } 
   catch (error) {
     console.error("Error scraping website:", error)
   }
 }
 
-export { scrapeWebsite };
+export { scrapeWebsite }
 /*
 node teamscraper.mjs
 */