grepsr-cli 0.7.4__py3-none-any.whl

grepsrcli/templates/node_boilerplate.jinja2

@@ -0,0 +1,100 @@
+/**
+ * Name: {{plugin_name}}
+ * Description: {{plugin_name}}
+ * PID: {{pid}} force
+ * PROP-mem: 512
+ */
+
+process.env['APIFY_MEMORY_MBYTES'] = 512;
+
+const Apify = require('apify');
+const cheerio = require('cheerio');
+const vortexNodebackEnd = require('vortex-node-backend');
+const GrepsrPuppeteerCrawler = vortexNodebackEnd.puppeteer_crawler;
+const options = vortexNodebackEnd.settings;
+const Proxy = vortexNodebackEnd.proxy;
+
+//pageName using dateformat
+let dateFormat = require("dateformat");
+const pageDate = dateFormat(new Date(), "yyyy-mm-dd");
+const page_name = `google_map_location_search-${pageDate}`;
+
+let crawler;
+const fs = require('fs');
+
+
+class Crawler extends GrepsrPuppeteerCrawler {
+    constructor(options) {
+        // Initialize parent.
+        //override settings here
+        super(options);
+        // Do any other initializations below.
+        this.handlePageFunction = this.handlePage;
+        this.handleFailedRequestFunction = this.handleFailedRequest;
+        this.gotoFunction = this.goto;
+
+    }
+
+
+    async goto({request, page}) {
+        await page.evaluateOnNewDocument(() => {
+            Object.defineProperty(navigator, 'webdriver', {
+                get: () => false,
+            });
+        });
+        await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36');
+        await page.setDefaultTimeout(60000);
+        // await page.setRequestInterception(true);
+
+        // page.on('request', (req) => {
+        //     if (req.resourceType() === 'stylesheet' || req.resourceType() === 'font' || req.resourceType() === 'image') {
+        //         req.abort();
+        //     } else {
+        //         req.continue();
+        //     }
+        // });
+        return await page.goto(request.url);
+    }
+
+
+    async handlePage({request, page}) {
+        console.log(`Processing ${request.url}`);
+  
+        let contents = await page.content();
+        const $ = cheerio.load(contents);
+        console.log(`data loaded ${request.url}`);
+
+
+    }
+
+    async handleFailedRequest() {
+        console.log(`Request ${request.url} failed too many times`);
+    }
+}
+
+
+Apify.main(async () => {
+    const requestQueue = await Apify.openRequestQueue('machinery');
+
+
+    let url = `https://www.tomtom.com/en_gb/traffic-index/pune-traffic/#statistics`;
+    console.log(url);
+    await requestQueue.addRequest({url: url});
+
+    //override settings here
+    //setting up proxy
+    let proxy = new Proxy();
+    let proxy_list = await proxy.getProxy({number: 500, type: 'default'});
+    // console.log(proxy_list);
+    // console.log('Proxy got', proxy_list.length);
+    // options.puppeteerPoolOptions.proxyUrls = proxy_list;
+    options.requestQueue = requestQueue;
+    options.launchPuppeteerOptions.headless = false;
+
+    crawler = new Crawler(options);
+    // Run the crawler and wait for it to finish.
+    console.log('begin');
+    await crawler.run();
+    await crawler.pipeline.close_spider();
+    console.log('Crawler finished.');
+});

grepsrcli/templates/php_boilerplate.jinja2

@@ -0,0 +1,63 @@
+<?php
+/**
+ * Name: {{plugin_name}}
+ * Description: {{plugin_name}}
+ * PID: {{pid}}
+ */
+//ini_set('memory_limit', '256M');
+
+class {{plugin_name}} extends Vtx_Service_Plugin
+{
+    private $colHeaders;
+    protected $baseUrl;
+    protected $domainUrl;
+    private $cache;
+
+    public function __construct()
+    {
+        parent::__construct();
+     
+        $this->colHeaders = array(
+        );
+        $this->baseUrl = "";
+        $this->domainUrl = "";
+        
+        //$this->cache = new Vtx_Service_Data_Cache(__CLASS__);
+        
+    }
+
+    private function newSession()
+    {
+        $this->resetCurl(true);
+        $this->resetHolaSuperProxy();
+
+        $this->useProxy = true;
+
+
+        $this->setHeaders('Accept-Encoding', 'gzip, deflate, br');
+        $this->setHeaders('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8');
+        $this->setHeaders('Accept-Language', 'en-US,en;q=0.5');
+        $this->setHeaders('Connection', 'keep-alive');
+        $this->setHeaders('Upgrade-Insecure-Requests', '1');
+        $this->setHeaders('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36');
+        //$this->setHeaders('Host',"%Host%");
+
+        //$this->setCurlOption(CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
+        //$this->setCurlOption(CURLOPT_CONNECTTIMEOUT, 180);
+        //$this->setCurlOption(CURLOPT_TIMEOUT, 180);
+    }
+
+    public function main($params)
+    {
+	    echo("Crawling : " . $this->domainUrl . PHP_EOL);
+	    $this->dataSet->setPageName(get_class($this) .'_%s' , [ date('Ymd') ]);
+        $this->dataSet->setColHeaders($this->colHeaders);
+	    $this->newSession();
+	    $this->switchProxy();
+
+        //$this->loadDocument($this->baseUrl);
+        echo("Happy Crawling ! \n-cli-helper \n");
+
+        return $this;
+    }
+}

grepsrcli/templates/php_brp_boilerplate.jinja2

@@ -0,0 +1,83 @@
+<?php
+/**
+ * Name: {{plugin_name}}
+ * Description: {{plugin_name}}
+ * PID: {{pid}}
+ */
+//ini_set('memory_limit', '256M');
+require_once __DIR__ . '/../brp_locations/brp_locations.php';
+class {{plugin_name}} extends brp_locations
+{
+    private $colHeaders;
+    protected $baseUrl;
+    protected $domainUrl;
+    private $cache;
+	
+    public function __construct()
+    {
+        parent::__construct();
+		$this->colHeaders = array("Universal_ID","Manufacturer","Product_Line","Product_Line_Standard","Product_Line_DeepDive","Dealer_Name","Dealer_ID","Address_1","Address_2","City","State","Zip","PO_Box","Country","Phone","Fax","Email","Website","Service_Level","Latitude","Longitude","Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday");
+        $this->baseUrl = "";
+        $this->domainUrl = "";
+        $this->cache = new Vtx_Service_Data_Cache(__CLASS__,time());
+    }
+    private function newSession()
+    {
+        $this->resetCurl(true);
+        $this->resetHolaSuperProxy();
+        $this->useProxy = true;
+        $this->setHeaders('Accept-Encoding', 'gzip, deflate, br');
+        $this->setHeaders('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8');
+        $this->setHeaders('Accept-Language', 'en-US,en;q=0.5');
+        $this->setHeaders('Connection', 'keep-alive');
+        $this->setHeaders('Upgrade-Insecure-Requests', '1');
+        $this->setHeaders('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36');
+    }
+    public function main($params)
+    {
+	    echo("Crawling : " . $this->domainUrl . PHP_EOL);
+	    $this->dataSet->setPageName(get_class($this) .'_%s' , [ date('Ymd') ]);
+        $this->dataSet->setColHeaders($this->colHeaders);
+	    $this->newSession();
+	    $this->switchProxy();
+	    // $zips = $this->db->fetchAll("SELECT min(postal_code) as zip FROM `us_postal_code` WHERE country = 'US' group by CONCAT(ROUND(latitude,1), ROUND(longitude,0)) UNION SELECT min(postal_code) as zip FROM `us_postal_code` WHERE country = 'US' group by CONCAT(ROUND(latitude,0), ROUND(longitude,1)) order by zip");
+		// $totalZips = count($zips);
+		$this->loadDocument($this->baseUrl);
+        return $this;
+    }
+    public function processItemDetail(){
+        $arr = $this->dataSet->getEmptyRow();
+	$arr['Universal_ID'] = '';
+	$arr['Manufacturer'] = '';
+	$arr['Product_Line'] = '';
+	$arr['Product_Line_Standard'] = '';
+	$arr['Product_Line_DeepDive'] = '';
+	$arr['Dealer_Name'] =  '';
+	$arr['Dealer_ID'] = '';
+	$arr['Address_1'] = '';
+	$arr['Address_2'] = '';
+	$arr['City'] = '';
+	$arr['State'] = '';
+	$arr['Zip'] = '';
+	$arr['PO_Box'] = '';
+	$arr['Country'] = '';
+	$arr['Phone'] = '';
+	$arr['Fax'] = '';
+	$arr['Email'] = '';
+	$arr['Website'] = '';
+	$arr['Latitude'] = '';
+	$arr['Longitude'] = '';
+	$arr['Sunday'] = '';
+	$arr['Monday'] = '';
+	$arr['Tuesday'] = '';
+	$arr['Wednesday'] = '';
+	$arr['Thursday'] = '';
+	$arr['Friday'] = '';
+	$arr['Saturday'] = '';
+
+	if($this->cache->checkIfNew($arr['Dealer_ID'])){
+		$this->addRow($arr);
+		//print_r($arr);
+	}
+    }
+}

grepsrcli/templates/php_vc_boilerplate.jinja2

@@ -0,0 +1,65 @@
+<?php
+/**
+ * Name: {{plugin_name}}
+ * Description: {{plugin_name}}
+ * PID: {{pid}}
+ */
+//ini_set('memory_limit', '256M');
+
+require_once __DIR__ . '/../vericred_service_main/vericred_service_main.php';
+
+class {{plugin_name}} extends vericred_service_main
+{
+    protected $baseUrl;
+    protected $domainUrl;
+    private $count;
+    private $limit;
+    private $cache;
+
+    public function __construct()
+    {
+        parent::__construct();
+        
+        $this->baseUrl = "";
+        $this->domainUrl = "";
+        $this->limit = 500;
+        $this->count = 0;
+
+        //$this->cache = new Vtx_Service_Data_Cache(__CLASS__);
+        
+    }
+
+    private function newSession()
+    {
+        $this->resetCurl(true);
+        $this->resetHolaSuperProxy();
+        
+        $this->useProxy = true;
+
+
+        $this->setHeaders('Accept-Encoding', 'gzip, deflate, br');
+        $this->setHeaders('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8');
+        $this->setHeaders('Accept-Language', 'en-US,en;q=0.5');
+        $this->setHeaders('Connection', 'keep-alive');
+        $this->setHeaders('Upgrade-Insecure-Requests', '1');
+        $this->setHeaders('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36');
+        //$this->setHeaders('Host', "%Host%");
+
+        //$this->setCurlOption(CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
+        //$this->setCurlOption(CURLOPT_CONNECTTIMEOUT, 180);
+        //$this->setCurlOption(CURLOPT_TIMEOUT, 180);
+    }
+
+    public function main($params)
+    {
+	    echo("Crawling : " . $this->domainUrl . PHP_EOL);
+        $this->dataSet->setPageName(get_class($this) . '_%s', [date('Ymd')]);
+        $this->newSession();
+        $this->switchProxy();
+
+        //$this->loadDocument($this->baseUrl);
+        echo("Happy Crawling ! \n-cli-helper \n");
+
+        return $this;
+    }
+}

grepsrcli/templates/py_boilerplate.jinja2

@@ -0,0 +1,56 @@
+'''
+Name: {{plugin_name}}
+Description: {{plugin_name}}
+PID:  {{pid}} force
+'''
+
+import os
+from scrapy.utils.project import get_project_settings
+from grepsr_crawler.lib.proxy import get_proxy
+from grepsr_crawler.crawler.spiders.base_crawler import GrepsrBaseCrawler
+from scrapy.crawler import CrawlerProcess
+from grepsr_crawler.lib.user_agent import get_user_agent
+
+class GrepsrCrawler(GrepsrBaseCrawler):
+   allowed_domains = [""]
+
+   custom_settings = {
+        'USER_AGENT' : get_user_agent(), # NOTE: impersonate as chrome browser by default 
+        'CONCURRENT_REQUESTS' : 5,
+        'CHANGE_PROXY_AFTER': 10,
+        'LOG_LEVEL' : 'DEBUG', # ERROR for prod and DEBUG for dev 
+        'JOBDIR' : './crawls',  # reduce memory by saving on a disk
+      #   'REDIRECT_ENABLED':  True, # NOTE: to enable redirect routing as redirect is disabled by default
+      #   'RETRY_HTTP_CODES' : [500, 503, 504, 400, 403, 408],
+    }
+
+   headers = {
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Upgrade-Insecure-Requests': '1',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Connection' : 'keep-alive'
+    }
+
+   '''
+    Template: 1
+   '''
+   # start_urls = ['list of urls']
+
+   # def parse(self, response):
+   #    pass
+
+   '''
+    Template: 2
+   '''
+   def start_requests(self):
+      pass
+      
+
+settings_file_path = 'grepsr_crawler.crawler.settings'
+os.environ.setdefault('SCRAPY_SETTINGS_MODULE', settings_file_path)
+settings = get_project_settings()
+settings['PROXY_LIST'] = get_proxy(100) # proxy pool of 100 
+process = CrawlerProcess(settings)
+process.crawl(GrepsrCrawler)
+process.start()  # the script will block here until the crawling is finished