powerdlz23 1.2.3 → 1.2.4

package/rubyretriever/lib/retriever/fetch.rb

@@ -0,0 +1,268 @@
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+require 'em-synchrony/fiber_iterator'
+require 'ruby-progressbar'
+require 'open-uri'
+require 'csv'
+require 'bloomfilter-rb'
+
+module Retriever
+  #
+  class Fetch
+    HR = '###############################'
+    attr_reader :max_pages, :t, :result
+    # given target URL and RR options, creates a fetch object.
+    # There is no direct output
+    # this is a parent class that the other fetch classes build off of.
+    def initialize(url, options)
+      @iterator = false
+      @result = []
+      @connection_tally = {
+        success: 0,
+        error: 0,
+        error_client: 0,
+        error_server: 0
+      }
+      setup_options(options)
+      setup_progress_bar if @progress
+      @t = Retriever::Target.new(url, @file_re)
+      @output = "rr-#{@t.host.split('.')[1]}" if @fileharvest && !@output
+      @already_crawled = setup_bloom_filter
+    end
+
+    def start
+      @page_one = crawl_page_one
+      @link_stack = create_link_stack
+      @temp_link_stack = []
+    end
+
+    def errlog(msg)
+      fail "ERROR: #{msg}"
+    end
+
+    def lg(msg)
+      puts "### #{msg}" if @verbose
+    end
+
+    # prints current data collection to STDOUT
+    def dump
+      puts HR
+      puts "Connection Tally:\n#{@connection_tally}\n#{HR}" if @verbose
+      puts "Target URL: #{@t.target}"
+      if @sitemap
+        puts 'Sitemap'
+      elsif @fileharvest
+        puts "File harvest by type: #{@fileharvest}"
+      elsif @seo
+        puts 'SEO Metrics'
+      end
+      puts "Data Dump -- Object Count: #{@result.size}"
+      puts HR
+      @result.each do |line|
+        puts line
+      end
+      puts
+    end
+
+    # writes current data collection out to CSV in current directory
+    def write
+      return false unless @output
+      i = 0
+      CSV.open("#{@output}.csv", 'w') do |csv|
+        if (i == 0) && @seo
+          csv << ['URL', 'Page Title', 'Meta Description', 'H1', 'H2']
+          i += 1
+        end
+        @result.each do |entry|
+          csv << entry
+        end
+      end
+      puts HR
+      puts "File Created: #{@output}.csv"
+      puts "Object Count: #{@result.size}"
+      puts HR
+      puts
+    end
+
+    # returns true is resp is ok to continue
+    def good_response?(resp, url)
+      return false unless resp
+      hdr = resp.response_header
+      if hdr.redirection?
+        loc = hdr.location
+        lg("#{url} Redirected to #{loc}")
+        if t.host_re =~ loc
+          @temp_link_stack.push(loc) unless @already_crawled.include?(loc)
+          lg('--Added to stack for later')
+          return false
+        end
+        lg("Redirection outside of target host. No - go. #{loc}")
+        return false
+      end
+      # lets not continue if unsuccessful connection
+      unless hdr.successful?
+        lg("UNSUCCESSFUL CONNECTION -- #{url}")
+        @connection_tally[:error] += 1
+        @connection_tally[:error_server] += 1 if hdr.server_error?
+        @connection_tally[:error_client] += 1 if hdr.client_error?
+        return false
+      end
+      # let's not continue if not text/html
+      unless hdr['CONTENT_TYPE'] =~ %r{(text/html|application/xhtml+xml)}
+        @already_crawled.insert(url)
+        lg("Page Not text/html -- #{url}")
+        return false
+      end
+      @connection_tally[:success] += 1
+      true
+    end
+
+    def filter_out_querystrings(path)
+      if path.include?('?')
+        uri = Addressable::URI.parse(path)
+        uri.query_values = {}
+        return uri.to_s.chomp('?')
+      end
+      path
+    end
+
+    private
+
+    def setup_options(options)
+      @progress     = options['progress']
+      @max_pages    = options['maxpages'] ? options['maxpages'].to_i : 100
+      @verbose      = options['verbose']
+      @output       = options['filename']
+      @fileharvest  = options['fileharvest']
+      @sitemap      = options['sitemap']
+      @seo          = options['seo']
+      @autodown     = options['autodown']
+      @file_re      = Regexp.new(/.#{@fileharvest}/).freeze if @fileharvest
+    end
+
+    def setup_bloom_filter
+      already_crawled = BloomFilter::Native.new(
+        size: 1_000_000,
+        hashes: 5,
+        seed: 1,
+        bucket: 8,
+        raise: false
+      )
+      already_crawled.insert(@t.target)
+      already_crawled
+    end
+
+    def setup_progress_bar
+      # verbose & progressbar conflict
+      errlog('CANNOT RUN VERBOSE & PROGRESSBAR AT SAME TIME') if @verbose
+      prgress_vars = {
+        title: 'Pages',
+        starting_at: 1,
+        total: @max_pages,
+        format: '%a |%b>%i| %c/%C %t'
+      }
+      @progressbar = ProgressBar.create(prgress_vars)
+    end
+
+    def crawl_page_one
+      page_one = Retriever::Page.new(@t.target, @t.source, @t)
+      lg("URL Crawled: #{@t.target}")
+      page_one
+    end
+
+    def create_link_stack
+      link_stack = @page_one.parse_internal_visitable
+      errlog("Bad URL -- #{@t.target}") unless link_stack
+      lg("#{link_stack.size - 1} links found")
+      link_stack.delete(@t.target)
+      link_stack.take(@max_pages) if (link_stack.size + 1) > @max_pages
+      link_stack
+    end
+
+    def end_crawl_notice
+      notice = "#{HR}\nENDING CRAWL\nCan't find any more links."
+      @progressbar.log(notice) if @progress
+      lg(notice)
+    end
+
+    # iterates over the existing @link_stack
+    # running until we reach the @max_pages value.
+    def async_crawl_and_collect(&block)
+      while @already_crawled.size < @max_pages
+        if @link_stack.empty?
+          end_crawl_notice
+          break
+        end
+        new_links_arr = process_link_stack(&block)
+        @temp_link_stack = []
+        next if new_links_arr.nil? || new_links_arr.empty?
+        @link_stack.concat(new_links_arr)
+        next unless @sitemap
+        @result.concat(new_links_arr)
+      end
+      @result.uniq!
+    end
+
+    def push_seo_to_result(url, new_page)
+      seos = [url]
+      seos.concat(new_page.parse_seo)
+      @result.push(seos)
+      lg('--page SEO scraped')
+    end
+
+    def push_files_to_result(new_page)
+      filez = new_page.parse_files(new_page.parse_internal)
+      @result.concat(filez) unless filez.empty?
+      lg("--#{filez.size} files found")
+    end
+
+    def page_from_response(url, response)
+      lg("Page Fetched: #{url}")
+      @already_crawled.insert(url)
+      if @progress && (@already_crawled.size < @max_pages)
+        @progressbar.increment
+      end
+      Retriever::Page.new(url, response, @t)
+    end
+
+    def new_visitable_links(current_page)
+      lg("--#{current_page.links.size} links found")
+      current_page.parse_internal_visitable
+    end
+
+    def push_custom_to_result(url, current_page, &block)
+      data = block.call current_page
+      @result.push(data) unless data.empty?
+      lg("-- PageIterator called on: #{url}")
+    end
+
+    # send a new wave of GET requests, using current @link_stack
+    # at end of the loop it empties link_stack
+    # puts new links into temporary stack
+    def process_link_stack(&block)
+      EM.synchrony do
+        concurrency = 10
+        EM::Synchrony::FiberIterator.new(@link_stack, concurrency).each do |url|
+          next if @already_crawled.size >= @max_pages
+          next if @already_crawled.include?(url)
+          resp = EventMachine::HttpRequest.new(url).get
+          next unless good_response?(resp, url)
+          current_page = page_from_response(url, resp.response)
+          # non-link dependent modes
+          push_seo_to_result(url, current_page) if @seo
+          push_custom_to_result(url, current_page, &block) if @iterator
+          next unless current_page.links.size > 0
+          @temp_link_stack.push(new_visitable_links(current_page))
+          # link dependent modes
+          next unless @fileharvest
+          push_files_to_result(current_page)
+        end
+        EventMachine.stop
+      end
+      # empty the stack. most clean way
+      @link_stack = []
+      # temp contains redirects + new visitable links
+      @temp_link_stack.flatten.uniq!
+    end
+  end
+end

package/rubyretriever/lib/retriever/fetchfiles.rb

@@ -0,0 +1,71 @@
+module Retriever
+  # receives target url and RR options
+  # returns an array of all unique files (based on given filetype)
+  #   found on the target site
+
+  class FetchFiles < Fetch
+    def initialize(url, options)
+      super
+      start
+      temp_file_collection = @page_one.parse_files(@page_one.parse_internal)
+      @result.concat(temp_file_collection) if temp_file_collection.size > 0
+      lg("#{@result.size} new files found")
+
+      async_crawl_and_collect
+      # done, make sure progress bar says we are done
+      @progressbar.finish if @progress
+      @result.sort_by! { |x| x.length }
+    end
+
+    def download_file(path)
+      path = filter_out_querystrings(path)
+      # given valid url, downloads file to current directory in /rr-downloads/
+      arr = path.split('/')
+      shortname = arr.pop
+      puts "Initiating Download of: #{shortname}"
+      File.open(shortname, 'wb') do |saved_file|
+        open(path) do |read_file|
+          saved_file.write(read_file.read)
+        end
+      end
+      puts '  SUCCESS: Download Complete'
+    end
+
+    def autodownload
+      # go through the fetched file URL collection and download each one.
+      puts HR
+      puts '### Initiating Autodownload...'
+      puts HR
+      puts "#{@result.count} - #{@file_ext}'s Located"
+      puts HR
+      move_to_download_dir
+      iterate_thru_collection_and_download
+      Dir.chdir('..')
+    end
+
+    private
+
+    def iterate_thru_collection_and_download
+      lenn = @result.count
+      @result.each_with_index do |entry, i|
+        begin
+          download_file(entry)
+        rescue StandardError
+          puts "ERROR: failed to download - #{entry}"
+        end
+        lg("    File [#{i + 1} of #{lenn}]\n")
+      end
+    end
+
+    def move_to_download_dir(dir_name = 'rr-downloads')
+      if File.directory?(dir_name)
+        Dir.chdir(dir_name)
+      else
+        puts "creating #{dir_name} Directory"
+        Dir.mkdir(dir_name)
+        Dir.chdir(dir_name)
+      end
+      puts "Downloading files to local directory: '/#{dir_name}/'"
+    end
+  end
+end

package/rubyretriever/lib/retriever/fetchseo.rb

@@ -0,0 +1,18 @@
+module Retriever
+  #
+  class FetchSEO < Fetch
+    # receives target url and RR options
+    # returns an array of onpage SEO related fields
+    #   on all unique pages found on the site
+    def initialize(url, options)
+      super
+      start
+      @result.push(@page_one.parse_seo)
+
+      async_crawl_and_collect
+      # done, make sure progress bar says we are done
+      @progressbar.finish if @progress
+      @result.sort_by! { |x| x[0].length }
+    end
+  end
+end

package/rubyretriever/lib/retriever/fetchsitemap.rb

@@ -0,0 +1,43 @@
+module Retriever
+  #
+  class FetchSitemap < Fetch
+    # receives target URL and RR options
+    # returns an array of all unique pages found on the site
+    def initialize(url, options)
+      super
+      start
+      @result.push(@t.target)
+      @result.concat(@link_stack)
+
+      async_crawl_and_collect
+      # done, make sure progress bar says we are done
+      @progressbar.finish if @progress
+      @result.sort_by! { |x| x.length } if @result.size > 1
+      @result.uniq!
+    end
+
+    # produces valid XML sitemap based on page collection fetched.
+    # Writes to current directory.
+    def gen_xml
+      filename = @t.host.split('.')[1]
+      f = File.open("sitemap-#{filename}.xml", 'w+')
+      f << "<?xml version='1.0' encoding='UTF-8'?>"
+      f << "<urlset xmlns='http://www.sitemaps.org/schemas/sitemap/0.9'>"
+      @result.each do |url|
+        f << "<url><loc>#{url}</loc></url>"
+      end
+      f << '</urlset>'
+      f.close
+      print_file_info(filename)
+    end
+
+    private
+
+    def print_file_info(filename)
+      puts HR
+      puts "File Created: sitemap-#{filename}.xml"
+      puts "Object Count: #{@result.size}"
+      puts HR + "\n"
+    end
+  end
+end

package/rubyretriever/lib/retriever/link.rb

@@ -0,0 +1,47 @@
+require 'addressable/uri'
+module Retriever
+  #
+  class Link
+    # HTTP_RE = Regexp.new(/^http/i).freeze
+    SLASH_RE = Regexp.new(%r(^/{1}[^/])).freeze
+    DOUBLE_SLASH_RE = Regexp.new(%r(^/{2}[^/])).freeze
+    WWW_DOT_RE = Regexp.new(/^www\./i).freeze
+
+    def initialize(target_scheme, target_host, this_link, current_url)
+      begin
+        #this_link = Addressable::URI.encode(this_link) //not necessary; and breaking links
+        @link_uri = Addressable::URI.parse(this_link)
+      rescue Addressable::URI::InvalidURIError
+        dummy = Retriever::Link.new(target_scheme, target_host, target_host, target_host)
+        @link_uri = Addressable::URI.parse(dummy.path)
+      end
+      @scheme = target_scheme
+      @host = target_host
+      @this_link = @link_uri.to_s
+      @current_page_url = current_url
+    end
+
+    def path
+      return this_link if link_uri.absolute?
+
+      return "#{@scheme}://#{this_link}" if WWW_DOT_RE =~ this_link
+
+      return "#{@scheme}://#{host}#{this_link}" if SLASH_RE =~ this_link
+
+      # link begins with '//'
+      return "#{@scheme}:#{this_link}" if DOUBLE_SLASH_RE =~ this_link
+
+      # link uses relative path with no slashes at all
+      if link_uri.relative?
+        if @current_page_url[-1, 1] == "/"
+          return "#{@current_page_url}#{this_link}"
+        end
+        return "#{@current_page_url}/#{this_link}"
+      end
+    end
+
+    private
+
+    attr_reader :this_link, :host, :link_uri, :current_page_url
+  end
+end

package/rubyretriever/lib/retriever/openuri_redirect_patch.rb

@@ -0,0 +1,8 @@
+#
+module OpenURI
+  # nesc patch otherwise OPENURI blocks redirects to and from https
+  def self.redirectable?(uri1, uri2)
+    uri1.scheme.downcase == uri2.scheme.downcase ||
+    (/\A(?:http|https)\z/i =~ uri1.scheme && /\A(?:http|https)\z/i =~ uri2.scheme)
+  end
+end

package/rubyretriever/lib/retriever/page.rb

@@ -0,0 +1,104 @@
+require 'nokogiri'
+require 'addressable/uri'
+#
+using SourceString
+module Retriever
+  #
+  class Page
+    HASH_RE   = Regexp.new(/^#/i).freeze
+    HTTP_RE   = Regexp.new(/^http/i).freeze
+    H1_RE     = Regexp.new(/<h1>(.*)<\/h1>/i).freeze
+    H2_RE     = Regexp.new(/<h2>(.*)<\/h2>/i).freeze
+    TITLE_RE  = Regexp.new(/<title>(.*)<\/title>/i).freeze
+    DESC_RE   = Regexp.new(/<meta[^>]*name=[\"|\']description[\"|\']
+                          [^>]*content=[\"]
+                          (
+                            [^\"]*
+                          )
+                          [\"]
+                          [^>]
+                          *>
+                          /ix).freeze
+    HREF_CONTENTS_RE = Regexp.new(/\shref=
+                                  ['|"]
+                                  (
+                                    [^\s]
+                                    [a-z0-9\.\/\:\-\%\+\?\!\=\&\,\:\;\~\_]+
+                                  )
+                                  ['|"]
+                                  [\s|\W]
+                                  /ix).freeze
+    NONPAGE_EXT_RE = Regexp.new(/\.
+                                (?:css|js|png|gif|jpg|mp4|
+                                wmv|flv|mp3|wav|doc|txt|ico|xml)
+                                /ix).freeze
+
+    attr_reader :links, :source, :t, :url
+
+    def initialize(url, source, t)
+      @url = url
+      @t = t
+      @source = source.encode_utf8_and_replace
+      @links = nil
+    end
+
+    # receives page source as string
+    # returns array of unique href links
+    def links
+      return @links if @links
+      return false unless @source
+      @links = @source.scan(HREF_CONTENTS_RE).map do |match|
+        # filter some malformed URLS that come in
+        # meant to be a loose filter to catch all reasonable HREF attributes.
+        link = match[0]
+        next if HASH_RE =~ link
+        Link.new(@t.scheme, host_with_port, link, @url).path
+      end.compact.uniq
+    end
+
+    def host_with_port
+      return @t.host if @t.port.nil?
+
+      @t.host + ':' + @t.port.to_s
+    end
+
+    def parse_internal
+      links.select do |x|
+        @t.host == Addressable::URI.parse(Addressable::URI.encode(x)).host
+      end
+    end
+
+    def parse_internal_visitable
+      parse_internal.select { |x| !(NONPAGE_EXT_RE =~ x) }
+    end
+
+    def parse_files(arr = parse_internal)
+      arr.select { |x| @t.file_re =~ x }
+    end
+
+    def parse_by_css(selector)
+      nokogiri_doc = Nokogiri::HTML(@source)
+      nokogiri_doc.css(selector).text
+    end
+
+    def title
+      TITLE_RE =~ @source ? @source.match(TITLE_RE)[1].decode_html : ''
+    end
+
+    def desc
+      DESC_RE =~ @source ? @source.match(DESC_RE)[1].decode_html  : ''
+    end
+
+    def h1
+      H1_RE =~ @source ? @source.match(H1_RE)[1].decode_html  : ''
+    end
+
+    def h2
+      H2_RE =~ @source ? @source.match(H2_RE)[1].decode_html  : ''
+    end
+
+    def parse_seo
+      [title, desc, h1, h2]
+    end
+  end
+end

package/rubyretriever/lib/retriever/page_iterator.rb

@@ -0,0 +1,21 @@
+module Retriever
+  #
+  class PageIterator < Fetch
+    # receives target url and RR options, and a block
+    # runs the block on all pages during crawl, pushing
+    #   the returned value of the block onto a result stack
+    #   the complete data returned from the crawl is accessible thru self.result
+    def initialize(url, options, &block)
+      super
+      start
+      fail 'block required for PageIterator' unless block_given?
+      @iterator = true
+      @result.push(block.call @page_one)
+      lg("-- PageIterator crawled- #{url}")
+      async_crawl_and_collect(&block)
+      # done, make sure progress bar says we are done
+      @progressbar.finish if @progress
+      @result.sort_by! { |x| x.length } if @result.size > 1
+    end
+  end
+end

package/rubyretriever/lib/retriever/target.rb

@@ -0,0 +1,47 @@
+require 'open-uri'
+require 'addressable/uri'
+
+module Retriever
+  #
+  class Target
+    HTTP_RE    = Regexp.new(/^http/i).freeze
+
+    attr_reader :host, :target, :host_re, :source, :file_re, :scheme, :port
+
+    def initialize(url, file_re = nil)
+      fail 'Bad URL' unless url.include?('.')
+      url         = "http://#{url}" unless HTTP_RE =~ url
+      target_uri  = Addressable::URI.parse(Addressable::URI.encode(url))
+      @target     = target_uri.to_s
+      @host       = target_uri.host
+      @host_re    = Regexp.new(@host.sub('www.', ''))
+      @file_re  ||= file_re
+      @scheme     = target_uri.scheme
+      @port       = target_uri.port
+    end
+
+    def source
+      resp = open(@target)
+      resp_url = resp.base_uri.to_s
+      if @target != resp_url
+        fail "Domain redirecting: #{resp_url}" unless @host_re =~ resp_url
+        # if redirect URL is same host, we want to re-sync @target
+        return resync_target_and_return_source(resp_url)
+      end
+      resp = resp.read
+      #
+      fail 'Domain is not working. Try the non-WWW version.' if resp == ''
+      fail 'Domain not working. Try HTTPS???' unless resp
+      # consider using scrub from ruby 2.1? this misses some things
+      resp.encode('UTF-8', 'binary', invalid: :replace, undef: :replace)
+    end
+
+    def resync_target_and_return_source(url)
+      new_t   = Retriever::Target.new(url)
+      @target = new_t.target
+      @host   = new_t.host
+      @scheme = new_t.scheme
+      new_t.source
+    end
+  end
+end

package/rubyretriever/lib/retriever/version.rb

@@ -0,0 +1,4 @@
+#
+module Retriever
+  VERSION = '1.4.5'
+end

package/rubyretriever/lib/retriever.rb

@@ -0,0 +1,15 @@
+require 'retriever/core_ext'
+require 'retriever/fetch'
+require 'retriever/fetchfiles'
+require 'retriever/fetchsitemap'
+require 'retriever/fetchseo'
+require 'retriever/page_iterator'
+require 'retriever/cli'
+require 'retriever/link'
+require 'retriever/target'
+require 'retriever/page'
+require 'retriever/openuri_redirect_patch'
+
+#
+module Retriever
+end