powerdlz23 1.2.4 → 1.2.5

package/grell/lib/grell/page.rb

@@ -0,0 +1,275 @@
+require 'forwardable'
+
+module Grell
+ # This class contains the logic related to work with each page we crawl. It is also the interface we use
+ # To access the information of each page.
+ # This information comes from result private classes below.
+  class Page
+    extend Forwardable
+
+    WAIT_TIME = 10
+    WAIT_INTERVAL = 0.5
+
+    attr_reader :url, :timestamp, :id, :parent_id, :rawpage
+
+    #Most of the interesting information accessed through this class is accessed by the methods below
+    def_delegators :@result_page, :headers, :body, :status, :links, :has_selector?, :host, :visited?
+
+    def initialize( url, id, parent_id)
+      @rawpage = RawPage.new
+      @url = url
+      @id = id
+      @parent_id = parent_id
+      @timestamp = nil
+      @times_visited = 0
+      @result_page = UnvisitedPage.new
+    end
+
+    def navigate
+      # We wait a maximum of WAIT_TIME seconds to get an HTML page. We try our best to workaround inconsistencies on poltergeist
+      Reader.wait_for(->{@rawpage.navigate(url)}, WAIT_TIME, WAIT_INTERVAL ) do
+        @rawpage.status && !@rawpage.headers.empty? &&
+          @rawpage.headers["Content-Type"] && @rawpage.headers["Content-Type"].include?('text/html').equal?(true)
+      end
+      @rawpage.wait_for_all_ajax_requests(WAIT_TIME, WAIT_INTERVAL)
+      @result_page = VisitedPage.new(@rawpage)
+      @timestamp = Time.now
+    rescue Capybara::Poltergeist::BrowserError, Capybara::Poltergeist::DeadClient,
+           Capybara::Poltergeist::JavascriptError, Capybara::Poltergeist::StatusFailError,
+           Capybara::Poltergeist::TimeoutError, Errno::ECONNRESET, URI::InvalidURIError => e
+      unavailable_page(404, e)
+    ensure
+      @times_visited += 1
+    end
+
+    # Number of times we have retried the current page
+    def retries
+      [@times_visited - 1, 0].max
+    end
+
+    # The current URL, this may be different from the URL we asked for if there was some redirect
+    def current_url
+      @rawpage.current_url
+    end
+
+    # True if we followed a redirect to get the current contents
+    def followed_redirects?
+      current_url != @url
+    end
+
+    # True if there page responded with an error
+    def error?
+      !!(status.to_s =~ /[4|5]\d\d/)
+    end
+
+    # Extracts the path (e.g. /actions/test_action) from the URL
+    def path
+      URI.parse(@url).path
+    rescue URI::InvalidURIError # Invalid URLs will be added and caught when we try to navigate to them
+      @url
+    end
+
+    def unavailable_page(status, exception)
+      Grell.logger.warn "The page with the URL #{@url} was not available. Exception #{exception}"
+      @result_page = ErroredPage.new(status, exception)
+      @timestamp = Time.now
+    end
+
+    private
+
+    # Private class.
+    # This is a result page when it has not been visited yet. Essentially empty of information
+    #
+    class UnvisitedPage
+      def status
+        nil
+      end
+
+      def body
+        ''
+      end
+
+      def headers
+        {grellStatus: 'NotVisited' }
+      end
+
+      def links
+        []
+      end
+
+      def host
+        ''
+      end
+
+      def visited?
+        false
+      end
+
+      def has_selector?(selector)
+        false
+      end
+
+    end
+
+    # Private class.
+    # This is a result page when some error happened. It provides some information about the error.
+    #
+    class ErroredPage
+      def initialize(error_code, exception)
+        @error_code = error_code
+        @exception = exception
+      end
+
+      def status
+        @error_code
+      end
+
+      def body
+        ''
+      end
+
+      def headers
+        message = begin
+          @exception.message
+        rescue StandardError
+          "Error message can not be accessed" #Poltergeist may try to access a nil object when accessing message
+        end
+
+        {
+          grellStatus: 'Error',
+          errorClass: @exception.class.to_s,
+          errorMessage: message
+        }
+      end
+
+      def links
+        []
+      end
+
+      def host
+        ''
+      end
+
+      def visited?
+        true
+      end
+
+      def has_selector?(selector)
+        false
+      end
+
+    end
+
+
+    # Private class.
+    # This is a result page when we successfully got some information back after visiting the page.
+    # It delegates most of the information to the @rawpage capybara page. But any transformation or logic is here
+    #
+    class VisitedPage
+      def initialize(rawpage)
+        @rawpage = rawpage
+      end
+
+      def status
+        @rawpage.status
+      end
+
+      def body
+        @rawpage.body
+      end
+
+      def headers
+        @rawpage.headers
+      rescue Capybara::Poltergeist::BrowserError => e #This may happen internally on Poltergeist, they claim is a bug.
+        {
+          grellStatus: 'Error',
+          errorClass: e.class.to_s,
+          errorMessage: e.message
+        }
+      end
+
+      def links
+        @links ||= all_links
+      end
+
+      def host
+        @rawpage.host
+      end
+
+      def visited?
+        true
+      end
+
+      def has_selector?(selector)
+        @rawpage.has_selector?(selector)
+      end
+
+      private
+      def all_links
+        links =  @rawpage.all_anchors.map { |anchor| Link.new(anchor) }
+        body_enabled_links = links.reject { |link| link.inside_header? || link.disabled? || link.js_href? }
+        body_enabled_links.map { |link| link.to_url(host) }.uniq.compact
+
+      rescue Capybara::Poltergeist::ObsoleteNode
+        Grell.logger.warn "We found an obsolete node in #{@url}. Ignoring all links"
+        # Sometimes Javascript and timing may screw this, we lose these links.
+        # TODO: Can we do something more intelligent here?
+        []
+      end
+
+      # Private class to group all the methods related to links.
+      class Link
+        def initialize(anchor)
+          @anchor = anchor
+        end
+
+        # <link> can only be used in the <head> as of: https://developer.mozilla.org/en/docs/Web/HTML/Element/link
+        def inside_header?
+          @anchor.tag_name == 'link'
+        end
+
+        # Is the link disabled by either Javascript or CSS?
+        def disabled?
+          @anchor.disabled? || !!@anchor.native.attributes['disabled']
+        end
+
+        # Does the href use javascript?
+        def js_href?
+          href.start_with?('javascript:')
+        end
+
+        # Some links may use data-href + javascript to do interesting things
+        def href
+          @anchor['href'] || @anchor['data-href']
+        end
+
+        # We only accept links in this same host that start with a path
+        def to_url(host)
+          uri = URI.parse(href)
+          if uri.absolute?
+            if uri.host != URI.parse(host).host
+              Grell.logger.debug "GRELL does not follow links to external hosts: #{href}"
+              nil
+            else
+              href # Absolute link to our own host
+            end
+          else
+            if uri.path.nil?
+              Grell.logger.debug "GRELL does not follow links without a path: #{uri}"
+              nil
+            end
+            if uri.path.start_with?('/')
+              host + href  # convert to full URL
+            else # links like href="google.com" the browser would go to http://google.com like "http://#{link}"
+              Grell.logger.debug "GRELL Bad formatted link: #{href}, assuming external"
+              nil
+            end
+          end
+        rescue URI::InvalidURIError # Invalid links propagating till we navigate to them
+          href
+        end
+      end
+
+    end
+  end
+end

package/grell/lib/grell/page_collection.rb

@@ -0,0 +1,62 @@
+module Grell
+  # Keeps a record of all the pages crawled.
+  # When a new url is found it is added to this collection, which makes sure it is unique.
+  # This page is part of the discovered pages. Eventually that page will be navigated to, then
+  # the page will be part of the visited pages.
+  class PageCollection
+    attr_reader :collection
+
+    # A block containing the logic that determines if a new URL should be added
+    # to the collection or if it is already present will be passed to the initializer.
+    def initialize(add_match_block)
+      @collection = []
+      @add_match_block = add_match_block || default_add_match
+    end
+
+    def create_page(url, parent_id)
+      page_id = next_id
+      page = Page.new(url, page_id, parent_id)
+      add(page)
+      page
+    end
+
+    def visited_pages
+      @collection.select {|page| page.visited?}
+    end
+
+    def discovered_pages
+      @collection - visited_pages
+    end
+
+    def next_page
+      discovered_pages.sort_by{|page| page.parent_id}.first
+    end
+
+    private
+
+    def next_id
+      @collection.size
+    end
+
+    def add(page)
+      # Although finding unique pages based on URL will add pages with different query parameters,
+      # in some cases we do link to different pages depending on the query parameters like when using proxies
+      new_url = @collection.none? do |collection_page|
+        @add_match_block.call(collection_page, page)
+      end
+
+      if new_url
+        @collection.push page
+      end
+    end
+
+    # If add_match_block is not provided, url matching to determine if a new page should be added
+    # to the page collection will default to this proc
+    def default_add_match
+      Proc.new do |collection_page, page|
+        collection_page.url.downcase == page.url.downcase
+      end
+    end
+
+  end
+end

package/grell/lib/grell/rawpage.rb

@@ -0,0 +1,62 @@
+module Grell
+  # This class depends heavily on Capybara but contains no logic.
+  class RawPage
+    include Capybara::DSL
+
+    def navigate(url)
+      visit(url)
+      follow_redirects!
+    end
+
+    def headers
+      page.response_headers
+    end
+
+    def status
+      page.status_code
+    end
+
+    def body
+      page.body
+    end
+
+    def all_anchors
+      # Some elements may not be "a" elements but still provide a link. This usually is done for Javascript
+      # to convert other elements which are not links to be able to be clicked naturally.
+      # Only return links which are visible.
+      all('[href]', visible: true).to_a + all('[data-href]', visible: true).to_a
+    end
+
+    def host
+      page.current_host
+    end
+
+    def has_selector?(selector)
+      page.has_selector?(selector)
+    end
+
+    def wait_for_all_ajax_requests(timeout, interval)
+      Timeout::timeout(timeout) do
+        (timeout / interval).ceil.times do
+          jquery_active = page.evaluate_script("typeof jQuery !== 'undefined' && jQuery.active;")
+          break if (!jquery_active || jquery_active.zero?)
+          sleep(interval)
+        end
+      end
+      true
+    end
+
+    private
+
+    def follow_redirects!
+      # Phantom is very weird, it will follow a redirect to provide the correct body but will not fill the
+      # status and the headers, if we are in that situation, revisit the page with the correct url this time.
+      # Note that we will still fail if we have more than 5 redirects on a row
+      redirects = 0
+      while(page.status_code == nil && redirects < 5)
+        visit( CGI.unescape(page.current_url))
+        redirects = redirects + 1
+      end
+    end
+  end
+end

package/grell/lib/grell/reader.rb

@@ -0,0 +1,18 @@
+module Grell
+  # A tooling class, it waits a maximum of max_waiting for an action to finish. If the action is not
+  # finished by then, it will continue anyway.
+  # The wait may be long but we want to finish it as soon as the action has finished
+  class Reader
+    def self.wait_for(action, max_waiting, sleeping_time)
+      time_start = Time.now
+      action.call()
+      return if yield
+      while (Time.now < time_start + max_waiting)
+        action.call()
+        break if yield
+        sleep(sleeping_time)
+      end
+    end
+
+  end
+end

package/grell/lib/grell/version.rb

@@ -0,0 +1,3 @@
+module Grell
+  VERSION = "2.1.2".freeze
+end

package/grell/lib/grell.rb

@@ -0,0 +1,11 @@
+require 'capybara/poltergeist'
+require 'capybara/dsl'
+
+require 'grell/grell_logger'
+require 'grell/capybara_driver'
+require 'grell/crawler_manager'
+require 'grell/crawler'
+require 'grell/rawpage'
+require 'grell/page'
+require 'grell/page_collection'
+require 'grell/reader'

package/grell/spec/lib/capybara_driver_spec.rb

@@ -0,0 +1,38 @@
+
+RSpec.describe Grell::CapybaraDriver do
+  let(:ts) { Time.now }
+  before do
+    Grell.logger = Logger.new(nil)
+  end
+
+  describe 'setup_capybara' do
+    it 'properly registers the poltergeist driver' do
+      Timecop.freeze(ts)
+      driver = Grell::CapybaraDriver.new.setup_capybara
+      expect(driver).to be_instance_of(Capybara::Poltergeist::Driver)
+    end
+
+    it 'raises an exception if the driver cannot be initialized' do
+      Timecop.freeze(ts + 60)
+
+      # Attempt to register twice with the same driver name
+      Grell::CapybaraDriver.new.setup_capybara
+      expect { Grell::CapybaraDriver.new.setup_capybara }.
+        to raise_error "Poltergeist Driver could not be properly initialized"
+    end
+
+    it 'can register the poltergeist driver multiple times in a row' do
+      Timecop.freeze(ts + 120)
+      driver = Grell::CapybaraDriver.new.setup_capybara
+      expect(driver).to be_instance_of(Capybara::Poltergeist::Driver)
+    end
+  end
+
+  after do
+    Timecop.return
+
+    # Reset Capybara so future tests can easily stub HTTP requests
+    Capybara.javascript_driver = :poltergeist_billy
+    Capybara.default_driver = :poltergeist_billy
+  end
+end

package/grell/spec/lib/crawler_manager_spec.rb

@@ -0,0 +1,174 @@
+RSpec.describe Grell::CrawlerManager do
+  let(:page) { Grell::Page.new(url, page_id, parent_page_id) }
+  let(:host) { 'http://www.example.com' }
+  let(:url) { 'http://www.example.com/test' }
+  let(:driver) { double(Grell::CapybaraDriver) }
+  let(:logger) { Logger.new(nil) }
+  let(:crawler_manager) do
+    described_class.new(logger: logger, driver: driver)
+  end
+
+  describe 'initialize' do
+    context 'provides a logger' do
+      let(:logger) { 33 }
+
+      it 'sets custom logger' do
+        crawler_manager
+        expect(Grell.logger).to eq(33)
+        Grell.logger = Logger.new(nil)
+      end
+    end
+
+    context 'does not provides a logger' do
+      let(:logger) { nil }
+
+      it 'sets default logger' do
+        crawler_manager
+        expect(Grell.logger).to be_instance_of(Logger)
+        Grell.logger = Logger.new(nil)
+      end
+    end
+
+    context 'does not provide a driver' do
+      let(:driver) { nil }
+
+      it 'setups a new Capybara driver' do
+        expect_any_instance_of(Grell::CapybaraDriver).to receive(:setup_capybara)
+        crawler_manager
+      end
+    end
+  end
+
+  describe '#quit' do
+    let(:driver) { double }
+
+    it 'quits the poltergeist driver' do
+      expect(logger).to receive(:info).with("GRELL. Driver quitting")
+      expect(driver).to receive(:quit)
+      crawler_manager.quit
+    end
+  end
+
+  describe '#restart' do
+    let(:driver) { double }
+
+    it 'restarts the poltergeist driver' do
+      expect(driver).to receive(:restart)
+      expect(logger).to receive(:info).with("GRELL. Driver restarted")
+      expect(logger).to receive(:info).with("GRELL. Driver restarting")
+      crawler_manager.restart
+    end
+  end
+
+  describe '#check_periodic_restart' do
+    let(:collection) { double }
+
+    context 'Periodic restart not setup' do
+      it 'does not restart' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { 100 }
+        expect(crawler_manager).not_to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+    end
+
+    context 'Periodic restart setup with default period' do
+      let(:do_something) { proc {} }
+      let(:crawler_manager) do
+        Grell::CrawlerManager.new(
+          logger: logger,
+          driver: driver,
+          on_periodic_restart: { do: do_something }
+        )
+      end
+
+      it 'does not restart after visiting 99 pages' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { 99 }
+        expect(crawler_manager).not_to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+
+      it 'restarts after visiting 100 pages' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { 100 }
+        expect(crawler_manager).to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+    end
+
+    context 'Periodic restart setup with custom period' do
+      let(:do_something) { proc {} }
+      let(:period) { 50 }
+      let(:crawler_manager) do
+        Grell::CrawlerManager.new(
+          logger: logger,
+          driver: driver,
+          on_periodic_restart: { do: do_something, each: period }
+        )
+      end
+
+      context 'restart option is not positive' do
+        let(:period) { 0 }
+
+        it 'logs a warning' do
+          message = 'GRELL. Restart option misconfigured with a negative period. Ignoring option.'
+          expect(logger).to receive(:warn).with(message)
+          crawler_manager
+        end
+      end
+
+      it 'does not restart after visiting a number different from custom period pages' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { period * 1.2 }
+        expect(crawler_manager).not_to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+
+      it 'restarts after visiting custom period pages' do
+        allow(collection).to receive_message_chain(:visited_pages, :size) { period }
+        expect(crawler_manager).to receive(:restart)
+        crawler_manager.check_periodic_restart(collection)
+      end
+    end
+  end
+
+  describe '.cleanup_all_processes' do
+    let(:driver) { double }
+
+    context 'There are some phantomjs processes running' do
+      let(:pids) { [10, 11] }
+      before do
+        allow_any_instance_of(Grell::CrawlerManager::PhantomJSManager)
+          .to receive(:running_phantomjs_pids).and_return(pids)
+      end
+
+      it 'logs processes pids' do
+        expect(Grell.logger).to receive(:warn).with('GRELL. Killing PhantomJS processes: [10, 11]')
+        expect(Grell.logger).to receive(:warn).with('GRELL. Sending KILL to PhantomJS process 10')
+        expect(Grell.logger).to receive(:warn).with('GRELL. Sending KILL to PhantomJS process 11')
+        described_class.cleanup_all_processes
+      end
+
+      it 'kills all phantomjs processes' do
+        expect_any_instance_of(Grell::CrawlerManager::PhantomJSManager).to receive(:kill_process).with(10)
+        expect_any_instance_of(Grell::CrawlerManager::PhantomJSManager).to receive(:kill_process).with(11)
+        described_class.cleanup_all_processes
+      end
+    end
+
+    context 'There are no phantomjs processes running' do
+      let(:pids) { [] }
+      before do
+        allow_any_instance_of(Grell::CrawlerManager::PhantomJSManager)
+          .to receive(:running_phantomjs_pids).and_return(pids)
+      end
+
+      it 'no warning is logged' do
+        expect(Grell.logger).not_to receive(:warn)
+        described_class.cleanup_all_processes
+      end
+
+      it 'No process is killed' do
+        expect_any_instance_of(Grell::CrawlerManager::PhantomJSManager).not_to receive(:kill_process)
+        described_class.cleanup_all_processes
+      end
+    end
+  end
+end