powerdlz23 1.2.4 → 1.2.5

package/grell/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

package/grell/.travis.yml

@@ -0,0 +1,28 @@
+language: ruby
+cache: bundler
+
+rvm:
+  - 2.2.4
+  - 2.3.0
+  - 2.4.2
+
+before_install:
+  - mkdir travis-phantomjs
+  - wget https://github.com/JordiPolo/phantomjs/blob/master/phantomjs-2.1.1-linux-x86_64.tar.bz2?raw=true
+    -O $PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64.tar.bz2
+  - tar -xvf $PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64.tar.bz2 -C $PWD/travis-phantomjs
+  - export PATH=$PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64/bin:$PATH
+
+install:
+  - bundle install --jobs=3 --retry=3
+
+script:
+  - bundle exec rspec
+
+deploy:
+  provider: rubygems
+  api_key:
+    secure: czStDI0W6MWL70sDwu53oNNCc8vKtT61pgvii+ZWIC9A41C2p7BzmbtosXsnLk2ApxmpWvFIgtQE0XIH7jkM5mY05cHinXDphtOTkNLFVjck3ZOMkx/cc+QRFW8K4FHkrzFsC+/Xx4t2/Psh35LpzhfJd0XzKKoCstXUVgJsfGcAK3DMpjXHSUbwLXGDZ4lzmsk52OLf0oL+in2447TJfVOvGXtYmfh1PjXRwDxKB0dan7w5mVgajS52b6wUhVPTaMe/JgCbMuV7BaQ1Goq8u7V4aaxU+liPAhzHWfMB6tF4TEW8yu2tvGLdOA0+1jmM8E9Q5saPWtwKiHvBxN8CzRpkiNDzyFAf8ljrWT5yKX3aRQCyPp3NNyhoumWap36b+O/zwZ3HxoAe22Yg0rjz8z8NxMR/ELPvjPYjCiF5zY7fO9PAzmIynMRUrxDnFj+/JGHdzx0ZMo3fEXgHHSaHPNxIzEffVVQk4XLVnFHDjBLY4mVp4sbHbja5qnui20RkdM/H9Yi/fQyl1ODhk+LUPoh45ZneDZq7GPrl+WKK06oEjXIXLU+1iEuqnSqybbmJMTUJlUV+7EJdtq2DgfDB4KXwLm2LLOR/IX63AzEav4NIxx3hIXifSKa9rp6D7nMTzdQwF0FFzIj/Y3qLrAe1WWt0gx3Vxq67pSwOJthk5Fc=
+  on:
+    tags: true
+    rvm: 2.4.2

package/grell/CHANGELOG.md

@@ -0,0 +1,111 @@
+# 2.1.2
+  * Change white/black lists to allow/deny lists
+
+# 2.1.1
+  * Update phantomjs_options to use 'TLSv1.2'
+
+# 2.1.0
+  * Delete `driver_options` configuration key as it was never used.
+  * `cleanup_all_processes` is a self method as intended to.
+
+# 2.0.0
+  * New configuration key `on_periodic_restart`.
+  * CrawlerManager.cleanup_all_processes method destroy all instances of phantomjs in this machine.
+
+  * Breaking changes
+    - Requires Ruby 2.1 or later.
+    - Crawler.start_crawling does not accept options anymore, all options are passed to Crawler.new.
+    - Crawler's methods `restart` and `quit` have been moved to CrawlerManager.
+    - Crawler gets whitelist and blacklist as configuration options instead of being set in specific methods.
+
+# 1.6.11
+  * Ensure all links are loaded by waiting for Ajax requests to complete
+  * Add '@evaluate_in_each_page' option to evaluate before extracting links (e.g. $('.dropdown').addClass('open');)
+
+# 1.6.10
+  * Avoid following JS href links, add missing dependencies to fix Travis build
+
+# 1.6.9
+  * Avoid following links when disabled by CSS (1.6.8 worked only for Javascript)
+
+# 1.6.8
+  * Avoid following disabled links
+
+# 1.6.7
+  * Increment '@times_visited' first to avoid infinite retries when rescuing errors
+
+# 1.6.6
+  * Updated phantomjs_logger not to open '/dev/null'
+
+# 1.6.5
+  * Added #quit to Crawler
+
+# 1.6.4
+  * Added #quit to Capybara driver
+
+# 1.6.3
+  * Only follow visible links
+
+# 1.6.2
+  * Reset Capybara driver to Puffing Billy (used to rewrite URL requests in specs)
+  * Use float timestamp for Poltergeist driver name to support fast test executions
+
+# 1.6.1
+  * Use non-static name to support registering Poltergeist crawler multiple times
+  * More exception handling, store redirected URLs in addition to original URL
+
+# 1.6
+  * Support custom URL comparison when adding new pages during crawling
+  * Don't rescue Timeout error, so that Delayed Job can properly terminate hanging jobs
+  * Fail early if Capybara doesn't initialize properly
+
+# 1.5.1
+  * Fixed deprecation warning (Thanks scott)
+  * Updated Poltergeist dependency
+
+# 1.5.0
+  * Grell will follow redirects.
+  * Added #followed_redirects? #error? #current_url methods to the Page class
+
+# 1.4.0
+  * Added crawler.restart to restart browser process
+  * The block of code can make grell retry any given page.
+
+# 1.3.2
+  * Rescue Timeout error and return an empty page when that happens
+
+# 1.3.1
+  * Added whitelisting and blacklisting
+  * Better info in gemspec
+
+# 1.3
+  * The Crawler object allows you to provide an external logger object.
+  * Clearer semantics when an error happens, special headers are returned so the user can inspect the error
+  * Caveats:
+    - The 'debug' option in the crawler does not have any affect anymore. Provide an external logger with 'logger' instead
+    - The errors provided in the headers by grell has changed from 'grell_status' to 'grellStatus'.
+    - The 'visited' property in the page was never supposed to be accesible. Use 'visited?' instead.
+
+# 1.2.1
+  * Solve bug: URLs are case insensitive
+
+# 1.2
+  * Grell now will consider two links to point to the same page only when the whole URL is exactly the same.
+    Versions previously would only consider two links to be the same when they shared the path.
+
+# 1.1.2
+  * Solve bug where we were adding links in heads as if there were normal links in the body
+
+# 1.1.1
+  * Solve bug with the new data-href functionality
+
+# 1.1
+  * Solve problem with randomly failing spec
+  * Search for elements with 'href' or 'data-href' to find links
+
+# 1.0.1
+  * Rescueing Javascript errors
+
+# 1.0
+  * Initial implementation
+  * Basic support to crawling pages.

package/grell/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Avoid ruby 2.1 to use Rack > 2.0 which is not compatible
+platform :ruby_21 do
+  gem 'rack', '~> 1.0'
+end
+gemspec

package/grell/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Medidata Solutions Worldwide
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/grell/README.md

@@ -0,0 +1,213 @@
+# Grell
+
+[![Build Status](https://travis-ci.org/mdsol/grell.svg?branch=develop)](https://travis-ci.org/mdsol/grell)
+
+Grell is a generic crawler for the web written in Ruby.
+It can be used to gather data, test pages in a given domain, etc.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'grell'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install grell
+
+Grell uses PhantomJS as a browser, you will need to download and install it in your
+system. Check for instructions in http://phantomjs.org/
+Grell has been tested with PhantomJS v2.1.x
+
+## Usage
+
+### Crawling an entire site
+
+The main entry point of the library is Grell::Crawler#start_crawling.
+Grell will yield to your code with each page it finds:
+
+```ruby
+require 'grell'
+
+crawler = Grell::Crawler.new
+crawler.start_crawling('http://www.google.com') do |page|
+  #Grell will keep iterating this block which each unique page it finds
+  puts "yes we crawled #{page.url}"
+  puts "status: #{page.status}"
+  puts "headers: #{page.headers}"
+  puts "body: #{page.body}"
+  puts "We crawled it at #{page.timestamp}"
+  puts "We found #{page.links.size} links"
+  puts "page id and parent_id #{page.id}, #{page.parent_id}"
+end
+
+```
+
+Grell keeps a list of pages previously crawled and do not visit the same page twice.
+This list is indexed by the complete url, including query parameters.
+
+### Re-retrieving a page
+If you want Grell to revisit a page and return the data to you again,
+return the symbol :retry in your block for the start_crawling method.
+For instance
+```ruby
+require 'grell'
+crawler = Grell::Crawler.new
+crawler.start_crawling('http://www.google.com') do |current_page|
+  if current_page.status == 500 && current_page.retries == 0
+    crawler.manager.restart
+    :retry
+  end
+end
+```
+
+### Pages' id
+
+Each page has an unique id, accessed by the property `id`. Also each page stores the id of the page from which we found this page, accessed by the property `parent_id`.
+The page object generated by accessing the first URL passed to the start_crawling(the root) has a `parent_id` equal to `nil` and an `id` equal to 0.
+Using this information it is possible to construct a directed graph.
+
+
+### Restart and quit
+
+Grell can be restarted. The current list of visited and yet-to-visit pages list are not modified when restarting
+but the browser is destroyed and recreated, all cookies and local storage are lost. After restarting, crawling is resumed with a
+new browser.
+To destroy the crawler, call the `quit` method. This will free the memory taken in Ruby and destroys the PhantomJS process.
+```ruby
+require 'grell'
+crawler = Grell::Crawler.new
+crawler.manager.restart # restarts the browser
+crawler.manager.quit # quits and destroys the crawler
+```
+
+### Options
+
+The `Grell:Crawler` class can be passed options to customize its behavior:
+- `logger`: Sets the logger object, for instance `Rails.logger`. Default: `Logger.new(STDOUT)`
+- `on_periodic_restart`: Sets periodic restarts of the crawler each certain number of visits. Default: 100 pages.
+- `allowlist`: Sets a allowlist filter for URLs to be visited. Default: all URLs are allowlisted.
+- `denylist`: Sets a denylist filter for URLs to be avoided. Default: no URL is denylisted.
+- `add_match_block`: Block evaluated to consider if a given page should be part of the pages to be visited. Default: add unique URLs.
+- `evaluate_in_each_page`: Javascript block to be evaluated on each page visited. Default: Nothing evaluated.
+
+Grell by default will follow all the links it finds in the site being crawled.
+It will never follow links linking outside your site.
+If you want to further limit the amount of links crawled, you can use
+allowlisting, denylisting or manual filtering.
+Below further details on these and other options.
+
+
+#### Automatically restarting PhantomJS
+If you are doing a long crawling it is possible that phantomJS gets into an inconsistent state or it starts leaking memory.
+The crawler can be restarted manually by calling `crawler.manager.restart` or automatically by using the
+`on_periodic_restart` configuration key as follows:
+
+ ```ruby
+ require 'grell'
+
+ crawler = Grell::Crawler.new(on_periodic_restart: { do: my_restart_procedure, each: 200 })
+
+ crawler.start_crawling('http://www.google.com') do |current_page|
+ ...
+ endd
+ ```
+
+ This code will setup the crawler to be restarted every 200 pages being crawled and to call `my_restart_procedure`
+ between restarts. A restart will destroy the cookies so for instance this custom block can be used to relogin.
+
+
+ #### Allowlisting
+
+ ```ruby
+ require 'grell'
+
+ crawler = Grell::Crawler.new(allowlist: [/games\/.*/, '/fun'])
+ crawler.start_crawling('http://www.google.com')
+ ```
+
+ Grell here will only follow links to games and '/fun' and ignore all
+ other links. You can provide a regexp, strings (if any part of the
+ string match is allowlisted) or an array with regexps and/or strings.
+
+ #### Denylisting
+
+ ```ruby
+ require 'grell'
+
+ crawler = Grell::Crawler.new(denylist: /games\/.*/)
+ crawler.start_crawling('http://www.google.com')
+ ```
+
+ Similar to allowlisting. But now Grell will follow every other link in
+ this site which does not go to /games/...
+
+ If you call both allowlist and denylist then both will apply, a link
+ has to fullfill both conditions to survive. If you do not call any, then
+ all links on this site will be crawled. Think of these methods as
+ filters.
+
+#### Manual link filtering
+
+If you have a more complex use-case, you can modify the list of links
+manually.
+Grell yields the page to you before it adds the links to the list of
+links to visit. So you can modify in your block of code "page.links" to
+add and delete links to instruct Grell to add them to the list of links
+to visit next.
+
+#### Custom URL Comparison
+By default, Grell will detect new URLs to visit by comparing the full URL
+with the URLs of the discovered and visited links. This functionality can
+be changed by passing a block of code to Grells `start_crawling` method.
+In the below example, the path of the URLs (instead of the full URL) will
+be compared.
+
+```ruby
+require 'grell'
+
+add_match_block = Proc.new do |collection_page, page|
+  collection_page.path == page.path
+end
+
+crawler = Grell::Crawler.new(add_match_block: add_match_block)
+
+crawler.start_crawling('http://www.google.com') do |current_page|
+...
+end
+```
+
+#### Evaluate script
+
+You can evalute a JavaScript snippet in each page before extracting links by passing the snippet to the 'evaluate_in_each_page' option:
+
+```ruby
+require 'grell'
+
+crawler = Grell::Crawler.new(evaluate_in_each_page: "typeof jQuery !== 'undefined' && $('.dropdown').addClass('open');")
+
+```
+
+### Errors
+When there is an error in the page or an internal error in the crawler (Javascript crashed the browser, etc). Grell will return with status 404 and the headers will have the following keys:
+- grellStatus: 'Error'
+- errorClass: The class of the error which broke this page.
+- errorMessage: A descriptive message with the information Grell could gather about the error.
+
+## Tests
+
+Run the tests with
+```ruby
+bundle exec rake ci
+```
+
+## Contributors
+Grell is (c) Medidata Solutions Worldwide and owned by its major contributors:
+* [Teruhide Hoshikawa](https://github.com/thoshikawa-mdsol)
+* [Jordi Polo Carres](https://github.com/jcarres-mdsol)

package/grell/Rakefile

@@ -0,0 +1,2 @@
+require 'kender/tasks'
+

package/grell/grell.gemspec

@@ -0,0 +1,36 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'grell/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "grell"
+  spec.version       = Grell::VERSION
+  spec.platform      = Gem::Platform::RUBY
+  spec.authors       = ["Jordi Polo Carres"]
+  spec.email         = ["jcarres@mdsol.com"]
+  spec.summary       = %q{Ruby web crawler}
+  spec.description   = %q{Ruby web crawler using PhantomJS}
+  spec.homepage      = "https://github.com/mdsol/grell"
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.1.8'
+
+  spec.add_dependency 'capybara', '~> 2.10'
+  spec.add_dependency 'poltergeist', '~> 1.11'
+
+  # spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'byebug', '~> 4.0'
+  spec.add_development_dependency 'kender', '~> 0.2'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'webmock', '~> 1.18'
+  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'puffing-billy', '~> 0.9'
+  spec.add_development_dependency 'timecop', '~> 0.8'
+  spec.add_development_dependency 'selenium-webdriver', '~> 2.53.4'
+end

package/grell/lib/grell/capybara_driver.rb

@@ -0,0 +1,44 @@
+module Grell
+  # This class setups the driver for capybara. Used internally by the CrawlerManager
+  # It uses Portelgeist to control PhantomJS
+  class CapybaraDriver
+    USER_AGENT = "Mozilla/5.0 (Grell Crawler)".freeze
+
+    # Returns a poltergeist driver
+    def setup_capybara
+      @poltergeist_driver = nil
+
+      # Capybara will not re-run the block if the driver name already exists, so the driver name
+      # will have a time integer appended to ensure uniqueness.
+      driver_name = "poltergeist_crawler_#{Time.now.to_f}".to_sym
+      Grell.logger.info "GRELL Registering poltergeist driver with name '#{driver_name}'"
+
+      Capybara.register_driver driver_name do |app|
+        @poltergeist_driver = Capybara::Poltergeist::Driver.new(app,
+          js_errors: false,
+          inspector: false,
+          phantomjs_logger: FakePoltergeistLogger,
+          phantomjs_options: ['--debug=no', '--load-images=no', '--ignore-ssl-errors=yes', '--ssl-protocol=TLSv1.2'])
+      end
+
+      Capybara.default_max_wait_time = 3
+      Capybara.run_server = false
+      Capybara.default_driver = driver_name
+      Capybara.current_session.driver.headers = { # The driver gets initialized when modified here
+        "DNT" => 1,
+        "User-Agent" => USER_AGENT
+      }
+
+      raise 'Poltergeist Driver could not be properly initialized' unless @poltergeist_driver
+
+      @poltergeist_driver
+    end
+
+    # Poltergeist driver needs a class with this signature. The javascript console.log is sent here.
+    # We just discard that information.
+    module FakePoltergeistLogger
+      def self.puts(*)
+      end
+    end
+  end
+end

package/grell/lib/grell/crawler.rb

@@ -0,0 +1,83 @@
+module Grell
+  # This is the class that starts and controls the crawling
+  class Crawler
+    attr_reader :collection, :manager
+
+    # Creates a crawler
+    # evaluate_in_each_page: javascript block to evaluate in each page we crawl
+    # add_match_block: block to evaluate to consider if a page is part of the collection
+    # manager_options: options passed to the manager class
+    # allowlist: Sets an allowlist filter, allows a regexp, string or array of either to be matched.
+    # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
+    def initialize(evaluate_in_each_page: nil, add_match_block: nil, allowlist: /.*/, denylist: /a^/, **manager_options)
+      @collection = nil
+      @manager = CrawlerManager.new(manager_options)
+      @evaluate_in_each_page = evaluate_in_each_page
+      @add_match_block = add_match_block
+      @allowlist_regexp = Regexp.union(allowlist)
+      @denylist_regexp = Regexp.union(denylist)
+    end
+
+    # Main method, it starts crawling on the given URL and calls a block for each of the pages found.
+    def start_crawling(url, &block)
+      Grell.logger.info "GRELL Started crawling"
+      @collection = PageCollection.new(@add_match_block)
+      @collection.create_page(url, nil)
+
+      while !@collection.discovered_pages.empty?
+        crawl(@collection.next_page, block)
+        @manager.check_periodic_restart(@collection)
+      end
+
+      Grell.logger.info "GRELL finished crawling"
+    end
+
+    def crawl(site, block)
+      Grell.logger.info "Visiting #{site.url}, visited_links: #{@collection.visited_pages.size}, discovered #{@collection.discovered_pages.size}"
+      crawl_site(site)
+
+      if block # The user of this block can send us a :retry to retry accessing the page
+        while crawl_block(block, site) == :retry
+          Grell.logger.info "Retrying our visit to #{site.url}"
+          crawl_site(site)
+        end
+      end
+
+      site.links.each do |url|
+        @collection.create_page(url, site.id)
+      end
+    end
+
+    private
+
+    def crawl_site(site)
+      site.navigate
+      site.rawpage.page.evaluate_script(@evaluate_in_each_page) if @evaluate_in_each_page
+      filter!(site.links)
+      add_redirect_url(site)
+    end
+
+    # Treat any exceptions from the block as an unavailable page
+    def crawl_block(block, site)
+      block.call(site)
+    rescue Capybara::Poltergeist::BrowserError, Capybara::Poltergeist::DeadClient,
+           Capybara::Poltergeist::JavascriptError, Capybara::Poltergeist::StatusFailError,
+           Capybara::Poltergeist::TimeoutError, Errno::ECONNRESET, URI::InvalidURIError => e
+      site.unavailable_page(404, e)
+    end
+
+    def filter!(links)
+      links.select! { |link| link =~ @allowlist_regexp } if @allowlist_regexp
+      links.delete_if { |link| link =~ @denylist_regexp } if @denylist_regexp
+    end
+
+    # Store the resulting redirected URL along with the original URL
+    def add_redirect_url(site)
+      if site.url != site.current_url
+        @collection.create_page(site.current_url, site.id)
+      end
+    end
+
+  end
+
+end

package/grell/lib/grell/crawler_manager.rb

@@ -0,0 +1,84 @@
+module Grell
+  # Manages the state of the process crawling, does not care about individual pages but about logging,
+  # restarting and quiting the crawler correctly.
+  class CrawlerManager
+    # logger: logger to use for Grell's messages
+    # on_periodic_restart: if set, the driver will restart every :each visits (100 default) and execute the :do block
+    # driver_options: Any extra options for the Capybara driver
+    def initialize(logger: nil, on_periodic_restart: {}, driver: nil)
+      Grell.logger = logger ? logger : Logger.new(STDOUT)
+      @periodic_restart_block = on_periodic_restart[:do]
+      @periodic_restart_period = on_periodic_restart[:each] || PAGES_TO_RESTART
+      @driver = driver || CapybaraDriver.new.setup_capybara
+      if @periodic_restart_period <= 0
+        Grell.logger.warn "GRELL. Restart option misconfigured with a negative period. Ignoring option."
+      end
+    end
+
+    # Restarts the PhantomJS process without modifying the state of visited and discovered pages.
+    def restart
+      Grell.logger.info "GRELL. Driver restarting"
+      @driver.restart
+      Grell.logger.info "GRELL. Driver restarted"
+    end
+
+    # Quits the poltergeist driver.
+    def quit
+      Grell.logger.info "GRELL. Driver quitting"
+      @driver.quit
+    end
+
+    # PhantomJS seems to consume memory increasingly as it crawls, periodic restart allows to restart
+    # the driver, potentially calling a block.
+    def check_periodic_restart(collection)
+      return unless @periodic_restart_block
+      return unless @periodic_restart_period > 0
+      return unless (collection.visited_pages.size % @periodic_restart_period).zero?
+      restart
+      @periodic_restart_block.call
+    end
+
+    def self.cleanup_all_processes
+      PhantomJSManager.new.cleanup_all_processes
+    end
+
+    private
+
+    PAGES_TO_RESTART = 100  # Default number of pages before we restart the driver.
+    KILL_TIMEOUT = 2        # Number of seconds we wait till we kill the process.
+
+    # Manages the PhantomJS process
+    class PhantomJSManager
+      def cleanup_all_processes
+        pids = running_phantomjs_pids
+        return if pids.empty?
+        Grell.logger.warn "GRELL. Killing PhantomJS processes: #{pids.inspect}"
+        pids.each do |pid|
+          Grell.logger.warn "GRELL. Sending KILL to PhantomJS process #{pid}"
+          kill_process(pid.to_i)
+        end
+      end
+
+      def running_phantomjs_pids
+        list_phantomjs_processes_cmd = "ps -ef | grep -E 'bin/phantomjs' | grep -v grep"
+        `#{list_phantomjs_processes_cmd} | awk '{print $2;}'`.split("\n")
+      end
+
+      def kill_process(pid)
+        Process.kill('TERM', pid)
+        force_kill(pid)
+      rescue Errno::ESRCH, Errno::ECHILD
+        # successfully terminated
+      rescue => e
+        Grell.logger.error ["GRELL. PhantomJS process could not be killed", e.message, *e.backtrace].join($/)
+      end
+
+      def force_kill(pid)
+        Timeout.timeout(KILL_TIMEOUT) { Process.wait(pid) }
+      rescue Timeout::Error
+        Process.kill('KILL', pid)
+        Process.wait(pid)
+      end
+    end
+  end
+end

package/grell/lib/grell/grell_logger.rb

@@ -0,0 +1,10 @@
+require 'logger'
+
+#Very simple global logger for our crawler.
+module Grell
+  class << self
+    attr_accessor :logger
+  end
+end
+
+Grell.logger = Logger.new(STDOUT)