powerdlz23 1.2.4 → 1.2.5

package/snapcrawl/lib/snapcrawl/page.rb

@@ -0,0 +1,118 @@
+require 'addressable/uri'
+require 'fileutils'
+require 'httparty'
+require 'lightly'
+require 'nokogiri'
+
+module Snapcrawl
+  class Page
+    using StringRefinements
+
+    attr_reader :url, :depth
+
+    EXTENSION_BLACKLIST = 'png|gif|jpg|pdf|zip'
+    PROTOCOL_BLACKLIST = 'mailto|tel'
+
+    def initialize(url, depth: 0)
+      @url = url.protocolize
+      @depth = depth
+    end
+
+    def valid?
+      http_response&.success?
+    end
+
+    def site
+      @site ||= Addressable::URI.parse(url).site
+    end
+
+    def path
+      @path ||= Addressable::URI.parse(url).request_uri
+    end
+
+    def links
+      return nil unless valid?
+
+      doc = Nokogiri::HTML http_response.body
+      normalize_links doc.css('a')
+    end
+
+    def pages
+      return nil unless valid?
+
+      links.map { |link| Page.new link, depth: depth + 1 }
+    end
+
+    def save_screenshot(outfile)
+      return false unless valid?
+
+      Screenshot.new(url).save outfile
+    end
+
+  private
+
+    def http_response
+      @http_response ||= http_response!
+    end
+
+    def http_response!
+      response = cache.get(url) { HTTParty.get url, httparty_options }
+
+      unless response.success?
+        $logger.warn "http error on mu`#{url}`, code: y`#{response.code}`, message: #{response.message.strip}"
+      end
+
+      response
+    rescue => e
+      $logger.error "http error on mu`#{url}` - r`#{e.class}`: #{e.message}"
+      nil
+    end
+
+    def httparty_options
+      Config.skip_ssl_verification ? { verify: false } : {}
+    end
+
+    def normalize_links(links)
+      result = []
+
+      links.each do |link|
+        valid_link = normalize_link link
+        result << valid_link if valid_link
+      end
+
+      result.uniq
+    end
+
+    def normalize_link(link)
+      link = link.attribute('href').to_s.dup
+
+      # Remove #hash
+      link.gsub!(/#.+$/, '')
+      return nil if link.empty?
+
+      # Remove links to specific extensions and protocols
+      return nil if /\.(#{EXTENSION_BLACKLIST})(\?.*)?$/o.match?(link)
+      return nil if /^(#{PROTOCOL_BLACKLIST}):/o.match?(link)
+
+      # Strip spaces
+      link.strip!
+
+      # Convert relative links to absolute
+      begin
+        link = Addressable::URI.join(url, link).to_s.dup
+      rescue => e
+        $logger.warn "r`#{e.class}`: #{e.message} on #{path} (link: #{link})"
+        return nil
+      end
+
+      # Keep only links in our base domain
+      return nil unless link.include? site
+
+      link
+    end
+
+    def cache
+      Lightly.new life: Config.cache_life
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/pretty_logger.rb

@@ -0,0 +1,11 @@
+require 'logger'
+
+module Snapcrawl
+  class PrettyLogger
+    extend LogHelpers
+
+    def self.new
+      Logger.new($stdout, formatter: log_formatter, level: Config.log_level)
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/refinements/pair_split.rb

@@ -0,0 +1,26 @@
+module Snapcrawl
+  module PairSplit
+    refine Array do
+      def pair_split
+        false_values = %w[no false]
+        true_values = %w[yes true]
+
+        to_h do |pair|
+          key, value = pair.split '='
+
+          value = if /^\d+$/.match?(value)
+            value.to_i
+          elsif false_values.include? value
+            false
+          elsif true_values.include? value
+            true
+          else
+            value
+          end
+
+          [key, value]
+        end
+      end
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/refinements/string_refinements.rb

@@ -0,0 +1,13 @@
+module Snapcrawl
+  module StringRefinements
+    refine String do
+      def to_slug
+        downcase.gsub(/[^a-z0-9]+/, '-')
+      end
+
+      def protocolize
+        /^http/.match?(self) ? self : "http://#{self}"
+      end
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/screenshot.rb

@@ -0,0 +1,73 @@
+require 'webshot'
+
+module Snapcrawl
+  class Screenshot
+    using StringRefinements
+
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+    end
+
+    def save(outfile = nil)
+      outfile ||= "#{url.to_slug}.png"
+      webshot_capture url, outfile
+    end
+
+  private
+
+    def webshot_capture(url, image_path)
+      webshot_capture! url, image_path
+    rescue => e
+      raise ScreenshotError, "#{e.class} #{e.message}"
+    end
+
+    def webshot_capture!(url, image_path)
+      hide_output do
+        webshot.capture url, image_path, webshot_options do |magick|
+          magick.combine_options do |c|
+            c.background 'white'
+            c.gravity 'north'
+            c.quality 100
+            c.extent Config.height.positive? ? "#{Config.width}x#{Config.height}" : "#{Config.width}x"
+          end
+        end
+      end
+    end
+
+    def webshot_options
+      result = { allowed_status_codes: [404, 401, 403] }
+
+      if Config.css_selector
+        result[:selector] = Config.css_selector
+        result[:full] = false
+      end
+
+      if Config.screenshot_delay
+        result[:timeout] = Config.screenshot_delay
+      end
+
+      result
+    end
+
+    def webshot
+      @webshot ||= Webshot::Screenshot.instance
+    end
+
+    # The webshot gem messes with stdout/stderr streams so we keep it in
+    # check by using this method. Also, in some sites (e.g. uown.co) it
+    # prints some output to stdout, this is why we override $stdout for
+    # the duration of the run.
+    def hide_output
+      keep_stdout = $stdout
+      keep_stderr = $stderr
+      $stdout = StringIO.new
+      $stderr = StringIO.new
+      yield
+    ensure
+      $stdout = keep_stdout
+      $stderr = keep_stderr
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/templates/config.yml

@@ -0,0 +1,49 @@
+# All values below are the default values
+
+# log level (0-4) 0=DEBUG 1=INFO 2=WARN 3=ERROR 4=FATAL
+log_level: 1
+
+# log_color (yes, no, auto)
+# yes  = always show log color
+# no   = never use colors
+# auto = only use colors when running in an interactive terminal
+log_color: auto
+
+# number of levels to crawl, 0 means capture only the root URL
+depth: 1
+
+# screenshot width in pixels
+width: 1280
+
+# screenshot height in pixels, 0 means the entire height
+height: 0
+
+# number of seconds to consider the page cache and its screenshot fresh
+cache_life: 86400
+
+# where to store the HTML page cache
+cache_dir: cache
+
+# where to store screenshots
+snaps_dir: snaps
+
+# screenshot filename template, where '%{url}' will be replaced with a 
+# slug version of the URL (no need to include the .png extension)
+name_template: '%{url}'
+
+# urls not matching this regular expression will be ignored
+url_whitelist: 
+
+# urls matching this regular expression will be ignored
+url_blacklist: 
+
+# take a screenshot of this CSS selector only
+css_selector: 
+
+# when true, ignore SSL related errors
+skip_ssl_verification: false
+
+# set to any number of seconds to wait for the page to load before taking
+# a screenshot, leave empty to not wait at all (only needed for pages with
+# animations or other post-load events).
+screenshot_delay: 

package/snapcrawl/lib/snapcrawl/templates/docopt.txt

@@ -0,0 +1,26 @@
+Snapcrawl
+
+Usage:
+  snapcrawl URL [--config FILE] [SETTINGS...]
+  snapcrawl -h | --help 
+  snapcrawl -v | --version
+
+Options:
+  -c, --config FILE
+    Path to config file, with or without the .yml extension.
+    A sample file will be created if not found.
+    The default filename is 'snapcrawl.yml'.
+
+  -h, --help
+    Show this screen
+
+  -v, --version
+    Show version number
+
+Settings:
+  Provide any of the options available in the config as 'key=value'.
+
+Examples:
+  snapcrawl example.com
+  snapcrawl example.com --config simple
+  snapcrawl example.com depth=1 log_level=2 width=768

package/snapcrawl/lib/snapcrawl/version.rb

@@ -0,0 +1,3 @@
+module Snapcrawl
+  VERSION = '0.5.4'
+end

package/snapcrawl/lib/snapcrawl.rb

@@ -0,0 +1,20 @@
+require 'snapcrawl/version'
+require 'snapcrawl/exceptions'
+require 'snapcrawl/refinements/pair_split'
+require 'snapcrawl/refinements/string_refinements'
+require 'snapcrawl/log_helpers'
+require 'snapcrawl/pretty_logger'
+require 'snapcrawl/dependencies'
+require 'snapcrawl/config'
+require 'snapcrawl/screenshot'
+require 'snapcrawl/page'
+require 'snapcrawl/crawler'
+require 'snapcrawl/cli'
+
+if ENV['BYEBUG']
+  require 'byebug'
+  require 'lp'
+end
+
+Snapcrawl::Config.load
+$logger = Snapcrawl::PrettyLogger.new

package/snapcrawl/snapcrawl.gemspec

@@ -0,0 +1,27 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'snapcrawl/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'snapcrawl'
+  s.version     = Snapcrawl::VERSION
+  s.summary     = 'Crawl a website and take screenshots (CLI + Library)'
+  s.description = 'Snapcrawl is a command line utility for crawling a website and saving screenshots.'
+  s.authors     = ['Danny Ben Shitrit']
+  s.email       = 'db@dannyben.com'
+  s.files       = Dir['README.md', 'lib/**/*']
+  s.executables = ['snapcrawl']
+  s.homepage    = 'https://github.com/DannyBen/snapcrawl'
+  s.license     = 'MIT'
+  s.required_ruby_version = '>= 3.0'
+
+  s.add_runtime_dependency 'addressable', '~> 2.7'
+  s.add_runtime_dependency 'colsole', '>= 0.8.1', '< 2'
+  s.add_runtime_dependency 'docopt', '~> 0.6'
+  s.add_runtime_dependency 'httparty', '~> 0.21'
+  s.add_runtime_dependency 'lightly', '~> 0.3'
+  s.add_runtime_dependency 'nokogiri', '~> 1.10'
+  s.add_runtime_dependency 'sting', '~> 0.4'
+  s.add_runtime_dependency 'webshot', '~> 0.1'
+  s.metadata['rubygems_mfa_required'] = 'true'
+end

package/snapcrawl/snapcrawl.yml

@@ -0,0 +1,41 @@
+# All values below are the default values
+
+# log level (0-4) 0=DEBUG 1=INFO 2=WARN 3=ERROR 4=FATAL
+log_level: 1
+
+# log_color (yes, no, auto)
+# yes  = always show log color
+# no   = never use colors
+# auto = only use colors when running in an interactive terminal
+log_color: auto
+
+# number of levels to crawl, 0 means capture only the root URL
+depth: 1
+
+# screenshot width in pixels
+width: 1280
+
+# screenshot height in pixels, 0 means the entire height
+height: 0
+
+# number of seconds to consider the page cache and its screenshot fresh
+cache_life: 86400
+
+# where to store the HTML page cache
+cache_dir: cache
+
+# where to store screenshots
+snaps_dir: snaps
+
+# screenshot filename template, where '%{url}' will be replaced with a 
+# slug version of the URL (no need to include the .png extension)
+name_template: '%{url}'
+
+# urls not matching this regular expression will be ignored
+url_whitelist: 
+
+# urls matching this regular expression will be ignored
+url_blacklist: 
+
+# take a screenshot of this CSS selector only
+css_selector: 

package/snapcrawl/spec/README.md

@@ -0,0 +1,16 @@
+Snapcrawl Tests
+==================================================
+
+Running Tests
+--------------------------------------------------
+
+```shell
+# Start a dummy sinatra server
+$ run server start -d
+
+# Run all tests
+$ run spec
+
+# Stop the server if you are done testing
+$ run server stop
+```

package/snapcrawl/spec/approvals/bin/help

@@ -0,0 +1,26 @@
+Snapcrawl
+
+Usage:
+  snapcrawl URL [--config FILE] [SETTINGS...]
+  snapcrawl -h | --help 
+  snapcrawl -v | --version
+
+Options:
+  -c, --config FILE
+    Path to config file, with or without the .yml extension.
+    A sample file will be created if not found.
+    The default filename is 'snapcrawl.yml'.
+
+  -h, --help
+    Show this screen
+
+  -v, --version
+    Show version number
+
+Settings:
+  Provide any of the options available in the config as 'key=value'.
+
+Examples:
+  snapcrawl example.com
+  snapcrawl example.com --config simple
+  snapcrawl example.com depth=1 log_level=2 width=768

package/snapcrawl/spec/approvals/bin/usage

@@ -0,0 +1,4 @@
+Usage:
+  snapcrawl URL [--config FILE] [SETTINGS...]
+  snapcrawl -h | --help 
+  snapcrawl -v | --version

package/snapcrawl/spec/approvals/cli/usage

@@ -0,0 +1,4 @@
+Usage:
+  snapcrawl URL [--config FILE] [SETTINGS...]
+  snapcrawl -h | --help 
+  snapcrawl -v | --version

package/snapcrawl/spec/approvals/config/defaults

@@ -0,0 +1,15 @@
+---
+depth: 1
+width: 1280
+height: 0
+cache_life: 86400
+cache_dir: cache
+snaps_dir: snaps
+name_template: "%{url}"
+url_whitelist:
+url_blacklist:
+css_selector:
+log_level: 1
+log_color: auto
+skip_ssl_verification: false
+screenshot_delay:

package/snapcrawl/spec/approvals/config/minimal

@@ -0,0 +1,15 @@
+---
+depth: 3
+width: 768
+height: 0
+cache_life: 86400
+cache_dir: cache
+snaps_dir: snaps
+name_template: "%{url}"
+url_whitelist:
+url_blacklist:
+css_selector:
+log_level: 3
+log_color: false
+skip_ssl_verification: false
+screenshot_delay:

package/snapcrawl/spec/approvals/integration/blacklist

@@ -0,0 +1,14 @@
+DEBUG : initializing cli
+DEBUG : initializing crawler with http://localhost:3000/filters
+DEBUG : config {"depth"=>1, "width"=>1280, "height"=>0, "cache_life"=>86400, "cache_dir"=>"cache", "snaps_dir"=>"snaps", "name_template"=>"%%{url}", "url_whitelist"=>nil, "url_blacklist"=>"exclude", "css_selector"=>nil, "log_level"=>0, "log_color"=>"auto", "skip_ssl_verification"=>false, "screenshot_delay"=>nil}
+DEBUG : processing queue: 1 remaining
+ INFO : processing http://localhost:3000/filters, depth: 0
+ INFO : screenshot for /filters already exists
+DEBUG : ignoring http://localhost:3000/filters/exclude-me/1, reason: blacklist
+DEBUG : ignoring http://localhost:3000/filters/exclude-me/2, reason: blacklist
+DEBUG : processing queue: 2 remaining
+ INFO : processing http://localhost:3000/filters/include-me/1, depth: 1
+ INFO : screenshot for /filters/include-me/1 already exists
+DEBUG : processing queue: 1 remaining
+ INFO : processing http://localhost:3000/filters/include-me/2, depth: 1
+ INFO : screenshot for /filters/include-me/2 already exists

package/snapcrawl/spec/approvals/integration/default-config

@@ -0,0 +1,14 @@
+DEBUG : verifying phantomjs is present
+DEBUG : verifying imagemagick is present
+DEBUG : initializing cli
+DEBUG : initializing crawler with http://localhost:3000
+DEBUG : config {"depth"=>1, "width"=>1280, "height"=>0, "cache_life"=>86400, "cache_dir"=>"cache", "snaps_dir"=>"snaps", "name_template"=>"%%{url}", "url_whitelist"=>nil, "url_blacklist"=>nil, "css_selector"=>nil, "log_level"=>1, "log_color"=>"auto", "skip_ssl_verification"=>false, "screenshot_delay"=>nil}
+DEBUG : processing queue: 1 remaining
+ INFO : processing http://localhost:3000, depth: 0
+ INFO : capturing screenshot for /
+DEBUG : processing queue: 2 remaining
+ INFO : processing http://localhost:3000/page, depth: 1
+ INFO : capturing screenshot for /page
+DEBUG : processing queue: 1 remaining
+ INFO : processing http://localhost:3000/errors, depth: 1
+ INFO : capturing screenshot for /errors

package/snapcrawl/spec/approvals/integration/depth-0

@@ -0,0 +1,6 @@
+DEBUG : initializing cli
+DEBUG : initializing crawler with http://localhost:3000
+DEBUG : config {"depth"=>0, "width"=>1280, "height"=>0, "cache_life"=>86400, "cache_dir"=>"cache", "snaps_dir"=>"snaps", "name_template"=>"%%{url}", "url_whitelist"=>nil, "url_blacklist"=>nil, "css_selector"=>nil, "log_level"=>1, "log_color"=>"auto", "skip_ssl_verification"=>false, "screenshot_delay"=>nil}
+DEBUG : processing queue: 1 remaining
+ INFO : processing http://localhost:3000, depth: 0
+ INFO : screenshot for / already exists

package/snapcrawl/spec/approvals/integration/depth-3

@@ -0,0 +1,6 @@
+ WARN : Addressable::URI::InvalidURIError: Invalid scheme format: '\problematic ' on /page (link: \problematic : link)
+ WARN : http error on http://localhost:3000/broken, code: 404, message: Not Found
+ WARN : http error on http://localhost:3000/secret, code: 401, message: Unauthorized
+ WARN : http error on http://localhost:3000/500, code: 500, message: Internal Server Error
+ WARN : http error on http://localhost:3000/401, code: 401, message: Unauthorized
+ WARN : http error on http://localhost:3000/403, code: 403, message: Forbidden

package/snapcrawl/spec/approvals/integration/log-color-no

@@ -0,0 +1,6 @@
+ INFO : processing http://localhost:3000, depth: 0
+ INFO : screenshot for / already exists
+ INFO : processing http://localhost:3000/page, depth: 1
+ INFO : screenshot for /page already exists
+ INFO : processing http://localhost:3000/errors, depth: 1
+ INFO : screenshot for /errors already exists

package/snapcrawl/spec/approvals/integration/screenshot-error

@@ -0,0 +1,3 @@
+ INFO : processing http://localhost:3000, depth: 0
+ INFO : capturing screenshot for /
+ERROR : screenshot error on / - Snapcrawl::ScreenshotError: Simulated error

package/snapcrawl/spec/approvals/integration/whitelist

@@ -0,0 +1,14 @@
+DEBUG : initializing cli
+DEBUG : initializing crawler with http://localhost:3000/filters
+DEBUG : config {"depth"=>1, "width"=>1280, "height"=>0, "cache_life"=>86400, "cache_dir"=>"cache", "snaps_dir"=>"snaps", "name_template"=>"%%{url}", "url_whitelist"=>"include", "url_blacklist"=>nil, "css_selector"=>nil, "log_level"=>0, "log_color"=>"auto", "skip_ssl_verification"=>false, "screenshot_delay"=>nil}
+DEBUG : processing queue: 1 remaining
+ INFO : processing http://localhost:3000/filters, depth: 0
+ INFO : capturing screenshot for /filters
+DEBUG : ignoring http://localhost:3000/filters/exclude-me/1, reason: whitelist
+DEBUG : ignoring http://localhost:3000/filters/exclude-me/2, reason: whitelist
+DEBUG : processing queue: 2 remaining
+ INFO : processing http://localhost:3000/filters/include-me/1, depth: 1
+ INFO : capturing screenshot for /filters/include-me/1
+DEBUG : processing queue: 1 remaining
+ INFO : processing http://localhost:3000/filters/include-me/2, depth: 1
+ INFO : capturing screenshot for /filters/include-me/2

package/snapcrawl/spec/approvals/models/pretty_logger/colors

@@ -0,0 +1 @@
+ INFO : Hello World

package/snapcrawl/spec/fixtures/config/minimal.yml

@@ -0,0 +1,4 @@
+log_level: 3
+log_color: no
+depth: 3
+width: 768