powerdlz23 1.2.3 → 1.2.5

package/snapcrawl/lib/snapcrawl/cli.rb

@@ -0,0 +1,52 @@
+require 'colsole'
+require 'docopt'
+require 'fileutils'
+
+module Snapcrawl
+  class CLI
+    include Colsole
+    using StringRefinements
+    using PairSplit
+
+    def call(args = [])
+      execute Docopt.docopt(docopt, version: VERSION, argv: args)
+    rescue Docopt::Exit => e
+      puts e.message
+    end
+
+  private
+
+    def execute(args)
+      config_file = args['--config']
+      Config.load config_file if config_file
+
+      tweaks = args['SETTINGS'].pair_split
+      apply_tweaks tweaks if tweaks
+
+      Dependencies.verify
+
+      $logger.debug 'initializing cli'
+      FileUtils.mkdir_p Config.snaps_dir
+
+      url = args['URL'].protocolize
+      crawler = Crawler.new url
+
+      crawler.crawl
+    end
+
+    def docopt
+      @docopt ||= File.read docopt_path
+    end
+
+    def docopt_path
+      File.expand_path 'templates/docopt.txt', __dir__
+    end
+
+    def apply_tweaks(tweaks)
+      tweaks.each do |key, value|
+        Config.settings[key] = value
+        $logger.level = value if key == 'log_level'
+      end
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/config.rb

@@ -0,0 +1,60 @@
+require 'sting'
+require 'fileutils'
+
+module Snapcrawl
+  class Config < Sting
+    class << self
+      def load(file = nil)
+        reset!
+        push defaults
+
+        return unless file
+
+        file = "#{file}.yml" unless /\.ya?ml$/.match?(file)
+
+        # FIXME: Cannot use logger here due to the "chicken and egg" with
+        #        Config. The $logger is available, but it was not yet fully
+        #        configured with log_level etc.
+        if File.exist? file
+          # $logger.debug "loading config file g`#{file}`"
+          push file
+        else
+          # $logger.debug "creating config file g`#{file}`"
+          create_config file
+        end
+      end
+
+    private
+
+      def defaults
+        {
+          depth:                 1,
+          width:                 1280,
+          height:                0,
+          cache_life:            86_400,
+          cache_dir:             'cache',
+          snaps_dir:             'snaps',
+          name_template:         '%{url}',
+          url_whitelist:         nil,
+          url_blacklist:         nil,
+          css_selector:          nil,
+          log_level:             1,
+          log_color:             'auto',
+          skip_ssl_verification: false,
+          screenshot_delay:      nil,
+        }
+      end
+
+      def create_config(file)
+        content = File.read config_template
+        dir = File.dirname file
+        FileUtils.mkdir_p dir
+        File.write file, content
+      end
+
+      def config_template
+        File.expand_path 'templates/config.yml', __dir__
+      end
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/crawler.rb

@@ -0,0 +1,98 @@
+require 'fileutils'
+
+module Snapcrawl
+  class Crawler
+    using StringRefinements
+
+    attr_reader :url
+
+    def initialize(url)
+      $logger.debug "initializing crawler with g`#{url}`"
+
+      config_for_display = Config.settings.dup
+      config_for_display['name_template'] = '%%{url}'
+
+      $logger.debug "config #{config_for_display}"
+      @url = url
+    end
+
+    def crawl
+      Dependencies.verify
+      todo[url] = Page.new url
+      process_todo while todo.any?
+    end
+
+  private
+
+    def process_todo
+      $logger.debug "processing queue: g`#{todo.count} remaining`"
+
+      url, page = todo.shift
+      done.push url
+
+      return unless process_page page
+
+      register_sub_pages page.pages if page.depth < Config.depth
+    end
+
+    def register_sub_pages(pages)
+      pages.each do |sub_page|
+        next if todo.has_key?(sub_page) || done.include?(sub_page)
+
+        if Config.url_whitelist && sub_page.path !~ (/#{Config.url_whitelist}/)
+          $logger.debug "ignoring mu`#{sub_page.url}`, reason: whitelist"
+          next
+        end
+
+        if Config.url_blacklist && sub_page.path =~ (/#{Config.url_blacklist}/)
+          $logger.debug "ignoring mu`#{sub_page.url}`, reason: blacklist"
+          next
+        end
+
+        todo[sub_page.url] = sub_page
+      end
+    end
+
+    def process_page(page)
+      outfile = "#{Config.snaps_dir}/#{Config.name_template}.png" % { url: page.url.to_slug }
+
+      $logger.info "processing mu`#{page.url}`, depth: #{page.depth}"
+
+      unless page.valid?
+        $logger.debug "page #{page.path} is invalid, aborting process"
+        return false
+      end
+
+      if file_fresh? outfile
+        $logger.info "screenshot for #{page.path} already exists"
+      else
+        $logger.info "gb`capturing screenshot for #{page.path}`"
+        save_screenshot page, outfile
+      end
+
+      true
+    end
+
+    def save_screenshot(page, outfile)
+      page.save_screenshot outfile
+    rescue => e
+      $logger.error "screenshot error on mu`#{page.path}` - r`#{e.class}`: #{e.message}"
+    end
+
+    def file_fresh?(file)
+      Config.cache_life.positive? and File.exist?(file) and file_age(file) < Config.cache_life
+    end
+
+    def file_age(file)
+      (Time.now - File.stat(file).mtime).to_i
+    end
+
+    def todo
+      @todo ||= {}
+    end
+
+    def done
+      @done ||= []
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/dependencies.rb

@@ -0,0 +1,21 @@
+require 'colsole'
+
+module Snapcrawl
+  class Dependencies
+    class << self
+      include Colsole
+
+      def verify
+        return if @verified
+
+        $logger.debug 'verifying g`phantomjs` is present'
+        raise MissingPhantomJS unless command_exist? 'phantomjs'
+
+        $logger.debug 'verifying g`imagemagick` is present'
+        raise MissingImageMagick unless command_exist? 'convert'
+
+        @verified = true
+      end
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/exceptions.rb

@@ -0,0 +1,5 @@
+module Snapcrawl
+  class MissingPhantomJS < StandardError; end
+  class MissingImageMagick < StandardError; end
+  class ScreenshotError < StandardError; end
+end

package/snapcrawl/lib/snapcrawl/log_helpers.rb

@@ -0,0 +1,36 @@
+require 'colsole'
+
+module Snapcrawl
+  module LogHelpers
+    include Colsole
+
+    SEVERITY_COLORS = {
+      'INFO'  => :b,
+      'WARN'  => :y,
+      'ERROR' => :r,
+      'FATAL' => :r,
+      'DEBUG' => :c,
+    }
+
+    def log_formatter
+      proc do |severity, _time, _prog, message|
+        severity_color = SEVERITY_COLORS[severity]
+        line = "#{severity_color}`#{severity.rjust 5}` : #{message}\n"
+        use_colors? ? colorize(line) : strip_colors(line)
+      end
+    end
+
+    def use_colors?
+      @use_colors ||= (Config.log_color == 'auto' ? tty? : Config.log_color)
+    end
+
+    def tty?
+      case ENV['TTY']
+      when 'on' then true
+      when 'off' then false
+      else
+        $stdout.tty?
+      end
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/page.rb

@@ -0,0 +1,118 @@
+require 'addressable/uri'
+require 'fileutils'
+require 'httparty'
+require 'lightly'
+require 'nokogiri'
+
+module Snapcrawl
+  class Page
+    using StringRefinements
+
+    attr_reader :url, :depth
+
+    EXTENSION_BLACKLIST = 'png|gif|jpg|pdf|zip'
+    PROTOCOL_BLACKLIST = 'mailto|tel'
+
+    def initialize(url, depth: 0)
+      @url = url.protocolize
+      @depth = depth
+    end
+
+    def valid?
+      http_response&.success?
+    end
+
+    def site
+      @site ||= Addressable::URI.parse(url).site
+    end
+
+    def path
+      @path ||= Addressable::URI.parse(url).request_uri
+    end
+
+    def links
+      return nil unless valid?
+
+      doc = Nokogiri::HTML http_response.body
+      normalize_links doc.css('a')
+    end
+
+    def pages
+      return nil unless valid?
+
+      links.map { |link| Page.new link, depth: depth + 1 }
+    end
+
+    def save_screenshot(outfile)
+      return false unless valid?
+
+      Screenshot.new(url).save outfile
+    end
+
+  private
+
+    def http_response
+      @http_response ||= http_response!
+    end
+
+    def http_response!
+      response = cache.get(url) { HTTParty.get url, httparty_options }
+
+      unless response.success?
+        $logger.warn "http error on mu`#{url}`, code: y`#{response.code}`, message: #{response.message.strip}"
+      end
+
+      response
+    rescue => e
+      $logger.error "http error on mu`#{url}` - r`#{e.class}`: #{e.message}"
+      nil
+    end
+
+    def httparty_options
+      Config.skip_ssl_verification ? { verify: false } : {}
+    end
+
+    def normalize_links(links)
+      result = []
+
+      links.each do |link|
+        valid_link = normalize_link link
+        result << valid_link if valid_link
+      end
+
+      result.uniq
+    end
+
+    def normalize_link(link)
+      link = link.attribute('href').to_s.dup
+
+      # Remove #hash
+      link.gsub!(/#.+$/, '')
+      return nil if link.empty?
+
+      # Remove links to specific extensions and protocols
+      return nil if /\.(#{EXTENSION_BLACKLIST})(\?.*)?$/o.match?(link)
+      return nil if /^(#{PROTOCOL_BLACKLIST}):/o.match?(link)
+
+      # Strip spaces
+      link.strip!
+
+      # Convert relative links to absolute
+      begin
+        link = Addressable::URI.join(url, link).to_s.dup
+      rescue => e
+        $logger.warn "r`#{e.class}`: #{e.message} on #{path} (link: #{link})"
+        return nil
+      end
+
+      # Keep only links in our base domain
+      return nil unless link.include? site
+
+      link
+    end
+
+    def cache
+      Lightly.new life: Config.cache_life
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/pretty_logger.rb

@@ -0,0 +1,11 @@
+require 'logger'
+
+module Snapcrawl
+  class PrettyLogger
+    extend LogHelpers
+
+    def self.new
+      Logger.new($stdout, formatter: log_formatter, level: Config.log_level)
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/refinements/pair_split.rb

@@ -0,0 +1,26 @@
+module Snapcrawl
+  module PairSplit
+    refine Array do
+      def pair_split
+        false_values = %w[no false]
+        true_values = %w[yes true]
+
+        to_h do |pair|
+          key, value = pair.split '='
+
+          value = if /^\d+$/.match?(value)
+            value.to_i
+          elsif false_values.include? value
+            false
+          elsif true_values.include? value
+            true
+          else
+            value
+          end
+
+          [key, value]
+        end
+      end
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/refinements/string_refinements.rb

@@ -0,0 +1,13 @@
+module Snapcrawl
+  module StringRefinements
+    refine String do
+      def to_slug
+        downcase.gsub(/[^a-z0-9]+/, '-')
+      end
+
+      def protocolize
+        /^http/.match?(self) ? self : "http://#{self}"
+      end
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/screenshot.rb

@@ -0,0 +1,73 @@
+require 'webshot'
+
+module Snapcrawl
+  class Screenshot
+    using StringRefinements
+
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+    end
+
+    def save(outfile = nil)
+      outfile ||= "#{url.to_slug}.png"
+      webshot_capture url, outfile
+    end
+
+  private
+
+    def webshot_capture(url, image_path)
+      webshot_capture! url, image_path
+    rescue => e
+      raise ScreenshotError, "#{e.class} #{e.message}"
+    end
+
+    def webshot_capture!(url, image_path)
+      hide_output do
+        webshot.capture url, image_path, webshot_options do |magick|
+          magick.combine_options do |c|
+            c.background 'white'
+            c.gravity 'north'
+            c.quality 100
+            c.extent Config.height.positive? ? "#{Config.width}x#{Config.height}" : "#{Config.width}x"
+          end
+        end
+      end
+    end
+
+    def webshot_options
+      result = { allowed_status_codes: [404, 401, 403] }
+
+      if Config.css_selector
+        result[:selector] = Config.css_selector
+        result[:full] = false
+      end
+
+      if Config.screenshot_delay
+        result[:timeout] = Config.screenshot_delay
+      end
+
+      result
+    end
+
+    def webshot
+      @webshot ||= Webshot::Screenshot.instance
+    end
+
+    # The webshot gem messes with stdout/stderr streams so we keep it in
+    # check by using this method. Also, in some sites (e.g. uown.co) it
+    # prints some output to stdout, this is why we override $stdout for
+    # the duration of the run.
+    def hide_output
+      keep_stdout = $stdout
+      keep_stderr = $stderr
+      $stdout = StringIO.new
+      $stderr = StringIO.new
+      yield
+    ensure
+      $stdout = keep_stdout
+      $stderr = keep_stderr
+    end
+  end
+end

package/snapcrawl/lib/snapcrawl/templates/config.yml

@@ -0,0 +1,49 @@
+# All values below are the default values
+
+# log level (0-4) 0=DEBUG 1=INFO 2=WARN 3=ERROR 4=FATAL
+log_level: 1
+
+# log_color (yes, no, auto)
+# yes  = always show log color
+# no   = never use colors
+# auto = only use colors when running in an interactive terminal
+log_color: auto
+
+# number of levels to crawl, 0 means capture only the root URL
+depth: 1
+
+# screenshot width in pixels
+width: 1280
+
+# screenshot height in pixels, 0 means the entire height
+height: 0
+
+# number of seconds to consider the page cache and its screenshot fresh
+cache_life: 86400
+
+# where to store the HTML page cache
+cache_dir: cache
+
+# where to store screenshots
+snaps_dir: snaps
+
+# screenshot filename template, where '%{url}' will be replaced with a 
+# slug version of the URL (no need to include the .png extension)
+name_template: '%{url}'
+
+# urls not matching this regular expression will be ignored
+url_whitelist: 
+
+# urls matching this regular expression will be ignored
+url_blacklist: 
+
+# take a screenshot of this CSS selector only
+css_selector: 
+
+# when true, ignore SSL related errors
+skip_ssl_verification: false
+
+# set to any number of seconds to wait for the page to load before taking
+# a screenshot, leave empty to not wait at all (only needed for pages with
+# animations or other post-load events).
+screenshot_delay: 

package/snapcrawl/lib/snapcrawl/templates/docopt.txt

@@ -0,0 +1,26 @@
+Snapcrawl
+
+Usage:
+  snapcrawl URL [--config FILE] [SETTINGS...]
+  snapcrawl -h | --help 
+  snapcrawl -v | --version
+
+Options:
+  -c, --config FILE
+    Path to config file, with or without the .yml extension.
+    A sample file will be created if not found.
+    The default filename is 'snapcrawl.yml'.
+
+  -h, --help
+    Show this screen
+
+  -v, --version
+    Show version number
+
+Settings:
+  Provide any of the options available in the config as 'key=value'.
+
+Examples:
+  snapcrawl example.com
+  snapcrawl example.com --config simple
+  snapcrawl example.com depth=1 log_level=2 width=768

package/snapcrawl/lib/snapcrawl/version.rb

@@ -0,0 +1,3 @@
+module Snapcrawl
+  VERSION = '0.5.4'
+end

package/snapcrawl/lib/snapcrawl.rb

@@ -0,0 +1,20 @@
+require 'snapcrawl/version'
+require 'snapcrawl/exceptions'
+require 'snapcrawl/refinements/pair_split'
+require 'snapcrawl/refinements/string_refinements'
+require 'snapcrawl/log_helpers'
+require 'snapcrawl/pretty_logger'
+require 'snapcrawl/dependencies'
+require 'snapcrawl/config'
+require 'snapcrawl/screenshot'
+require 'snapcrawl/page'
+require 'snapcrawl/crawler'
+require 'snapcrawl/cli'
+
+if ENV['BYEBUG']
+  require 'byebug'
+  require 'lp'
+end
+
+Snapcrawl::Config.load
+$logger = Snapcrawl::PrettyLogger.new

package/snapcrawl/snapcrawl.gemspec

@@ -0,0 +1,27 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'snapcrawl/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'snapcrawl'
+  s.version     = Snapcrawl::VERSION
+  s.summary     = 'Crawl a website and take screenshots (CLI + Library)'
+  s.description = 'Snapcrawl is a command line utility for crawling a website and saving screenshots.'
+  s.authors     = ['Danny Ben Shitrit']
+  s.email       = 'db@dannyben.com'
+  s.files       = Dir['README.md', 'lib/**/*']
+  s.executables = ['snapcrawl']
+  s.homepage    = 'https://github.com/DannyBen/snapcrawl'
+  s.license     = 'MIT'
+  s.required_ruby_version = '>= 3.0'
+
+  s.add_runtime_dependency 'addressable', '~> 2.7'
+  s.add_runtime_dependency 'colsole', '>= 0.8.1', '< 2'
+  s.add_runtime_dependency 'docopt', '~> 0.6'
+  s.add_runtime_dependency 'httparty', '~> 0.21'
+  s.add_runtime_dependency 'lightly', '~> 0.3'
+  s.add_runtime_dependency 'nokogiri', '~> 1.10'
+  s.add_runtime_dependency 'sting', '~> 0.4'
+  s.add_runtime_dependency 'webshot', '~> 0.1'
+  s.metadata['rubygems_mfa_required'] = 'true'
+end

package/snapcrawl/snapcrawl.yml

@@ -0,0 +1,41 @@
+# All values below are the default values
+
+# log level (0-4) 0=DEBUG 1=INFO 2=WARN 3=ERROR 4=FATAL
+log_level: 1
+
+# log_color (yes, no, auto)
+# yes  = always show log color
+# no   = never use colors
+# auto = only use colors when running in an interactive terminal
+log_color: auto
+
+# number of levels to crawl, 0 means capture only the root URL
+depth: 1
+
+# screenshot width in pixels
+width: 1280
+
+# screenshot height in pixels, 0 means the entire height
+height: 0
+
+# number of seconds to consider the page cache and its screenshot fresh
+cache_life: 86400
+
+# where to store the HTML page cache
+cache_dir: cache
+
+# where to store screenshots
+snaps_dir: snaps
+
+# screenshot filename template, where '%{url}' will be replaced with a 
+# slug version of the URL (no need to include the .png extension)
+name_template: '%{url}'
+
+# urls not matching this regular expression will be ignored
+url_whitelist: 
+
+# urls matching this regular expression will be ignored
+url_blacklist: 
+
+# take a screenshot of this CSS selector only
+css_selector: