powerdlz23 1.2.4 → 1.2.5

package/snapcrawl/spec/server/config.ru

@@ -0,0 +1,97 @@
+require 'sinatra/base'
+
+class Public < Sinatra::Base
+  get '/' do
+    "<a href='/page'>Hello World</a><br>" \
+      "<a href='/errors'>Errors</a>"
+  end
+
+  get '/page' do
+    <<~HTML
+      Some Page with links:<br>
+      <a href='/broken'>Broken Link</a><br>
+      <a href='/ok'>Valid Link</a><br>
+      <a href='https://www.example.com/'>External</a><br>
+      <a href='https://www.\u0105.com/'>Unicode</a><br>
+      <a href='\\problematic : link'>problematic link</a><br>
+      <a name='anchor-without-href'>without href</a><br>
+      <a href='#id'>with hash only href</a><br>
+    HTML
+  end
+
+  get '/errors' do
+    <<~HTML
+      <a href='/secret'>Basic Auth Dialog Link</a><br>
+      <a href='/500'>500 Link</a><br>
+      <a href='/401'>401 Link</a><br>
+      <a href='/403'>403 Link</a><br>
+    HTML
+  end
+
+  get '/ok' do
+    "<a href='/deeper/ok'>Depper Valid Link</a><br>OK"
+  end
+
+  get '/deeper/ok' do
+    'ALSO OK'
+  end
+
+  get '/selector' do
+    <<~HTML
+      <p>
+      Outside section with a long text:
+      Where could they be? Close the blast doors! Open the blast doors! Open the
+      blast doors! I've been waiting for you, Obi-Wan. We meet again, at last.
+      The circle is now complete. When I left you, I was but the learner,
+      now I am the master. Only a master of evil, Darth. Your powers are weak,
+      old man. You can't win, Darth. If you strike me down, I shall become more
+      powerful than you can possibly imagine.
+      </p>
+      <p class='select-me'>And a small one</p>
+    HTML
+  end
+
+  get '/filters' do
+    <<~HTML
+      <a href='/filters/include-me/1'>Included</a><br>
+      <a href='/filters/include-me/2'>Included</a><br>
+      <a href='/filters/exclude-me/1'>Excluded</a><br>
+      <a href='/filters/exclude-me/2'>Excluded</a><br>
+    HTML
+  end
+
+  get '/filters/include-me/:id' do
+    "include-me #{params[:id]}"
+  end
+
+  get '/filters/exclude-me/:id' do
+    "exclude-me #{params[:id]}"
+  end
+
+  get '/500' do
+    raise 'server error'
+  end
+
+  get '/401' do
+    status 401
+    '401 Unauthorized'
+  end
+
+  get '/403' do
+    status 403
+    '403 Forbidden'
+  end
+end
+
+class Protected < Sinatra::Base
+  use Rack::Auth::Basic, 'Protected Area' do |username, password|
+    username == 'user' && password == 'pass'
+  end
+
+  get('/') { 'secret' }
+end
+
+run Rack::URLMap.new({
+  '/'       => Public,
+  '/secret' => Protected,
+})

package/snapcrawl/spec/snapcrawl/bin_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe 'bin/snapcrawl' do
+  it 'shows usage patterns' do
+    expect(`bin/snapcrawl`).to match_approval 'bin/usage'
+  end
+
+  it 'shows help' do
+    expect(`bin/snapcrawl --help`).to match_approval 'bin/help'
+  end
+
+  it 'shows correct version' do
+    expect(`bin/snapcrawl -v`).to match VERSION
+  end
+end

package/snapcrawl/spec/snapcrawl/cli_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe CLI do
+  subject { described_class.new }
+
+  it 'shows usage' do
+    expect { subject.call }.to output_approval('cli/usage')
+  end
+end

package/snapcrawl/spec/snapcrawl/config_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Config do
+  subject { described_class }
+
+  describe '#load' do
+    it 'has defaults' do
+      subject.load
+      expect(subject.settings.to_yaml).to match_approval('config/defaults').diff(4)
+    end
+
+    it 'loads file if it exists and merges it with the defaults' do
+      subject.load 'spec/fixtures/config/minimal'
+      expect(subject.settings.to_yaml).to match_approval('config/minimal').diff(4)
+    end
+
+    context 'when the config file is not found' do
+      before { system 'rm -f tmp/config.yml' }
+
+      it 'creates it' do
+        subject.load 'tmp/config.yml'
+        expect(File.read 'tmp/config.yml').to eq File.read('lib/snapcrawl/templates/config.yml')
+      end
+    end
+  end
+end

package/snapcrawl/spec/snapcrawl/integration_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+
+describe 'integration' do
+  subject { CLI.new }
+
+  let(:url) { 'http://localhost:3000' }
+  let(:log) { @logger.string }
+
+  before do
+    @logger = fresh_logger
+    Config.load   # reload defaults
+  end
+
+  it 'executes successfully' do
+    subject.call [url]
+    expect(log).to match_approval('integration/default-config')
+  end
+
+  context 'with depth=0' do
+    it 'captures the first page only' do
+      subject.call [url, 'depth=0']
+      expect(log).to match_approval('integration/depth-0')
+    end
+  end
+
+  context 'with depth=3 log_level=2' do
+    it 'captures 4 levels and shows warnings and above' do
+      subject.call [url, 'depth=3', 'log_level=2']
+      expect(log).to match_approval('integration/depth-3')
+    end
+  end
+
+  context 'with log_color=no' do
+    it 'outputs without colors' do
+      subject.call [url, 'log_color=no', 'log_level=1']
+      expect(log).to match_approval('integration/log-color-no')
+    end
+  end
+
+  context 'with url_whitelist' do
+    let(:url) { 'http://localhost:3000/filters' }
+
+    it 'only processes urls that match the regex' do
+      subject.call [url, 'url_whitelist=include', 'log_level=0']
+      expect(log).to match_approval('integration/whitelist')
+    end
+  end
+
+  context 'with url_blacklist' do
+    let(:url) { 'http://localhost:3000/filters' }
+
+    it 'ignores urls that match the regex' do
+      subject.call [url, 'url_blacklist=exclude', 'log_level=0']
+      expect(log).to match_approval('integration/blacklist')
+    end
+  end
+
+  context 'when screenshot errors' do
+    it 'logs the error and continues' do
+      expect_any_instance_of(Screenshot).to receive(:save).and_raise(ScreenshotError, 'Simulated error')
+      subject.call [url, 'cache_life=0', 'depth=0', 'log_level=1']
+      expect(log).to match_approval('integration/screenshot-error')
+    end
+  end
+end

package/snapcrawl/spec/snapcrawl/page_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe Page do
+  subject { described_class.new url }
+
+  let(:url) { 'http://localhost:3000/page' }
+
+  describe '#valid?' do
+    context 'when the page is valid' do
+      it 'returns true' do
+        expect(subject).to be_valid
+      end
+    end
+
+    context 'when the page is not found' do
+      let(:url) { 'http://localhost:3000/not-found' }
+
+      it 'returns false and logs a warning' do
+        expect($logger).to receive(:warn).with(/code.*404/)
+        expect(subject).not_to be_valid
+      end
+    end
+
+    context "when the url can't be reached" do
+      let(:url) { 'http://localhost:1111/' }
+
+      it 'returns false and logs an error' do
+        expect($logger).to receive(:error).with(/connection refused/i)
+        expect(subject).not_to be_valid
+      end
+    end
+
+    context 'when the url has a bad SSL certificate' do
+      let(:url) { 'https://untrusted-root.badssl.com/' }
+
+      it 'returns false and logs an error' do
+        expect($logger).to receive(:error).with(/certificate verify failed/i)
+        expect(subject).not_to be_valid
+      end
+
+      context 'when skip_ssl_verification is true' do
+        before { Config.skip_ssl_verification = true }
+        after { Config.skip_ssl_verification = false }
+
+        it 'returns true' do
+          expect(subject).to be_valid
+        end
+      end
+    end
+  end
+
+  describe '#site' do
+    it 'returns the site name' do
+      expect(subject.site).to eq 'http://localhost:3000'
+    end
+  end
+
+  describe '#path' do
+    it 'returns the path' do
+      expect(subject.path).to eq '/page'
+    end
+  end
+
+  describe '#links' do
+    it 'returns an array of links on the page and logs warnings' do
+      expect($logger).to receive(:warn).with(/problematic/)
+      expect(subject.links).to eq ['http://localhost:3000/broken', 'http://localhost:3000/ok']
+    end
+  end
+
+  describe '#pages' do
+    it 'returns an array of Page objects from the links' do
+      expect($logger).to receive(:warn).with(/problematic/)
+      pages = subject.pages
+      expect(pages.count).to eq 2
+      expect(pages.first).to be_a described_class
+    end
+  end
+
+  describe '#save_screenshot' do
+    let(:double) { Screenshot.new subject.url }
+
+    it 'delegates to Screenshot' do
+      allow(Screenshot).to receive(:new).with(subject.url).and_return(double)
+      expect(double).to receive(:save).with('outfile')
+      subject.save_screenshot 'outfile'
+    end
+  end
+end

package/snapcrawl/spec/snapcrawl/pretty_logger_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe PrettyLogger do
+  before { Config.load }
+
+  describe '::new' do
+    it 'returns a Logger instance' do
+      expect(subject).to be_a Logger
+    end
+  end
+
+  describe 'log formatting' do
+    let(:message) { 'g`Hello World`' }
+
+    it 'logs with colors' do
+      expect { subject.info message }.to output_approval('models/pretty_logger/colors')
+    end
+  end
+end

package/snapcrawl/spec/snapcrawl/refinements/pair_split_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe PairSplit do
+  using described_class
+
+  subject { ['key=value', 'key2=value2'] }
+
+  it 'splits an array of key=value elements into a hash' do
+    expect(subject.pair_split).to eq({ 'key' => 'value', 'key2' => 'value2' })
+  end
+
+  context 'when the value is integer-like' do
+    subject { ['cakes=3'] }
+
+    it 'convertts it to integer' do
+      expect(subject.pair_split['cakes']).to eq 3
+    end
+  end
+
+  context 'when the value is boolean-like' do
+    subject { ['pizza=yes', 'burger=true', 'broccoli=no', 'eggplant=false'] }
+
+    it 'convertts it to boolean' do
+      expect(subject.pair_split).to eq({ 'pizza' => true, 'burger' => true, 'broccoli' => false, 'eggplant' => false })
+    end
+  end
+end

package/snapcrawl/spec/snapcrawl/refinements/string_refinements_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe StringRefinements do
+  using described_class
+
+  describe '#to_slug' do
+    it 'converts to hyphen-delimited string' do
+      expect('http://comain.com/index.html'.to_slug).to eq 'http-comain-com-index-html'
+    end
+  end
+
+  describe '#protocolize' do
+    context 'when the string does not start with http' do
+      subject { 'example.com' }
+
+      it 'adds http://' do
+        expect(subject.protocolize).to eq "http://#{subject}"
+      end
+    end
+
+    context 'when the string starts with http' do
+      subject { 'https://example.com' }
+
+      it 'does nothing' do
+        expect(subject.protocolize).to eq subject
+      end
+    end
+  end
+end

package/snapcrawl/spec/snapcrawl/screenshot_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+describe Screenshot do
+  subject { described_class.new url }
+
+  let(:url) { 'http://localhost:3000/page' }
+
+  describe '#save' do
+    let(:outfile) { 'tmp/screenshot.png' }
+
+    before do
+      system 'rm -f tmp/*.png'
+      expect(Dir['tmp/*.png'].count).to eq 0
+    end
+
+    it 'saves a screenshot' do
+      subject.save outfile
+      expect(File).to exist outfile
+      expect(File.size outfile).to be > 22_000
+    end
+
+    context 'when Config.css_selector is set' do
+      let(:url) { 'http://localhost:3000/selector' }
+
+      before do
+        Config.css_selector = nil
+        subject.save 'tmp/full-page.png'
+      end
+
+      after do
+        Config.css_selector = nil
+      end
+
+      it 'only captures the selected area' do
+        Config.css_selector = '.select-me'
+        subject.save 'tmp/selected-area.png'
+        full_size = File.size('tmp/full-page.png')
+        selected_size = File.size('tmp/selected-area.png')
+
+        expect(selected_size).to be < full_size
+      end
+    end
+
+    context 'when Config.screenshot_delay is set' do
+      before { Config.screenshot_delay = 3 }
+      after { Config.screenshot_delay = nil }
+
+      it 'sends the timeout argument to webshot' do
+        expect(Webshot::Screenshot.instance).to receive(:capture).with(String, String, hash_including({ timeout: 3 }))
+        subject.save 'tmp/delay.png'
+      end
+    end
+
+    context 'when there is an error' do
+      let(:url) { 'http://localhost:1111' }
+
+      it 'raises ScreenshotError' do
+        expect { subject.save outfile }.to raise_error(ScreenshotError)
+      end
+    end
+  end
+end

package/snapcrawl/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+require 'simplecov'
+
+SimpleCov.start
+
+require 'rubygems'
+require 'bundler'
+Bundler.require :default, :development
+
+require 'stringio'
+include Snapcrawl
+
+# Consistent rspec fixtures output
+ENV['TTY'] = 'on'
+
+system 'rm -rf snaps'
+system 'mkdir -p tmp'
+
+require_relative 'spec_mixin'
+RSpec.configure do |config|
+  config.include SpecMixin
+  config.example_status_persistence_file_path = 'spec/status.txt'
+end

package/snapcrawl/spec/spec_mixin.rb

@@ -0,0 +1,10 @@
+module SpecMixin
+  include Snapcrawl::LogHelpers
+
+  def fresh_logger
+    spec_logger = StringIO.new
+    $logger = Logger.new spec_logger
+    $logger.formatter = log_formatter
+    spec_logger
+  end
+end