npm - powerdlz23 - Versions diffs - 1.2.3 → 1.2.5 - Mend

powerdlz23 1.2.3 → 1.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (207) hide show

package/Spider/README.md +19 -0
package/Spider/domain.py +18 -0
package/Spider/general.py +51 -0
package/Spider/link_finder.py +25 -0
package/Spider/main.py +50 -0
package/Spider/spider.py +74 -0
package/crawler/.formatter.exs +5 -0
package/crawler/.github/workflows/ci.yml +29 -0
package/crawler/.recode.exs +33 -0
package/crawler/.tool-versions +2 -0
package/crawler/CHANGELOG.md +82 -0
package/crawler/README.md +198 -0
package/crawler/architecture.svg +4 -0
package/crawler/config/config.exs +9 -0
package/crawler/config/dev.exs +5 -0
package/crawler/config/test.exs +5 -0
package/crawler/examples/google_search/scraper.ex +37 -0
package/crawler/examples/google_search/url_filter.ex +11 -0
package/crawler/examples/google_search.ex +77 -0
package/crawler/lib/crawler/dispatcher/worker.ex +14 -0
package/crawler/lib/crawler/dispatcher.ex +20 -0
package/crawler/lib/crawler/fetcher/header_preparer.ex +60 -0
package/crawler/lib/crawler/fetcher/modifier.ex +45 -0
package/crawler/lib/crawler/fetcher/policer.ex +77 -0
package/crawler/lib/crawler/fetcher/recorder.ex +55 -0
package/crawler/lib/crawler/fetcher/requester.ex +32 -0
package/crawler/lib/crawler/fetcher/retrier.ex +43 -0
package/crawler/lib/crawler/fetcher/url_filter.ex +26 -0
package/crawler/lib/crawler/fetcher.ex +81 -0
package/crawler/lib/crawler/http.ex +7 -0
package/crawler/lib/crawler/linker/path_builder.ex +71 -0
package/crawler/lib/crawler/linker/path_expander.ex +59 -0
package/crawler/lib/crawler/linker/path_finder.ex +106 -0
package/crawler/lib/crawler/linker/path_offliner.ex +59 -0
package/crawler/lib/crawler/linker/path_prefixer.ex +46 -0
package/crawler/lib/crawler/linker.ex +173 -0
package/crawler/lib/crawler/options.ex +127 -0
package/crawler/lib/crawler/parser/css_parser.ex +37 -0
package/crawler/lib/crawler/parser/guarder.ex +38 -0
package/crawler/lib/crawler/parser/html_parser.ex +41 -0
package/crawler/lib/crawler/parser/link_parser/link_expander.ex +32 -0
package/crawler/lib/crawler/parser/link_parser.ex +50 -0
package/crawler/lib/crawler/parser.ex +122 -0
package/crawler/lib/crawler/queue_handler.ex +45 -0
package/crawler/lib/crawler/scraper.ex +28 -0
package/crawler/lib/crawler/snapper/dir_maker.ex +45 -0
package/crawler/lib/crawler/snapper/link_replacer.ex +95 -0
package/crawler/lib/crawler/snapper.ex +82 -0
package/crawler/lib/crawler/store/counter.ex +19 -0
package/crawler/lib/crawler/store/page.ex +7 -0
package/crawler/lib/crawler/store.ex +87 -0
package/crawler/lib/crawler/worker.ex +62 -0
package/crawler/lib/crawler.ex +91 -0
package/crawler/mix.exs +78 -0
package/crawler/mix.lock +40 -0
package/crawler/test/fixtures/introducing-elixir.jpg +0 -0
package/crawler/test/integration_test.exs +135 -0
package/crawler/test/lib/crawler/dispatcher/worker_test.exs +7 -0
package/crawler/test/lib/crawler/dispatcher_test.exs +5 -0
package/crawler/test/lib/crawler/fetcher/header_preparer_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher/policer_test.exs +71 -0
package/crawler/test/lib/crawler/fetcher/recorder_test.exs +9 -0
package/crawler/test/lib/crawler/fetcher/requester_test.exs +9 -0
package/crawler/test/lib/crawler/fetcher/retrier_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher/url_filter_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher_test.exs +153 -0
package/crawler/test/lib/crawler/http_test.exs +47 -0
package/crawler/test/lib/crawler/linker/path_builder_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_expander_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_finder_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_offliner_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_prefixer_test.exs +7 -0
package/crawler/test/lib/crawler/linker_test.exs +7 -0
package/crawler/test/lib/crawler/options_test.exs +7 -0
package/crawler/test/lib/crawler/parser/css_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser/guarder_test.exs +7 -0
package/crawler/test/lib/crawler/parser/html_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser/link_parser/link_expander_test.exs +7 -0
package/crawler/test/lib/crawler/parser/link_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser_test.exs +8 -0
package/crawler/test/lib/crawler/queue_handler_test.exs +7 -0
package/crawler/test/lib/crawler/scraper_test.exs +7 -0
package/crawler/test/lib/crawler/snapper/dir_maker_test.exs +7 -0
package/crawler/test/lib/crawler/snapper/link_replacer_test.exs +7 -0
package/crawler/test/lib/crawler/snapper_test.exs +9 -0
package/crawler/test/lib/crawler/worker_test.exs +5 -0
package/crawler/test/lib/crawler_test.exs +295 -0
package/crawler/test/support/test_case.ex +24 -0
package/crawler/test/support/test_helpers.ex +28 -0
package/crawler/test/test_helper.exs +7 -0
package/grell/.rspec +2 -0
package/grell/.travis.yml +28 -0
package/grell/CHANGELOG.md +111 -0
package/grell/Gemfile +7 -0
package/grell/LICENSE.txt +22 -0
package/grell/README.md +213 -0
package/grell/Rakefile +2 -0
package/grell/grell.gemspec +36 -0
package/grell/lib/grell/capybara_driver.rb +44 -0
package/grell/lib/grell/crawler.rb +83 -0
package/grell/lib/grell/crawler_manager.rb +84 -0
package/grell/lib/grell/grell_logger.rb +10 -0
package/grell/lib/grell/page.rb +275 -0
package/grell/lib/grell/page_collection.rb +62 -0
package/grell/lib/grell/rawpage.rb +62 -0
package/grell/lib/grell/reader.rb +18 -0
package/grell/lib/grell/version.rb +3 -0
package/grell/lib/grell.rb +11 -0
package/grell/spec/lib/capybara_driver_spec.rb +38 -0
package/grell/spec/lib/crawler_manager_spec.rb +174 -0
package/grell/spec/lib/crawler_spec.rb +361 -0
package/grell/spec/lib/page_collection_spec.rb +159 -0
package/grell/spec/lib/page_spec.rb +418 -0
package/grell/spec/lib/reader_spec.rb +43 -0
package/grell/spec/spec_helper.rb +66 -0
package/heartmagic/config.py +1 -0
package/heartmagic/heart.py +3 -0
package/heartmagic/pytransform/__init__.py +483 -0
package/heartmagic/pytransform/_pytransform.dll +0 -0
package/heartmagic/pytransform/_pytransform.so +0 -0
package/httpStatusCode/README.md +2 -0
package/httpStatusCode/httpStatusCode.js +4 -0
package/httpStatusCode/reasonPhrases.js +344 -0
package/httpStatusCode/statusCodes.js +344 -0
package/package.json +1 -1
package/rubyretriever/.rspec +2 -0
package/rubyretriever/.travis.yml +7 -0
package/rubyretriever/Gemfile +3 -0
package/rubyretriever/Gemfile.lock +64 -0
package/rubyretriever/LICENSE +20 -0
package/rubyretriever/Rakefile +7 -0
package/rubyretriever/bin/rr +79 -0
package/rubyretriever/lib/retriever/cli.rb +25 -0
package/rubyretriever/lib/retriever/core_ext.rb +13 -0
package/rubyretriever/lib/retriever/fetch.rb +268 -0
package/rubyretriever/lib/retriever/fetchfiles.rb +71 -0
package/rubyretriever/lib/retriever/fetchseo.rb +18 -0
package/rubyretriever/lib/retriever/fetchsitemap.rb +43 -0
package/rubyretriever/lib/retriever/link.rb +47 -0
package/rubyretriever/lib/retriever/openuri_redirect_patch.rb +8 -0
package/rubyretriever/lib/retriever/page.rb +104 -0
package/rubyretriever/lib/retriever/page_iterator.rb +21 -0
package/rubyretriever/lib/retriever/target.rb +47 -0
package/rubyretriever/lib/retriever/version.rb +4 -0
package/rubyretriever/lib/retriever.rb +15 -0
package/rubyretriever/readme.md +166 -0
package/rubyretriever/rubyretriever.gemspec +41 -0
package/rubyretriever/spec/link_spec.rb +77 -0
package/rubyretriever/spec/page_spec.rb +94 -0
package/rubyretriever/spec/retriever_spec.rb +84 -0
package/rubyretriever/spec/spec_helper.rb +17 -0
package/rubyretriever/spec/target_spec.rb +55 -0
package/snapcrawl/.changelog.old.md +157 -0
package/snapcrawl/.gitattributes +1 -0
package/snapcrawl/.github/workflows/test.yml +41 -0
package/snapcrawl/.rspec +3 -0
package/snapcrawl/.rubocop.yml +23 -0
package/snapcrawl/CHANGELOG.md +182 -0
package/snapcrawl/Gemfile +15 -0
package/snapcrawl/LICENSE +21 -0
package/snapcrawl/README.md +135 -0
package/snapcrawl/Runfile +35 -0
package/snapcrawl/bin/snapcrawl +25 -0
package/snapcrawl/lib/snapcrawl/cli.rb +52 -0
package/snapcrawl/lib/snapcrawl/config.rb +60 -0
package/snapcrawl/lib/snapcrawl/crawler.rb +98 -0
package/snapcrawl/lib/snapcrawl/dependencies.rb +21 -0
package/snapcrawl/lib/snapcrawl/exceptions.rb +5 -0
package/snapcrawl/lib/snapcrawl/log_helpers.rb +36 -0
package/snapcrawl/lib/snapcrawl/page.rb +118 -0
package/snapcrawl/lib/snapcrawl/pretty_logger.rb +11 -0
package/snapcrawl/lib/snapcrawl/refinements/pair_split.rb +26 -0
package/snapcrawl/lib/snapcrawl/refinements/string_refinements.rb +13 -0
package/snapcrawl/lib/snapcrawl/screenshot.rb +73 -0
package/snapcrawl/lib/snapcrawl/templates/config.yml +49 -0
package/snapcrawl/lib/snapcrawl/templates/docopt.txt +26 -0
package/snapcrawl/lib/snapcrawl/version.rb +3 -0
package/snapcrawl/lib/snapcrawl.rb +20 -0
package/snapcrawl/snapcrawl.gemspec +27 -0
package/snapcrawl/snapcrawl.yml +41 -0
package/snapcrawl/spec/README.md +16 -0
package/snapcrawl/spec/approvals/bin/help +26 -0
package/snapcrawl/spec/approvals/bin/usage +4 -0
package/snapcrawl/spec/approvals/cli/usage +4 -0
package/snapcrawl/spec/approvals/config/defaults +15 -0
package/snapcrawl/spec/approvals/config/minimal +15 -0
package/snapcrawl/spec/approvals/integration/blacklist +14 -0
package/snapcrawl/spec/approvals/integration/default-config +14 -0
package/snapcrawl/spec/approvals/integration/depth-0 +6 -0
package/snapcrawl/spec/approvals/integration/depth-3 +6 -0
package/snapcrawl/spec/approvals/integration/log-color-no +6 -0
package/snapcrawl/spec/approvals/integration/screenshot-error +3 -0
package/snapcrawl/spec/approvals/integration/whitelist +14 -0
package/snapcrawl/spec/approvals/models/pretty_logger/colors +1 -0
package/snapcrawl/spec/fixtures/config/minimal.yml +4 -0
package/snapcrawl/spec/server/config.ru +97 -0
package/snapcrawl/spec/snapcrawl/bin_spec.rb +15 -0
package/snapcrawl/spec/snapcrawl/cli_spec.rb +9 -0
package/snapcrawl/spec/snapcrawl/config_spec.rb +26 -0
package/snapcrawl/spec/snapcrawl/integration_spec.rb +65 -0
package/snapcrawl/spec/snapcrawl/page_spec.rb +89 -0
package/snapcrawl/spec/snapcrawl/pretty_logger_spec.rb +19 -0
package/snapcrawl/spec/snapcrawl/refinements/pair_split_spec.rb +27 -0
package/snapcrawl/spec/snapcrawl/refinements/string_refinements_spec.rb +29 -0
package/snapcrawl/spec/snapcrawl/screenshot_spec.rb +62 -0
package/snapcrawl/spec/spec_helper.rb +22 -0
package/snapcrawl/spec/spec_mixin.rb +10 -0

package/grell/spec/lib/crawler_spec.rb ADDED Viewed

@@ -0,0 +1,361 @@
+RSpec.describe Grell::Crawler do
+  let(:page_id) { rand(10).floor + 10 }
+  let(:parent_page_id) { rand(10).floor }
+  let(:page) { Grell::Page.new(url, page_id, parent_page_id) }
+  let(:host) { 'http://www.example.com' }
+  let(:url) { 'http://www.example.com/test' }
+  let(:add_match_block) { nil }
+  let(:denylist) { /a^/ }
+  let(:allowlist) { /.*/ }
+  let(:crawler) do
+    Grell::Crawler.new(
+      logger: Logger.new(nil),
+      driver: double(nil),
+      evaluate_in_each_page: script,
+      add_match_block: add_match_block,
+      denylist: denylist,
+      allowlist: allowlist)
+  end
+  let(:script) { nil }
+  let(:body) { 'body' }
+  let(:custom_add_match) do
+    Proc.new do |collection_page, page|
+      collection_page.path == page.path
+    end
+  end
+  before do
+    proxy.stub(url).and_return(body: body, code: 200)
+  end
+  describe '#crawl' do
+    before do
+      crawler.instance_variable_set('@collection', Grell::PageCollection.new(custom_add_match))
+    end
+    it 'yields the result if a block is given' do
+      result = []
+      block = Proc.new { |n| result.push(n) }
+      crawler.crawl(page, block)
+      expect(result.size).to eq(1)
+      expect(result.first.url).to eq(url)
+      expect(result.first.visited?).to eq(true)
+    end
+    it 'rescues any specified exceptions raised during the block execution' do
+      block = Proc.new { |n| raise Capybara::Poltergeist::BrowserError, 'Exception' }
+      expect{ crawler.crawl(page, block) }.to_not raise_error
+      expect(page.status).to eq(404)
+    end
+    it 'logs interesting information' do
+      crawler
+      expect(Grell.logger).to receive(:info).with(/Visiting #{url}, visited_links: 0, discovered 0/)
+      crawler.crawl(page, nil)
+    end
+    it 'retries when the block returns :retry' do
+      counter = 0
+      times_retrying = 2
+      block = Proc.new do |n|
+        if counter < times_retrying
+          counter += 1
+          :retry
+        end
+      end
+      crawler.crawl(page, block)
+      expect(counter).to eq(times_retrying)
+    end
+    it 'handles redirects by adding the current_url to the page collection' do
+      redirect_url = 'http://www.example.com/test/landing_page'
+      allow(page).to receive(:current_url).and_return(redirect_url)
+      expect_any_instance_of(Grell::PageCollection).to receive(:create_page).with(redirect_url, page_id)
+      crawler.crawl(page, nil)
+    end
+    context 'without script' do
+      it 'does not evaluate a script' do
+        expect_any_instance_of(Capybara::Session).not_to receive(:evaluate_script)
+        crawler.crawl(page, nil)
+      end
+    end
+    context 'with script' do
+      let(:script) { "(typeof(jQuery)!='undefined') && $('.dropdown').addClass('open');" }
+      it 'evaluates a script' do
+        expect_any_instance_of(Capybara::Session).to receive(:evaluate_script).with(script)
+        crawler.crawl(page, nil)
+      end
+    end
+  end
+  context '#start_crawling' do
+    let(:body) do
+      <<-EOS
+      <html><head></head><body>
+      <a href="/musmis.html">trusmis</a>
+      Hello world!
+      </body></html>
+      EOS
+    end
+    let(:url_visited) { "http://www.example.com/musmis.html" }
+    before do
+      proxy.stub(url_visited).and_return(body: 'body', code: 200)
+    end
+    it 'calls the block we used to start_crawling' do
+      result = []
+      block = Proc.new { |n| result.push(n) }
+      crawler.start_crawling(url, &block)
+      expect(result.size).to eq(2)
+      expect(result[0].url).to eq(url)
+      expect(result[1].url).to eq(url_visited)
+    end
+  end
+  shared_examples_for 'visits all available pages' do
+    it 'visits all the pages' do
+      crawler.start_crawling(url)
+      expect(crawler.collection.visited_pages.size).to eq(visited_pages_count)
+    end
+    it 'has no more pages to discover' do
+      crawler.start_crawling(url)
+      expect(crawler.collection.discovered_pages.size).to eq(0)
+    end
+    it 'contains the allowlisted page and the base page only' do
+      crawler.start_crawling(url)
+      expect(crawler.collection.visited_pages.map(&:url)).
+        to eq(visited_pages)
+    end
+  end
+  context 'the url has no links' do
+    let(:body) do
+      "<html><head></head><body>
+      Hello world!
+      </body></html>"
+    end
+    let(:visited_pages_count) { 1 }
+    let(:visited_pages) { ['http://www.example.com/test'] }
+    it_behaves_like 'visits all available pages'
+  end
+  context 'the url has several links' do
+    let(:visited_pages_count) { 3 }
+    let(:visited_pages) do
+      ['http://www.example.com/test', 'http://www.example.com/trusmis.html', 'http://www.example.com/help.html']
+    end
+    let(:body) do
+      "<html><head></head><body>
+      <a href=\"/trusmis.html\">trusmis</a>
+      <a href=\"/help.html\">help</a>
+      Hello world!
+      </body></html>"
+    end
+    before do
+      proxy.stub('http://www.example.com/trusmis.html').and_return(body: 'body', code: 200)
+      proxy.stub('http://www.example.com/help.html').and_return(body: 'body', code: 200)
+    end
+    it_behaves_like 'visits all available pages'
+  end
+  describe '#allowlist' do
+    let(:body) do
+      "<html><head></head><body>
+      <a href=\"/trusmis.html\">trusmis</a>
+      <a href=\"/help.html\">help</a>
+      Hello world!
+      </body></html>"
+    end
+    before do
+      proxy.stub('http://www.example.com/trusmis.html').and_return(body: 'body', code: 200)
+      proxy.stub('http://www.example.com/help.html').and_return(body: 'body', code: 200)
+    end
+    context 'using a single string' do
+      let(:allowlist) { '/trusmis.html' }
+      let(:visited_pages_count) { 2 } # my own page + trusmis
+      let(:visited_pages) do
+        ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'using an array of strings' do
+      let(:allowlist) { ['/trusmis.html', '/nothere', 'another.html'] }
+      let(:visited_pages_count) { 2 }
+      let(:visited_pages) do
+        ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'using a regexp' do
+      let(:allowlist) { /\/trusmis\.html/ }
+      let(:visited_pages_count) { 2 }
+      let(:visited_pages) do
+        ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'using an array of regexps' do
+      let(:allowlist) { [/\/trusmis\.html/] }
+      let(:visited_pages_count) { 2 }
+      let(:visited_pages) do
+        ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'using an empty array' do
+      let(:allowlist) { [] }
+      let(:visited_pages_count) { 1 } # my own page only
+      let(:visited_pages) do
+        ['http://www.example.com/test']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'adding all links to the allowlist' do
+      let(:allowlist) { ['/trusmis', '/help'] }
+      let(:visited_pages_count) { 3 } # all links
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/trusmis.html', 'http://www.example.com/help.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+  end
+  describe '#denylist' do
+    let(:body) do
+      "<html><head></head><body>
+      <a href=\"/trusmis.html\">trusmis</a>
+      <a href=\"/help.html\">help</a>
+      Hello world!
+      </body></html>"
+    end
+    before do
+      proxy.stub('http://www.example.com/trusmis.html').and_return(body: 'body', code: 200)
+      proxy.stub('http://www.example.com/help.html').and_return(body: 'body', code: 200)
+    end
+    context 'using a single string' do
+      let(:denylist) { '/trusmis.html' }
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/help.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'using an array of strings' do
+      let(:denylist) { ['/trusmis.html', '/nothere', 'another.html'] }
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/help.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'using a regexp' do
+      let(:denylist) { /\/trusmis\.html/ }
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/help.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'using an array of regexps' do
+      let(:denylist) { [/\/trusmis\.html/] }
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/help.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'using an empty array' do
+      let(:denylist) { [] }
+      let(:visited_pages_count) { 3 } # all links
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/trusmis.html', 'http://www.example.com/help.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'adding all links to the denylist' do
+      let(:denylist) { ['/trusmis', '/help'] }
+      let(:visited_pages_count) { 1 }
+      let(:visited_pages) do
+        ['http://www.example.com/test']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+  end
+  describe 'allowlisting and denylisting' do
+    let(:body) do
+      "<html><head></head><body>
+      <a href=\"/trusmis.html\">trusmis</a>
+      <a href=\"/help.html\">help</a>
+      Hello world!
+      </body></html>"
+    end
+    before do
+      proxy.stub('http://www.example.com/trusmis.html').and_return(body: 'body', code: 200)
+      proxy.stub('http://www.example.com/help.html').and_return(body: 'body', code: 200)
+    end
+    context 'we denylist the only allowlisted page' do
+      let(:allowlist) { '/trusmis.html' }
+      let(:denylist) { '/trusmis.html' }
+      let(:visited_pages_count) { 1 }
+      let(:visited_pages) do
+        ['http://www.example.com/test']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+    context 'we denylist none of the allowlisted pages' do
+      let(:allowlist) { '/trusmis.html' }
+      let(:denylist) { '/raistlin.html' }
+      let(:visited_pages_count) { 2 }
+      let(:visited_pages) do
+        ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
+      end
+      it_behaves_like 'visits all available pages'
+    end
+  end
+end

package/grell/spec/lib/page_collection_spec.rb ADDED Viewed

@@ -0,0 +1,159 @@
+RSpec.describe Grell::PageCollection do
+  let(:add_match_block) do
+    Proc.new do |collection_page, page|
+      collection_page.url.downcase == page.url.downcase
+    end
+  end
+  let(:collection) { Grell::PageCollection.new(add_match_block) }
+  let(:url) { 'http://www.github.com/SomeUser/dragonlance?search=false' }
+  let(:url2) { 'http://www.github.com/OtherUser/forgotten?search=false' }
+  context 'empty collection' do
+    it 'has no visited pages' do
+      expect(collection.visited_pages).to be_empty
+    end
+    it 'has no discovered pages' do
+      expect(collection.discovered_pages).to be_empty
+    end
+    it 'next page is nil' do
+      expect(collection.next_page).to be_nil
+    end
+  end
+  context 'one unvisited page' do
+    let(:page) { collection.create_page(url, 0) }
+    before do
+      allow(page).to receive(:visited?).and_return(false)
+    end
+    it 'has no visited pages' do
+      expect(collection.visited_pages).to be_empty
+    end
+    it 'has one discovered page' do
+      expect(collection.discovered_pages).to eq([page])
+    end
+    it 'next page is the unvisited page' do
+      expect(collection.next_page).to eq(page)
+    end
+  end
+  context 'one visited page' do
+    let(:page) { collection.create_page(url, 0) }
+    before do
+      allow(page).to receive(:visited?).and_return(true)
+    end
+    it 'has one visited page' do
+      expect(collection.visited_pages).to eq([page])
+    end
+    it 'has no discovered pages' do
+      expect(collection.discovered_pages).to be_empty
+    end
+    it 'next page is nil' do
+      expect(collection.next_page).to be_nil
+    end
+  end
+  context 'one visited and one unvisited page with the same url' do
+    let(:page) { collection.create_page(url, 0) }
+    let(:unvisited) { collection.create_page(url.upcase, 0) }
+    before do
+      allow(page).to receive(:visited?).and_return(true)
+      allow(unvisited).to receive(:visited?).and_return(false)
+    end
+    it 'first page has id 0' do
+      expect(page.id).to eq(0)
+    end
+    it 'second page has id 1' do
+      expect(unvisited.id).to eq(1)
+    end
+    it 'has one visited page' do
+      expect(collection.visited_pages).to eq([page])
+    end
+    it 'has no discovered pages' do
+      expect(collection.discovered_pages).to be_empty
+    end
+    it 'next page is nil' do
+      expect(collection.next_page).to be_nil
+    end
+  end
+  context 'one visited and one unvisited page with different URLs' do
+    let(:page) { collection.create_page(url, 0) }
+    let(:unvisited) { collection.create_page(url2, 0) }
+    before do
+      allow(page).to receive(:visited?).and_return(true)
+      allow(unvisited).to receive(:visited?).and_return(false)
+    end
+    it 'has one visited page' do
+      expect(collection.visited_pages).to eq([page])
+    end
+    it 'has one discovered page' do
+      expect(collection.discovered_pages).to eq([unvisited])
+    end
+    it 'next page is the unvisited page' do
+      expect(collection.next_page).to eq(unvisited)
+    end
+  end
+  context 'one visited and one unvisited page with different URLs only different by the query' do
+    let(:page) { collection.create_page(url, 0) }
+    let(:url3) { 'http://www.github.com/SomeUser/dragonlance?search=true' }
+    let(:unvisited) { collection.create_page(url3, 0) }
+    before do
+      allow(page).to receive(:visited?).and_return(true)
+      allow(unvisited).to receive(:visited?).and_return(false)
+    end
+    it 'has one visited page' do
+      expect(collection.visited_pages).to eq([page])
+    end
+    it 'has one discovered page' do
+      expect(collection.discovered_pages).to eq([unvisited])
+    end
+    it 'next page is the unvisited page' do
+      expect(collection.next_page).to eq(unvisited)
+    end
+  end
+  context 'several unvisited pages' do
+    let(:page) { collection.create_page(url, 2) }
+    let(:page2) { collection.create_page(url2, 0) }
+    before do
+      allow(page).to receive(:visited?).and_return(true)
+      allow(page2).to receive(:visited?).and_return(false)
+    end
+    it 'returns the page which has an earlier parent' do
+      expect(collection.next_page).to eq(page2)
+    end
+  end
+end