npm - powerdlz23 - Versions diffs - 1.2.3 → 1.2.5 - Mend

powerdlz23 1.2.3 → 1.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (207) hide show

package/Spider/README.md +19 -0
package/Spider/domain.py +18 -0
package/Spider/general.py +51 -0
package/Spider/link_finder.py +25 -0
package/Spider/main.py +50 -0
package/Spider/spider.py +74 -0
package/crawler/.formatter.exs +5 -0
package/crawler/.github/workflows/ci.yml +29 -0
package/crawler/.recode.exs +33 -0
package/crawler/.tool-versions +2 -0
package/crawler/CHANGELOG.md +82 -0
package/crawler/README.md +198 -0
package/crawler/architecture.svg +4 -0
package/crawler/config/config.exs +9 -0
package/crawler/config/dev.exs +5 -0
package/crawler/config/test.exs +5 -0
package/crawler/examples/google_search/scraper.ex +37 -0
package/crawler/examples/google_search/url_filter.ex +11 -0
package/crawler/examples/google_search.ex +77 -0
package/crawler/lib/crawler/dispatcher/worker.ex +14 -0
package/crawler/lib/crawler/dispatcher.ex +20 -0
package/crawler/lib/crawler/fetcher/header_preparer.ex +60 -0
package/crawler/lib/crawler/fetcher/modifier.ex +45 -0
package/crawler/lib/crawler/fetcher/policer.ex +77 -0
package/crawler/lib/crawler/fetcher/recorder.ex +55 -0
package/crawler/lib/crawler/fetcher/requester.ex +32 -0
package/crawler/lib/crawler/fetcher/retrier.ex +43 -0
package/crawler/lib/crawler/fetcher/url_filter.ex +26 -0
package/crawler/lib/crawler/fetcher.ex +81 -0
package/crawler/lib/crawler/http.ex +7 -0
package/crawler/lib/crawler/linker/path_builder.ex +71 -0
package/crawler/lib/crawler/linker/path_expander.ex +59 -0
package/crawler/lib/crawler/linker/path_finder.ex +106 -0
package/crawler/lib/crawler/linker/path_offliner.ex +59 -0
package/crawler/lib/crawler/linker/path_prefixer.ex +46 -0
package/crawler/lib/crawler/linker.ex +173 -0
package/crawler/lib/crawler/options.ex +127 -0
package/crawler/lib/crawler/parser/css_parser.ex +37 -0
package/crawler/lib/crawler/parser/guarder.ex +38 -0
package/crawler/lib/crawler/parser/html_parser.ex +41 -0
package/crawler/lib/crawler/parser/link_parser/link_expander.ex +32 -0
package/crawler/lib/crawler/parser/link_parser.ex +50 -0
package/crawler/lib/crawler/parser.ex +122 -0
package/crawler/lib/crawler/queue_handler.ex +45 -0
package/crawler/lib/crawler/scraper.ex +28 -0
package/crawler/lib/crawler/snapper/dir_maker.ex +45 -0
package/crawler/lib/crawler/snapper/link_replacer.ex +95 -0
package/crawler/lib/crawler/snapper.ex +82 -0
package/crawler/lib/crawler/store/counter.ex +19 -0
package/crawler/lib/crawler/store/page.ex +7 -0
package/crawler/lib/crawler/store.ex +87 -0
package/crawler/lib/crawler/worker.ex +62 -0
package/crawler/lib/crawler.ex +91 -0
package/crawler/mix.exs +78 -0
package/crawler/mix.lock +40 -0
package/crawler/test/fixtures/introducing-elixir.jpg +0 -0
package/crawler/test/integration_test.exs +135 -0
package/crawler/test/lib/crawler/dispatcher/worker_test.exs +7 -0
package/crawler/test/lib/crawler/dispatcher_test.exs +5 -0
package/crawler/test/lib/crawler/fetcher/header_preparer_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher/policer_test.exs +71 -0
package/crawler/test/lib/crawler/fetcher/recorder_test.exs +9 -0
package/crawler/test/lib/crawler/fetcher/requester_test.exs +9 -0
package/crawler/test/lib/crawler/fetcher/retrier_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher/url_filter_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher_test.exs +153 -0
package/crawler/test/lib/crawler/http_test.exs +47 -0
package/crawler/test/lib/crawler/linker/path_builder_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_expander_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_finder_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_offliner_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_prefixer_test.exs +7 -0
package/crawler/test/lib/crawler/linker_test.exs +7 -0
package/crawler/test/lib/crawler/options_test.exs +7 -0
package/crawler/test/lib/crawler/parser/css_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser/guarder_test.exs +7 -0
package/crawler/test/lib/crawler/parser/html_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser/link_parser/link_expander_test.exs +7 -0
package/crawler/test/lib/crawler/parser/link_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser_test.exs +8 -0
package/crawler/test/lib/crawler/queue_handler_test.exs +7 -0
package/crawler/test/lib/crawler/scraper_test.exs +7 -0
package/crawler/test/lib/crawler/snapper/dir_maker_test.exs +7 -0
package/crawler/test/lib/crawler/snapper/link_replacer_test.exs +7 -0
package/crawler/test/lib/crawler/snapper_test.exs +9 -0
package/crawler/test/lib/crawler/worker_test.exs +5 -0
package/crawler/test/lib/crawler_test.exs +295 -0
package/crawler/test/support/test_case.ex +24 -0
package/crawler/test/support/test_helpers.ex +28 -0
package/crawler/test/test_helper.exs +7 -0
package/grell/.rspec +2 -0
package/grell/.travis.yml +28 -0
package/grell/CHANGELOG.md +111 -0
package/grell/Gemfile +7 -0
package/grell/LICENSE.txt +22 -0
package/grell/README.md +213 -0
package/grell/Rakefile +2 -0
package/grell/grell.gemspec +36 -0
package/grell/lib/grell/capybara_driver.rb +44 -0
package/grell/lib/grell/crawler.rb +83 -0
package/grell/lib/grell/crawler_manager.rb +84 -0
package/grell/lib/grell/grell_logger.rb +10 -0
package/grell/lib/grell/page.rb +275 -0
package/grell/lib/grell/page_collection.rb +62 -0
package/grell/lib/grell/rawpage.rb +62 -0
package/grell/lib/grell/reader.rb +18 -0
package/grell/lib/grell/version.rb +3 -0
package/grell/lib/grell.rb +11 -0
package/grell/spec/lib/capybara_driver_spec.rb +38 -0
package/grell/spec/lib/crawler_manager_spec.rb +174 -0
package/grell/spec/lib/crawler_spec.rb +361 -0
package/grell/spec/lib/page_collection_spec.rb +159 -0
package/grell/spec/lib/page_spec.rb +418 -0
package/grell/spec/lib/reader_spec.rb +43 -0
package/grell/spec/spec_helper.rb +66 -0
package/heartmagic/config.py +1 -0
package/heartmagic/heart.py +3 -0
package/heartmagic/pytransform/__init__.py +483 -0
package/heartmagic/pytransform/_pytransform.dll +0 -0
package/heartmagic/pytransform/_pytransform.so +0 -0
package/httpStatusCode/README.md +2 -0
package/httpStatusCode/httpStatusCode.js +4 -0
package/httpStatusCode/reasonPhrases.js +344 -0
package/httpStatusCode/statusCodes.js +344 -0
package/package.json +1 -1
package/rubyretriever/.rspec +2 -0
package/rubyretriever/.travis.yml +7 -0
package/rubyretriever/Gemfile +3 -0
package/rubyretriever/Gemfile.lock +64 -0
package/rubyretriever/LICENSE +20 -0
package/rubyretriever/Rakefile +7 -0
package/rubyretriever/bin/rr +79 -0
package/rubyretriever/lib/retriever/cli.rb +25 -0
package/rubyretriever/lib/retriever/core_ext.rb +13 -0
package/rubyretriever/lib/retriever/fetch.rb +268 -0
package/rubyretriever/lib/retriever/fetchfiles.rb +71 -0
package/rubyretriever/lib/retriever/fetchseo.rb +18 -0
package/rubyretriever/lib/retriever/fetchsitemap.rb +43 -0
package/rubyretriever/lib/retriever/link.rb +47 -0
package/rubyretriever/lib/retriever/openuri_redirect_patch.rb +8 -0
package/rubyretriever/lib/retriever/page.rb +104 -0
package/rubyretriever/lib/retriever/page_iterator.rb +21 -0
package/rubyretriever/lib/retriever/target.rb +47 -0
package/rubyretriever/lib/retriever/version.rb +4 -0
package/rubyretriever/lib/retriever.rb +15 -0
package/rubyretriever/readme.md +166 -0
package/rubyretriever/rubyretriever.gemspec +41 -0
package/rubyretriever/spec/link_spec.rb +77 -0
package/rubyretriever/spec/page_spec.rb +94 -0
package/rubyretriever/spec/retriever_spec.rb +84 -0
package/rubyretriever/spec/spec_helper.rb +17 -0
package/rubyretriever/spec/target_spec.rb +55 -0
package/snapcrawl/.changelog.old.md +157 -0
package/snapcrawl/.gitattributes +1 -0
package/snapcrawl/.github/workflows/test.yml +41 -0
package/snapcrawl/.rspec +3 -0
package/snapcrawl/.rubocop.yml +23 -0
package/snapcrawl/CHANGELOG.md +182 -0
package/snapcrawl/Gemfile +15 -0
package/snapcrawl/LICENSE +21 -0
package/snapcrawl/README.md +135 -0
package/snapcrawl/Runfile +35 -0
package/snapcrawl/bin/snapcrawl +25 -0
package/snapcrawl/lib/snapcrawl/cli.rb +52 -0
package/snapcrawl/lib/snapcrawl/config.rb +60 -0
package/snapcrawl/lib/snapcrawl/crawler.rb +98 -0
package/snapcrawl/lib/snapcrawl/dependencies.rb +21 -0
package/snapcrawl/lib/snapcrawl/exceptions.rb +5 -0
package/snapcrawl/lib/snapcrawl/log_helpers.rb +36 -0
package/snapcrawl/lib/snapcrawl/page.rb +118 -0
package/snapcrawl/lib/snapcrawl/pretty_logger.rb +11 -0
package/snapcrawl/lib/snapcrawl/refinements/pair_split.rb +26 -0
package/snapcrawl/lib/snapcrawl/refinements/string_refinements.rb +13 -0
package/snapcrawl/lib/snapcrawl/screenshot.rb +73 -0
package/snapcrawl/lib/snapcrawl/templates/config.yml +49 -0
package/snapcrawl/lib/snapcrawl/templates/docopt.txt +26 -0
package/snapcrawl/lib/snapcrawl/version.rb +3 -0
package/snapcrawl/lib/snapcrawl.rb +20 -0
package/snapcrawl/snapcrawl.gemspec +27 -0
package/snapcrawl/snapcrawl.yml +41 -0
package/snapcrawl/spec/README.md +16 -0
package/snapcrawl/spec/approvals/bin/help +26 -0
package/snapcrawl/spec/approvals/bin/usage +4 -0
package/snapcrawl/spec/approvals/cli/usage +4 -0
package/snapcrawl/spec/approvals/config/defaults +15 -0
package/snapcrawl/spec/approvals/config/minimal +15 -0
package/snapcrawl/spec/approvals/integration/blacklist +14 -0
package/snapcrawl/spec/approvals/integration/default-config +14 -0
package/snapcrawl/spec/approvals/integration/depth-0 +6 -0
package/snapcrawl/spec/approvals/integration/depth-3 +6 -0
package/snapcrawl/spec/approvals/integration/log-color-no +6 -0
package/snapcrawl/spec/approvals/integration/screenshot-error +3 -0
package/snapcrawl/spec/approvals/integration/whitelist +14 -0
package/snapcrawl/spec/approvals/models/pretty_logger/colors +1 -0
package/snapcrawl/spec/fixtures/config/minimal.yml +4 -0
package/snapcrawl/spec/server/config.ru +97 -0
package/snapcrawl/spec/snapcrawl/bin_spec.rb +15 -0
package/snapcrawl/spec/snapcrawl/cli_spec.rb +9 -0
package/snapcrawl/spec/snapcrawl/config_spec.rb +26 -0
package/snapcrawl/spec/snapcrawl/integration_spec.rb +65 -0
package/snapcrawl/spec/snapcrawl/page_spec.rb +89 -0
package/snapcrawl/spec/snapcrawl/pretty_logger_spec.rb +19 -0
package/snapcrawl/spec/snapcrawl/refinements/pair_split_spec.rb +27 -0
package/snapcrawl/spec/snapcrawl/refinements/string_refinements_spec.rb +29 -0
package/snapcrawl/spec/snapcrawl/screenshot_spec.rb +62 -0
package/snapcrawl/spec/spec_helper.rb +22 -0
package/snapcrawl/spec/spec_mixin.rb +10 -0

package/crawler/test/lib/crawler_test.exs ADDED Viewed

@@ -0,0 +1,295 @@
+defmodule CrawlerTest do
+  use Crawler.TestCase, async: false
+  alias Crawler.Store
+  @moduletag capture_log: true
+  doctest Crawler
+  test ".crawl", %{bypass: bypass, url: url} do
+    Store.ops_reset()
+    url = "#{url}/crawler"
+    linked_url1 = "#{url}/link1"
+    linked_url2 = "#{url}/link2"
+    linked_url3 = "#{url}/link3"
+    linked_url4 = "#{url}/link4"
+    Bypass.expect_once(bypass, "GET", "/crawler", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url1}">1</a></html>
+        <html><a href="#{linked_url2}">2</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler/link1", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url2}">2</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler/link2", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler/link3", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url4}">4</a></html>
+      """)
+    end)
+    {:ok, opts} = Crawler.crawl(url, max_depths: 3, workers: 3, interval: 100, store: Store)
+    assert Crawler.running?(opts)
+    Crawler.pause(opts)
+    refute Crawler.running?(opts)
+    assert opts[:workers] == 3
+    Crawler.resume(opts)
+    assert Crawler.running?(opts)
+    wait(fn ->
+      assert Store.ops_count() == 4
+    end)
+    wait(fn ->
+      assert %Store.Page{url: ^url, opts: %{workers: 3}} = Store.find_processed({url, nil})
+      assert Store.find_processed({linked_url1, nil})
+      assert Store.find_processed({linked_url2, nil})
+      assert Store.find_processed({linked_url3, nil})
+      refute Store.find({linked_url4, nil})
+      urls = Crawler.Store.all_urls()
+      assert Enum.member?(urls, {url, nil})
+      assert Enum.member?(urls, {linked_url1, nil})
+      assert Enum.member?(urls, {linked_url2, nil})
+      assert Enum.member?(urls, {linked_url3, nil})
+      refute Enum.member?(urls, {linked_url4, nil})
+    end)
+    wait(fn ->
+      refute Crawler.running?(opts)
+      assert OPQ.info(opts[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 3}
+    end)
+  end
+  test ".crawl without a store", %{bypass: bypass, url: url} do
+    url = "#{url}/crawler_without_store"
+    Bypass.expect_once(bypass, "GET", "/crawler_without_store", fn conn ->
+      Plug.Conn.resp(conn, 200, "200")
+    end)
+    {:ok, opts} = Crawler.crawl(url, max_depths: 1, workers: 1, interval: 100, store: nil)
+    wait(fn ->
+      assert %Store.Page{url: ^url, body: nil, opts: nil} = Store.find_processed({url, nil})
+    end)
+    wait(fn ->
+      assert OPQ.info(opts[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 1}
+    end)
+  end
+  test ".crawl with max_pages", %{bypass: bypass, url: url} do
+    Store.ops_reset()
+    url = "#{url}/crawler_with_max_pages"
+    linked_url1 = "#{url}/link1"
+    linked_url2 = "#{url}/link2"
+    linked_url3 = "#{url}/link3"
+    linked_url4 = "#{url}/link4"
+    linked_url5 = "#{url}/link5"
+    Bypass.expect_once(bypass, "GET", "/crawler_with_max_pages", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url1}">1</a></html>
+        <html><a href="#{linked_url2}">2</a></html>
+        <html><a href="#{linked_url3}">3</a></html>
+        <html><a href="#{linked_url4}">4</a></html>
+        <html><a href="#{linked_url5}">5</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler_with_max_pages/link1", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url2}">2</a></html>
+        <html><a href="#{linked_url3}">3</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler_with_max_pages/link2", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+        <html><a href="#{linked_url4}">4</a></html>
+        <html><a href="#{linked_url5}">5</a></html>
+      """)
+    end)
+    Bypass.stub(bypass, "GET", "/crawler_with_max_pages/link3", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+        <html><a href="#{linked_url4}">4</a></html>
+        <html><a href="#{linked_url5}">5</a></html>
+      """)
+    end)
+    Bypass.stub(bypass, "GET", "/crawler_with_max_pages/link4", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+        <html><a href="#{linked_url4}">4</a></html>
+        <html><a href="#{linked_url5}">5</a></html>
+      """)
+    end)
+    {:ok, opts} = Crawler.crawl(url, max_depths: 3, force: true, workers: 4, max_pages: 3, interval: 100)
+    wait(fn ->
+      assert Store.ops_count() == 4
+    end)
+    wait(fn ->
+      assert Store.find_processed({url, opts[:scope]})
+      assert Store.find_processed({linked_url1, opts[:scope]})
+      assert Store.find_processed({linked_url2, opts[:scope]})
+      assert Store.find_processed({linked_url3, opts[:scope]})
+      refute Store.find({linked_url4, opts[:scope]})
+      refute Store.find({linked_url5, opts[:scope]})
+    end)
+    wait(fn ->
+      assert OPQ.info(opts[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 4}
+    end)
+  end
+  test ".crawl with an existing queue", %{bypass: bypass, url: url} do
+    Store.ops_reset()
+    url = "#{url}/crawler_with_queue"
+    linked_url1 = "#{url}/link1"
+    linked_url2 = "#{url}/link2"
+    linked_url3 = "#{url}/link3"
+    linked_url4 = "#{url}/link4"
+    Bypass.expect_once(bypass, "GET", "/crawler_with_queue/link1", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url2}">2</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler_with_queue/link2", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler_with_queue/link3", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html>ok</html>
+      """)
+    end)
+    {:ok, queue} = OPQ.init(worker: Crawler.Dispatcher.Worker, workers: 2, interval: 100)
+    {:ok, opts1} = Crawler.crawl(linked_url1, store: Store, queue: queue)
+    {:ok, opts2} = Crawler.crawl(linked_url2, store: Store, queue: queue)
+    wait(fn ->
+      assert Store.ops_count() == 3
+    end)
+    wait(fn ->
+      assert Store.find_processed({linked_url1, nil})
+      assert Store.find_processed({linked_url2, nil})
+      assert Store.find_processed({linked_url3, nil})
+      refute Store.find_processed({linked_url4, nil})
+      urls = Crawler.Store.all_urls()
+      assert Enum.member?(urls, {linked_url1, nil})
+      assert Enum.member?(urls, {linked_url2, nil})
+      assert Enum.member?(urls, {linked_url3, nil})
+      refute Enum.member?(urls, {linked_url4, nil})
+    end)
+    wait(fn ->
+      assert OPQ.info(opts1[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 2}
+      assert OPQ.info(opts2[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 2}
+    end)
+  end
+  test ".crawl forced", %{bypass: bypass, url: url} do
+    Store.ops_reset()
+    url = "#{url}/crawler_forced"
+    linked_url1 = "#{url}/link1"
+    linked_url2 = "#{url}/link2"
+    Bypass.expect(bypass, "GET", "/crawler_forced", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url1}">1</a></html>
+        <html><a href="#{linked_url1}">1</a></html>
+      """)
+    end)
+    Bypass.expect(bypass, "GET", "/crawler_forced/link1", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url2}">2</a></html>
+      """)
+    end)
+    Bypass.expect(bypass, "GET", "/crawler_forced/link2", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html>ok</html>
+      """)
+    end)
+    {:ok, opts1} = Crawler.crawl(url, force: true, workers: 1, interval: 100)
+    {:ok, opts2} = Crawler.crawl(url, force: true, workers: 2, interval: 100)
+    refute opts1[:scope] == opts2[:scope]
+    wait(fn ->
+      assert Store.find_processed({url, opts1[:scope]})
+      assert Store.find_processed({url, opts2[:scope]})
+      assert Store.find_processed({linked_url1, opts1[:scope]})
+      assert Store.find_processed({linked_url1, opts2[:scope]})
+      assert Store.find_processed({linked_url2, opts1[:scope]})
+      assert Store.find_processed({linked_url2, opts2[:scope]})
+      assert Store.ops_count() >= 6
+      assert Store.ops_count() <= 10
+      assert OPQ.info(opts1[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 1}
+      assert OPQ.info(opts2[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 2}
+    end)
+  end
+  test ".crawl stopped", %{bypass: bypass, url: url} do
+    url = "#{url}/stop"
+    linked_url = "#{url}/stop1"
+    Bypass.expect_once(bypass, "GET", "/stop", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url}">1</a></html>
+      """)
+    end)
+    {:ok, opts} = Crawler.crawl(url, workers: 1, interval: 500)
+    Process.sleep(200)
+    Crawler.stop(opts)
+    refute Store.find({linked_url, nil})
+  end
+end

package/crawler/test/support/test_case.ex ADDED Viewed

@@ -0,0 +1,24 @@
+defmodule Crawler.TestCase do
+  use ExUnit.CaseTemplate
+  using do
+    quote do
+      import Crawler.TestHelpers
+    end
+  end
+  setup_all do
+    bypass = Bypass.open()
+    url = "http://localhost:#{bypass.port}"
+    path = "localhost-#{bypass.port}"
+    bypass2 = Bypass.open()
+    url2 = "http://localhost:#{bypass2.port}"
+    path2 = "localhost-#{bypass2.port}"
+    {
+      :ok,
+      bypass: bypass, url: url, path: path, bypass2: bypass2, url2: url2, path2: path2
+    }
+  end
+end

package/crawler/test/support/test_helpers.ex ADDED Viewed

@@ -0,0 +1,28 @@
+# Credit: https://gist.github.com/cblavier/5e15791387a6e22b98d8
+defmodule Crawler.TestHelpers do
+  def wait(fun), do: wait(500, fun)
+  def wait(0, fun), do: fun.()
+  def wait(timeout, fun) do
+    try do
+      fun.()
+    rescue
+      _ ->
+        :timer.sleep(10)
+        wait(max(0, timeout - 10), fun)
+    end
+  end
+  def tmp(path \\ "", filename \\ "") do
+    tmp_path = Path.join([File.cwd!(), "test", "tmp", path])
+    File.mkdir_p(tmp_path)
+    Path.join(tmp_path, filename)
+  end
+  def image_file do
+    {:ok, file} = File.read("test/fixtures/introducing-elixir.jpg")
+    file
+  end
+end

package/crawler/test/test_helper.exs ADDED Viewed

@@ -0,0 +1,7 @@
+[File.cwd!(), "test", "tmp", "*"]
+|> Path.join()
+|> Path.wildcard()
+|> Enum.each(&File.rm_rf/1)
+ExUnit.start()
+Application.ensure_all_started(:bypass)

package/grell/.rspec ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ --color
2	+ --require spec_helper

package/grell/.travis.yml ADDED Viewed

@@ -0,0 +1,28 @@
+language: ruby
+cache: bundler
+rvm:
+  - 2.2.4
+  - 2.3.0
+  - 2.4.2
+before_install:
+  - mkdir travis-phantomjs
+  - wget https://github.com/JordiPolo/phantomjs/blob/master/phantomjs-2.1.1-linux-x86_64.tar.bz2?raw=true
+    -O $PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64.tar.bz2
+  - tar -xvf $PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64.tar.bz2 -C $PWD/travis-phantomjs
+  - export PATH=$PWD/travis-phantomjs/phantomjs-2.1.1-linux-x86_64/bin:$PATH
+install:
+  - bundle install --jobs=3 --retry=3
+script:
+  - bundle exec rspec
+deploy:
+  provider: rubygems
+  api_key:
+    secure: czStDI0W6MWL70sDwu53oNNCc8vKtT61pgvii+ZWIC9A41C2p7BzmbtosXsnLk2ApxmpWvFIgtQE0XIH7jkM5mY05cHinXDphtOTkNLFVjck3ZOMkx/cc+QRFW8K4FHkrzFsC+/Xx4t2/Psh35LpzhfJd0XzKKoCstXUVgJsfGcAK3DMpjXHSUbwLXGDZ4lzmsk52OLf0oL+in2447TJfVOvGXtYmfh1PjXRwDxKB0dan7w5mVgajS52b6wUhVPTaMe/JgCbMuV7BaQ1Goq8u7V4aaxU+liPAhzHWfMB6tF4TEW8yu2tvGLdOA0+1jmM8E9Q5saPWtwKiHvBxN8CzRpkiNDzyFAf8ljrWT5yKX3aRQCyPp3NNyhoumWap36b+O/zwZ3HxoAe22Yg0rjz8z8NxMR/ELPvjPYjCiF5zY7fO9PAzmIynMRUrxDnFj+/JGHdzx0ZMo3fEXgHHSaHPNxIzEffVVQk4XLVnFHDjBLY4mVp4sbHbja5qnui20RkdM/H9Yi/fQyl1ODhk+LUPoh45ZneDZq7GPrl+WKK06oEjXIXLU+1iEuqnSqybbmJMTUJlUV+7EJdtq2DgfDB4KXwLm2LLOR/IX63AzEav4NIxx3hIXifSKa9rp6D7nMTzdQwF0FFzIj/Y3qLrAe1WWt0gx3Vxq67pSwOJthk5Fc=
+  on:
+    tags: true
+    rvm: 2.4.2

package/grell/CHANGELOG.md ADDED Viewed

@@ -0,0 +1,111 @@
+# 2.1.2
+  * Change white/black lists to allow/deny lists
+# 2.1.1
+  * Update phantomjs_options to use 'TLSv1.2'
+# 2.1.0
+  * Delete `driver_options` configuration key as it was never used.
+  * `cleanup_all_processes` is a self method as intended to.
+# 2.0.0
+  * New configuration key `on_periodic_restart`.
+  * CrawlerManager.cleanup_all_processes method destroy all instances of phantomjs in this machine.
+  * Breaking changes
+    - Requires Ruby 2.1 or later.
+    - Crawler.start_crawling does not accept options anymore, all options are passed to Crawler.new.
+    - Crawler's methods `restart` and `quit` have been moved to CrawlerManager.
+    - Crawler gets whitelist and blacklist as configuration options instead of being set in specific methods.
+# 1.6.11
+  * Ensure all links are loaded by waiting for Ajax requests to complete
+  * Add '@evaluate_in_each_page' option to evaluate before extracting links (e.g. $('.dropdown').addClass('open');)
+# 1.6.10
+  * Avoid following JS href links, add missing dependencies to fix Travis build
+# 1.6.9
+  * Avoid following links when disabled by CSS (1.6.8 worked only for Javascript)
+# 1.6.8
+  * Avoid following disabled links
+# 1.6.7
+  * Increment '@times_visited' first to avoid infinite retries when rescuing errors
+# 1.6.6
+  * Updated phantomjs_logger not to open '/dev/null'
+# 1.6.5
+  * Added #quit to Crawler
+# 1.6.4
+  * Added #quit to Capybara driver
+# 1.6.3
+  * Only follow visible links
+# 1.6.2
+  * Reset Capybara driver to Puffing Billy (used to rewrite URL requests in specs)
+  * Use float timestamp for Poltergeist driver name to support fast test executions
+# 1.6.1
+  * Use non-static name to support registering Poltergeist crawler multiple times
+  * More exception handling, store redirected URLs in addition to original URL
+# 1.6
+  * Support custom URL comparison when adding new pages during crawling
+  * Don't rescue Timeout error, so that Delayed Job can properly terminate hanging jobs
+  * Fail early if Capybara doesn't initialize properly
+# 1.5.1
+  * Fixed deprecation warning (Thanks scott)
+  * Updated Poltergeist dependency
+# 1.5.0
+  * Grell will follow redirects.
+  * Added #followed_redirects? #error? #current_url methods to the Page class
+# 1.4.0
+  * Added crawler.restart to restart browser process
+  * The block of code can make grell retry any given page.
+# 1.3.2
+  * Rescue Timeout error and return an empty page when that happens
+# 1.3.1
+  * Added whitelisting and blacklisting
+  * Better info in gemspec
+# 1.3
+  * The Crawler object allows you to provide an external logger object.
+  * Clearer semantics when an error happens, special headers are returned so the user can inspect the error
+  * Caveats:
+    - The 'debug' option in the crawler does not have any affect anymore. Provide an external logger with 'logger' instead
+    - The errors provided in the headers by grell has changed from 'grell_status' to 'grellStatus'.
+    - The 'visited' property in the page was never supposed to be accesible. Use 'visited?' instead.
+# 1.2.1
+  * Solve bug: URLs are case insensitive
+# 1.2
+  * Grell now will consider two links to point to the same page only when the whole URL is exactly the same.
+    Versions previously would only consider two links to be the same when they shared the path.
+# 1.1.2
+  * Solve bug where we were adding links in heads as if there were normal links in the body
+# 1.1.1
+  * Solve bug with the new data-href functionality
+# 1.1
+  * Solve problem with randomly failing spec
+  * Search for elements with 'href' or 'data-href' to find links
+# 1.0.1
+  * Rescueing Javascript errors
+# 1.0
+  * Initial implementation
+  * Basic support to crawling pages.

package/grell/Gemfile ADDED Viewed

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+# Avoid ruby 2.1 to use Rack > 2.0 which is not compatible
+platform :ruby_21 do
+  gem 'rack', '~> 1.0'
+end
+gemspec

package/grell/LICENSE.txt ADDED Viewed

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Medidata Solutions Worldwide
+MIT License
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.