npm - powerdlz23 - Versions diffs - 1.2.3 → 1.2.5 - Mend

powerdlz23 1.2.3 → 1.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (207) hide show

package/Spider/README.md +19 -0
package/Spider/domain.py +18 -0
package/Spider/general.py +51 -0
package/Spider/link_finder.py +25 -0
package/Spider/main.py +50 -0
package/Spider/spider.py +74 -0
package/crawler/.formatter.exs +5 -0
package/crawler/.github/workflows/ci.yml +29 -0
package/crawler/.recode.exs +33 -0
package/crawler/.tool-versions +2 -0
package/crawler/CHANGELOG.md +82 -0
package/crawler/README.md +198 -0
package/crawler/architecture.svg +4 -0
package/crawler/config/config.exs +9 -0
package/crawler/config/dev.exs +5 -0
package/crawler/config/test.exs +5 -0
package/crawler/examples/google_search/scraper.ex +37 -0
package/crawler/examples/google_search/url_filter.ex +11 -0
package/crawler/examples/google_search.ex +77 -0
package/crawler/lib/crawler/dispatcher/worker.ex +14 -0
package/crawler/lib/crawler/dispatcher.ex +20 -0
package/crawler/lib/crawler/fetcher/header_preparer.ex +60 -0
package/crawler/lib/crawler/fetcher/modifier.ex +45 -0
package/crawler/lib/crawler/fetcher/policer.ex +77 -0
package/crawler/lib/crawler/fetcher/recorder.ex +55 -0
package/crawler/lib/crawler/fetcher/requester.ex +32 -0
package/crawler/lib/crawler/fetcher/retrier.ex +43 -0
package/crawler/lib/crawler/fetcher/url_filter.ex +26 -0
package/crawler/lib/crawler/fetcher.ex +81 -0
package/crawler/lib/crawler/http.ex +7 -0
package/crawler/lib/crawler/linker/path_builder.ex +71 -0
package/crawler/lib/crawler/linker/path_expander.ex +59 -0
package/crawler/lib/crawler/linker/path_finder.ex +106 -0
package/crawler/lib/crawler/linker/path_offliner.ex +59 -0
package/crawler/lib/crawler/linker/path_prefixer.ex +46 -0
package/crawler/lib/crawler/linker.ex +173 -0
package/crawler/lib/crawler/options.ex +127 -0
package/crawler/lib/crawler/parser/css_parser.ex +37 -0
package/crawler/lib/crawler/parser/guarder.ex +38 -0
package/crawler/lib/crawler/parser/html_parser.ex +41 -0
package/crawler/lib/crawler/parser/link_parser/link_expander.ex +32 -0
package/crawler/lib/crawler/parser/link_parser.ex +50 -0
package/crawler/lib/crawler/parser.ex +122 -0
package/crawler/lib/crawler/queue_handler.ex +45 -0
package/crawler/lib/crawler/scraper.ex +28 -0
package/crawler/lib/crawler/snapper/dir_maker.ex +45 -0
package/crawler/lib/crawler/snapper/link_replacer.ex +95 -0
package/crawler/lib/crawler/snapper.ex +82 -0
package/crawler/lib/crawler/store/counter.ex +19 -0
package/crawler/lib/crawler/store/page.ex +7 -0
package/crawler/lib/crawler/store.ex +87 -0
package/crawler/lib/crawler/worker.ex +62 -0
package/crawler/lib/crawler.ex +91 -0
package/crawler/mix.exs +78 -0
package/crawler/mix.lock +40 -0
package/crawler/test/fixtures/introducing-elixir.jpg +0 -0
package/crawler/test/integration_test.exs +135 -0
package/crawler/test/lib/crawler/dispatcher/worker_test.exs +7 -0
package/crawler/test/lib/crawler/dispatcher_test.exs +5 -0
package/crawler/test/lib/crawler/fetcher/header_preparer_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher/policer_test.exs +71 -0
package/crawler/test/lib/crawler/fetcher/recorder_test.exs +9 -0
package/crawler/test/lib/crawler/fetcher/requester_test.exs +9 -0
package/crawler/test/lib/crawler/fetcher/retrier_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher/url_filter_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher_test.exs +153 -0
package/crawler/test/lib/crawler/http_test.exs +47 -0
package/crawler/test/lib/crawler/linker/path_builder_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_expander_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_finder_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_offliner_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_prefixer_test.exs +7 -0
package/crawler/test/lib/crawler/linker_test.exs +7 -0
package/crawler/test/lib/crawler/options_test.exs +7 -0
package/crawler/test/lib/crawler/parser/css_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser/guarder_test.exs +7 -0
package/crawler/test/lib/crawler/parser/html_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser/link_parser/link_expander_test.exs +7 -0
package/crawler/test/lib/crawler/parser/link_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser_test.exs +8 -0
package/crawler/test/lib/crawler/queue_handler_test.exs +7 -0
package/crawler/test/lib/crawler/scraper_test.exs +7 -0
package/crawler/test/lib/crawler/snapper/dir_maker_test.exs +7 -0
package/crawler/test/lib/crawler/snapper/link_replacer_test.exs +7 -0
package/crawler/test/lib/crawler/snapper_test.exs +9 -0
package/crawler/test/lib/crawler/worker_test.exs +5 -0
package/crawler/test/lib/crawler_test.exs +295 -0
package/crawler/test/support/test_case.ex +24 -0
package/crawler/test/support/test_helpers.ex +28 -0
package/crawler/test/test_helper.exs +7 -0
package/grell/.rspec +2 -0
package/grell/.travis.yml +28 -0
package/grell/CHANGELOG.md +111 -0
package/grell/Gemfile +7 -0
package/grell/LICENSE.txt +22 -0
package/grell/README.md +213 -0
package/grell/Rakefile +2 -0
package/grell/grell.gemspec +36 -0
package/grell/lib/grell/capybara_driver.rb +44 -0
package/grell/lib/grell/crawler.rb +83 -0
package/grell/lib/grell/crawler_manager.rb +84 -0
package/grell/lib/grell/grell_logger.rb +10 -0
package/grell/lib/grell/page.rb +275 -0
package/grell/lib/grell/page_collection.rb +62 -0
package/grell/lib/grell/rawpage.rb +62 -0
package/grell/lib/grell/reader.rb +18 -0
package/grell/lib/grell/version.rb +3 -0
package/grell/lib/grell.rb +11 -0
package/grell/spec/lib/capybara_driver_spec.rb +38 -0
package/grell/spec/lib/crawler_manager_spec.rb +174 -0
package/grell/spec/lib/crawler_spec.rb +361 -0
package/grell/spec/lib/page_collection_spec.rb +159 -0
package/grell/spec/lib/page_spec.rb +418 -0
package/grell/spec/lib/reader_spec.rb +43 -0
package/grell/spec/spec_helper.rb +66 -0
package/heartmagic/config.py +1 -0
package/heartmagic/heart.py +3 -0
package/heartmagic/pytransform/__init__.py +483 -0
package/heartmagic/pytransform/_pytransform.dll +0 -0
package/heartmagic/pytransform/_pytransform.so +0 -0
package/httpStatusCode/README.md +2 -0
package/httpStatusCode/httpStatusCode.js +4 -0
package/httpStatusCode/reasonPhrases.js +344 -0
package/httpStatusCode/statusCodes.js +344 -0
package/package.json +1 -1
package/rubyretriever/.rspec +2 -0
package/rubyretriever/.travis.yml +7 -0
package/rubyretriever/Gemfile +3 -0
package/rubyretriever/Gemfile.lock +64 -0
package/rubyretriever/LICENSE +20 -0
package/rubyretriever/Rakefile +7 -0
package/rubyretriever/bin/rr +79 -0
package/rubyretriever/lib/retriever/cli.rb +25 -0
package/rubyretriever/lib/retriever/core_ext.rb +13 -0
package/rubyretriever/lib/retriever/fetch.rb +268 -0
package/rubyretriever/lib/retriever/fetchfiles.rb +71 -0
package/rubyretriever/lib/retriever/fetchseo.rb +18 -0
package/rubyretriever/lib/retriever/fetchsitemap.rb +43 -0
package/rubyretriever/lib/retriever/link.rb +47 -0
package/rubyretriever/lib/retriever/openuri_redirect_patch.rb +8 -0
package/rubyretriever/lib/retriever/page.rb +104 -0
package/rubyretriever/lib/retriever/page_iterator.rb +21 -0
package/rubyretriever/lib/retriever/target.rb +47 -0
package/rubyretriever/lib/retriever/version.rb +4 -0
package/rubyretriever/lib/retriever.rb +15 -0
package/rubyretriever/readme.md +166 -0
package/rubyretriever/rubyretriever.gemspec +41 -0
package/rubyretriever/spec/link_spec.rb +77 -0
package/rubyretriever/spec/page_spec.rb +94 -0
package/rubyretriever/spec/retriever_spec.rb +84 -0
package/rubyretriever/spec/spec_helper.rb +17 -0
package/rubyretriever/spec/target_spec.rb +55 -0
package/snapcrawl/.changelog.old.md +157 -0
package/snapcrawl/.gitattributes +1 -0
package/snapcrawl/.github/workflows/test.yml +41 -0
package/snapcrawl/.rspec +3 -0
package/snapcrawl/.rubocop.yml +23 -0
package/snapcrawl/CHANGELOG.md +182 -0
package/snapcrawl/Gemfile +15 -0
package/snapcrawl/LICENSE +21 -0
package/snapcrawl/README.md +135 -0
package/snapcrawl/Runfile +35 -0
package/snapcrawl/bin/snapcrawl +25 -0
package/snapcrawl/lib/snapcrawl/cli.rb +52 -0
package/snapcrawl/lib/snapcrawl/config.rb +60 -0
package/snapcrawl/lib/snapcrawl/crawler.rb +98 -0
package/snapcrawl/lib/snapcrawl/dependencies.rb +21 -0
package/snapcrawl/lib/snapcrawl/exceptions.rb +5 -0
package/snapcrawl/lib/snapcrawl/log_helpers.rb +36 -0
package/snapcrawl/lib/snapcrawl/page.rb +118 -0
package/snapcrawl/lib/snapcrawl/pretty_logger.rb +11 -0
package/snapcrawl/lib/snapcrawl/refinements/pair_split.rb +26 -0
package/snapcrawl/lib/snapcrawl/refinements/string_refinements.rb +13 -0
package/snapcrawl/lib/snapcrawl/screenshot.rb +73 -0
package/snapcrawl/lib/snapcrawl/templates/config.yml +49 -0
package/snapcrawl/lib/snapcrawl/templates/docopt.txt +26 -0
package/snapcrawl/lib/snapcrawl/version.rb +3 -0
package/snapcrawl/lib/snapcrawl.rb +20 -0
package/snapcrawl/snapcrawl.gemspec +27 -0
package/snapcrawl/snapcrawl.yml +41 -0
package/snapcrawl/spec/README.md +16 -0
package/snapcrawl/spec/approvals/bin/help +26 -0
package/snapcrawl/spec/approvals/bin/usage +4 -0
package/snapcrawl/spec/approvals/cli/usage +4 -0
package/snapcrawl/spec/approvals/config/defaults +15 -0
package/snapcrawl/spec/approvals/config/minimal +15 -0
package/snapcrawl/spec/approvals/integration/blacklist +14 -0
package/snapcrawl/spec/approvals/integration/default-config +14 -0
package/snapcrawl/spec/approvals/integration/depth-0 +6 -0
package/snapcrawl/spec/approvals/integration/depth-3 +6 -0
package/snapcrawl/spec/approvals/integration/log-color-no +6 -0
package/snapcrawl/spec/approvals/integration/screenshot-error +3 -0
package/snapcrawl/spec/approvals/integration/whitelist +14 -0
package/snapcrawl/spec/approvals/models/pretty_logger/colors +1 -0
package/snapcrawl/spec/fixtures/config/minimal.yml +4 -0
package/snapcrawl/spec/server/config.ru +97 -0
package/snapcrawl/spec/snapcrawl/bin_spec.rb +15 -0
package/snapcrawl/spec/snapcrawl/cli_spec.rb +9 -0
package/snapcrawl/spec/snapcrawl/config_spec.rb +26 -0
package/snapcrawl/spec/snapcrawl/integration_spec.rb +65 -0
package/snapcrawl/spec/snapcrawl/page_spec.rb +89 -0
package/snapcrawl/spec/snapcrawl/pretty_logger_spec.rb +19 -0
package/snapcrawl/spec/snapcrawl/refinements/pair_split_spec.rb +27 -0
package/snapcrawl/spec/snapcrawl/refinements/string_refinements_spec.rb +29 -0
package/snapcrawl/spec/snapcrawl/screenshot_spec.rb +62 -0
package/snapcrawl/spec/spec_helper.rb +22 -0
package/snapcrawl/spec/spec_mixin.rb +10 -0

package/crawler/test/lib/crawler/fetcher/policer_test.exs ADDED Viewed

@@ -0,0 +1,71 @@
+defmodule Crawler.Fetcher.PolicerTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Fetcher.Policer
+  alias Crawler.Fetcher.UrlFilter
+  alias Crawler.Store
+  @moduletag capture_log: true
+  doctest Policer
+  setup do
+    Store.ops_reset()
+    :ok
+  end
+  test "max_pages ok" do
+    Store.ops_inc()
+    Store.ops_inc()
+    assert {:ok, %{max_pages: :infinity}} = Policer.police(%{max_pages: :infinity})
+  end
+  test "max_pages error" do
+    Store.ops_inc()
+    Store.ops_inc()
+    assert {:warn, "Fetch failed check 'within_max_pages?', with opts: " <> _} =
+             Policer.police(%{max_pages: 1})
+  end
+  test "max_depths ok" do
+    assert {:ok, %{depth: 1, max_depths: 2}} = Policer.police(%{depth: 1, max_depths: 2})
+  end
+  test "max_depths error" do
+    assert {:warn, "Fetch failed check 'within_fetch_depth?', with opts: " <> _} =
+             Policer.police(%{
+               depth: 2,
+               max_depths: 2,
+               html_tag: "a"
+             })
+  end
+  test "uri_scheme ok" do
+    assert {:ok,
+            %{
+              html_tag: "img",
+              url: "http://policer/hi.jpg",
+              url_filter: UrlFilter
+            }} =
+             Policer.police(%{
+               html_tag: "img",
+               url: "http://policer/hi.jpg",
+               url_filter: UrlFilter
+             })
+  end
+  test "uri_scheme error" do
+    assert {:warn, "Fetch failed check 'acceptable_uri_scheme?', with opts: " <> _} =
+             Policer.police(%{url: "ftp://hello.world"})
+  end
+  test "fetched error" do
+    Crawler.Store.add({"http://policer/exist/", nil})
+    assert {:warn, "Fetch failed check 'not_fetched_yet?', with opts: " <> _} =
+             Policer.police(%{url: "http://policer/exist/", scope: nil})
+  end
+end

package/crawler/test/lib/crawler/fetcher/recorder_test.exs ADDED Viewed

@@ -0,0 +1,9 @@
+defmodule Crawler.Fetcher.RecorderTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Fetcher.Recorder
+  alias Crawler.Store
+  alias Crawler.Store.Page
+  doctest Recorder
+end

package/crawler/test/lib/crawler/fetcher/requester_test.exs ADDED Viewed

@@ -0,0 +1,9 @@
+defmodule Crawler.Fetcher.RequesterTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Fetcher.Requester
+  @moduletag capture_log: true
+  doctest Requester
+end

package/crawler/test/lib/crawler/fetcher/retrier_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Fetcher.RetrierTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Fetcher.Retrier
+  doctest Retrier
+end

package/crawler/test/lib/crawler/fetcher/url_filter_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Fetcher.UrlFilterTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Fetcher.UrlFilter
+  doctest UrlFilter
+end

package/crawler/test/lib/crawler/fetcher_test.exs ADDED Viewed

@@ -0,0 +1,153 @@
+defmodule Crawler.FetcherTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Fetcher
+  alias Crawler.Fetcher.Modifier
+  alias Crawler.Fetcher.Retrier
+  alias Crawler.Fetcher.UrlFilter
+  alias Crawler.Store
+  @moduletag capture_log: true
+  doctest Fetcher
+  defmodule DummyRetrier do
+    @behaviour Retrier.Spec
+    def perform(fetch_url, _opts), do: fetch_url.()
+  end
+  @defaults %{
+    depth: 0,
+    retries: 2,
+    url_filter: UrlFilter,
+    modifier: Modifier,
+    retrier: DummyRetrier,
+    store: Store,
+    html_tag: "a"
+  }
+  test "success", %{bypass: bypass, url: url} do
+    url = "#{url}/fetcher/200"
+    Bypass.expect_once(bypass, "GET", "/fetcher/200", fn conn ->
+      Plug.Conn.resp(conn, 200, "<html>200</html>")
+    end)
+    @defaults
+    |> Map.merge(%{url: url})
+    |> Fetcher.fetch()
+    page = Store.find({url, nil})
+    assert page.url == url
+    assert page.body == "<html>200</html>"
+  end
+  test "success: 301", %{bypass: bypass, url: url} do
+    Bypass.expect_once(bypass, "GET", "/fetcher/301", fn conn ->
+      conn
+      |> Plug.Conn.merge_resp_headers([{"location", "#{url}/fetcher/301_200"}])
+      |> Plug.Conn.resp(301, "")
+    end)
+    Bypass.expect_once(bypass, "GET", "/fetcher/301_200", fn conn ->
+      Plug.Conn.resp(conn, 200, "<html>301_200</html>")
+    end)
+    url = "#{url}/fetcher/301"
+    @defaults
+    |> Map.merge(%{url: url})
+    |> Fetcher.fetch()
+    page = Store.find({url, nil})
+    assert page.url == url
+    assert page.body == "<html>301_200</html>"
+  end
+  test "failure: 500", %{bypass: bypass, url: url} do
+    url = "#{url}/fetcher/500"
+    Bypass.expect_once(bypass, "GET", "/fetcher/500", fn conn ->
+      Plug.Conn.resp(conn, 500, "<html>500</html>")
+    end)
+    fetcher =
+      @defaults
+      |> Map.merge(%{url: url})
+      |> Fetcher.fetch()
+    assert fetcher == {:warn, "Failed to fetch #{url}, status code: 500"}
+    refute Store.find({url, nil}).body
+  end
+  test "failure: timeout", %{bypass: bypass, url: url} do
+    url = "#{url}/fetcher/timeout"
+    Bypass.expect_once(bypass, "GET", "/fetcher/timeout", fn conn ->
+      Process.flag(:trap_exit, true)
+      :timer.sleep(100)
+      Plug.Conn.resp(conn, 200, "<html>200</html>")
+    end)
+    fetcher =
+      @defaults
+      |> Map.merge(%{url: url, timeout: 50})
+      |> Fetcher.fetch()
+    assert fetcher == {:warn, "Failed to fetch #{url}, reason: :timeout"}
+    refute Store.find({url, nil}).body
+  end
+  test "failure: retries", %{bypass: bypass, url: url} do
+    url = "#{url}/fetcher/retries"
+    Bypass.expect(bypass, "GET", "/fetcher/retries", fn conn ->
+      Plug.Conn.resp(conn, 500, "<html>500</html>")
+    end)
+    wait(fn ->
+      fetcher =
+        @defaults
+        |> Map.merge(%{url: url, timeout: 100, retrier: Retrier})
+        |> Fetcher.fetch()
+      assert fetcher == {:warn, "Failed to fetch #{url}, status code: 500"}
+      refute Store.find({url, nil}).body
+    end)
+  end
+  test "failure: unable to write", %{bypass: bypass, url: url, path: path} do
+    url = "#{url}/fetcher/fail.html"
+    Bypass.expect_once(bypass, "GET", "/fetcher/fail.html", fn conn ->
+      Plug.Conn.resp(conn, 200, "<html>200</html>")
+    end)
+    fetcher =
+      @defaults
+      |> Map.merge(%{url: url, save_to: "nope"})
+      |> Fetcher.fetch()
+    assert {:error, "Cannot write to file nope/#{path}/fetcher/fail.html, reason: enoent"} ==
+             fetcher
+  end
+  test "snap /fetcher/page.html", %{bypass: bypass, url: url, path: path} do
+    url = "#{url}/fetcher/page.html"
+    Bypass.expect_once(bypass, "GET", "/fetcher/page.html", fn conn ->
+      Plug.Conn.resp(conn, 200, "<html>200</html>")
+    end)
+    @defaults
+    |> Map.merge(%{url: url, save_to: tmp("fetcher")})
+    |> Fetcher.fetch()
+    wait(fn ->
+      assert {:ok, "<html>200</html>"} == File.read(tmp("fetcher/#{path}/fetcher", "page.html"))
+    end)
+  end
+end

package/crawler/test/lib/crawler/http_test.exs ADDED Viewed

@@ -0,0 +1,47 @@
+defmodule Crawler.HTTPTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.HTTP
+  @moduletag capture_log: true
+  doctest HTTP
+  test "default user agent", %{bypass: bypass, url: url} do
+    Agent.start_link(fn -> "" end, name: HTTP.DefaultUA)
+    Bypass.expect_once(bypass, "GET", "/http/default_ua", fn conn ->
+      {_, ua} = Enum.find(conn.req_headers, fn {header, _} -> header == "user-agent" end)
+      Agent.update(HTTP.DefaultUA, fn _ -> ua end)
+      Plug.Conn.resp(conn, 200, "")
+    end)
+    Crawler.crawl("#{url}/http/default_ua")
+    wait(fn ->
+      assert String.match?(
+               Agent.get(HTTP.DefaultUA, & &1),
+               ~r{Crawler/\d\.\d\.\d \(https://github\.com/fredwu/crawler\)}
+             )
+    end)
+  end
+  test "custom user agent", %{bypass: bypass, url: url} do
+    Agent.start_link(fn -> "" end, name: HTTP.CustomUA)
+    Bypass.expect_once(bypass, "GET", "/http/custom_ua", fn conn ->
+      {_, ua} = Enum.find(conn.req_headers, fn {header, _} -> header == "user-agent" end)
+      Agent.update(HTTP.CustomUA, fn _ -> ua end)
+      Plug.Conn.resp(conn, 200, "")
+    end)
+    Crawler.crawl("#{url}/http/custom_ua", user_agent: "Hello World")
+    wait(fn ->
+      assert Agent.get(HTTP.CustomUA, & &1) == "Hello World"
+    end)
+  end
+end

package/crawler/test/lib/crawler/linker/path_builder_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Linker.PathBuilderTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Linker.PathBuilder
+  doctest PathBuilder
+end

package/crawler/test/lib/crawler/linker/path_expander_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Linker.PathExpanderTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Linker.PathExpander
+  doctest PathExpander
+end

package/crawler/test/lib/crawler/linker/path_finder_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Linker.PathFinderTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Linker.PathFinder
+  doctest PathFinder
+end

package/crawler/test/lib/crawler/linker/path_offliner_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Linker.PathOfflinerTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Linker.PathOffliner
+  doctest PathOffliner
+end

package/crawler/test/lib/crawler/linker/path_prefixer_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Linker.PathPrefixerTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Linker.PathPrefixer
+  doctest PathPrefixer
+end

package/crawler/test/lib/crawler/linker_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.LinkerTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Linker
+  doctest Linker
+end

package/crawler/test/lib/crawler/options_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.OptionsTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Options
+  doctest Options
+end

package/crawler/test/lib/crawler/parser/css_parser_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Parser.CssParserTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Parser.CssParser
+  doctest CssParser
+end

package/crawler/test/lib/crawler/parser/guarder_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Parser.GuarderTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Parser.Guarder
+  doctest Guarder
+end

package/crawler/test/lib/crawler/parser/html_parser_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Parser.HtmlParserTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Parser.HtmlParser
+  doctest HtmlParser
+end

package/crawler/test/lib/crawler/parser/link_parser/link_expander_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Parser.LinkParser.LinkExpanderTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Parser.LinkParser.LinkExpander
+  doctest LinkExpander
+end

package/crawler/test/lib/crawler/parser/link_parser_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Parser.LinkParserTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Parser.LinkParser
+  doctest LinkParser
+end

package/crawler/test/lib/crawler/parser_test.exs ADDED Viewed

@@ -0,0 +1,8 @@
+defmodule Crawler.ParserTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Parser
+  alias Crawler.Store.Page
+  doctest Parser
+end

package/crawler/test/lib/crawler/queue_handler_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.QueueHandlerTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.QueueHandler
+  doctest QueueHandler
+end

package/crawler/test/lib/crawler/scraper_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.ScraperTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Scraper
+  doctest Scraper
+end

package/crawler/test/lib/crawler/snapper/dir_maker_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Snapper.DirMakerTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Snapper.DirMaker
+  doctest DirMaker
+end

package/crawler/test/lib/crawler/snapper/link_replacer_test.exs ADDED Viewed

@@ -0,0 +1,7 @@
+defmodule Crawler.Snapper.LinkReplacerTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Snapper.LinkReplacer
+  doctest LinkReplacer
+end

package/crawler/test/lib/crawler/snapper_test.exs ADDED Viewed

@@ -0,0 +1,9 @@
+defmodule Crawler.SnapperTest do
+  use Crawler.TestCase, async: true
+  alias Crawler.Snapper
+  @moduletag capture_log: true
+  doctest Snapper
+end

package/crawler/test/lib/crawler/worker_test.exs ADDED Viewed

@@ -0,0 +1,5 @@
+defmodule Crawler.WorkerTest do
+  use Crawler.TestCase, async: true
+  doctest Crawler.Worker
+end