powerdlz23 1.2.3 → 1.2.5

package/crawler/lib/crawler/snapper.ex

@@ -0,0 +1,82 @@
+defmodule Crawler.Snapper do
+  @moduledoc """
+  Stores crawled pages offline.
+  """
+
+  require Logger
+
+  alias Crawler.Snapper.DirMaker
+  alias Crawler.Snapper.LinkReplacer
+
+  @doc """
+  In order to store pages offline, it provides the following functionalities:
+
+  - replaces all URLs to their equivalent relative paths
+  - creates directories when necessary to store the files
+
+  ## Examples
+
+      iex> Snapper.snap("hello", %{save_to: tmp("snapper"), url: "http://hello-world.local"})
+      iex> File.read(tmp("snapper/hello-world.local", "index.html"))
+      {:ok, "hello"}
+
+      iex> Snapper.snap("hello", %{save_to: tmp("snapper"), url: "http://snapper.local/index.html"})
+      iex> File.read(tmp("snapper/snapper.local", "index.html"))
+      {:ok, "hello"}
+
+      iex> Snapper.snap("hello", %{save_to: "nope", url: "http://snapper.local/index.html"})
+      {:error, "Cannot write to file nope/snapper.local/index.html, reason: enoent"}
+
+      iex> Snapper.snap("hello", %{save_to: tmp("snapper"), url: "http://snapper.local/hello"})
+      iex> File.read(tmp("snapper/snapper.local/hello", "index.html"))
+      {:ok, "hello"}
+
+      iex> Snapper.snap("hello", %{save_to: tmp("snapper"), url: "http://snapper.local/hello1/"})
+      iex> File.read(tmp("snapper/snapper.local/hello1", "index.html"))
+      {:ok, "hello"}
+
+      iex> Snapper.snap(
+      iex>   "<a href='http://another.domain/page'></a>",
+      iex>   %{
+      iex>     save_to: tmp("snapper"),
+      iex>     url: "http://snapper.local/depth0",
+      iex>     depth: 1,
+      iex>     max_depths: 2,
+      iex>     html_tag: "a",
+      iex>     content_type: "text/html",
+      iex>   }
+      iex> )
+      iex> File.read(tmp("snapper/snapper.local/depth0", "index.html"))
+      {:ok, "<a href='../../another.domain/page/index.html'></a>"}
+
+      iex> Snapper.snap(
+      iex>   "<a href='https://another.domain:8888/page'></a>",
+      iex>   %{
+      iex>     save_to: tmp("snapper"),
+      iex>     url: "http://snapper.local:7777/dir/depth1",
+      iex>     depth: 1,
+      iex>     max_depths: 2,
+      iex>     html_tag: "a",
+      iex>     content_type: "text/html",
+      iex>   }
+      iex> )
+      iex> File.read(tmp("snapper/snapper.local-7777/dir/depth1", "index.html"))
+      {:ok, "<a href='../../../another.domain-8888/page/index.html'></a>"}
+  """
+  def snap(body, opts) do
+    {:ok, body} = LinkReplacer.replace_links(body, opts)
+    file_path = DirMaker.make_dir(opts)
+
+    case File.write(file_path, body) do
+      :ok ->
+        {:ok, opts}
+
+      {:error, reason} ->
+        msg = "Cannot write to file #{file_path}, reason: #{reason}"
+
+        Logger.error(msg)
+
+        {:error, msg}
+    end
+  end
+end

package/crawler/lib/crawler/store/counter.ex

@@ -0,0 +1,19 @@
+defmodule Crawler.Store.Counter do
+  use Agent
+
+  def start_link(_args) do
+    Agent.start_link(fn -> 0 end, name: __MODULE__)
+  end
+
+  def value do
+    Agent.get(__MODULE__, & &1)
+  end
+
+  def inc do
+    Agent.update(__MODULE__, &(&1 + 1))
+  end
+
+  def reset do
+    Agent.update(__MODULE__, fn _ -> 0 end)
+  end
+end

package/crawler/lib/crawler/store/page.ex

@@ -0,0 +1,7 @@
+defmodule Crawler.Store.Page do
+  @moduledoc """
+  An internal struct for keeping the url and content of a crawled page.
+  """
+
+  defstruct [:url, :body, :opts, :processed]
+end

package/crawler/lib/crawler/store.ex

@@ -0,0 +1,87 @@
+defmodule Crawler.Store do
+  @moduledoc """
+  An internal data store for information related to each crawl.
+  """
+
+  alias Crawler.Store.Counter
+  alias Crawler.Store.DB
+  alias Crawler.Store.Page
+
+  use GenServer
+
+  def start_link(opts) do
+    children = [
+      {Registry, keys: :unique, name: DB},
+      Counter
+    ]
+
+    Supervisor.start_link(
+      children,
+      [strategy: :one_for_one, name: __MODULE__] ++ opts
+    )
+  end
+
+  @doc """
+  Initialises a new `Registry` named `Crawler.Store.DB`.
+  """
+  def init(args) do
+    {:ok, args}
+  end
+
+  @doc """
+  Finds a stored URL and returns its page data.
+  """
+  def find({url, scope}) do
+    case Registry.lookup(DB, {url, scope}) do
+      [{_, page}] -> page
+      _ -> nil
+    end
+  end
+
+  @doc """
+  Finds a stored URL and returns its page data only if it's processed.
+  """
+  def find_processed({url, scope}) do
+    case Registry.match(DB, {url, scope}, %{processed: true}) do
+      [{_, page}] -> page
+      _ -> nil
+    end
+  end
+
+  @doc """
+  Adds a URL to the registry.
+  """
+  def add({url, scope}) do
+    Registry.register(DB, {url, scope}, %Page{url: url})
+  end
+
+  @doc """
+  Adds the page data for a URL to the registry.
+  """
+  def add_page_data({url, scope}, body, opts) do
+    {_new, _old} = Registry.update_value(DB, {url, scope}, &%{&1 | body: body, opts: opts})
+  end
+
+  @doc """
+  Marks a URL as processed in the registry.
+  """
+  def processed({url, scope}) do
+    {_new, _old} = Registry.update_value(DB, {url, scope}, &%{&1 | processed: true})
+  end
+
+  def all_urls do
+    Registry.select(DB, [{{:"$1", :_, :_}, [], [:"$1"]}])
+  end
+
+  def ops_inc do
+    Counter.inc()
+  end
+
+  def ops_count do
+    Counter.value()
+  end
+
+  def ops_reset do
+    Counter.reset()
+  end
+end

package/crawler/lib/crawler/worker.ex

@@ -0,0 +1,62 @@
+defmodule Crawler.Worker do
+  @moduledoc """
+  Handles the crawl tasks.
+  """
+
+  require Logger
+
+  alias Crawler.Fetcher
+  alias Crawler.Store
+  alias Crawler.Store.Page
+
+  use GenServer
+
+  def init(args) do
+    :timer.send_after(args[:timeout], :stop)
+
+    {:ok, args}
+  end
+
+  @doc """
+  Runs the worker that casts data to itself to kick off the crawl workflow.
+  """
+  def run(opts) do
+    Logger.debug("Running worker with opts: #{inspect(opts)}")
+
+    {:ok, pid} = GenServer.start_link(__MODULE__, opts, hibernate_after: 0)
+
+    GenServer.cast(pid, opts)
+  end
+
+  @doc """
+  A crawl workflow that delegates responsibilities to:
+
+  - `Crawler.Fetcher.fetch/1`
+  - `Crawler.Parser.parse/1` (or a custom parser)
+  """
+  def handle_cast(_req, state) do
+    Logger.debug("Running worker with opts: #{inspect(state)}")
+
+    state
+    |> Fetcher.fetch()
+    |> state[:parser].parse()
+    |> mark_processed()
+
+    {:noreply, state, :hibernate}
+  end
+
+  def handle_info(:stop, state) do
+    {:stop, :normal, state}
+  end
+
+  def handle_info(_msg, state) do
+    {:noreply, state}
+  end
+
+  defp mark_processed({:ok, %Page{url: url, opts: opts}}) do
+    Store.ops_inc()
+    Store.processed({url, opts[:scope]})
+  end
+
+  defp mark_processed(_), do: nil
+end

package/crawler/lib/crawler.ex

@@ -0,0 +1,91 @@
+defmodule Crawler do
+  @moduledoc """
+  A high performance web crawler in Elixir.
+  """
+
+  alias Crawler.Options
+  alias Crawler.QueueHandler
+  alias Crawler.Store
+  alias Crawler.Worker
+
+  use Application
+
+  @doc """
+  Crawler is an application that gets started automatically with:
+
+  - a `Crawler.Store` that initiates a `Registry` for keeping internal data
+  """
+  def start(_type, _args) do
+    children = [
+      Store,
+      {DynamicSupervisor, name: Crawler.QueueSupervisor, strategy: :one_for_one}
+    ]
+
+    Supervisor.start_link(children, strategy: :one_for_one, name: Crawler)
+  end
+
+  @doc """
+  Enqueues a crawl, via `Crawler.QueueHandler.enqueue/1`.
+
+  This is the default crawl behaviour as the queue determines when an actual
+  crawl should happen based on the available workers and the rate limit. The
+  queue kicks off `Crawler.Dispatcher.Worker` which in turn calls
+  `Crawler.crawl_now/1`.
+  """
+  def crawl(url, opts \\ []) do
+    opts =
+      opts
+      |> Enum.into(%{})
+      |> Options.assign_defaults()
+      |> Options.assign_scope()
+      |> Options.assign_url(url)
+      |> Options.perform_default_actions()
+
+    if Store.ops_count() < opts[:max_pages] do
+      QueueHandler.enqueue(opts)
+    end
+  end
+
+  @doc """
+  Stops the crawler.
+  """
+  def stop(opts) do
+    Process.flag(:trap_exit, true)
+    OPQ.stop(opts[:queue])
+  end
+
+  @doc """
+  Pauses the crawler.
+  """
+  def pause(opts), do: OPQ.pause(opts[:queue])
+
+  @doc """
+  Resumes the crawler after it was paused.
+  """
+  def resume(opts), do: OPQ.resume(opts[:queue])
+
+  @doc """
+  Checks whether the crawler is still crawling.
+  """
+  def running?(opts) do
+    Process.sleep(10)
+
+    cond do
+      opts[:queue] |> OPQ.info() |> elem(0) == :paused -> false
+      Store.ops_count() <= 1 -> true
+      OPQ.queue(opts[:queue]) |> Enum.any?() -> true
+      true -> false
+    end
+  end
+
+  @doc """
+  Crawls immediately, this is used by `Crawler.Dispatcher.Worker.start_link/1`.
+
+  For general purpose use cases, always use `Crawler.crawl/2` instead.
+  """
+  def crawl_now(opts) do
+    if Store.ops_count() < opts[:max_pages] do
+      Worker.run(opts)
+    end
+  end
+end

package/crawler/mix.exs

@@ -0,0 +1,78 @@
+defmodule Crawler.Mixfile do
+  use Mix.Project
+
+  @source_url "https://github.com/fredwu/crawler"
+  @version "1.5.0"
+
+  def project do
+    [
+      app: :crawler,
+      version: @version,
+      elixir: "~> 1.13",
+      elixirc_paths: elixirc_paths(Mix.env()),
+      package: package(),
+      name: "Crawler",
+      description: "A high performance web crawler in Elixir.",
+      start_permanent: Mix.env() == :prod,
+      deps: deps(),
+      docs: docs(),
+      test_coverage: [tool: ExCoveralls],
+      preferred_cli_env: [coveralls: :test],
+      aliases: [publish: ["hex.publish", &git_tag/1]],
+      dialyzer: [
+        plt_add_apps: [:crawler],
+        flags: [:error_handling, :race_conditions, :underspecs]
+      ]
+    ]
+  end
+
+  def application do
+    [
+      mod: {Crawler, []},
+      extra_applications: [:logger, :runtime_tools, :observer, :wx]
+    ]
+  end
+
+  defp elixirc_paths(:test), do: ["lib", "test/support"]
+  defp elixirc_paths(:dev), do: ["lib", "examples"]
+  defp elixirc_paths(_), do: ["lib"]
+
+  defp deps do
+    [
+      {:httpoison, "~> 2.1"},
+      {:floki, "~> 0.30"},
+      {:opq, "~> 4.0"},
+      {:retry, "~> 0.10"},
+      {:recode, "~> 0.6", only: :dev},
+      {:ex_doc, ">= 0.0.0", only: :dev},
+      {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false},
+      {:plug_cowboy, "~> 2.0", only: :test},
+      {:bypass, "~> 2.1", only: :test},
+      {:excoveralls, "~> 0.7", only: :test}
+    ]
+  end
+
+  defp package do
+    [
+      maintainers: ["Fred Wu"],
+      licenses: ["MIT"],
+      links: %{"GitHub" => @source_url}
+    ]
+  end
+
+  defp git_tag(_args) do
+    System.cmd("git", ["tag", "v" <> Mix.Project.config()[:version]])
+    System.cmd("git", ["push"])
+    System.cmd("git", ["push", "--tags"])
+  end
+
+  defp docs do
+    [
+      extras: ["CHANGELOG.md": [title: "Changelog"], "README.md": [title: "Overview"]],
+      main: "readme",
+      source_url: @source_url,
+      source_ref: "v#{@version}",
+      formatters: ["html"]
+    ]
+  end
+end

package/crawler/mix.lock

@@ -0,0 +1,40 @@
+%{
+  "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"},
+  "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},
+  "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
+  "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"},
+  "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
+  "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
+  "dialyxir": {:hex, :dialyxir, "1.4.1", "a22ed1e7bd3a3e3f197b68d806ef66acb61ee8f57b3ac85fc5d57354c5482a93", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "84b795d6d7796297cca5a3118444b80c7d94f7ce247d49886e7c291e1ae49801"},
+  "earmark_parser": {:hex, :earmark_parser, "1.4.36", "487ea8ef9bdc659f085e6e654f3c3feea1d36ac3943edf9d2ef6c98de9174c13", [:mix], [], "hexpm", "a524e395634bdcf60a616efe77fd79561bec2e930d8b82745df06ab4e844400a"},
+  "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
+  "ex_doc": {:hex, :ex_doc, "0.30.6", "5f8b54854b240a2b55c9734c4b1d0dd7bdd41f71a095d42a70445c03cf05a281", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bd48f2ddacf4e482c727f9293d9498e0881597eae6ddc3d9562bd7923375109f"},
+  "excoveralls": {:hex, :excoveralls, "0.17.1", "83fa7906ef23aa7fc8ad7ee469c357a63b1b3d55dd701ff5b9ce1f72442b2874", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "95bc6fda953e84c60f14da4a198880336205464e75383ec0f570180567985ae0"},
+  "floki": {:hex, :floki, "0.34.3", "5e2dcaec5d7c228ce5b1d3501502e308b2d79eb655e4191751a1fe491c37feac", [:mix], [], "hexpm", "9577440eea5b97924b4bf3c7ea55f7b8b6dce589f9b28b096cc294a8dc342341"},
+  "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"},
+  "glob_ex": {:hex, :glob_ex, "0.1.4", "fc69cb3f6df9138a1e36e9aa041ef2eab0d4dfe916331425f6bac290d1977e79", [:mix], [], "hexpm", "583d35559dc5b17f14612f7153aaaf6dcc13edf2e383126e2dfb5f2d19c78b89"},
+  "hackney": {:hex, :hackney, "1.19.1", "59de4716e985dd2b5cbd4954fa1ae187e2b610a9c4520ffcb0b1653c3d6e5559", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "8aa08234bdefc269995c63c2282cf3cd0e36febe3a6bfab11b610572fdd1cad0"},
+  "httpoison": {:hex, :httpoison, "2.1.0", "655fd9a7b0b95ee3e9a3b535cf7ac8e08ef5229bab187fa86ac4208b122d934b", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "fc455cb4306b43827def4f57299b2d5ac8ac331cb23f517e734a4b78210a160c"},
+  "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
+  "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
+  "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"},
+  "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"},
+  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
+  "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
+  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"},
+  "opq": {:hex, :opq, "4.0.3", "04fd4bc42d8de8ea0175a1758d2f88fd22c2ead5342cbcb7777d899ba7c8b44f", [:mix], [{:gen_stage, "~> 1.1", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm", "78cb240d11ceeb72008df5613d59ebdc1dc7c1a896a35a500c59075b0931f9c8"},
+  "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"},
+  "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"},
+  "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"},
+  "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},
+  "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
+  "recode": {:hex, :recode, "0.6.4", "a3370bda63376953fb5e4698593c64388f27efd60f6b06763dbcd37e869c159e", [:mix], [{:bunt, "~> 0.2", [hex: :bunt, repo: "hexpm", optional: false]}, {:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:rewrite, "~> 0.9", [hex: :rewrite, repo: "hexpm", optional: false]}], "hexpm", "46700acb62d1f585a25cd3c3f5b19377911911d34107c9a5879d5e0bc6544995"},
+  "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"},
+  "rewrite": {:hex, :rewrite, "0.9.0", "90f2108ee657705bea00fa30d56dc26b8113ddfe1481487b8f6687871316a131", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 0.13", [hex: :sourceror, repo: "hexpm", optional: false]}], "hexpm", "5ee26ba5ab0ae3c1155b2ba8093d2bbf78346b85c8493bc0bb4b49a3d6b3330f"},
+  "sourceror": {:hex, :sourceror, "0.14.0", "b6b8552d0240400d66b6f107c1bab7ac1726e998efc797f178b7b517e928e314", [:mix], [], "hexpm", "809c71270ad48092d40bbe251a133e49ae229433ce103f762a2373b7a10a8d8b"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
+  "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
+}

package/crawler/test/integration_test.exs

@@ -0,0 +1,135 @@
+defmodule IntegrationTest do
+  use Crawler.TestCase, async: false
+
+  import Plug.Conn
+
+  @moduletag capture_log: true
+
+  test "integration", %{
+    bypass: bypass,
+    url: url,
+    path: path,
+    bypass2: bypass2,
+    url2: url2,
+    path2: path2
+  } do
+    linked_url1 = "#{url}/page1.html"
+    linked_url2 = "#{url}/dir/page2.html"
+    linked_url3 = "#{url2}/page3.html"
+
+    page1_raw = "<html><a href='#{linked_url2}'>2</a> <a href='#{linked_url3}'>3</a></html>"
+    page2_raw = "<html><a href='#{linked_url3}'>3</a></html>"
+    page3_raw = "<html><a href='dir/page4'>4</a> <a href='/dir/page4'>4</a></html>"
+
+    page4_raw =
+      "<html><head><script type='text/javascript' src='/javascript.js' /><link rel='stylesheet' href='../styles.css' /></head><a href='../page5.html'>5</a> <img src='../image1.png' /></html>"
+
+    page5_raw = "<html><a href='/page6'>6</a> <img src='/image2.png' /></html>"
+    css_raw = "img { url(image3.png); }"
+
+    Bypass.expect_once(
+      bypass,
+      "GET",
+      "/page1.html",
+      &(&1 |> put_resp_header("content-type", "text/html") |> resp(200, page1_raw))
+    )
+
+    Bypass.expect_once(
+      bypass,
+      "GET",
+      "/dir/page2.html",
+      &(&1 |> put_resp_header("content-type", "text/html") |> resp(200, page2_raw))
+    )
+
+    Bypass.expect_once(
+      bypass2,
+      "GET",
+      "/page3.html",
+      &(&1 |> put_resp_header("content-type", "text/html") |> resp(200, page3_raw))
+    )
+
+    Bypass.expect_once(
+      bypass2,
+      "GET",
+      "/dir/page4",
+      &(&1 |> put_resp_header("content-type", "text/html") |> resp(200, page4_raw))
+    )
+
+    Bypass.expect_once(
+      bypass2,
+      "GET",
+      "/page5.html",
+      &(&1 |> put_resp_header("content-type", "text/html") |> resp(200, page5_raw))
+    )
+
+    Bypass.expect_once(
+      bypass2,
+      "GET",
+      "/image1.png",
+      &(&1 |> put_resp_header("content-type", "image/png") |> resp(200, "png"))
+    )
+
+    Bypass.expect_once(
+      bypass2,
+      "GET",
+      "/image2.png",
+      &(&1 |> put_resp_header("content-type", "image/png") |> resp(200, "png"))
+    )
+
+    Bypass.expect_once(
+      bypass2,
+      "GET",
+      "/image3.png",
+      &(&1 |> put_resp_header("content-type", "image/png") |> resp(200, "png"))
+    )
+
+    Bypass.expect_once(
+      bypass2,
+      "GET",
+      "/styles.css",
+      &(&1 |> put_resp_header("content-type", "text/css") |> resp(200, css_raw))
+    )
+
+    Bypass.expect_once(
+      bypass2,
+      "GET",
+      "/javascript.js",
+      &(&1 |> put_resp_header("content-type", "application/javascript") |> resp(200, "js"))
+    )
+
+    Crawler.crawl(linked_url1,
+      save_to: tmp("integration"),
+      max_depths: 4,
+      assets: ["js", "css", "images"]
+    )
+
+    page1 =
+      "<html><a href='../#{path}/dir/page2.html'>2</a> <a href='../#{path2}/page3.html'>3</a></html>"
+
+    page2 = "<html><a href='../../#{path2}/page3.html'>3</a></html>"
+
+    page3 =
+      "<html><a href='../#{path2}/dir/page4/index.html'>4</a> <a href='../#{path2}/dir/page4/index.html'>4</a></html>"
+
+    page4 =
+      "<html><head><script type='text/javascript' src='../../../#{path2}/javascript.js' /><link rel='stylesheet' href='../../../#{path2}/styles.css' /></head><a href='../../../#{path2}/page5.html'>5</a> <img src='../../../#{path2}/image1.png' /></html>"
+
+    page5 =
+      "<html><a href='../#{path2}/page6/index.html'>6</a> <img src='../#{path2}/image2.png' /></html>"
+
+    css = "img { url(../#{path2}/image3.png); }"
+
+    wait(fn ->
+      assert {:ok, page1} == File.read(tmp("integration/#{path}", "page1.html"))
+      assert {:ok, page2} == File.read(tmp("integration/#{path}/dir", "page2.html"))
+      assert {:ok, page3} == File.read(tmp("integration/#{path2}", "page3.html"))
+      assert {:ok, page4} == File.read(tmp("integration/#{path2}/dir/page4", "index.html"))
+      assert {:ok, page5} == File.read(tmp("integration/#{path2}", "page5.html"))
+      assert {:ok, "png"} == File.read(tmp("integration/#{path2}", "image1.png"))
+      assert {:ok, "png"} == File.read(tmp("integration/#{path2}", "image2.png"))
+      assert {:ok, "png"} == File.read(tmp("integration/#{path2}", "image3.png"))
+      assert {:ok, css} == File.read(tmp("integration/#{path2}", "styles.css"))
+      assert {:ok, "js"} == File.read(tmp("integration/#{path2}", "javascript.js"))
+    end)
+  end
+end

package/crawler/test/lib/crawler/dispatcher/worker_test.exs

@@ -0,0 +1,7 @@
+defmodule Crawler.Dispatcher.WorkerTest do
+  use Crawler.TestCase, async: true
+
+  alias Crawler.Dispatcher.Worker
+
+  doctest Worker
+end

package/crawler/test/lib/crawler/dispatcher_test.exs

@@ -0,0 +1,5 @@
+defmodule Crawler.DispatcherTest do
+  use Crawler.TestCase, async: true
+
+  doctest Crawler.Dispatcher
+end

package/crawler/test/lib/crawler/fetcher/header_preparer_test.exs

@@ -0,0 +1,7 @@
+defmodule Crawler.Fetcher.HeaderPreparerTest do
+  use Crawler.TestCase, async: true
+
+  alias Crawler.Fetcher.HeaderPreparer
+
+  doctest HeaderPreparer
+end