powerdlz23 1.2.3 → 1.2.5

package/crawler/lib/crawler/fetcher.ex

@@ -0,0 +1,81 @@
+defmodule Crawler.Fetcher do
+  @moduledoc """
+  Fetches pages and perform tasks on them.
+  """
+
+  require Logger
+
+  alias Crawler.Fetcher.HeaderPreparer
+  alias Crawler.Fetcher.Policer
+  alias Crawler.Fetcher.Recorder
+  alias Crawler.Fetcher.Requester
+  alias Crawler.Snapper
+  alias Crawler.Store.Page
+
+  @doc """
+  Fetches a URL by:
+
+  - verifying whether the URL needs fetching through `Crawler.Fetcher.Policer.police/1`
+  - recording data for internal use through `Crawler.Fetcher.Recorder.record/1`
+  - fetching the URL
+  - performing retries upon failed fetches through `Crawler.Fetcher.Retrier.perform/2`
+  """
+  def fetch(opts) do
+    with {:ok, opts} <- Policer.police(opts),
+         {:ok, opts} <- Recorder.record(opts) do
+      opts[:retrier].perform(fn -> fetch_url(opts) end, opts)
+    end
+  end
+
+  defp fetch_url(opts) do
+    case Requester.make(opts) do
+      {:ok, %HTTPoison.Response{status_code: 200, body: body, headers: headers}} ->
+        fetch_url_200(body, headers, opts)
+
+      {:ok, %HTTPoison.Response{status_code: status_code}} ->
+        fetch_url_non_200(status_code, opts)
+
+      {:error, %HTTPoison.Error{reason: reason}} ->
+        fetch_url_failed(reason, opts)
+    end
+  end
+
+  defp fetch_url_200(body, headers, opts) do
+    with opts <- HeaderPreparer.prepare(headers, opts),
+         {:ok, _} <- Recorder.maybe_store_page(body, opts),
+         {:ok, opts} <- record_referrer_url(opts),
+         {:ok, _} <- snap_page(body, opts) do
+      Logger.debug("Fetched #{opts[:url]}")
+
+      %Page{url: opts[:url], body: body, opts: opts}
+    end
+  end
+
+  defp fetch_url_non_200(status_code, opts) do
+    msg = "Failed to fetch #{opts[:url]}, status code: #{status_code}"
+
+    Logger.debug(msg)
+
+    {:warn, msg}
+  end
+
+  defp fetch_url_failed(reason, opts) do
+    msg = "Failed to fetch #{opts[:url]}, reason: #{inspect(reason)}"
+
+    Logger.debug(msg)
+
+    {:warn, msg}
+  end
+
+  defp record_referrer_url(opts) do
+    {:ok, Map.put(opts, :referrer_url, opts[:url])}
+  end
+
+  defp snap_page(body, opts) do
+    if opts[:save_to] do
+      Snapper.snap(body, opts)
+    else
+      {:ok, ""}
+    end
+  end
+end

package/crawler/lib/crawler/http.ex

@@ -0,0 +1,7 @@
+defmodule Crawler.HTTP do
+  @moduledoc """
+  Custom HTTPoison base module for potential customisation.
+  """
+
+  use HTTPoison.Base
+end

package/crawler/lib/crawler/linker/path_builder.ex

@@ -0,0 +1,71 @@
+defmodule Crawler.Linker.PathBuilder do
+  @moduledoc """
+  Builds a path for a link (can be a URL itself or a relative link) based on
+  the input string which is a URL with or without its protocol.
+  """
+
+  alias Crawler.Linker.PathExpander
+  alias Crawler.Linker.PathFinder
+
+  @doc """
+  Builds a path for a link (can be a URL itself or a relative link) based on
+  the input string which is a URL with or without its protocol.
+
+  ## Examples
+
+      iex> PathBuilder.build_path(
+      iex>   "https://cool.beans:7777/dir/page1",
+      iex>   "https://hello.world:8888/remote/page2"
+      iex> )
+      "hello.world-8888/remote/page2"
+
+      iex> PathBuilder.build_path(
+      iex>   "https://cool.beans:7777/dir/page1",
+      iex>   "local/page2"
+      iex> )
+      "cool.beans-7777/dir/local/page2"
+
+      iex> PathBuilder.build_path(
+      iex>   "https://cool.beans:7777/dir/page1",
+      iex>   "/local/page2"
+      iex> )
+      "cool.beans-7777/local/page2"
+
+      iex> PathBuilder.build_path(
+      iex>   "https://cool.beans:7777/parent/dir/page1",
+      iex>   "../local/page2"
+      iex> )
+      "cool.beans-7777/parent/local/page2"
+
+      iex> PathBuilder.build_path(
+      iex>   "https://cool.beans:7777/parent/dir/page1",
+      iex>   "../../local/page2"
+      iex> )
+      "cool.beans-7777/local/page2"
+  """
+  def build_path(current_url, link, safe \\ true) do
+    current_url
+    |> base_path(link, safe)
+    |> build(link, safe)
+  end
+
+  defp base_path(url, "/" <> _link, safe), do: PathFinder.find_domain(url, safe)
+  defp base_path(url, _link, safe), do: PathFinder.find_base_path(url, safe)
+
+  defp build(path, link, safe) do
+    link
+    |> normalise(path)
+    |> PathFinder.find_path(safe)
+    |> PathExpander.expand_dot()
+  end
+
+  defp normalise(link, path) do
+    link
+    |> String.split("://", parts: 2)
+    |> Enum.count()
+    |> join_path(link, path)
+  end
+
+  defp join_path(2, link, _path), do: link
+  defp join_path(1, link, path), do: Path.join(path, link)
+end

package/crawler/lib/crawler/linker/path_expander.ex

@@ -0,0 +1,59 @@
+defmodule Crawler.Linker.PathExpander do
+  @moduledoc """
+  Expands the path by expanding any `.` and `..` characters.
+
+  See [this pull request](https://github.com/elixir-lang/elixir/pull/6486).
+  """
+
+  @doc """
+  Expands the path by expanding any `.` and `..` characters.
+
+  Note this function only expands any `.` and `..` characters within the given
+  path, and it does not take into account the absolute or relative nature of
+  the path itself, for that please use `expand/1` or `expand/2`.
+
+  ## Examples
+
+      ### Intended use case
+
+      iex> PathExpander.expand_dot("foo/bar/../baz")
+      "foo/baz"
+
+      iex> PathExpander.expand_dot("/foo/bar/../baz")
+      "/foo/baz"
+
+      ### Non-intended use cases are ignored
+
+      iex> PathExpander.expand_dot("foo/bar/./baz")
+      "foo/bar/baz"
+
+      iex> PathExpander.expand_dot("../foo/bar")
+      "foo/bar"
+  """
+  def expand_dot(<<"/", rest::binary>>),
+    do: "/" <> do_expand_dot(rest)
+
+  def expand_dot(path),
+    do: do_expand_dot(path)
+
+  defp do_expand_dot(path),
+    do: do_expand_dot(:binary.split(path, "/", [:global]), [])
+
+  defp do_expand_dot([".." | t], [_, _ | acc]),
+    do: do_expand_dot(t, acc)
+
+  defp do_expand_dot([".." | t], []),
+    do: do_expand_dot(t, [])
+
+  defp do_expand_dot(["." | t], acc),
+    do: do_expand_dot(t, acc)
+
+  defp do_expand_dot([h | t], acc),
+    do: do_expand_dot(t, ["/", h | acc])
+
+  defp do_expand_dot([], []),
+    do: ""
+
+  defp do_expand_dot([], ["/" | acc]),
+    do: IO.iodata_to_binary(:lists.reverse(acc))
+end

package/crawler/lib/crawler/linker/path_finder.ex

@@ -0,0 +1,106 @@
+defmodule Crawler.Linker.PathFinder do
+  @moduledoc """
+  Finds different components of a given URL, e.g. its domain name, directory
+  path, or full path.
+
+  The `safe` option in some the functions indicates whether the return value
+  should be transformed in order to be safely used as folder and file names.
+  """
+
+  @doc """
+  Finds the URL scheme (e.g. `https://`).
+
+  ## Examples
+
+      iex> PathFinder.find_scheme("http://hi.hello")
+      "http://"
+
+      iex> PathFinder.find_scheme("https://hi.hello:8888/")
+      "https://"
+  """
+  def find_scheme(url) do
+    (url
+     |> String.split("://", part: 2)
+     |> Kernel.hd()) <> "://"
+  end
+
+  @doc """
+  Finds the domain name with port number (e.g. `example.org:8080`).
+
+  ## Examples
+
+      iex> PathFinder.find_domain("http://hi.hello")
+      "hi.hello"
+
+      iex> PathFinder.find_domain("https://hi.hello:8888/world")
+      "hi.hello-8888"
+
+      iex> PathFinder.find_domain("https://hi.hello:8888/world", false)
+      "hi.hello:8888"
+  """
+  def find_domain(url, safe \\ true) do
+    url
+    |> find_path(safe)
+    |> String.split("/", parts: 2)
+    |> Kernel.hd()
+  end
+
+  @doc """
+  Finds the base path of a given page.
+
+  ## Examples
+
+      iex> PathFinder.find_base_path("http://hi.hello")
+      "hi.hello"
+
+      iex> PathFinder.find_base_path("https://hi.hello:8888/dir/world")
+      "hi.hello-8888/dir"
+
+      iex> PathFinder.find_base_path("https://hi.hello:8888/dir/world", false)
+      "hi.hello:8888/dir"
+  """
+  def find_base_path(url, safe \\ true) do
+    url
+    |> find_path(safe)
+    |> String.split("/")
+    |> base_path()
+  end
+
+  defp base_path([path]), do: path
+
+  defp base_path(list) do
+    [_head | tail] = Enum.reverse(list)
+
+    tail
+    |> Enum.reverse()
+    |> Path.join()
+  end
+
+  @doc """
+  Finds the full path of a given page.
+
+  ## Examples
+
+      iex> PathFinder.find_path("http://hi.hello")
+      "hi.hello"
+
+      iex> PathFinder.find_path("https://hi.hello:8888/world")
+      "hi.hello-8888/world"
+
+      iex> PathFinder.find_path("https://hi.hello:8888/world", false)
+      "hi.hello:8888/world"
+  """
+  def find_path(url, safe \\ true)
+
+  def find_path(url, false) do
+    url
+    |> String.split("://", parts: 2)
+    |> Enum.at(-1)
+  end
+
+  def find_path(url, true) do
+    url
+    |> find_path(false)
+    |> String.replace(":", "-")
+  end
+end

package/crawler/lib/crawler/linker/path_offliner.ex

@@ -0,0 +1,59 @@
+defmodule Crawler.Linker.PathOffliner do
+  @moduledoc """
+  Transforms a link to be storable and linkable offline.
+  """
+
+  alias Crawler.Linker.PathFinder
+
+  @doc """
+  Transforms a given link so that it can be stored and linked to by other pages.
+
+  When a page does not have a file extension (e.g. html) it is treated as the
+  index page for a directory, therefore `index.html` is appended to the link.
+
+  ## Examples
+
+      iex> PathOffliner.transform("http://hello.world")
+      "http://hello.world/index.html"
+
+      iex> PathOffliner.transform("hello.world")
+      "hello.world/index.html"
+
+      iex> PathOffliner.transform("hello.world/")
+      "hello.world/index.html"
+
+      iex> PathOffliner.transform("hello/world")
+      "hello/world/index.html"
+
+      iex> PathOffliner.transform("hello/world.html")
+      "hello/world.html"
+  """
+  def transform(link) do
+    link
+    |> PathFinder.find_path()
+    |> String.split("/", trim: true)
+    |> Enum.count()
+    |> last_segment(link)
+  end
+
+  defp last_segment(1, link) do
+    transform_link(false, link)
+  end
+
+  defp last_segment(_count, link) do
+    link
+    |> String.split("/")
+    |> Enum.take(-1)
+    |> Kernel.hd()
+    |> transform_segment(link)
+  end
+
+  defp transform_segment(segment, link) do
+    segment
+    |> String.contains?(".")
+    |> transform_link(link)
+  end
+
+  defp transform_link(true, link), do: link
+  defp transform_link(false, link), do: Path.join(link, "index.html")
+end

package/crawler/lib/crawler/linker/path_prefixer.ex

@@ -0,0 +1,46 @@
+defmodule Crawler.Linker.PathPrefixer do
+  @moduledoc """
+  Returns prefixes (`../`s) according to the given URL's structure.
+  """
+
+  alias Crawler.Linker.PathFinder
+  alias Crawler.Linker.PathOffliner
+
+  @doc """
+  Returns prefixes (`../`s) according to the given URL's structure.
+
+  ## Examples
+
+      iex> PathPrefixer.prefix("https://hello.world/")
+      "../"
+
+      iex> PathPrefixer.prefix("https://hello.world/page.html")
+      "../"
+
+      iex> PathPrefixer.prefix("https://hello.world/page")
+      "../../"
+
+      iex> PathPrefixer.prefix("https://hello.world/dir/page.html")
+      "../../"
+
+      iex> PathPrefixer.prefix("https://hello.world/dir/page")
+      "../../../"
+  """
+  def prefix(current_url) do
+    current_url
+    |> PathFinder.find_path()
+    |> PathOffliner.transform()
+    |> count_depth()
+    |> make_prefix()
+  end
+
+  defp count_depth(string, token \\ "/") do
+    (string
+     |> String.split(token)
+     |> Enum.count()) - 1
+  end
+
+  defp make_prefix(depth) do
+    String.duplicate("../", depth)
+  end
+end

package/crawler/lib/crawler/linker.ex

@@ -0,0 +1,173 @@
+defmodule Crawler.Linker do
+  @moduledoc """
+  A set of high level functions for making online and offline URLs and links.
+  """
+
+  alias Crawler.Linker.PathBuilder
+  alias Crawler.Linker.PathFinder
+  alias Crawler.Linker.PathOffliner
+  alias Crawler.Linker.PathPrefixer
+
+  @doc """
+  Given the `current_link`, it works out what the offline URL should be for
+  `link`.
+
+  ## Examples
+
+      iex> Linker.offline_url(
+      iex>   "http://hello.world/dir/page",
+      iex>   "page1"
+      iex> )
+      "http://hello.world/dir/page1/index.html"
+
+      iex> Linker.offline_url(
+      iex>   "http://hello.world/dir/page",
+      iex>   "page1.html"
+      iex> )
+      "http://hello.world/dir/page1.html"
+
+      iex> Linker.offline_url(
+      iex>   "http://hello.world/dir/page",
+      iex>   "../page1"
+      iex> )
+      "http://hello.world/page1/index.html"
+
+      iex> Linker.offline_url(
+      iex>   "http://hello.world/dir/page",
+      iex>   "../page1.html"
+      iex> )
+      "http://hello.world/page1.html"
+
+      iex> Linker.offline_url(
+      iex>   "http://hello.world/dir/page",
+      iex>   "http://thank.you/page1"
+      iex> )
+      "http://thank.you/page1/index.html"
+
+      iex> Linker.offline_url(
+      iex>   "http://hello.world/dir/page",
+      iex>   "http://thank.you/page1.html"
+      iex> )
+      "http://thank.you/page1.html"
+
+      iex> Linker.offline_url(
+      iex>   "http://hello.world/dir/page",
+      iex>   "http://thank.you/"
+      iex> )
+      "http://thank.you/index.html"
+  """
+  def offline_url(current_url, link) do
+    current_url
+    |> url(link)
+    |> PathOffliner.transform()
+  end
+
+  @doc """
+  Given the `current_link`, it works out what the relative
+  offline link should be for `link`.
+
+  ## Examples
+
+      iex> Linker.offline_link(
+      iex>   "http://hello.world/dir/page",
+      iex>   "page1"
+      iex> )
+      "../../../hello.world/dir/page1/index.html"
+
+      iex> Linker.offline_link(
+      iex>   "http://hello.world/dir/page",
+      iex>   "page1.html"
+      iex> )
+      "../../../hello.world/dir/page1.html"
+
+      iex> Linker.offline_link(
+      iex>   "http://hello.world/dir/page",
+      iex>   "../page1"
+      iex> )
+      "../../../hello.world/page1/index.html"
+
+      iex> Linker.offline_link(
+      iex>   "http://hello.world/dir/page",
+      iex>   "../page1.html"
+      iex> )
+      "../../../hello.world/page1.html"
+
+      iex> Linker.offline_link(
+      iex>   "http://hello.world/dir/page",
+      iex>   "http://thank.you/page1"
+      iex> )
+      "../../../thank.you/page1/index.html"
+
+      iex> Linker.offline_link(
+      iex>   "http://hello.world/dir/page",
+      iex>   "http://thank.you/page1.html"
+      iex> )
+      "../../../thank.you/page1.html"
+  """
+  def offline_link(current_url, link) do
+    current_url
+    |> link(link)
+    |> PathOffliner.transform()
+  end
+
+  @doc """
+  Given the `current_link`, it works out what the URL should be for `link`.
+
+  ## Examples
+
+      iex> Linker.url(
+      iex>   "http://another.domain:8888/page",
+      iex>   "/dir/page2"
+      iex> )
+      "http://another.domain:8888/dir/page2"
+
+      iex> Linker.url(
+      iex>   "http://another.domain:8888/parent/page",
+      iex>   "dir/page2"
+      iex> )
+      "http://another.domain:8888/parent/dir/page2"
+  """
+  def url(current_url, link) do
+    Path.join(
+      PathFinder.find_scheme(current_url),
+      PathBuilder.build_path(current_url, link, false)
+    )
+  end
+
+  @doc """
+  Given the `current_link`, it works out what the relative link should be for
+  `link`.
+
+  ## Examples
+
+      iex> Linker.link(
+      iex>   "http://another.domain/page.html",
+      iex>   "/dir/page2"
+      iex> )
+      "../another.domain/dir/page2"
+
+      iex> Linker.link(
+      iex>   "http://another.domain/page",
+      iex>   "/dir/page2"
+      iex> )
+      "../../another.domain/dir/page2"
+
+      iex> Linker.link(
+      iex>   "http://another.domain/parent/page",
+      iex>   "dir/page2"
+      iex> )
+      "../../../another.domain/parent/dir/page2"
+
+      iex> Linker.link(
+      iex>   "http://another.domain/parent/page",
+      iex>   "../dir/page2"
+      iex> )
+      "../../../another.domain/dir/page2"
+  """
+  def link(current_url, link) do
+    Path.join(
+      PathPrefixer.prefix(current_url),
+      PathBuilder.build_path(current_url, link)
+    )
+  end
+end