powerdlz23 1.2.3 → 1.2.4

package/crawler/lib/crawler/options.ex

@@ -0,0 +1,127 @@
+defmodule Crawler.Options do
+  @moduledoc """
+  Options for the crawler.
+  """
+
+  alias Crawler.Mixfile
+  alias Crawler.Store
+
+  @assets []
+  @save_to nil
+  @workers 10
+  @interval 0
+  @max_depths 3
+  @max_pages :infinity
+  @timeout 5_000
+  @retries 2
+  @store nil
+  @force false
+  @scope nil
+  @user_agent "Crawler/#{Mixfile.project()[:version]} (https://github.com/fredwu/crawler)"
+  @url_filter Crawler.Fetcher.UrlFilter
+  @retrier Crawler.Fetcher.Retrier
+  @modifier Crawler.Fetcher.Modifier
+  @scraper Crawler.Scraper
+  @parser Crawler.Parser
+  @encode_uri false
+  @queue nil
+
+  @doc """
+  Assigns default option values.
+
+  ## Examples
+
+      iex> Options.assign_defaults(%{}) |> Map.has_key?(:depth)
+      true
+
+      iex> Options.assign_defaults(%{}) |> Map.get(:max_depths)
+      3
+
+      iex> Options.assign_defaults(%{max_depths: 4}) |> Map.get(:max_depths)
+      4
+  """
+  def assign_defaults(opts) do
+    Map.merge(
+      %{
+        depth: 0,
+        html_tag: "a",
+        assets: assets(),
+        save_to: save_to(),
+        workers: workers(),
+        interval: interval(),
+        max_depths: max_depths(),
+        max_pages: max_pages(),
+        timeout: timeout(),
+        retries: retries(),
+        store: store(),
+        force: force(),
+        scope: scope(),
+        user_agent: user_agent(),
+        url_filter: url_filter(),
+        retrier: retrier(),
+        modifier: modifier(),
+        scraper: scraper(),
+        parser: parser(),
+        encode_uri: encode_uri(),
+        queue: queue()
+      },
+      opts
+    )
+  end
+
+  @doc """
+  Takes the `url` argument and puts it in the `opts`.
+
+  The `opts` map gets passed around internally and eventually gets stored in
+  the registry.
+
+  ## Examples
+
+      iex> Options.assign_url(%{}, "http://options/")
+      %{url: "http://options/"}
+
+      iex> Options.assign_url(%{url: "http://example.com/"}, "http://options/")
+      %{url: "http://options/"}
+  """
+  def assign_url(%{encode_uri: true} = opts, url) do
+    Map.merge(opts, %{url: URI.encode(url)})
+  end
+
+  def assign_url(opts, url) do
+    Map.merge(opts, %{url: url})
+  end
+
+  def assign_scope(%{force: true, scope: nil} = opts) do
+    Map.merge(opts, %{scope: System.unique_integer()})
+  end
+
+  def assign_scope(opts), do: opts
+
+  def perform_default_actions(%{depth: 0} = opts) do
+    Store.ops_reset()
+
+    opts
+  end
+
+  def perform_default_actions(opts), do: opts
+
+  defp assets, do: Application.get_env(:crawler, :assets, @assets)
+  defp save_to, do: Application.get_env(:crawler, :save_to, @save_to)
+  defp workers, do: Application.get_env(:crawler, :workers, @workers)
+  defp interval, do: Application.get_env(:crawler, :interval, @interval)
+  defp max_depths, do: Application.get_env(:crawler, :max_depths, @max_depths)
+  defp max_pages, do: Application.get_env(:crawler, :max_pages, @max_pages)
+  defp timeout, do: Application.get_env(:crawler, :timeout, @timeout)
+  defp retries, do: Application.get_env(:crawler, :retries, @retries)
+  defp store, do: Application.get_env(:crawler, :store, @store)
+  defp force, do: Application.get_env(:crawler, :force, @force)
+  defp scope, do: Application.get_env(:crawler, :scope, @scope)
+  defp user_agent, do: Application.get_env(:crawler, :user_agent, @user_agent)
+  defp url_filter, do: Application.get_env(:crawler, :url_filter, @url_filter)
+  defp retrier, do: Application.get_env(:crawler, :retrier, @retrier)
+  defp modifier, do: Application.get_env(:crawler, :modifier, @modifier)
+  defp scraper, do: Application.get_env(:crawler, :scraper, @scraper)
+  defp parser, do: Application.get_env(:crawler, :parser, @parser)
+  defp encode_uri, do: Application.get_env(:crawler, :encode_uri, @encode_uri)
+  defp queue, do: Application.get_env(:crawler, :queue, @queue)
+end

package/crawler/lib/crawler/parser/css_parser.ex

@@ -0,0 +1,37 @@
+defmodule Crawler.Parser.CssParser do
+  @moduledoc """
+  Parses CSS files.
+  """
+
+  @url_unsafe_chars ")'\""
+
+  @doc """
+  Parses CSS files.
+
+  ## Examples
+
+      iex> CssParser.parse(
+      iex>   "img { url(http://hello.world) }"
+      iex> )
+      [{"link", [{"href", "http://hello.world"}], []}]
+
+      iex> CssParser.parse(
+      iex>   "@font-face { src: url('icons.ttf') format('truetype'); }"
+      iex> )
+      [{"link", [{"href", "icons.ttf"}], []}]
+
+      iex> CssParser.parse(
+      iex>   "@font-face { src: url('data:application/blah'); }"
+      iex> )
+      []
+  """
+  def parse(body) do
+    ~r{url\(['"]?((?!data:)[^#{@url_unsafe_chars}]+)['"]?\)}
+    |> Regex.scan(body, capture: :all_but_first)
+    |> Enum.map(&prep_css_element/1)
+  end
+
+  defp prep_css_element([link]) do
+    {"link", [{"href", link}], []}
+  end
+end

package/crawler/lib/crawler/parser/guarder.ex

@@ -0,0 +1,38 @@
+defmodule Crawler.Parser.Guarder do
+  @moduledoc """
+  Detects whether a page is parsable.
+  """
+
+  @doc """
+  Detects whether a page is parsable.
+
+  ## Examples
+
+      iex> Guarder.pass?(
+      iex>   %{html_tag: "link", content_type: "text/css"}
+      iex> )
+      true
+
+      iex> Guarder.pass?(
+      iex>   %{html_tag: "img", content_type: "text/css"}
+      iex> )
+      false
+
+      iex> Guarder.pass?(
+      iex>   %{html_tag: "link", content_type: "text/css"}
+      iex> )
+      true
+
+      iex> Guarder.pass?(
+      iex>   %{html_tag: "link", content_type: "image/png"}
+      iex> )
+      false
+  """
+  def pass?(opts) do
+    is_text_link?(opts[:html_tag]) && is_text_file?(opts[:content_type])
+  end
+
+  defp is_text_link?(html_tag), do: Enum.member?(["a", "link"], html_tag)
+
+  defp is_text_file?(content_type), do: String.starts_with?(content_type, "text")
+end

package/crawler/lib/crawler/parser/html_parser.ex

@@ -0,0 +1,41 @@
+defmodule Crawler.Parser.HtmlParser do
+  @moduledoc """
+  Parses HTML files.
+  """
+
+  @tag_selectors %{
+    "pages" => "a",
+    "js" => "script[type='text/javascript'][src]",
+    "css" => "link[rel='stylesheet']",
+    "images" => "img"
+  }
+
+  @doc """
+  Parses HTML files.
+
+  ## Examples
+
+      iex> HtmlParser.parse(
+      iex>   "<a href='http://hello.world'>Link</a>",
+      iex>   %{}
+      iex> )
+      [{"a", [{"href", "http://hello.world"}], ["Link"]}]
+
+      iex> HtmlParser.parse(
+      iex>   "<script type='text/javascript'>js</script>",
+      iex>   %{assets: ["js"]}
+      iex> )
+      []
+  """
+  def parse(body, opts) do
+    {:ok, document} = Floki.parse_document(body)
+    Floki.find(document, selectors(opts))
+  end
+
+  defp selectors(opts) do
+    @tag_selectors
+    |> Map.take(["pages"] ++ (opts[:assets] || []))
+    |> Map.values()
+    |> Enum.join(", ")
+  end
+end

package/crawler/lib/crawler/parser/link_parser/link_expander.ex

@@ -0,0 +1,32 @@
+defmodule Crawler.Parser.LinkParser.LinkExpander do
+  @moduledoc """
+  Expands a link into a full URL.
+  """
+
+  alias Crawler.Linker
+
+  @doc """
+  Expands a link into a full URL.
+
+  ## Examples
+
+      iex> LinkExpander.expand({"href", "http://hello.world"}, %{})
+      {"href", "http://hello.world"}
+
+      iex> LinkExpander.expand({"href", "page"}, %{referrer_url: "http://hello.world"})
+      {"link", "page", "href", "http://hello.world/page"}
+  """
+  def expand({_src, link} = element, opts) do
+    link
+    |> is_url?()
+    |> transform_link(element, opts)
+  end
+
+  defp is_url?(link), do: String.contains?(link, "://")
+
+  defp transform_link(true, element, _opts), do: element
+
+  defp transform_link(false, {src, link}, opts) do
+    {"link", link, src, Linker.url(opts[:referrer_url], link)}
+  end
+end

package/crawler/lib/crawler/parser/link_parser.ex

@@ -0,0 +1,50 @@
+defmodule Crawler.Parser.LinkParser do
+  @moduledoc """
+  Parses links and transforms them if necessary.
+  """
+
+  alias Crawler.Parser.LinkParser.LinkExpander
+
+  @tag_attr %{
+    "a" => "href",
+    "link" => "href",
+    "script" => "src",
+    "img" => "src"
+  }
+
+  @doc """
+  Parses links and transforms them if necessary.
+
+  ## Examples
+
+      iex> LinkParser.parse(
+      iex>   {"a", [{"hello", "world"}, {"href", "http://hello.world"}], []},
+      iex>   %{},
+      iex>   &Kernel.inspect(&1, Enum.into(&2, []))
+      iex> )
+      "{\\\"href\\\", \\\"http://hello.world\\\"}"
+
+      iex> LinkParser.parse(
+      iex>   {"img", [{"hello", "world"}, {"src", "http://hello.world"}], []},
+      iex>   %{},
+      iex>   &Kernel.inspect(&1, Enum.into(&2, []))
+      iex> )
+      "{\\\"src\\\", \\\"http://hello.world\\\"}"
+  """
+  def parse({tag, attrs, _}, opts, link_handler) do
+    src = @tag_attr[tag]
+
+    with {_tag, link} <- detect_link(src, attrs),
+         element <- LinkExpander.expand({src, link}, opts) do
+      opts = Map.merge(opts, %{html_tag: tag})
+
+      link_handler.(element, opts)
+    end
+  end
+
+  defp detect_link(src, attrs) do
+    Enum.find(attrs, fn attr ->
+      Kernel.match?({^src, _link}, attr)
+    end)
+  end
+end

package/crawler/lib/crawler/parser.ex

@@ -0,0 +1,122 @@
+defmodule Crawler.Parser do
+  @moduledoc """
+  Parses pages and calls a link handler to handle the detected links.
+  """
+
+  alias Crawler.Dispatcher
+  alias Crawler.Parser.CssParser
+  alias Crawler.Parser.Guarder
+  alias Crawler.Parser.HtmlParser
+  alias Crawler.Parser.LinkParser
+
+  require Logger
+
+  defmodule Spec do
+    @moduledoc """
+    Spec for defining a parser.
+    """
+
+    alias Crawler.Store.Page
+
+    @type url :: String.t()
+    @type body :: String.t()
+    @type opts :: map
+    @type page :: %Page{url: url, body: body, opts: opts}
+
+    @callback parse(page) :: {:ok, page}
+    @callback parse({:error, term}) :: :ok
+  end
+
+  @behaviour __MODULE__.Spec
+
+  @doc """
+  Parses the links and returns the page.
+
+  There are two hooks:
+
+  - `link_handler` is useful when a custom parser calls this default parser and
+  utilises a different link handler for processing links.
+  - `scraper` is useful for scraping content immediately as the parser parses
+  the page, alternatively you can simply access the crawled data
+  asynchronously, refer to the [README](https://github.com/fredwu/crawler#usage)
+
+  ## Examples
+
+      iex> {:ok, page} = Parser.parse(%Page{
+      iex>   body: "Body",
+      iex>   opts: %{scraper: Crawler.Scraper, html_tag: "a", content_type: "text/html"}
+      iex> })
+      iex> page.body
+      "Body"
+
+      iex> {:ok, page} = Parser.parse(%Page{
+      iex>   body: "<a href='http://parser/1'>Link</a>",
+      iex>   opts: %{scraper: Crawler.Scraper, html_tag: "a", content_type: "text/html"}
+      iex> })
+      iex> page.body
+      "<a href='http://parser/1'>Link</a>"
+
+      iex> {:ok, page} = Parser.parse(%Page{
+      iex>   body: "<a name='hello'>Link</a>",
+      iex>   opts: %{scraper: Crawler.Scraper, html_tag: "a", content_type: "text/html"}
+      iex> })
+      iex> page.body
+      "<a name='hello'>Link</a>"
+
+      iex> {:ok, page} = Parser.parse(%Page{
+      iex>   body: "<a href='http://parser/2' target='_blank'>Link</a>",
+      iex>   opts: %{scraper: Crawler.Scraper, html_tag: "a", content_type: "text/html"}
+      iex> })
+      iex> page.body
+      "<a href='http://parser/2' target='_blank'>Link</a>"
+
+      iex> {:ok, page} = Parser.parse(%Page{
+      iex>   body: "<a href='parser/2'>Link</a>",
+      iex>   opts: %{scraper: Crawler.Scraper, html_tag: "a", content_type: "text/html", referrer_url: "http://hello"}
+      iex> })
+      iex> page.body
+      "<a href='parser/2'>Link</a>"
+
+      iex> {:ok, page} = Parser.parse(%Page{
+      iex>   body: "<a href='../parser/2'>Link</a>",
+      iex>   opts: %{scraper: Crawler.Scraper, html_tag: "a", content_type: "text/html", referrer_url: "http://hello"}
+      iex> })
+      iex> page.body
+      "<a href='../parser/2'>Link</a>"
+
+      iex> {:ok, page} = Parser.parse(%Page{
+      iex>   body: image_file(),
+      iex>   opts: %{scraper: Crawler.Scraper, html_tag: "img", content_type: "image/png"}
+      iex> })
+      iex> page.body
+      "\#{image_file()}"
+  """
+  def parse(input)
+
+  def parse({:warn, reason}), do: Logger.debug(fn -> "#{inspect(reason)}" end)
+  def parse({:error, reason}), do: Logger.error(fn -> "#{inspect(reason)}" end)
+
+  def parse(%{body: body, opts: opts} = page) do
+    parse_links(body, opts, &Dispatcher.dispatch(&1, &2))
+
+    {:ok, _page} = opts[:scraper].scrape(page)
+  end
+
+  def parse_links(body, opts, link_handler) do
+    opts
+    |> Guarder.pass?()
+    |> do_parse_links(body, opts, link_handler)
+  end
+
+  defp do_parse_links(false, _body, _opts, _link_handler), do: []
+
+  defp do_parse_links(true, body, opts, link_handler) do
+    Enum.map(
+      parse_file(body, opts),
+      &LinkParser.parse(&1, opts, link_handler)
+    )
+  end
+
+  defp parse_file(body, %{content_type: "text/css"}), do: CssParser.parse(body)
+  defp parse_file(body, opts), do: HtmlParser.parse(body, opts)
+end

package/crawler/lib/crawler/queue_handler.ex

@@ -0,0 +1,45 @@
+defmodule Crawler.QueueHandler do
+  @moduledoc """
+  Handles the queueing of crawl requests.
+  """
+
+  alias Crawler.Dispatcher.Worker
+
+  @doc """
+  Enqueues a crawl request.
+
+  Also initialises the queue if it's not already initialised, this is necessary
+  so that consumer apps don't have to manually handle the queue initialisation.
+  """
+  def enqueue(opts) do
+    opts = init_queue(opts[:queue], opts)
+
+    OPQ.enqueue(opts[:queue], opts)
+
+    {:ok, opts}
+  end
+
+  defp init_queue(nil, opts) do
+    {:ok, _} =
+      DynamicSupervisor.start_child(
+        Crawler.QueueSupervisor,
+        {OPQ,
+         [
+           worker: Worker,
+           workers: opts[:workers],
+           interval: opts[:interval],
+           timeout: opts[:timeout]
+         ]}
+      )
+
+    pid =
+      Crawler.QueueSupervisor
+      |> Supervisor.which_children()
+      |> List.last()
+      |> elem(1)
+
+    Map.merge(opts, %{queue: pid})
+  end
+
+  defp init_queue(_queue, opts), do: opts
+end

package/crawler/lib/crawler/scraper.ex

@@ -0,0 +1,28 @@
+defmodule Crawler.Scraper do
+  @moduledoc """
+  A placeholder module that demonstrates the scraping interface.
+  """
+
+  defmodule Spec do
+    @moduledoc """
+    Spec for defining a scraper.
+    """
+
+    alias Crawler.Store.Page
+
+    @type url :: String.t()
+    @type body :: String.t()
+    @type opts :: map
+    @type page :: %Page{url: url, body: body, opts: opts}
+
+    @callback scrape(page) :: {:ok, page}
+  end
+
+  alias Crawler.Store.Page
+
+  @behaviour __MODULE__.Spec
+
+  @doc """
+  """
+  def scrape(%Page{url: _url, body: _body, opts: _opts} = page), do: {:ok, page}
+end

package/crawler/lib/crawler/snapper/dir_maker.ex

@@ -0,0 +1,45 @@
+defmodule Crawler.Snapper.DirMaker do
+  @moduledoc """
+  Makes a new (nested) folder according to the options provided.
+  """
+
+  alias Crawler.Linker
+  alias Crawler.Linker.PathFinder
+
+  @doc """
+  Makes a new (nested) folder according to the options provided.
+
+  ## Examples
+
+      iex> DirMaker.make_dir(
+      iex>   save_to: tmp("snapper/dir_creator"),
+      iex>   url: "http://hello-world.local"
+      iex> )
+      iex> |> Path.relative_to_cwd
+      "test/tmp/snapper/dir_creator/hello-world.local/index.html"
+  """
+  def make_dir(opts) do
+    opts[:url]
+    |> prep_filepath()
+    |> build_save_path(opts[:save_to])
+    |> make_save_path(opts[:save_to])
+  end
+
+  defp prep_filepath(url) do
+    url
+    |> Linker.offline_url(url)
+    |> PathFinder.find_path()
+  end
+
+  defp build_save_path(path, save_to) do
+    Path.join(save_to, path)
+  end
+
+  defp make_save_path(path, save_to) do
+    if File.exists?(save_to) do
+      File.mkdir_p(Path.dirname(path))
+    end
+
+    path
+  end
+end

package/crawler/lib/crawler/snapper/link_replacer.ex

@@ -0,0 +1,95 @@
+defmodule Crawler.Snapper.LinkReplacer do
+  @moduledoc """
+  Replaces links found in a page so they work offline.
+  """
+
+  alias Crawler.Linker
+  alias Crawler.Parser
+
+  @doc """
+  Replaces links found in a page so they work offline.
+
+  ## Examples
+
+      iex> LinkReplacer.replace_links(
+      iex>   "<a href='http://another.domain/page.html'></a>",
+      iex>   %{
+      iex>     url: "http://main.domain/dir/page",
+      iex>     depth: 1,
+      iex>     max_depths: 2,
+      iex>     html_tag: "a",
+      iex>     content_type: "text/html",
+      iex>   }
+      iex> )
+      {:ok, "<a href='../../../another.domain/page.html'></a>"}
+
+      iex> LinkReplacer.replace_links(
+      iex>   "<a href='http://another.domain/dir/page.html'></a>",
+      iex>   %{
+      iex>     url: "http://main.domain/page",
+      iex>     depth: 1,
+      iex>     max_depths: 2,
+      iex>     html_tag: "a",
+      iex>     content_type: "text/html",
+      iex>   }
+      iex> )
+      {:ok, "<a href='../../another.domain/dir/page.html'></a>"}
+
+      iex> LinkReplacer.replace_links(
+      iex>   "<a href='http://another.domain/dir/page'></a>",
+      iex>   %{
+      iex>     url: "http://main.domain/dir/page",
+      iex>     depth: 1,
+      iex>     max_depths: 2,
+      iex>     html_tag: "a",
+      iex>     content_type: "text/html",
+      iex>   }
+      iex> )
+      {:ok, "<a href='../../../another.domain/dir/page/index.html'></a>"}
+
+      iex> LinkReplacer.replace_links(
+      iex>   "<a href='/dir/page2.html'></a>",
+      iex>   %{
+      iex>     url: "http://main.domain/dir/page",
+      iex>     referrer_url: "http://main.domain/dir/page",
+      iex>     depth: 1,
+      iex>     max_depths: 2,
+      iex>     html_tag: "a",
+      iex>     content_type: "text/html",
+      iex>   }
+      iex> )
+      {:ok, "<a href='../../../main.domain/dir/page2.html'></a>"}
+  """
+  def replace_links(body, opts) do
+    new_body =
+      body
+      |> Parser.parse_links(opts, &get_link/2)
+      |> List.flatten()
+      |> Enum.reject(&(&1 == nil))
+      |> Enum.reduce(body, &modify_body(opts[:content_type], &2, opts[:url], &1))
+
+    {:ok, new_body}
+  end
+
+  defp get_link({_, url}, _opts), do: url
+  defp get_link({_, link, _, url}, _opts), do: [link, url]
+
+  defp modify_body(content_type, body, current_url, link) do
+    String.replace(
+      body,
+      regexes(content_type, link),
+      modify_link(current_url, link)
+    )
+  end
+
+  defp regexes(content_type, link) do
+    case content_type do
+      "text/css" -> ~r{((?!url)\(['"]?)#{link}(['"]?\))}
+      _ -> ~r{((?!src|href)=['"])#{link}(['"])}
+    end
+  end
+
+  defp modify_link(current_url, link) do
+    "\\1" <> Linker.offline_link(current_url, link) <> "\\2"
+  end
+end