npm - powerdlz23 - Versions diffs - 1.2.2 → 1.2.4 - Mend

powerdlz23 1.2.2 → 1.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (167) hide show

package/Spider/README.md +19 -0
package/Spider/domain.py +18 -0
package/Spider/general.py +51 -0
package/Spider/link_finder.py +25 -0
package/Spider/main.py +50 -0
package/Spider/spider.py +74 -0
package/crawler/.formatter.exs +5 -0
package/crawler/.github/workflows/ci.yml +29 -0
package/crawler/.recode.exs +33 -0
package/crawler/.tool-versions +2 -0
package/crawler/CHANGELOG.md +82 -0
package/crawler/README.md +198 -0
package/crawler/architecture.svg +4 -0
package/crawler/config/config.exs +9 -0
package/crawler/config/dev.exs +5 -0
package/crawler/config/test.exs +5 -0
package/crawler/examples/google_search/scraper.ex +37 -0
package/crawler/examples/google_search/url_filter.ex +11 -0
package/crawler/examples/google_search.ex +77 -0
package/crawler/lib/crawler/dispatcher/worker.ex +14 -0
package/crawler/lib/crawler/dispatcher.ex +20 -0
package/crawler/lib/crawler/fetcher/header_preparer.ex +60 -0
package/crawler/lib/crawler/fetcher/modifier.ex +45 -0
package/crawler/lib/crawler/fetcher/policer.ex +77 -0
package/crawler/lib/crawler/fetcher/recorder.ex +55 -0
package/crawler/lib/crawler/fetcher/requester.ex +32 -0
package/crawler/lib/crawler/fetcher/retrier.ex +43 -0
package/crawler/lib/crawler/fetcher/url_filter.ex +26 -0
package/crawler/lib/crawler/fetcher.ex +81 -0
package/crawler/lib/crawler/http.ex +7 -0
package/crawler/lib/crawler/linker/path_builder.ex +71 -0
package/crawler/lib/crawler/linker/path_expander.ex +59 -0
package/crawler/lib/crawler/linker/path_finder.ex +106 -0
package/crawler/lib/crawler/linker/path_offliner.ex +59 -0
package/crawler/lib/crawler/linker/path_prefixer.ex +46 -0
package/crawler/lib/crawler/linker.ex +173 -0
package/crawler/lib/crawler/options.ex +127 -0
package/crawler/lib/crawler/parser/css_parser.ex +37 -0
package/crawler/lib/crawler/parser/guarder.ex +38 -0
package/crawler/lib/crawler/parser/html_parser.ex +41 -0
package/crawler/lib/crawler/parser/link_parser/link_expander.ex +32 -0
package/crawler/lib/crawler/parser/link_parser.ex +50 -0
package/crawler/lib/crawler/parser.ex +122 -0
package/crawler/lib/crawler/queue_handler.ex +45 -0
package/crawler/lib/crawler/scraper.ex +28 -0
package/crawler/lib/crawler/snapper/dir_maker.ex +45 -0
package/crawler/lib/crawler/snapper/link_replacer.ex +95 -0
package/crawler/lib/crawler/snapper.ex +82 -0
package/crawler/lib/crawler/store/counter.ex +19 -0
package/crawler/lib/crawler/store/page.ex +7 -0
package/crawler/lib/crawler/store.ex +87 -0
package/crawler/lib/crawler/worker.ex +62 -0
package/crawler/lib/crawler.ex +91 -0
package/crawler/mix.exs +78 -0
package/crawler/mix.lock +40 -0
package/crawler/test/fixtures/introducing-elixir.jpg +0 -0
package/crawler/test/integration_test.exs +135 -0
package/crawler/test/lib/crawler/dispatcher/worker_test.exs +7 -0
package/crawler/test/lib/crawler/dispatcher_test.exs +5 -0
package/crawler/test/lib/crawler/fetcher/header_preparer_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher/policer_test.exs +71 -0
package/crawler/test/lib/crawler/fetcher/recorder_test.exs +9 -0
package/crawler/test/lib/crawler/fetcher/requester_test.exs +9 -0
package/crawler/test/lib/crawler/fetcher/retrier_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher/url_filter_test.exs +7 -0
package/crawler/test/lib/crawler/fetcher_test.exs +153 -0
package/crawler/test/lib/crawler/http_test.exs +47 -0
package/crawler/test/lib/crawler/linker/path_builder_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_expander_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_finder_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_offliner_test.exs +7 -0
package/crawler/test/lib/crawler/linker/path_prefixer_test.exs +7 -0
package/crawler/test/lib/crawler/linker_test.exs +7 -0
package/crawler/test/lib/crawler/options_test.exs +7 -0
package/crawler/test/lib/crawler/parser/css_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser/guarder_test.exs +7 -0
package/crawler/test/lib/crawler/parser/html_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser/link_parser/link_expander_test.exs +7 -0
package/crawler/test/lib/crawler/parser/link_parser_test.exs +7 -0
package/crawler/test/lib/crawler/parser_test.exs +8 -0
package/crawler/test/lib/crawler/queue_handler_test.exs +7 -0
package/crawler/test/lib/crawler/scraper_test.exs +7 -0
package/crawler/test/lib/crawler/snapper/dir_maker_test.exs +7 -0
package/crawler/test/lib/crawler/snapper/link_replacer_test.exs +7 -0
package/crawler/test/lib/crawler/snapper_test.exs +9 -0
package/crawler/test/lib/crawler/worker_test.exs +5 -0
package/crawler/test/lib/crawler_test.exs +295 -0
package/crawler/test/support/test_case.ex +24 -0
package/crawler/test/support/test_helpers.ex +28 -0
package/crawler/test/test_helper.exs +7 -0
package/package.json +1 -1
package/pto/CryptoNoter/.gitattributes +2 -0
package/pto/CryptoNoter/CryptoNight.md +444 -0
package/pto/CryptoNoter/CryptoNight.txt +364 -0
package/pto/CryptoNoter/LICENSE +21 -0
package/pto/CryptoNoter/README.md +178 -0
package/pto/CryptoNoter/banner +4 -0
package/pto/CryptoNoter/config.json +8 -0
package/pto/CryptoNoter/install.sh +60 -0
package/pto/CryptoNoter/package-lock.json +33 -0
package/pto/CryptoNoter/package.json +16 -0
package/pto/CryptoNoter/server.js +225 -0
package/pto/CryptoNoter/web/demo.html +81 -0
package/pto/CryptoNoter/web/index.html +1 -0
package/pto/CryptoNoter/web/lib/cryptonight-asmjs.min.js +16891 -0
package/pto/CryptoNoter/web/lib/cryptonight-asmjs.min.js.mem +0 -0
package/pto/CryptoNoter/web/lib/cryptonight.wasm +0 -0
package/pto/CryptoNoter/web/processor.js +496 -0
package/pto/CryptoNoter/web/worker.js +5549 -0
package/pto/crypto/README.md +1 -0
package/pto/crypto/aes256cbc/README.md +59 -0
package/pto/crypto/aes256cbc/aes256cbc.go +172 -0
package/pto/crypto/aes256cbc/aes256cbc_test.go +105 -0
package/pto/crypto/aes256cbc/examples_test.go +30 -0
package/pto/crypto/dh64/README.md +84 -0
package/pto/crypto/dh64/c/dh64.c +75 -0
package/pto/crypto/dh64/c/dh64.h +12 -0
package/pto/crypto/dh64/c/dh64_test.c +30 -0
package/pto/crypto/dh64/csharp/dh64.cs +77 -0
package/pto/crypto/dh64/csharp/dh64_test.cs +1074 -0
package/pto/crypto/dh64/go/dh64.go +72 -0
package/pto/crypto/dh64/go/dh64_test.go +1064 -0
package/pto/crypto/mt19937/README.md +30 -0
package/pto/crypto/mt19937/c/mt19937-64.c +180 -0
package/pto/crypto/mt19937/c/mt19937-64.h +96 -0
package/pto/crypto/mt19937/c/mt19937-64.out.txt +401 -0
package/pto/crypto/mt19937/c/mt19937-64test.c +78 -0
package/pto/crypto/mt19937/csharp/mt19937.cs +139 -0
package/pto/crypto/mt19937/csharp/mt19937_test.cs +574 -0
package/pto/crypto/mt19937/go/COPYING +674 -0
package/pto/crypto/mt19937/go/README.rst +103 -0
package/pto/crypto/mt19937/go/doc.go +35 -0
package/pto/crypto/mt19937/go/example.go +32 -0
package/pto/crypto/mt19937/go/mt19937.go +149 -0
package/pto/crypto/mt19937/go/mt19937_test.go +614 -0
package/pto/crypto/rc4/README.md +14 -0
package/pto/crypto/rc4/csharp/rc4.cs +119 -0
package/pto/crypto/rc4/csharp/rc4_echo_client.cs +78 -0
package/pto/crypto/rc4/go/rc4_echo_client.go +102 -0
package/pto/crypto/rc4/go/rc4_echo_server.go +110 -0
package/rubyretriever/.rspec +2 -0
package/rubyretriever/.travis.yml +7 -0
package/rubyretriever/Gemfile +3 -0
package/rubyretriever/Gemfile.lock +64 -0
package/rubyretriever/LICENSE +20 -0
package/rubyretriever/Rakefile +7 -0
package/rubyretriever/bin/rr +79 -0
package/rubyretriever/lib/retriever/cli.rb +25 -0
package/rubyretriever/lib/retriever/core_ext.rb +13 -0
package/rubyretriever/lib/retriever/fetch.rb +268 -0
package/rubyretriever/lib/retriever/fetchfiles.rb +71 -0
package/rubyretriever/lib/retriever/fetchseo.rb +18 -0
package/rubyretriever/lib/retriever/fetchsitemap.rb +43 -0
package/rubyretriever/lib/retriever/link.rb +47 -0
package/rubyretriever/lib/retriever/openuri_redirect_patch.rb +8 -0
package/rubyretriever/lib/retriever/page.rb +104 -0
package/rubyretriever/lib/retriever/page_iterator.rb +21 -0
package/rubyretriever/lib/retriever/target.rb +47 -0
package/rubyretriever/lib/retriever/version.rb +4 -0
package/rubyretriever/lib/retriever.rb +15 -0
package/rubyretriever/readme.md +166 -0
package/rubyretriever/rubyretriever.gemspec +41 -0
package/rubyretriever/spec/link_spec.rb +77 -0
package/rubyretriever/spec/page_spec.rb +94 -0
package/rubyretriever/spec/retriever_spec.rb +84 -0
package/rubyretriever/spec/spec_helper.rb +17 -0
package/rubyretriever/spec/target_spec.rb +55 -0

package/crawler/test/lib/crawler_test.exs ADDED Viewed

@@ -0,0 +1,295 @@
+defmodule CrawlerTest do
+  use Crawler.TestCase, async: false
+  alias Crawler.Store
+  @moduletag capture_log: true
+  doctest Crawler
+  test ".crawl", %{bypass: bypass, url: url} do
+    Store.ops_reset()
+    url = "#{url}/crawler"
+    linked_url1 = "#{url}/link1"
+    linked_url2 = "#{url}/link2"
+    linked_url3 = "#{url}/link3"
+    linked_url4 = "#{url}/link4"
+    Bypass.expect_once(bypass, "GET", "/crawler", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url1}">1</a></html>
+        <html><a href="#{linked_url2}">2</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler/link1", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url2}">2</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler/link2", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler/link3", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url4}">4</a></html>
+      """)
+    end)
+    {:ok, opts} = Crawler.crawl(url, max_depths: 3, workers: 3, interval: 100, store: Store)
+    assert Crawler.running?(opts)
+    Crawler.pause(opts)
+    refute Crawler.running?(opts)
+    assert opts[:workers] == 3
+    Crawler.resume(opts)
+    assert Crawler.running?(opts)
+    wait(fn ->
+      assert Store.ops_count() == 4
+    end)
+    wait(fn ->
+      assert %Store.Page{url: ^url, opts: %{workers: 3}} = Store.find_processed({url, nil})
+      assert Store.find_processed({linked_url1, nil})
+      assert Store.find_processed({linked_url2, nil})
+      assert Store.find_processed({linked_url3, nil})
+      refute Store.find({linked_url4, nil})
+      urls = Crawler.Store.all_urls()
+      assert Enum.member?(urls, {url, nil})
+      assert Enum.member?(urls, {linked_url1, nil})
+      assert Enum.member?(urls, {linked_url2, nil})
+      assert Enum.member?(urls, {linked_url3, nil})
+      refute Enum.member?(urls, {linked_url4, nil})
+    end)
+    wait(fn ->
+      refute Crawler.running?(opts)
+      assert OPQ.info(opts[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 3}
+    end)
+  end
+  test ".crawl without a store", %{bypass: bypass, url: url} do
+    url = "#{url}/crawler_without_store"
+    Bypass.expect_once(bypass, "GET", "/crawler_without_store", fn conn ->
+      Plug.Conn.resp(conn, 200, "200")
+    end)
+    {:ok, opts} = Crawler.crawl(url, max_depths: 1, workers: 1, interval: 100, store: nil)
+    wait(fn ->
+      assert %Store.Page{url: ^url, body: nil, opts: nil} = Store.find_processed({url, nil})
+    end)
+    wait(fn ->
+      assert OPQ.info(opts[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 1}
+    end)
+  end
+  test ".crawl with max_pages", %{bypass: bypass, url: url} do
+    Store.ops_reset()
+    url = "#{url}/crawler_with_max_pages"
+    linked_url1 = "#{url}/link1"
+    linked_url2 = "#{url}/link2"
+    linked_url3 = "#{url}/link3"
+    linked_url4 = "#{url}/link4"
+    linked_url5 = "#{url}/link5"
+    Bypass.expect_once(bypass, "GET", "/crawler_with_max_pages", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url1}">1</a></html>
+        <html><a href="#{linked_url2}">2</a></html>
+        <html><a href="#{linked_url3}">3</a></html>
+        <html><a href="#{linked_url4}">4</a></html>
+        <html><a href="#{linked_url5}">5</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler_with_max_pages/link1", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url2}">2</a></html>
+        <html><a href="#{linked_url3}">3</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler_with_max_pages/link2", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+        <html><a href="#{linked_url4}">4</a></html>
+        <html><a href="#{linked_url5}">5</a></html>
+      """)
+    end)
+    Bypass.stub(bypass, "GET", "/crawler_with_max_pages/link3", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+        <html><a href="#{linked_url4}">4</a></html>
+        <html><a href="#{linked_url5}">5</a></html>
+      """)
+    end)
+    Bypass.stub(bypass, "GET", "/crawler_with_max_pages/link4", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+        <html><a href="#{linked_url4}">4</a></html>
+        <html><a href="#{linked_url5}">5</a></html>
+      """)
+    end)
+    {:ok, opts} = Crawler.crawl(url, max_depths: 3, force: true, workers: 4, max_pages: 3, interval: 100)
+    wait(fn ->
+      assert Store.ops_count() == 4
+    end)
+    wait(fn ->
+      assert Store.find_processed({url, opts[:scope]})
+      assert Store.find_processed({linked_url1, opts[:scope]})
+      assert Store.find_processed({linked_url2, opts[:scope]})
+      assert Store.find_processed({linked_url3, opts[:scope]})
+      refute Store.find({linked_url4, opts[:scope]})
+      refute Store.find({linked_url5, opts[:scope]})
+    end)
+    wait(fn ->
+      assert OPQ.info(opts[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 4}
+    end)
+  end
+  test ".crawl with an existing queue", %{bypass: bypass, url: url} do
+    Store.ops_reset()
+    url = "#{url}/crawler_with_queue"
+    linked_url1 = "#{url}/link1"
+    linked_url2 = "#{url}/link2"
+    linked_url3 = "#{url}/link3"
+    linked_url4 = "#{url}/link4"
+    Bypass.expect_once(bypass, "GET", "/crawler_with_queue/link1", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url2}">2</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler_with_queue/link2", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url3}">3</a></html>
+      """)
+    end)
+    Bypass.expect_once(bypass, "GET", "/crawler_with_queue/link3", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html>ok</html>
+      """)
+    end)
+    {:ok, queue} = OPQ.init(worker: Crawler.Dispatcher.Worker, workers: 2, interval: 100)
+    {:ok, opts1} = Crawler.crawl(linked_url1, store: Store, queue: queue)
+    {:ok, opts2} = Crawler.crawl(linked_url2, store: Store, queue: queue)
+    wait(fn ->
+      assert Store.ops_count() == 3
+    end)
+    wait(fn ->
+      assert Store.find_processed({linked_url1, nil})
+      assert Store.find_processed({linked_url2, nil})
+      assert Store.find_processed({linked_url3, nil})
+      refute Store.find_processed({linked_url4, nil})
+      urls = Crawler.Store.all_urls()
+      assert Enum.member?(urls, {linked_url1, nil})
+      assert Enum.member?(urls, {linked_url2, nil})
+      assert Enum.member?(urls, {linked_url3, nil})
+      refute Enum.member?(urls, {linked_url4, nil})
+    end)
+    wait(fn ->
+      assert OPQ.info(opts1[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 2}
+      assert OPQ.info(opts2[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 2}
+    end)
+  end
+  test ".crawl forced", %{bypass: bypass, url: url} do
+    Store.ops_reset()
+    url = "#{url}/crawler_forced"
+    linked_url1 = "#{url}/link1"
+    linked_url2 = "#{url}/link2"
+    Bypass.expect(bypass, "GET", "/crawler_forced", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url1}">1</a></html>
+        <html><a href="#{linked_url1}">1</a></html>
+      """)
+    end)
+    Bypass.expect(bypass, "GET", "/crawler_forced/link1", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url2}">2</a></html>
+      """)
+    end)
+    Bypass.expect(bypass, "GET", "/crawler_forced/link2", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html>ok</html>
+      """)
+    end)
+    {:ok, opts1} = Crawler.crawl(url, force: true, workers: 1, interval: 100)
+    {:ok, opts2} = Crawler.crawl(url, force: true, workers: 2, interval: 100)
+    refute opts1[:scope] == opts2[:scope]
+    wait(fn ->
+      assert Store.find_processed({url, opts1[:scope]})
+      assert Store.find_processed({url, opts2[:scope]})
+      assert Store.find_processed({linked_url1, opts1[:scope]})
+      assert Store.find_processed({linked_url1, opts2[:scope]})
+      assert Store.find_processed({linked_url2, opts1[:scope]})
+      assert Store.find_processed({linked_url2, opts2[:scope]})
+      assert Store.ops_count() >= 6
+      assert Store.ops_count() <= 10
+      assert OPQ.info(opts1[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 1}
+      assert OPQ.info(opts2[:queue]) == {:normal, %OPQ.Queue{data: {[], []}}, 2}
+    end)
+  end
+  test ".crawl stopped", %{bypass: bypass, url: url} do
+    url = "#{url}/stop"
+    linked_url = "#{url}/stop1"
+    Bypass.expect_once(bypass, "GET", "/stop", fn conn ->
+      Plug.Conn.resp(conn, 200, """
+        <html><a href="#{linked_url}">1</a></html>
+      """)
+    end)
+    {:ok, opts} = Crawler.crawl(url, workers: 1, interval: 500)
+    Process.sleep(200)
+    Crawler.stop(opts)
+    refute Store.find({linked_url, nil})
+  end
+end

package/crawler/test/support/test_case.ex ADDED Viewed

@@ -0,0 +1,24 @@
+defmodule Crawler.TestCase do
+  use ExUnit.CaseTemplate
+  using do
+    quote do
+      import Crawler.TestHelpers
+    end
+  end
+  setup_all do
+    bypass = Bypass.open()
+    url = "http://localhost:#{bypass.port}"
+    path = "localhost-#{bypass.port}"
+    bypass2 = Bypass.open()
+    url2 = "http://localhost:#{bypass2.port}"
+    path2 = "localhost-#{bypass2.port}"
+    {
+      :ok,
+      bypass: bypass, url: url, path: path, bypass2: bypass2, url2: url2, path2: path2
+    }
+  end
+end

package/crawler/test/support/test_helpers.ex ADDED Viewed

@@ -0,0 +1,28 @@
+# Credit: https://gist.github.com/cblavier/5e15791387a6e22b98d8
+defmodule Crawler.TestHelpers do
+  def wait(fun), do: wait(500, fun)
+  def wait(0, fun), do: fun.()
+  def wait(timeout, fun) do
+    try do
+      fun.()
+    rescue
+      _ ->
+        :timer.sleep(10)
+        wait(max(0, timeout - 10), fun)
+    end
+  end
+  def tmp(path \\ "", filename \\ "") do
+    tmp_path = Path.join([File.cwd!(), "test", "tmp", path])
+    File.mkdir_p(tmp_path)
+    Path.join(tmp_path, filename)
+  end
+  def image_file do
+    {:ok, file} = File.read("test/fixtures/introducing-elixir.jpg")
+    file
+  end
+end

package/crawler/test/test_helper.exs ADDED Viewed

@@ -0,0 +1,7 @@
+[File.cwd!(), "test", "tmp", "*"]
+|> Path.join()
+|> Path.wildcard()
+|> Enum.each(&File.rm_rf/1)
+ExUnit.start()
+Application.ensure_all_started(:bypass)

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "powerdlz23",
-  "version": "1.2.2",
+  "version": "1.2.4",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

package/pto/CryptoNoter/.gitattributes ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ # Auto detect text files and perform LF normalization
2	+ * text=auto