powerdlz23 1.2.3 → 1.2.4

package/Spider/README.md

@@ -0,0 +1,19 @@
+![](http://i.imgur.com/wYi2CkD.png)
+
+
+# Overview
+
+This is an open source, multi-threaded website crawler written in Python. There is still a lot of work to do, so feel free to help out with development.
+
+***
+
+Note: This is part of an open source search engine. The purpose of this tool is to gather links **only**. The analytics, data harvesting, and search algorithms are being created as separate programs. 
+
+### Links
+
+- [Support thenewboston](https://www.patreon.com/thenewboston)
+- [thenewboston.com](https://thenewboston.com/)
+- [Facebook](https://www.facebook.com/TheNewBoston-464114846956315/)
+- [Twitter](https://twitter.com/bucky_roberts)
+- [Google+](https://plus.google.com/+BuckyRoberts)
+- [reddit](https://www.reddit.com/r/thenewboston/)

package/Spider/domain.py

@@ -0,0 +1,18 @@
+from urllib.parse import urlparse
+
+
+# Get domain name (example.com)
+def get_domain_name(url):
+    try:
+        results = get_sub_domain_name(url).split('.')
+        return results[-2] + '.' + results[-1]
+    except:
+        return ''
+
+
+# Get sub domain name (name.example.com)
+def get_sub_domain_name(url):
+    try:
+        return urlparse(url).netloc
+    except:
+        return ''

package/Spider/general.py

@@ -0,0 +1,51 @@
+import os
+
+
+# Each website is a separate project (folder)
+def create_project_dir(directory):
+    if not os.path.exists(directory):
+        print('Creating directory ' + directory)
+        os.makedirs(directory)
+
+
+# Create queue and crawled files (if not created)
+def create_data_files(project_name, base_url):
+    queue = os.path.join(project_name , 'queue.txt')
+    crawled = os.path.join(project_name,"crawled.txt")
+    if not os.path.isfile(queue):
+        write_file(queue, base_url)
+    if not os.path.isfile(crawled):
+        write_file(crawled, '')
+
+
+# Create a new file
+def write_file(path, data):
+    with open(path, 'w') as f:
+        f.write(data)
+
+
+# Add data onto an existing file
+def append_to_file(path, data):
+    with open(path, 'a') as file:
+        file.write(data + '\n')
+
+
+# Delete the contents of a file
+def delete_file_contents(path):
+    open(path, 'w').close()
+
+
+# Read a file and convert each line to set items
+def file_to_set(file_name):
+    results = set()
+    with open(file_name, 'rt') as f:
+        for line in f:
+            results.add(line.replace('\n', ''))
+    return results
+
+
+# Iterate through a set, each item will be a line in a file
+def set_to_file(links, file_name):
+    with open(file_name,"w") as f:
+        for l in sorted(links):
+            f.write(l+"\n")

package/Spider/link_finder.py

@@ -0,0 +1,25 @@
+from html.parser import HTMLParser
+from urllib import parse
+
+
+class LinkFinder(HTMLParser):
+
+    def __init__(self, base_url, page_url):
+        super().__init__()
+        self.base_url = base_url
+        self.page_url = page_url
+        self.links = set()
+
+    # When we call HTMLParser feed() this function is called when it encounters an opening tag <a>
+    def handle_starttag(self, tag, attrs):
+        if tag == 'a':
+            for (attribute, value) in attrs:
+                if attribute == 'href':
+                    url = parse.urljoin(self.base_url, value)
+                    self.links.add(url)
+
+    def page_links(self):
+        return self.links
+
+    def error(self, message):
+        pass

package/Spider/main.py

@@ -0,0 +1,50 @@
+import threading
+from queue import Queue
+from spider import Spider
+from domain import *
+from general import *
+
+PROJECT_NAME = 'viper-seo'
+HOMEPAGE = 'http://viper-seo.com/'
+DOMAIN_NAME = get_domain_name(HOMEPAGE)
+QUEUE_FILE = PROJECT_NAME + '/queue.txt'
+CRAWLED_FILE = PROJECT_NAME + '/crawled.txt'
+NUMBER_OF_THREADS = 8
+queue = Queue()
+Spider(PROJECT_NAME, HOMEPAGE, DOMAIN_NAME)
+
+
+# Create worker threads (will die when main exits)
+def create_workers():
+    for _ in range(NUMBER_OF_THREADS):
+        t = threading.Thread(target=work)
+        t.daemon = True
+        t.start()
+
+
+# Do the next job in the queue
+def work():
+    while True:
+        url = queue.get()
+        Spider.crawl_page(threading.current_thread().name, url)
+        queue.task_done()
+
+
+# Each queued link is a new job
+def create_jobs():
+    for link in file_to_set(QUEUE_FILE):
+        queue.put(link)
+    queue.join()
+    crawl()
+
+
+# Check if there are items in the queue, if so crawl them
+def crawl():
+    queued_links = file_to_set(QUEUE_FILE)
+    if len(queued_links) > 0:
+        print(str(len(queued_links)) + ' links in the queue')
+        create_jobs()
+
+
+create_workers()
+crawl()

package/Spider/spider.py

@@ -0,0 +1,74 @@
+from urllib.request import urlopen
+from link_finder import LinkFinder
+from domain import *
+from general import *
+
+
+class Spider:
+
+    project_name = ''
+    base_url = ''
+    domain_name = ''
+    queue_file = ''
+    crawled_file = ''
+    queue = set()
+    crawled = set()
+
+    def __init__(self, project_name, base_url, domain_name):
+        Spider.project_name = project_name
+        Spider.base_url = base_url
+        Spider.domain_name = domain_name
+        Spider.queue_file = Spider.project_name + '/queue.txt'
+        Spider.crawled_file = Spider.project_name + '/crawled.txt'
+        self.boot()
+        self.crawl_page('First spider', Spider.base_url)
+
+    # Creates directory and files for project on first run and starts the spider
+    @staticmethod
+    def boot():
+        create_project_dir(Spider.project_name)
+        create_data_files(Spider.project_name, Spider.base_url)
+        Spider.queue = file_to_set(Spider.queue_file)
+        Spider.crawled = file_to_set(Spider.crawled_file)
+
+    # Updates user display, fills queue and updates files
+    @staticmethod
+    def crawl_page(thread_name, page_url):
+        if page_url not in Spider.crawled:
+            print(thread_name + ' now crawling ' + page_url)
+            print('Queue ' + str(len(Spider.queue)) + ' | Crawled  ' + str(len(Spider.crawled)))
+            Spider.add_links_to_queue(Spider.gather_links(page_url))
+            Spider.queue.remove(page_url)
+            Spider.crawled.add(page_url)
+            Spider.update_files()
+
+    # Converts raw response data into readable information and checks for proper html formatting
+    @staticmethod
+    def gather_links(page_url):
+        html_string = ''
+        try:
+            response = urlopen(page_url)
+            if 'text/html' in response.getheader('Content-Type'):
+                html_bytes = response.read()
+                html_string = html_bytes.decode("utf-8")
+            finder = LinkFinder(Spider.base_url, page_url)
+            finder.feed(html_string)
+        except Exception as e:
+            print(str(e))
+            return set()
+        return finder.page_links()
+
+    # Saves queue data to project files
+    @staticmethod
+    def add_links_to_queue(links):
+        for url in links:
+            if (url in Spider.queue) or (url in Spider.crawled):
+                continue
+            if Spider.domain_name != get_domain_name(url):
+                continue
+            Spider.queue.add(url)
+
+    @staticmethod
+    def update_files():
+        set_to_file(Spider.queue, Spider.queue_file)
+        set_to_file(Spider.crawled, Spider.crawled_file)

package/crawler/.formatter.exs

@@ -0,0 +1,5 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"],
+  plugins: [Recode.FormatterPlugin]
+]

package/crawler/.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+on: push
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Elixir
+        uses: erlef/setup-beam@v1
+        with:
+          version-type: strict
+          version-file: .tool-versions
+      - name: Restore dependencies cache
+        id: mix-cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            deps
+            _build
+          key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
+          restore-keys: ${{ runner.os }}-mix-
+      - name: Install dependencies
+        if: steps.mix-cache.outputs.cache-hit != 'true'
+        run: |
+          mix local.rebar --force
+          mix local.hex --force
+          mix deps.get
+      - name: Run tests
+        run: mix test

package/crawler/.recode.exs

@@ -0,0 +1,33 @@
+[
+  version: "0.6.4",
+  # Can also be set/reset with `--autocorrect`/`--no-autocorrect`.
+  autocorrect: true,
+  # With "--dry" no changes will be written to the files.
+  # Can also be set/reset with `--dry`/`--no-dry`.
+  # If dry is true then verbose is also active.
+  dry: false,
+  # Can also be set/reset with `--verbose`/`--no-verbose`.
+  verbose: false,
+  # Can be overwritten by calling `mix recode "lib/**/*.ex"`.
+  inputs: ["{mix,.formatter}.exs", "{apps,config,lib,test}/**/*.{ex,exs}"],
+  formatter: {Recode.Formatter, []},
+  tasks: [
+    # Tasks could be added by a tuple of the tasks module name and an options
+    # keyword list. A task can be deactivated by `active: false`. The execution of
+    # a deactivated task can be forced by calling `mix recode --task ModuleName`.
+    {Recode.Task.AliasExpansion, []},
+    {Recode.Task.AliasOrder, []},
+    {Recode.Task.Dbg, [autocorrect: false]},
+    {Recode.Task.EnforceLineLength, [active: false]},
+    {Recode.Task.FilterCount, []},
+    {Recode.Task.IOInspect, [autocorrect: false]},
+    {Recode.Task.Nesting, []},
+    {Recode.Task.PipeFunOne, []},
+    {Recode.Task.SinglePipe, []},
+    {Recode.Task.Specs, [active: false, exclude: "test/**/*.{ex,exs}", config: [only: :visible]]},
+    {Recode.Task.TagFIXME, [exit_code: 2]},
+    {Recode.Task.TagTODO, [exit_code: 4]},
+    {Recode.Task.TestFileExt, []},
+    {Recode.Task.UnusedVariable, [active: false]}
+  ]
+]

package/crawler/.tool-versions

@@ -0,0 +1,2 @@
+erlang 26.1.1
+elixir 1.15.6

package/crawler/CHANGELOG.md

@@ -0,0 +1,82 @@
+# Crawler Changelog
+
+## master
+
+- [Added] Add `:retries` option
+
+## v1.5.0 [2023-10-10]
+
+- [Added] Add `:force` option
+- [Added] Add `:scope` option
+
+## v1.4.0 [2023-10-07]
+
+- [Added] Allow multiple instances of Crawler sharing the same queue
+- [Improved] Logger will now log entries as `debug` or `warn`
+
+## v1.3.0 [2023-09-30]
+
+- [Added] `:store` option, defaults to `nil` to save memory usage
+- [Added] `:max_pages` option
+- [Added] `Crawler.running?/1` to check whether Crawler is running
+- [Improved] The queue is being supervised now
+
+## v1.2.0 [2023-09-29]
+
+- [Added] `Crawler.Store.all_urls/0` to find all scraped URLs
+- [Improved] Memory usage optimisations
+
+## v1.1.2 [2021-10-14]
+
+- [Improved] Documentation improvements (thanks @kianmeng)
+
+## v1.1.1 [2020-05-15]
+
+- [Improved] Updated `floki` and other dependencies
+
+## v1.1.0 [2019-02-25]
+
+- [Added] `:modifier` option
+- [Added] `:encode_uri` option
+- [Improved] Varies small fixes and improvements
+
+## v1.0.0 [2017-08-31]
+
+- [Added] Pause / resume / stop Crawler
+- [Improved] Varies small fixes and improvements
+
+## v0.4.0 [2017-08-28]
+
+- [Added] `:scraper` option to allow scraping content
+- [Improved] Varies small fixes and improvements
+
+## v0.3.1 [2017-08-28]
+
+- [Improved] `Crawler.Store.DB` now stores the `opts` meta data
+- [Improved] Code documentation
+- [Improved] Varies small fixes and improvements
+
+## v0.3.0 [2017-08-27]
+
+- [Added] `:retrier` option to allow custom fetch retrying logic
+- [Added] `:url_filter` option to allow custom url filtering logic
+- [Improved] Parser is now more stable and skips unparsable files
+- [Improved] Varies small fixes and improvements
+
+## v0.2.0 [2017-08-21]
+
+- [Added] `:workers` option
+- [Added] `:interval` option
+- [Added] `:timeout` option
+- [Added] `:user_agent` option
+- [Added] `:save_to` option
+- [Added] `:assets` option
+- [Added] `:parser` option to allow custom parsing logic
+- [Improved] Renamed `:max_levels` to `:max_depths`
+- [Improved] Varies small fixes and improvements
+
+## v0.1.0 [2017-07-30]
+
+- [Added] A semi-functioning prototype
+- [Added] Finished the very basic crawling function
+- [Added] `:max_levels` option

package/crawler/README.md

@@ -0,0 +1,198 @@
+# Crawler
+
+[![Build Status](https://github.com/fredwu/crawler/actions/workflows/ci.yml/badge.svg)](https://github.com/fredwu/crawler/actions)
+[![CodeBeat](https://codebeat.co/badges/76916047-5b66-466d-91d3-7131a269899a)](https://codebeat.co/projects/github-com-fredwu-crawler-master)
+[![Coverage](https://img.shields.io/coveralls/fredwu/crawler.svg)](https://coveralls.io/github/fredwu/crawler?branch=master)
+[![Module Version](https://img.shields.io/hexpm/v/crawler.svg)](https://hex.pm/packages/crawler)
+[![Hex Docs](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/crawler/)
+[![Total Download](https://img.shields.io/hexpm/dt/crawler.svg)](https://hex.pm/packages/crawler)
+[![License](https://img.shields.io/hexpm/l/crawler.svg)](https://github.com/fredwu/crawler/blob/master/LICENSE.md)
+[![Last Updated](https://img.shields.io/github/last-commit/fredwu/crawler.svg)](https://github.com/fredwu/crawler/commits/master)
+
+A high performance web crawler / scraper in Elixir, with worker pooling and rate limiting via [OPQ](https://github.com/fredwu/opq).
+
+## Features
+
+- Crawl assets (javascript, css and images).
+- Save to disk.
+- Hook for scraping content.
+- Restrict crawlable domains, paths or content types.
+- Limit concurrent crawlers.
+- Limit rate of crawling.
+- Set the maximum crawl depth.
+- Set timeouts.
+- Set retries strategy.
+- Set crawler's user agent.
+- Manually pause/resume/stop the crawler.
+
+See [Hex documentation](https://hexdocs.pm/crawler/).
+
+## Architecture
+
+Below is a very high level architecture diagram demonstrating how Crawler works.
+
+![](architecture.svg)
+
+## Usage
+
+```elixir
+Crawler.crawl("http://elixir-lang.org", max_depths: 2)
+```
+
+There are several ways to access the crawled page data:
+
+1. Use [`Crawler.Store`](https://hexdocs.pm/crawler/Crawler.Store.html)
+2. Tap into the registry([?](https://hexdocs.pm/elixir/Registry.html)) [`Crawler.Store.DB`](lib/crawler/store.ex)
+3. Use your own [scraper](#custom-modules)
+4. If the `:save_to` option is set, pages will be saved to disk in addition to the above mentioned places
+5. Provide your own [custom parser](#custom-modules) and manage how data is stored and accessed yourself
+
+## Configurations
+
+| Option        | Type    | Default Value               | Description                                                                                                                                                                               |
+| ------------- | ------- | --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `:assets`     | list    | `[]`                        | Whether to fetch any asset files, available options: `"css"`, `"js"`, `"images"`.                                                                                                         |
+| `:save_to`    | string  | `nil`                       | When provided, the path for saving crawled pages.                                                                                                                                         |
+| `:workers`    | integer | `10`                        | Maximum number of concurrent workers for crawling.                                                                                                                                        |
+| `:interval`   | integer | `0`                         | Rate limit control - number of milliseconds before crawling more pages, defaults to `0` which is effectively no rate limit.                                                               |
+| `:max_depths` | integer | `3`                         | Maximum nested depth of pages to crawl.                                                                                                                                                   |
+| `:max_pages`  | integer | `:infinity`                 | Maximum amount of pages to crawl.                                                                                                                                                         |
+| `:timeout`    | integer | `5000`                      | Timeout value for fetching a page, in ms. Can also be set to `:infinity`, useful when combined with `Crawler.pause/1`.                                                                    |
+| `:retries`    | integer | `2`                         | Number of times to retry a fetch.                                                                                                                                                         |
+| `:store`      | module  | `nil`                       | Module for storing the crawled page data and crawling metadata. You can set it to `Crawler.Store` or use your own module, see `Crawler.Store.add_page_data/3` for implementation details. |
+| `:force`      | boolean | `false`                     | Force crawling URLs even if they have already been crawled, useful if you want to refresh the crawled data.                                                                               |
+| `:scope`      | term    | `nil`                       | Similar to `:force`, but you can pass a custom `:scope` to determine how Crawler should perform on links already seen.                                                                    |
+| `:user_agent` | string  | `Crawler/x.x.x (...)`       | User-Agent value sent by the fetch requests.                                                                                                                                              |
+| `:url_filter` | module  | `Crawler.Fetcher.UrlFilter` | Custom URL filter, useful for restricting crawlable domains, paths or content types.                                                                                                      |
+| `:retrier`    | module  | `Crawler.Fetcher.Retrier`   | Custom fetch retrier, useful for retrying failed crawls, nullifies the `:retries` option.                                                                                                 |
+| `:modifier`   | module  | `Crawler.Fetcher.Modifier`  | Custom modifier, useful for adding custom request headers or options.                                                                                                                     |
+| `:scraper`    | module  | `Crawler.Scraper`           | Custom scraper, useful for scraping content as soon as the parser parses it.                                                                                                              |
+| `:parser`     | module  | `Crawler.Parser`            | Custom parser, useful for handling parsing differently or to add extra functionalities.                                                                                                   |
+| `:encode_uri` | boolean | `false`                     | When set to `true` apply the `URI.encode` to the URL to be crawled.                                                                                                                       |
+| `:queue`      | pid     | `nil`                       | You can pass in an `OPQ` pid so that multiple crawlers can share the same queue.                                                                                                          |
+
+## Custom Modules
+
+It is possible to swap in your custom logic as shown in the configurations section. Your custom modules need to conform to their respective behaviours:
+
+### Retrier
+
+See [`Crawler.Fetcher.Retrier`](lib/crawler/fetcher/retrier.ex).
+
+Crawler uses [ElixirRetry](https://github.com/safwank/ElixirRetry)'s exponential backoff strategy by default.
+
+```elixir
+defmodule CustomRetrier do
+  @behaviour Crawler.Fetcher.Retrier.Spec
+end
+```
+
+### URL Filter
+
+See [`Crawler.Fetcher.UrlFilter`](lib/crawler/fetcher/url_filter.ex).
+
+```elixir
+defmodule CustomUrlFilter do
+  @behaviour Crawler.Fetcher.UrlFilter.Spec
+end
+```
+
+### Scraper
+
+See [`Crawler.Scraper`](lib/crawler/scraper.ex).
+
+```elixir
+defmodule CustomScraper do
+  @behaviour Crawler.Scraper.Spec
+end
+```
+
+### Parser
+
+See [`Crawler.Parser`](lib/crawler/parser.ex).
+
+```elixir
+defmodule CustomParser do
+  @behaviour Crawler.Parser.Spec
+end
+```
+
+### Modifier
+
+See [`Crawler.Fetcher.Modifier`](lib/crawler/fetcher/modifier.ex).
+
+```elixir
+defmodule CustomModifier do
+  @behaviour Crawler.Fetcher.Modifier.Spec
+end
+```
+
+## Pause / Resume / Stop Crawler
+
+Crawler provides `pause/1`, `resume/1` and `stop/1`, see below.
+
+```elixir
+{:ok, opts} = Crawler.crawl("https://elixir-lang.org")
+
+Crawler.running?(opts) # => true
+
+Crawler.pause(opts)
+
+Crawler.running?(opts) # => false
+
+Crawler.resume(opts)
+
+Crawler.running?(opts) # => true
+
+Crawler.stop(opts)
+
+Crawler.running?(opts) # => false
+```
+
+Please note that when pausing Crawler, you would need to set a large enough `:timeout` (or even set it to `:infinity`) otherwise parser would timeout due to unprocessed links.
+
+## Multiple Crawlers
+
+It is possible to start multiple crawlers sharing the same queue.
+
+```elixir
+{:ok, queue} = OPQ.init(worker: Crawler.Dispatcher.Worker, workers: 2)
+
+Crawler.crawl("https://elixir-lang.org", queue: queue)
+Crawler.crawl("https://github.com", queue: queue)
+```
+
+## Find All Scraped URLs
+
+```elixir
+Crawler.Store.all_urls() # => ["https://elixir-lang.org", "https://google.com", ...]
+```
+
+## Examples
+
+### Google Search + Github
+
+This example performs a Google search, then scrapes the results to find Github projects and output their name and description.
+
+See the [source code](examples/google_search.ex).
+
+You can run the example by cloning the repo and run the command:
+
+```shell
+mix run -e "Crawler.Example.GoogleSearch.run()"
+```
+
+## API Reference
+
+Please see https://hexdocs.pm/crawler.
+
+## Changelog
+
+Please see [CHANGELOG.md](CHANGELOG.md).
+
+## Copyright and License
+
+Copyright (c) 2016 Fred Wu
+
+This work is free. You can redistribute it and/or modify it under the
+terms of the [MIT License](http://fredwu.mit-license.org/).