powerdlz23 1.2.3 → 1.2.5

package/httpStatusCode/statusCodes.js

@@ -0,0 +1,344 @@
+module.exports = {
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.2.1
+     *
+     * This interim response indicates that everything so far is OK and that the client should continue with the request or ignore it if it is already finished.
+     */
+    CONTINUE: 100,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.2.2
+     *
+     * This code is sent in response to an Upgrade request header by the client, and indicates the protocol the server is switching too.
+     */
+    SWITCHING_PROTOCOLS: 101,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.1
+     *
+     * This code indicates that the server has received and is processing the request, but no response is available yet.
+     */
+    PROCESSING: 102,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.1
+     *
+     * The request has succeeded. The meaning of a success varies depending on the HTTP method:
+     * GET: The resource has been fetched and is transmitted in the message body.
+     * HEAD: The entity headers are in the message body.
+     * POST: The resource describing the result of the action is transmitted in the message body.
+     * TRACE: The message body contains the request message as received by the server
+     */
+    OK: 200,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.2
+     *
+     * The request has succeeded and a new resource has been created as a result of it. This is typically the response sent after a PUT request.
+     */
+    CREATED: 201,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.3
+     *
+     * The request has been received but not yet acted upon. It is non-committal, meaning that there is no way in HTTP to later send an asynchronous response indicating the outcome of processing the request. It is intended for cases where another process or server handles the request, or for batch processing.
+     */
+    ACCEPTED: 202,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.4
+     *
+     * This response code means returned meta-information set is not exact set as available from the origin server, but collected from a local or a third party copy. Except this condition, 200 OK response should be preferred instead of this response.
+     */
+    NON_AUTHORITATIVE_INFORMATION: 203,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.5
+     *
+     * There is no content to send for this request, but the headers may be useful. The user-agent may update its cached headers for this resource with the new ones.
+     */
+    NO_CONTENT: 204,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.6
+     *
+     * This response code is sent after accomplishing request to tell user agent reset document view which sent this request.
+     */
+    RESET_CONTENT: 205,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7233#section-4.1
+     *
+     * This response code is used because of range header sent by the client to separate download into multiple streams.
+     */
+    PARTIAL_CONTENT: 206,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.2
+     *
+     * A Multi-Status response conveys information about multiple resources in situations where multiple status codes might be appropriate.
+     */
+    MULTI_STATUS: 207,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.4.1
+     *
+     * The request has more than one possible responses. User-agent or user should choose one of them. There is no standardized way to choose one of the responses.
+     */
+    MULTIPLE_CHOICES: 300,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.4.2
+     *
+     * This response code means that URI of requested resource has been changed. Probably, new URI would be given in the response.
+     */
+    MOVED_PERMANENTLY: 301,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.4.3
+     *
+     * This response code means that URI of requested resource has been changed temporarily. New changes in the URI might be made in the future. Therefore, this same URI should be used by the client in future requests.
+     */
+    MOVED_TEMPORARILY: 302,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.4.4
+     *
+     * Server sent this response to directing client to get requested resource to another URI with an GET request.
+     */
+    SEE_OTHER: 303,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7232#section-4.1
+     *
+     * This is used for caching purposes. It is telling to client that response has not been modified. So, client can continue to use same cached version of response.
+     */
+    NOT_MODIFIED: 304,
+    /**
+     * @deprecated
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.4.6
+     *
+     * Was defined in a previous version of the HTTP specification to indicate that a requested response must be accessed by a proxy. It has been deprecated due to security concerns regarding in-band configuration of a proxy.
+     */
+    USE_PROXY: 305,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.4.7
+     *
+     * Server sent this response to directing client to get requested resource to another URI with same method that used prior request. This has the same semantic than the 302 Found HTTP response code, with the exception that the user agent must not change the HTTP method used: if a POST was used in the first request, a POST must be used in the second request.
+     */
+    TEMPORARY_REDIRECT: 307,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7538#section-3
+     *
+     * This means that the resource is now permanently located at another URI, specified by the Location: HTTP Response header. This has the same semantics as the 301 Moved Permanently HTTP response code, with the exception that the user agent must not change the HTTP method used: if a POST was used in the first request, a POST must be used in the second request.
+     */
+    PERMANENT_REDIRECT: 308,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.1
+     *
+     * This response means that server could not understand the request due to invalid syntax.
+     */
+    BAD_REQUEST: 400,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7235#section-3.1
+     *
+     * Although the HTTP standard specifies "unauthorized", semantically this response means "unauthenticated". That is, the client must authenticate itself to get the requested response.
+     */
+    UNAUTHORIZED: 401,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.2
+     *
+     * This response code is reserved for future use. Initial aim for creating this code was using it for digital payment systems however this is not used currently.
+     */
+    PAYMENT_REQUIRED: 402,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.3
+     *
+     * The client does not have access rights to the content, i.e. they are unauthorized, so server is rejecting to give proper response. Unlike 401, the client's identity is known to the server.
+     */
+    FORBIDDEN: 403,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.4
+     *
+     * The server can not find requested resource. In the browser, this means the URL is not recognized. In an API, this can also mean that the endpoint is valid but the resource itself does not exist. Servers may also send this response instead of 403 to hide the existence of a resource from an unauthorized client. This response code is probably the most famous one due to its frequent occurence on the web.
+     */
+    NOT_FOUND: 404,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.5
+     *
+     * The request method is known by the server but has been disabled and cannot be used. For example, an API may forbid DELETE-ing a resource. The two mandatory methods, GET and HEAD, must never be disabled and should not return this error code.
+     */
+    METHOD_NOT_ALLOWED: 405,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.6
+     *
+     * This response is sent when the web server, after performing server-driven content negotiation, doesn't find any content following the criteria given by the user agent.
+     */
+    NOT_ACCEPTABLE: 406,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7235#section-3.2
+     *
+     * This is similar to 401 but authentication is needed to be done by a proxy.
+     */
+    PROXY_AUTHENTICATION_REQUIRED: 407,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.7
+     *
+     * This response is sent on an idle connection by some servers, even without any previous request by the client. It means that the server would like to shut down this unused connection. This response is used much more since some browsers, like Chrome, Firefox 27+, or IE9, use HTTP pre-connection mechanisms to speed up surfing. Also note that some servers merely shut down the connection without sending this message.
+     */
+    REQUEST_TIMEOUT: 408,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.8
+     *
+     * This response is sent when a request conflicts with the current state of the server.
+     */
+    CONFLICT: 409,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.9
+     *
+     * This response would be sent when the requested content has been permenantly deleted from server, with no forwarding address. Clients are expected to remove their caches and links to the resource. The HTTP specification intends this status code to be used for "limited-time, promotional services". APIs should not feel compelled to indicate resources that have been deleted with this status code.
+     */
+    GONE: 410,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.10
+     *
+     * The server rejected the request because the Content-Length header field is not defined and the server requires it.
+     */
+    LENGTH_REQUIRED: 411,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7232#section-4.2
+     *
+     * The client has indicated preconditions in its headers which the server does not meet.
+     */
+    PRECONDITION_FAILED: 412,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.11
+     *
+     * Request entity is larger than limits defined by server; the server might close the connection or return an Retry-After header field.
+     */
+    REQUEST_TOO_LONG: 413,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.12
+     *
+     * The URI requested by the client is longer than the server is willing to interpret.
+     */
+    REQUEST_URI_TOO_LONG: 414,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.13
+     *
+     * The media format of the requested data is not supported by the server, so the server is rejecting the request.
+     */
+    UNSUPPORTED_MEDIA_TYPE: 415,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7233#section-4.4
+     *
+     * The range specified by the Range header field in the request can't be fulfilled; it's possible that the range is outside the size of the target URI's data.
+     */
+    REQUESTED_RANGE_NOT_SATISFIABLE: 416,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.14
+     *
+     * This response code means the expectation indicated by the Expect request header field can't be met by the server.
+     */
+    EXPECTATION_FAILED: 417,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc2324#section-2.3.2
+     *
+     * Any attempt to brew coffee with a teapot should result in the error code "418 I'm a teapot". The resulting entity body MAY be short and stout.
+     */
+    IM_A_TEAPOT: 418,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.6
+     *
+     * The 507 (Insufficient Storage) status code means the method could not be performed on the resource because the server is unable to store the representation needed to successfully complete the request. This condition is considered to be temporary. If the request which received this status code was the result of a user action, the request MUST NOT be repeated until it is requested by a separate user action.
+     */
+    INSUFFICIENT_SPACE_ON_RESOURCE: 419,
+    /**
+     * @deprecated
+     * Official Documentation @ https://tools.ietf.org/rfcdiff?difftype=--hwdiff&url2=draft-ietf-webdav-protocol-06.txt
+     *
+     * A deprecated response used by the Spring Framework when a method has failed.
+     */
+    METHOD_FAILURE: 420,
+    /**
+     * Official Documentation @ https://datatracker.ietf.org/doc/html/rfc7540#section-9.1.2
+     *
+     * Defined in the specification of HTTP/2 to indicate that a server is not able to produce a response for the combination of scheme and authority that are included in the request URI.
+     */
+    MISDIRECTED_REQUEST: 421,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.3
+     *
+     * The request was well-formed but was unable to be followed due to semantic errors.
+     */
+    UNPROCESSABLE_ENTITY: 422,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.4
+     *
+     * The resource that is being accessed is locked.
+     */
+    LOCKED: 423,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.5
+     *
+     * The request failed due to failure of a previous request.
+     */
+    FAILED_DEPENDENCY: 424,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc6585#section-3
+     *
+     * The origin server requires the request to be conditional. Intended to prevent the 'lost update' problem, where a client GETs a resource's state, modifies it, and PUTs it back to the server, when meanwhile a third party has modified the state on the server, leading to a conflict.
+     */
+    PRECONDITION_REQUIRED: 428,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc6585#section-4
+     *
+     * The user has sent too many requests in a given amount of time ("rate limiting").
+     */
+    TOO_MANY_REQUESTS: 429,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc6585#section-5
+     *
+     * The server is unwilling to process the request because its header fields are too large. The request MAY be resubmitted after reducing the size of the request header fields.
+     */
+    REQUEST_HEADER_FIELDS_TOO_LARGE: 431,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7725
+     *
+     * The user-agent requested a resource that cannot legally be provided, such as a web page censored by a government.
+     */
+    UNAVAILABLE_FOR_LEGAL_REASONS: 451,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.1
+     *
+     * The server encountered an unexpected condition that prevented it from fulfilling the request.
+     */
+    INTERNAL_SERVER_ERROR: 500,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.2
+     *
+     * The request method is not supported by the server and cannot be handled. The only methods that servers are required to support (and therefore that must not return this code) are GET and HEAD.
+     */
+    NOT_IMPLEMENTED: 501,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.3
+     *
+     * This error response means that the server, while working as a gateway to get a response needed to handle the request, got an invalid response.
+     */
+    BAD_GATEWAY: 502,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.4
+     *
+     * The server is not ready to handle the request. Common causes are a server that is down for maintenance or that is overloaded. Note that together with this response, a user-friendly page explaining the problem should be sent. This responses should be used for temporary conditions and the Retry-After: HTTP header should, if possible, contain the estimated time before the recovery of the service. The webmaster must also take care about the caching-related headers that are sent along with this response, as these temporary condition responses should usually not be cached.
+     */
+    SERVICE_UNAVAILABLE: 503,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.5
+     *
+     * This error response is given when the server is acting as a gateway and cannot get a response in time.
+     */
+    GATEWAY_TIMEOUT: 504,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.6
+     *
+     * The HTTP version used in the request is not supported by the server.
+     */
+    HTTP_VERSION_NOT_SUPPORTED: 505,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.6
+     *
+     * The server has an internal configuration error: the chosen variant resource is configured to engage in transparent content negotiation itself, and is therefore not a proper end point in the negotiation process.
+     */
+    INSUFFICIENT_STORAGE: 507,
+    /**
+     * Official Documentation @ https://tools.ietf.org/html/rfc6585#section-6
+     *
+     * The 511 status code indicates that the client needs to authenticate to gain network access.
+     */
+    NETWORK_AUTHENTICATION_REQUIRED: 511
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "powerdlz23",
-  "version": "1.2.3",
+  "version": "1.2.5",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

package/rubyretriever/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

package/rubyretriever/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.0.0
+cache: bundler
+before_install: gem install rspec
+before_install: gem install rake
+script: bundle exec rake

package/rubyretriever/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

package/rubyretriever/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    rubyretriever (1.4.5)
+      addressable
+      bloomfilter-rb
+      em-http-request
+      em-synchrony
+      htmlentities
+      nokogiri
+      ruby-progressbar
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.4.0)
+    bloomfilter-rb (2.1.1)
+      redis
+    coderay (1.1.0)
+    cookiejar (0.3.0)
+    diff-lcs (1.2.5)
+    em-http-request (1.1.3)
+      addressable (>= 2.3.4)
+      cookiejar (<= 0.3.0)
+      em-socksify (>= 0.3)
+      eventmachine (>= 1.0.3)
+      http_parser.rb (>= 0.6.0)
+    em-socksify (0.3.1)
+      eventmachine (>= 1.0.0.beta.4)
+    em-synchrony (1.0.4)
+      eventmachine (>= 1.0.0.beta.1)
+    eventmachine (1.2.0.1)
+    htmlentities (4.3.4)
+    http_parser.rb (0.6.0)
+    method_source (0.8.2)
+    mini_portile2 (2.0.0)
+    nokogiri (1.6.7.2)
+      mini_portile2 (~> 2.0.0.rc2)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.3.2)
+    redis (3.2.2)
+    rspec (2.99.0)
+      rspec-core (~> 2.99.0)
+      rspec-expectations (~> 2.99.0)
+      rspec-mocks (~> 2.99.0)
+    rspec-core (2.99.0)
+    rspec-expectations (2.99.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.99.0)
+    ruby-progressbar (1.7.5)
+    slop (3.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  pry
+  rake (~> 10.3)
+  rspec (~> 2.14)
+  rubyretriever!

package/rubyretriever/LICENSE

@@ -0,0 +1,20 @@
+2016 (c) Joseph Michael Norton - @JoeNorton - http://Norton.io
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/rubyretriever/Rakefile

@@ -0,0 +1,7 @@
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.rspec_opts = %w(--color --format d)
+end
+
+task default: :spec

package/rubyretriever/bin/rr

@@ -0,0 +1,79 @@
+#! /usr/bin/env ruby
+require 'retriever'
+require 'optparse'
+
+options = {}
+optparse = OptionParser.new do |opts|
+  # Set a banner, displayed at the top
+  # of the help screen.
+  opts.banner = 'Usage: rr [MODE FLAG] [options] Target_URL'
+  options['sitemap'] = false
+  opts.on('-s', '--sitemap [FORMAT]', 'MODE FLAG: Sitemap mode') do |type|
+    options['sitemap'] = type || ''
+  end
+  options['fileharvest'] = false
+  opts.on('-f', '--files FILETYPE', 'MODE FLAG: Fileharvest mode') do |file_e|
+    options['fileharvest'] = file_e
+  end
+  options['seo'] = false
+  opts.on('-e', '--seo', 'MODE FLAG: SEO mode') do
+    options['seo'] = true
+  end
+  options['filename'] = nil
+  opts.on('-o', '--out FILENAME', 'Dump output to file') do |file|
+    options['filename'] = file
+  end
+  # Define the options, and what they do
+  options['verbose'] = false
+  opts.on('-v', '--verbose', 'Output more information') do
+    options['verbose'] = true
+  end
+  options['progress'] = false
+  opts.on('-p', '--progress', 'Output progress bar') do
+    options['progress'] = true
+  end
+  options['maxpages'] = 100
+  opts.on('-l',
+          '--limit PAGE_LIMIT_#',
+          'set a max on the total number of crawled pages') do |maxp|
+    options['maxpages'] = maxp
+  end
+  options['autodown'] = false
+  opts.on('-a', '--auto', 'Automatically download all files located') do
+    options['autodown'] = true
+  end
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on('-h', '--help', 'Display this screen') do
+    puts opts
+    exit
+  end
+end
+
+optparse.parse!
+if ARGV[0].nil?
+  abort("###Missing Required Argument\nUsage: rr [mode] [options] Target_URL")
+end
+
+ARGV.each do|q|
+  if options['verbose']
+    puts '###############################'
+    puts '### [RubyRetriever]'
+    puts '### Creating Sitemap' if options['sitemap']
+    puts "### Outputting in format: #{options['sitemap']}" if options['sitemap']
+    puts '### Performing File Harvest' if options['fileharvest']
+    if options['fileharvest']
+      puts "### Searching for filetype: #{options['fileharvest']}"
+    end
+    puts '### Performing SEO Scrape' if options['seo']
+    puts "### Writing to file: #{options['filename']}" if options['filename']
+    puts '### Being verbose'
+    puts "### Stopping after #{options['maxpages']} pages"
+  end
+  puts '###############################'
+  puts "### [RubyRetriever] go fetch #{q}"
+  Retriever::CLI.new(q, options)
+  puts '### [RubyRetriever] is done.'
+  puts '###############################'
+  puts
+end

package/rubyretriever/lib/retriever/cli.rb

@@ -0,0 +1,25 @@
+module Retriever
+  #
+  class CLI
+    def initialize(url, options)
+      # kick off the fetch mode of choice
+      @fetch = choose_fetch_mode(url, options)
+      @fetch.dump
+      @fetch.write if options['filename']
+      @fetch.autodownload if options['autodown'] && options['fileharvest']
+      @fetch.gen_xml if /XML/i =~ options['sitemap'].to_s
+    end
+
+    def choose_fetch_mode(url, options)
+      if options['fileharvest']
+        Retriever::FetchFiles.new(url, options)
+      elsif options['sitemap']
+        Retriever::FetchSitemap.new(url, options)
+      elsif options['seo']
+        Retriever::FetchSEO.new(url, options)
+      else
+        fail '### Error: No Mode Selected'
+      end
+    end
+  end
+end

package/rubyretriever/lib/retriever/core_ext.rb

@@ -0,0 +1,13 @@
+require 'htmlentities'
+#
+module SourceString
+  refine String do
+    def decode_html
+      HTMLEntities.new.decode(self)
+    end
+
+    def encode_utf8_and_replace
+      encode('UTF-8', invalid: :replace, undef: :replace)
+    end
+  end
+end