terrier-engine 4.67.3 → 4.67.4

package/lib/terrier/scripts/script_executor.rb

@@ -1,263 +0,0 @@
-require 'terrier/io/tabular_io'
-
-class ScriptExecutor
-  # Doesn't need to be Loggable, it already has all the methods
-
-  attr_reader :cache, :each_count, :each_total, :log_lines, :email_settings
-  attr_accessor :me, :params, :script
-
-  def should_soft_destroy
-    true
-  end
-
-  def initialize(script, cache=nil, params=nil)
-    @script = script
-    @cache = cache
-    @each_count = 0
-    @each_total = 0
-    @field_values = {}
-    @log_lines = []
-    @output_files = []
-    @params = params
-    @email_settings = ActiveSupport::HashWithIndifferentAccess.new({
-      file_output: 'attachment', # link | attachment
-      disable: false, # true | false
-      body: '' # insert a body into the email
-    })
-  end
-
-  def set_email_settings(settings)
-    @email_settings.update(settings)
-  end
-
-  def set_field_values(values)
-    @field_values = ActiveSupport::HashWithIndifferentAccess.new @script.compute_field_values(values)
-  end
-
-  # returns a ScriptRun object describing the run
-  def init_run
-    script_run = ScriptRun.new script_id: @script.id, status: 'running', created_at: Time.now, duration: 0
-
-    if script_run.respond_to?(:fields)
-      file_fields = @script
-                     .script_fields
-                     .select { |field| %w[csv xlsx].include?(field.field_type) }
-                     .map(&:name)
-      script_run.fields = @field_values.reject { |k, _| file_fields.include?(k) }
-    end
-
-    script_run
-  end
-
-  # actually executes the script
-  def run(script_run, stream)
-    @stream = stream
-    t = Time.now
-    begin
-      if @stream
-        @stream.write '['
-      end
-      res = execute_body @script.body
-      if res && res.is_a?(String) && res.present? # we probably don't need to print random crap that's returned
-        info "DONE: #{res}"
-      end
-      script_run.status = 'success'
-      script_run.duration = Time.now - t
-      if @script.persisted? # we can't save the run if it's a temporary script
-        script_run.write_log @log_lines.join("\n")
-        script_email_log = @script.send_email_if_necessary @email_settings, me, @output_files, script_run.log_url
-        puts script_email_log if script_email_log
-      end
-      true
-    rescue Exception => ex
-      line = ex.backtrace[0].split(':')[1].to_i
-      write_raw 'error', "Error on line #{line}: #{ex.message}"
-      script_run.status = 'error'
-      script_run.exception = ex.message
-      script_run.backtrace = ex.backtrace.join("\n")
-      script_run.duration = Time.now - t
-      @log_lines << ex.message
-      error ex
-      if @script.persisted? # we can't save the run if it's a temporary script
-        script_run.write_log @log_lines.join("\n")
-      end
-      false
-    ensure
-      if @stream
-        @stream.write '{}]'
-      end
-      @stream.close if @stream
-    end
-  end
-
-  # executes the body code and returns the result
-  # this needs to be wrapped so that returning from the script doesn't interrupt the run method
-  def execute_body(body)
-    escaped_body = body.gsub('\"', '"')
-    filename = "script"
-    filename += "<#{@script.title}>" if @script&.title.present?
-    eval(escaped_body, binding, filename, 1)
-  end
-
-  def write_raw(type, body, extra={})
-    return unless @stream
-    extra[:type] = type
-    extra[:body] = CGI.escapeHTML(body)
-    @stream.write(extra.to_json + ',')
-  end
-
-  def puts(message)
-    write_raw 'print', message.to_s
-    Rails.logger.debug "(ScriptExecutor) #{message}"
-    @log_lines << message.to_s
-  end
-
-  def log(level, message)
-    write_raw level, message.to_s
-    s = "#{level.upcase}: #{message}"
-    level = 'info' if level == 'success'
-    Rails.logger.send level, "(ScriptExecutor) #{s}"
-    @log_lines << s
-  end
-
-  # make it work like a logger
-  %w(debug info warn success).each do |level|
-    define_method level do |message|
-      log level, message
-    end
-  end
-
-  def error(ex)
-    if ex.is_a? Exception
-      log 'error', ex.message
-      if ex.backtrace
-        ex.backtrace[0..6].each do |line|
-          log 'error', line
-        end
-      end
-    else
-      log 'error', ex.to_s
-    end
-  end
-
-  def puts_count(message='')
-    puts "[#{@each_count} of #{@each_total}] #{message}"
-  end
-
-  def dump_xls(data, rel_path, options={})
-    dump_file data, rel_path, options
-  end
-
-  def dump_csv(data, rel_path, options={})
-    dump_file data, rel_path, options
-  end
-
-  # either dumps a csv or xls using TabularIo.save
-  def dump_file(data, rel_path, options={})
-    abs_path = TabularIo.save data, rel_path, options
-    @output_files << abs_path.to_s
-    write_raw 'file', TabularIo.abs_to_rel_path(abs_path)
-    puts "Wrote #{data.count} records to #{rel_path}"
-  end
-
-  def puts_file(path)
-    @output_files << path
-    if path.index(Rails.root.to_s)
-      path = TabularIo.abs_to_rel_path path
-    end
-    write_raw 'file', path
-    file_name = File.basename path
-    puts "Showing #{file_name}"
-  end
-
-  # loads either a csv or xlsx file using TabularIo.load
-  def load_file(rel_path, options={})
-    TabularIo.load rel_path, options
-  end
-
-  def get_field(name)
-    @field_values[name]
-  end
-
-  def get_params
-    @params
-  end
-
-  def each(collection)
-    @each_total = collection.count
-    @each_count = 0
-    result = collection.map do |item|
-      @each_count += 1
-      yield item
-    end
-    @each_count = 0
-    @each_total = 0
-    result
-  end
-
-  def read_input(name=nil)
-    if name && name.length > 0
-      input = @script.script_inputs.where(name: name).first
-      unless input
-        raise "No input named '#{name}'"
-      end
-      input.read
-    else # default to the first input
-      input = @script.script_inputs.first
-      unless input
-        raise 'This script does not have any inputs!'
-      end
-      input.read
-    end
-  end
-
-  def parse_dollars(dollars)
-    if dollars.instance_of? String
-      dollars = dollars.to_f
-    end
-    return 0 if dollars.nil? || dollars==0
-    (dollars * 100).round.to_i # fix floating point truncating error
-  end
-
-  def format_cents(cents)
-    "$#{'%.2f' % (cents / 100.0).round(2)}"
-  end
-
-  def email_recipients
-    if Rails.env == 'production'
-      if @script.email_recipients.blank?
-        raise 'No e-mail recipients for this script!'
-      end
-      @script.email_recipients
-    else
-      ['clypboardtesting@gmail.com']
-    end
-  end
-
-  def raw_sql(query)
-    ActiveRecord::Base.connection.execute(query).to_a
-  end
-
-  # No longer supported, see ScriptBase::send_email_if_necessary or send_explicit_email
-  def send_email(options)
-    puts "send_email function is no longer supported"
-  end
-
-  def send_explicit_email(options)
-    puts @script.send_explicit_email options
-  end
-
-  def self_destruct(countdown=5)
-    if countdown.present? && countdown > 0
-      puts "This script will self destruct in..."
-      countdown.downto(1) do |i|
-        puts i
-        sleep(1)
-      end
-    end
-    puts "Bang!"
-    self.script._state = 2
-    self.script.save_by_user!(self.me)
-  end
-
-end

package/lib/terrier/scripts/script_field.rb

@@ -1,68 +0,0 @@
-class ScriptField
-  include Terrier::Embedded
-
-  field :name, type: String, required: true
-
-  field :default_value, type: String, required: true
-
-  field :values, type: String
-
-  field :required, type: String
-
-  enum_field :field_type, %w(date string select csv xlsx hidden)
-
-  def self.compute_date_value(s)
-    if s.nil?
-      s
-    elsif s =~ /\d{4}-\d{2}-\d{2}/
-      Time.parse s
-    else
-      eval(s)&.to_time
-    end
-  end
-
-  def self.compute_value(type, s)
-    case type
-    when 'date'
-      compute_date_value s
-    when 'string'
-      s
-    when 'select'
-      s
-    when 'hidden'
-      s
-    when 'csv'
-      if s.blank?
-        []
-      else
-        TabularIo.parse_csv s
-      end
-    when 'xlsx'
-      if s.blank?
-        {}
-      else
-        TabularIo.load_xlsx nil, {raw: StringIO.new(Base64.decode64(s))}
-      end
-    else
-      raise "Unknown field type #{type}"
-    end
-  end
-
-  def compute_options
-    return [] unless self.field_type == 'select'
-    opts = eval(self.values)
-    opts = opts.is_a?(Array) ? opts : []
-    # don't add a blank space if there are no other options
-    # this way, the field will appear as a hidden field
-    if opts.size > 0 
-      if opts.first.is_a? Array
-        [['', nil]] + opts
-      else
-        [''] + opts
-      end
-    else
-      opts
-    end
-  end
-
-end

package/lib/terrier/scripts/script_searcher.rb

@@ -1,23 +0,0 @@
-class ScriptSearcher
-  def search(query)
-    
-    query= SqlBuilder.new
-      .select('body.id, t.total')
-      .from("body")
-      .left_join('total', 't', 'TRUE')
-      .with("body AS (SELECT id FROM scripts WHERE body ILIKE '%#{ query.strip }%'),
-             total AS (SELECT COUNT(*) total FROM body)")
-
-    t = Time.now
-    results = query.exec
-    total= results.present? ? results.first['total'] : 0
-    ids = results.present? ? results.to_a.pluck('id') : []
-    dt = Time.now - t
-
-    OpenStruct.new({
-      'took' => dt,
-      'ids' => ids,
-      'total' => total,
-    })
-  end
-end

package/lib/terrier/strings.rb

@@ -1,289 +0,0 @@
-
-EMAIL_SPLIT_REGEX = /[\s,;]+/
-
-SLUG_REGEX = /([[:lower:]]|[0-9]+-?[[:lower:]])(-[[:lower:]0-9]+|[[:lower:]0-9])*/
-
-# add values here that should always be upcased when displayed to the user
-UPCASE_VALUES = %w[cod ace bce csr dot pca tpp it qal html pdf ach eft wdi wdo]
-
-# these shouldn't be upcased, even though they're only two characters
-UPCASE_BLACKLIST = %w[at by to is or]
-
-GIGA = 1024 * 1024 * 1024
-MEGA = 1024 * 1024
-KILO = 1024
-
-# needed to parse WKB geo points
-if defined? RGeo
-  RGEO_FACTORY = RGeo::Geographic.spherical_factory(
-      wkb_parser: {support_ewkb: true}, wkb_generator: {hex_format: true, emit_ewkb_srid: true})
-else
-  RGEO_FACTORY = nil
-end
-
-class NilClass
-
-  def is_true?
-    false
-  end
-
-  def is_false?
-    true
-  end
-
-  def is_float?
-    false
-  end
-
-end
-
-TRUTHY_STRINGS = %w[true t on 1]
-FALSY_STRINGS = %w[false f off 0]
-
-ANSI_CODE_CLASSES = {
-  1 => 'bold',
-  3 => 'italic',
-  31 => 'red',
-  32 => 'green',
-  33 => 'yellow',
-  34 => 'blue',
-  36 => 'cyan'
-}
-
-class String
-
-  def uncapitalize
-    self[0, 1].downcase + self[1..-1]
-  end
-
-  def slugify
-    self.strip.split(/[\s[[:punct:]]]+/).join('_').gsub(/[^\w\-.]/, '').downcase
-  end
-
-  def parse_postgres_array
-    self.gsub('{', '').gsub('}', '').gsub('"', '').split(',')
-  end
-
-  def is_float?
-    !!Float(self) rescue false
-  end
-
-  def is_true?
-    !(TRUTHY_STRINGS.index(self.downcase.strip)).nil?
-  end
-
-  def is_false?
-    !(FALSY_STRINGS.index(self.downcase.strip)).nil?
-  end
-
-  def to_bool
-    TRUTHY_STRINGS.index self.downcase.strip
-  end
-
-  def smart_strip
-    self.strip.gsub(/\A[[:space:]]+/, '').gsub(/[[:space:]]+\z/, '')
-  end
-
-  def smart_title
-    lower_s = self.downcase
-    if (self.length<3 and !UPCASE_BLACKLIST.index(lower_s)) or UPCASE_VALUES.index(lower_s)
-      self.upcase
-    else
-      self.titleize
-    end
-  end
-
-  def parse_geo_point
-    if self.index('POINT')
-      self.gsub('POINT (', '').gsub(')', '').strip.split(' ').map(&:to_f)
-    else
-      RGEO_FACTORY.parse_wkb self
-    end
-  end
-
-  # returns the string with anything between parentheses
-  def without_parens
-    self.gsub(/\(.+\)/, '').strip
-  end
-
-  # converts a string containing terminal colors codes to an HTML string with .code-* spans
-  def terminal_to_html
-    self.gsub("\e[0m", '</span>')
-             .gsub(/\e\[([\d;]+)m/) do
-      codes = $1.split(';')
-      classes = codes.map { |code| ANSI_CODE_CLASSES[code.to_i] }.compact.map { |c| "code-#{c}" }
-      "<span class='#{classes.join(' ')}'>"
-    end
-  end
-
-  # creates a data URL of the string of the given mime type
-  def to_base64_data_url(mime_type = "image/svg+xml")
-    base64_data = Base64.strict_encode64(self)
-    "data:#{mime_type};base64,#{base64_data}"
-  end
-
-
-  def self.random_string(len)
-    rand(36 ** len).to_s(36)
-  end
-end
-
-
-class Integer
-
-  def cents(include_cents = true)
-    precision = include_cents ? 2 : 0
-    ActionController::Base.helpers.number_to_currency self / 100.0, precision: precision
-  end
-
-  def dollars(include_cents = true)
-    precision = include_cents ? 2 : 0
-    ActionController::Base.helpers.number_to_currency self, precision: precision
-  end
-
-  # @return [String] a string representing the byte size of self
-  def to_human_size
-    ActiveSupport::NumberHelper.number_to_human_size self
-  end
-
-  # @return [String] a string representing the number with comma delimiters for the thousands
-  def to_delimited
-    ActiveSupport::NumberHelper.number_to_delimited self
-  end
-
-  # @return [String] the number formatted as a string with an explicit '+' prefic for positive values
-  def to_signed_s
-    self > 0 ? "+#{self}" : self.to_s
-  end
-
-  # @return [String] the bolded string value of the number
-  def bold
-    self.to_s.bold
-  end
-end
-
-class Float
-
-  # @return [String] the number formatted as a string with an explicit '+' prefic for positive values
-  def to_signed_s
-    self > 0 ? "+#{self}" : self.to_s
-  end
-
-  # @return [String] the bolded string value of the number
-  def bold
-    self.to_s.bold
-  end
-
-  # @return [String] the floating point number formatted as a percentage with the given number of decimals
-  def to_percentage(decimals = 1)
-    "#{(self * 100).round(decimals)}%"
-  end
-
-  def to_ms
-    (self*1000).round(1)
-  end
-
-  # format the floating point seconds value for human consumption
-  def seconds_s
-    "#{self.round(1)}s"
-  end
-
-  def is_float?
-    true
-  end
-
-  def dollars(include_cents=true)
-    precision = include_cents ? 2 : 0
-    ActionController::Base.helpers.number_to_currency self, precision: precision
-  end
-
-  def percent
-    '%g%%' % (self*100.0)
-  end
-
-end
-
-
-class Array
-
-  # wraps each element in single quotes, joins them with commas, and wraps the whole thing in parentheses
-  def to_postgres_array
-    '(' + self.map{|a| "'#{a}'"}.join(',') + ')'
-  end
-
-  def to_postgres_array_literal
-    str = self.join('", "')
-    str = "{\"#{str}\"}"
-    str
-  end
-
-  def mean
-    self.sum / self.size.to_f
-  end
-
-  # remove unnecessary leading directories from backtrace lines
-  def filter_backtrace
-    dir = Dir.pwd.split('/')[0..-2].join('/') + '/'
-    self.map do |line|
-      if line.starts_with? dir
-        line.gsub dir, ''
-      elsif line.index '/gems/'
-        'gems/' + line.split('/gems/').last
-      elsif line.index '/rubygems/'
-        'rubygems/' + line.split('/rubygems/').last
-      elsif line.index '/ruby/'
-        'ruby/' + line.split('/ruby/').last
-      else
-        line
-      end
-    end
-  end
-
-  def sort_by_key(key)
-    self.sort_by do |v|
-      if v.nil?
-        raise "Can't sort nil element"
-      end
-      unless v.is_a?(Hash) && v.has_key?(key)
-        raise "Element is a #{v.class.name} and does not have a #{key} value"
-      end
-      v[key]
-    end
-  end
-
-  def map_key(key)
-    self.map{|v| v[key]}
-  end
-
-end
-
-
-class Hash
-
-  # creates a new hash with all keys as symbols
-  def symbolize
-    h = {}
-    self.each do |k, v|
-      h[k.to_sym] = v
-    end
-    h
-  end
-end
-
-class TrueClass
-  def is_true?
-    true
-  end
-  def is_false?
-    false
-  end
-end
-
-class FalseClass
-  def is_true?
-    false
-  end
-  def is_false?
-    true
-  end
-end

package/lib/terrier/system/ssh_key_manager.rb

@@ -1,74 +0,0 @@
-require 'http'
-
-# manages SSH public keys distributed by terrier.tech
-class SshKeyManager
-  include Loggable
-
-  def initialize
-    @env = Rails.env&.to_s || 'development'
-  end
-
-  # updates the keys in the authorized_keys file based on the given clyp from terrier.tech
-  def update_public_keys
-    info "Updating public keys"
-
-    # compute the authorized_keys path
-    if @env == 'development'
-      dir = Rails.root.join('tmp/ssh').to_s
-      FileUtils.mkdir_p(dir) unless File.exist?(dir)
-      file_path = "#{dir}/authorized_keys"
-      info "Using dummy authorized_keys path in development: #{file_path}"
-    else
-      file_path = File.expand_path "~/.ssh/authorized_keys"
-      info "Using authorized_keys path: #{file_path}"
-    end
-
-    File.open(file_path, "r+:US-ASCII") do |file|
-      # lock the file to avoid contention with other apps on the same machine editing this file simultaneously
-      lock = file.flock(File::LOCK_EX | File::LOCK_NB)
-      unless lock
-        info "Another process is already updating keys!"
-        return
-      end
-
-      # get the keys
-      res = HTTP.get "https://terrier.tech/public_keys.json"
-      raw = JSON.parse res.to_s
-      unless raw['status'] == 'success'
-        ap raw
-        raise "Error getting public keys: #{res['message']}"
-      end
-      new_keys = raw['public_keys'].sort
-      info "Found #{new_keys.count} keys"
-      new_keys.each do |key|
-        puts key
-      end
-
-      # read existing keys
-      existing_keys = file.read.split("\n").compact.map(&:strip).sort
-      info "#{existing_keys.count} existing keys in #{file_path}"
-
-      # abort if the existing keys are the same as the new ones
-      if existing_keys.join == new_keys.join
-        info "Existing keys are the same as the new ones, nothing to do"
-        return
-      end
-
-      # write the existing keys to a backup file
-      if existing_keys.present?
-        backup_path = "#{file_path}_#{Time.new.strftime(TIMESTAMP_FORMAT)}"
-        info "Existing keys are different than the new ones, writing them to #{backup_path}"
-        File.write backup_path, existing_keys.join("\n")
-      else
-        info "No existing keys, nothing to back up"
-      end
-
-      # write the new keys
-      info "Writing new keys to #{file_path}"
-      file.pos = 0
-      file.truncate(0)
-      file.write(new_keys.join("\n"))
-    end
-  end
-
-end