terrier-engine 4.67.7 → 4.67.10

package/lib/terrier/scripts/script_executor.rb

@@ -0,0 +1,263 @@
+require 'terrier/io/tabular_io'
+
+class ScriptExecutor
+  # Doesn't need to be Loggable, it already has all the methods
+
+  attr_reader :cache, :each_count, :each_total, :log_lines, :email_settings
+  attr_accessor :me, :params, :script
+
+  def should_soft_destroy
+    true
+  end
+
+  def initialize(script, cache=nil, params=nil)
+    @script = script
+    @cache = cache
+    @each_count = 0
+    @each_total = 0
+    @field_values = {}
+    @log_lines = []
+    @output_files = []
+    @params = params
+    @email_settings = ActiveSupport::HashWithIndifferentAccess.new({
+      file_output: 'attachment', # link | attachment
+      disable: false, # true | false
+      body: '' # insert a body into the email
+    })
+  end
+
+  def set_email_settings(settings)
+    @email_settings.update(settings)
+  end
+
+  def set_field_values(values)
+    @field_values = ActiveSupport::HashWithIndifferentAccess.new @script.compute_field_values(values)
+  end
+
+  # returns a ScriptRun object describing the run
+  def init_run
+    script_run = ScriptRun.new script_id: @script.id, status: 'running', created_at: Time.now, duration: 0
+
+    if script_run.respond_to?(:fields)
+      file_fields = @script
+                     .script_fields
+                     .select { |field| %w[csv xlsx].include?(field.field_type) }
+                     .map(&:name)
+      script_run.fields = @field_values.reject { |k, _| file_fields.include?(k) }
+    end
+
+    script_run
+  end
+
+  # actually executes the script
+  def run(script_run, stream)
+    @stream = stream
+    t = Time.now
+    begin
+      if @stream
+        @stream.write '['
+      end
+      res = execute_body @script.body
+      if res && res.is_a?(String) && res.present? # we probably don't need to print random crap that's returned
+        info "DONE: #{res}"
+      end
+      script_run.status = 'success'
+      script_run.duration = Time.now - t
+      if @script.persisted? # we can't save the run if it's a temporary script
+        script_run.write_log @log_lines.join("\n")
+        script_email_log = @script.send_email_if_necessary @email_settings, me, @output_files, script_run.log_url
+        puts script_email_log if script_email_log
+      end
+      true
+    rescue Exception => ex
+      line = ex.backtrace[0].split(':')[1].to_i
+      write_raw 'error', "Error on line #{line}: #{ex.message}"
+      script_run.status = 'error'
+      script_run.exception = ex.message
+      script_run.backtrace = ex.backtrace.join("\n")
+      script_run.duration = Time.now - t
+      @log_lines << ex.message
+      error ex
+      if @script.persisted? # we can't save the run if it's a temporary script
+        script_run.write_log @log_lines.join("\n")
+      end
+      false
+    ensure
+      if @stream
+        @stream.write '{}]'
+      end
+      @stream.close if @stream
+    end
+  end
+
+  # executes the body code and returns the result
+  # this needs to be wrapped so that returning from the script doesn't interrupt the run method
+  def execute_body(body)
+    escaped_body = body.gsub('\"', '"')
+    filename = "script"
+    filename += "<#{@script.title}>" if @script&.title.present?
+    eval(escaped_body, binding, filename, 1)
+  end
+
+  def write_raw(type, body, extra={})
+    return unless @stream
+    extra[:type] = type
+    extra[:body] = CGI.escapeHTML(body)
+    @stream.write(extra.to_json + ',')
+  end
+
+  def puts(message)
+    write_raw 'print', message.to_s
+    Rails.logger.debug "(ScriptExecutor) #{message}"
+    @log_lines << message.to_s
+  end
+
+  def log(level, message)
+    write_raw level, message.to_s
+    s = "#{level.upcase}: #{message}"
+    level = 'info' if level == 'success'
+    Rails.logger.send level, "(ScriptExecutor) #{s}"
+    @log_lines << s
+  end
+
+  # make it work like a logger
+  %w(debug info warn success).each do |level|
+    define_method level do |message|
+      log level, message
+    end
+  end
+
+  def error(ex)
+    if ex.is_a? Exception
+      log 'error', ex.message
+      if ex.backtrace
+        ex.backtrace[0..6].each do |line|
+          log 'error', line
+        end
+      end
+    else
+      log 'error', ex.to_s
+    end
+  end
+
+  def puts_count(message='')
+    puts "[#{@each_count} of #{@each_total}] #{message}"
+  end
+
+  def dump_xls(data, rel_path, options={})
+    dump_file data, rel_path, options
+  end
+
+  def dump_csv(data, rel_path, options={})
+    dump_file data, rel_path, options
+  end
+
+  # either dumps a csv or xls using TabularIo.save
+  def dump_file(data, rel_path, options={})
+    abs_path = TabularIo.save data, rel_path, options
+    @output_files << abs_path.to_s
+    write_raw 'file', TabularIo.abs_to_rel_path(abs_path)
+    puts "Wrote #{data.count} records to #{rel_path}"
+  end
+
+  def puts_file(path)
+    @output_files << path
+    if path.index(Rails.root.to_s)
+      path = TabularIo.abs_to_rel_path path
+    end
+    write_raw 'file', path
+    file_name = File.basename path
+    puts "Showing #{file_name}"
+  end
+
+  # loads either a csv or xlsx file using TabularIo.load
+  def load_file(rel_path, options={})
+    TabularIo.load rel_path, options
+  end
+
+  def get_field(name)
+    @field_values[name]
+  end
+
+  def get_params
+    @params
+  end
+
+  def each(collection)
+    @each_total = collection.count
+    @each_count = 0
+    result = collection.map do |item|
+      @each_count += 1
+      yield item
+    end
+    @each_count = 0
+    @each_total = 0
+    result
+  end
+
+  def read_input(name=nil)
+    if name && name.length > 0
+      input = @script.script_inputs.where(name: name).first
+      unless input
+        raise "No input named '#{name}'"
+      end
+      input.read
+    else # default to the first input
+      input = @script.script_inputs.first
+      unless input
+        raise 'This script does not have any inputs!'
+      end
+      input.read
+    end
+  end
+
+  def parse_dollars(dollars)
+    if dollars.instance_of? String
+      dollars = dollars.to_f
+    end
+    return 0 if dollars.nil? || dollars==0
+    (dollars * 100).round.to_i # fix floating point truncating error
+  end
+
+  def format_cents(cents)
+    "$#{'%.2f' % (cents / 100.0).round(2)}"
+  end
+
+  def email_recipients
+    if Rails.env == 'production'
+      if @script.email_recipients.blank?
+        raise 'No e-mail recipients for this script!'
+      end
+      @script.email_recipients
+    else
+      ['clypboardtesting@gmail.com']
+    end
+  end
+
+  def raw_sql(query)
+    ActiveRecord::Base.connection.execute(query).to_a
+  end
+
+  # No longer supported, see ScriptBase::send_email_if_necessary or send_explicit_email
+  def send_email(options)
+    puts "send_email function is no longer supported"
+  end
+
+  def send_explicit_email(options)
+    puts @script.send_explicit_email options
+  end
+
+  def self_destruct(countdown=5)
+    if countdown.present? && countdown > 0
+      puts "This script will self destruct in..."
+      countdown.downto(1) do |i|
+        puts i
+        sleep(1)
+      end
+    end
+    puts "Bang!"
+    self.script._state = 2
+    self.script.save_by_user!(self.me)
+  end
+
+end

package/lib/terrier/scripts/script_field.rb

@@ -0,0 +1,68 @@
+class ScriptField
+  include Terrier::Embedded
+
+  field :name, type: String, required: true
+
+  field :default_value, type: String, required: true
+
+  field :values, type: String
+
+  field :required, type: String
+
+  enum_field :field_type, %w(date string select csv xlsx hidden)
+
+  def self.compute_date_value(s)
+    if s.nil?
+      s
+    elsif s =~ /\d{4}-\d{2}-\d{2}/
+      Time.parse s
+    else
+      eval(s)&.to_time
+    end
+  end
+
+  def self.compute_value(type, s)
+    case type
+    when 'date'
+      compute_date_value s
+    when 'string'
+      s
+    when 'select'
+      s
+    when 'hidden'
+      s
+    when 'csv'
+      if s.blank?
+        []
+      else
+        TabularIo.parse_csv s
+      end
+    when 'xlsx'
+      if s.blank?
+        {}
+      else
+        TabularIo.load_xlsx nil, {raw: StringIO.new(Base64.decode64(s))}
+      end
+    else
+      raise "Unknown field type #{type}"
+    end
+  end
+
+  def compute_options
+    return [] unless self.field_type == 'select'
+    opts = eval(self.values)
+    opts = opts.is_a?(Array) ? opts : []
+    # don't add a blank space if there are no other options
+    # this way, the field will appear as a hidden field
+    if opts.size > 0 
+      if opts.first.is_a? Array
+        [['', nil]] + opts
+      else
+        [''] + opts
+      end
+    else
+      opts
+    end
+  end
+
+end

package/lib/terrier/scripts/script_searcher.rb

@@ -0,0 +1,23 @@
+class ScriptSearcher
+  def search(query)
+    
+    query= SqlBuilder.new
+      .select('body.id, t.total')
+      .from("body")
+      .left_join('total', 't', 'TRUE')
+      .with("body AS (SELECT id FROM scripts WHERE body ILIKE '%#{ query.strip }%'),
+             total AS (SELECT COUNT(*) total FROM body)")
+
+    t = Time.now
+    results = query.exec
+    total= results.present? ? results.first['total'] : 0
+    ids = results.present? ? results.to_a.pluck('id') : []
+    dt = Time.now - t
+
+    OpenStruct.new({
+      'took' => dt,
+      'ids' => ids,
+      'total' => total,
+    })
+  end
+end

package/lib/terrier/strings.rb

@@ -0,0 +1,289 @@
+
+EMAIL_SPLIT_REGEX = /[\s,;]+/
+
+SLUG_REGEX = /([[:lower:]]|[0-9]+-?[[:lower:]])(-[[:lower:]0-9]+|[[:lower:]0-9])*/
+
+# add values here that should always be upcased when displayed to the user
+UPCASE_VALUES = %w[cod ace bce csr dot pca tpp it qal html pdf ach eft wdi wdo]
+
+# these shouldn't be upcased, even though they're only two characters
+UPCASE_BLACKLIST = %w[at by to is or]
+
+GIGA = 1024 * 1024 * 1024
+MEGA = 1024 * 1024
+KILO = 1024
+
+# needed to parse WKB geo points
+if defined? RGeo
+  RGEO_FACTORY = RGeo::Geographic.spherical_factory(
+      wkb_parser: {support_ewkb: true}, wkb_generator: {hex_format: true, emit_ewkb_srid: true})
+else
+  RGEO_FACTORY = nil
+end
+
+class NilClass
+
+  def is_true?
+    false
+  end
+
+  def is_false?
+    true
+  end
+
+  def is_float?
+    false
+  end
+
+end
+
+TRUTHY_STRINGS = %w[true t on 1]
+FALSY_STRINGS = %w[false f off 0]
+
+ANSI_CODE_CLASSES = {
+  1 => 'bold',
+  3 => 'italic',
+  31 => 'red',
+  32 => 'green',
+  33 => 'yellow',
+  34 => 'blue',
+  36 => 'cyan'
+}
+
+class String
+
+  def uncapitalize
+    self[0, 1].downcase + self[1..-1]
+  end
+
+  def slugify
+    self.strip.split(/[\s[[:punct:]]]+/).join('_').gsub(/[^\w\-.]/, '').downcase
+  end
+
+  def parse_postgres_array
+    self.gsub('{', '').gsub('}', '').gsub('"', '').split(',')
+  end
+
+  def is_float?
+    !!Float(self) rescue false
+  end
+
+  def is_true?
+    !(TRUTHY_STRINGS.index(self.downcase.strip)).nil?
+  end
+
+  def is_false?
+    !(FALSY_STRINGS.index(self.downcase.strip)).nil?
+  end
+
+  def to_bool
+    TRUTHY_STRINGS.index self.downcase.strip
+  end
+
+  def smart_strip
+    self.strip.gsub(/\A[[:space:]]+/, '').gsub(/[[:space:]]+\z/, '')
+  end
+
+  def smart_title
+    lower_s = self.downcase
+    if (self.length<3 and !UPCASE_BLACKLIST.index(lower_s)) or UPCASE_VALUES.index(lower_s)
+      self.upcase
+    else
+      self.titleize
+    end
+  end
+
+  def parse_geo_point
+    if self.index('POINT')
+      self.gsub('POINT (', '').gsub(')', '').strip.split(' ').map(&:to_f)
+    else
+      RGEO_FACTORY.parse_wkb self
+    end
+  end
+
+  # returns the string with anything between parentheses
+  def without_parens
+    self.gsub(/\(.+\)/, '').strip
+  end
+
+  # converts a string containing terminal colors codes to an HTML string with .code-* spans
+  def terminal_to_html
+    self.gsub("\e[0m", '</span>')
+             .gsub(/\e\[([\d;]+)m/) do
+      codes = $1.split(';')
+      classes = codes.map { |code| ANSI_CODE_CLASSES[code.to_i] }.compact.map { |c| "code-#{c}" }
+      "<span class='#{classes.join(' ')}'>"
+    end
+  end
+
+  # creates a data URL of the string of the given mime type
+  def to_base64_data_url(mime_type = "image/svg+xml")
+    base64_data = Base64.strict_encode64(self)
+    "data:#{mime_type};base64,#{base64_data}"
+  end
+
+
+  def self.random_string(len)
+    rand(36 ** len).to_s(36)
+  end
+end
+
+
+class Integer
+
+  def cents(include_cents = true)
+    precision = include_cents ? 2 : 0
+    ActionController::Base.helpers.number_to_currency self / 100.0, precision: precision
+  end
+
+  def dollars(include_cents = true)
+    precision = include_cents ? 2 : 0
+    ActionController::Base.helpers.number_to_currency self, precision: precision
+  end
+
+  # @return [String] a string representing the byte size of self
+  def to_human_size
+    ActiveSupport::NumberHelper.number_to_human_size self
+  end
+
+  # @return [String] a string representing the number with comma delimiters for the thousands
+  def to_delimited
+    ActiveSupport::NumberHelper.number_to_delimited self
+  end
+
+  # @return [String] the number formatted as a string with an explicit '+' prefic for positive values
+  def to_signed_s
+    self > 0 ? "+#{self}" : self.to_s
+  end
+
+  # @return [String] the bolded string value of the number
+  def bold
+    self.to_s.bold
+  end
+end
+
+class Float
+
+  # @return [String] the number formatted as a string with an explicit '+' prefic for positive values
+  def to_signed_s
+    self > 0 ? "+#{self}" : self.to_s
+  end
+
+  # @return [String] the bolded string value of the number
+  def bold
+    self.to_s.bold
+  end
+
+  # @return [String] the floating point number formatted as a percentage with the given number of decimals
+  def to_percentage(decimals = 1)
+    "#{(self * 100).round(decimals)}%"
+  end
+
+  def to_ms
+    (self*1000).round(1)
+  end
+
+  # format the floating point seconds value for human consumption
+  def seconds_s
+    "#{self.round(1)}s"
+  end
+
+  def is_float?
+    true
+  end
+
+  def dollars(include_cents=true)
+    precision = include_cents ? 2 : 0
+    ActionController::Base.helpers.number_to_currency self, precision: precision
+  end
+
+  def percent
+    '%g%%' % (self*100.0)
+  end
+
+end
+
+
+class Array
+
+  # wraps each element in single quotes, joins them with commas, and wraps the whole thing in parentheses
+  def to_postgres_array
+    '(' + self.map{|a| "'#{a}'"}.join(',') + ')'
+  end
+
+  def to_postgres_array_literal
+    str = self.join('", "')
+    str = "{\"#{str}\"}"
+    str
+  end
+
+  def mean
+    self.sum / self.size.to_f
+  end
+
+  # remove unnecessary leading directories from backtrace lines
+  def filter_backtrace
+    dir = Dir.pwd.split('/')[0..-2].join('/') + '/'
+    self.map do |line|
+      if line.starts_with? dir
+        line.gsub dir, ''
+      elsif line.index '/gems/'
+        'gems/' + line.split('/gems/').last
+      elsif line.index '/rubygems/'
+        'rubygems/' + line.split('/rubygems/').last
+      elsif line.index '/ruby/'
+        'ruby/' + line.split('/ruby/').last
+      else
+        line
+      end
+    end
+  end
+
+  def sort_by_key(key)
+    self.sort_by do |v|
+      if v.nil?
+        raise "Can't sort nil element"
+      end
+      unless v.is_a?(Hash) && v.has_key?(key)
+        raise "Element is a #{v.class.name} and does not have a #{key} value"
+      end
+      v[key]
+    end
+  end
+
+  def map_key(key)
+    self.map{|v| v[key]}
+  end
+
+end
+
+
+class Hash
+
+  # creates a new hash with all keys as symbols
+  def symbolize
+    h = {}
+    self.each do |k, v|
+      h[k.to_sym] = v
+    end
+    h
+  end
+end
+
+class TrueClass
+  def is_true?
+    true
+  end
+  def is_false?
+    false
+  end
+end
+
+class FalseClass
+  def is_true?
+    false
+  end
+  def is_false?
+    true
+  end
+end

package/lib/terrier/system/ssh_key_manager.rb

@@ -0,0 +1,74 @@
+require 'http'
+
+# manages SSH public keys distributed by terrier.tech
+class SshKeyManager
+  include Loggable
+
+  def initialize
+    @env = Rails.env&.to_s || 'development'
+  end
+
+  # updates the keys in the authorized_keys file based on the given clyp from terrier.tech
+  def update_public_keys
+    info "Updating public keys"
+
+    # compute the authorized_keys path
+    if @env == 'development'
+      dir = Rails.root.join('tmp/ssh').to_s
+      FileUtils.mkdir_p(dir) unless File.exist?(dir)
+      file_path = "#{dir}/authorized_keys"
+      info "Using dummy authorized_keys path in development: #{file_path}"
+    else
+      file_path = File.expand_path "~/.ssh/authorized_keys"
+      info "Using authorized_keys path: #{file_path}"
+    end
+
+    File.open(file_path, "r+:US-ASCII") do |file|
+      # lock the file to avoid contention with other apps on the same machine editing this file simultaneously
+      lock = file.flock(File::LOCK_EX | File::LOCK_NB)
+      unless lock
+        info "Another process is already updating keys!"
+        return
+      end
+
+      # get the keys
+      res = HTTP.get "https://terrier.tech/public_keys.json"
+      raw = JSON.parse res.to_s
+      unless raw['status'] == 'success'
+        ap raw
+        raise "Error getting public keys: #{res['message']}"
+      end
+      new_keys = raw['public_keys'].sort
+      info "Found #{new_keys.count} keys"
+      new_keys.each do |key|
+        puts key
+      end
+
+      # read existing keys
+      existing_keys = file.read.split("\n").compact.map(&:strip).sort
+      info "#{existing_keys.count} existing keys in #{file_path}"
+
+      # abort if the existing keys are the same as the new ones
+      if existing_keys.join == new_keys.join
+        info "Existing keys are the same as the new ones, nothing to do"
+        return
+      end
+
+      # write the existing keys to a backup file
+      if existing_keys.present?
+        backup_path = "#{file_path}_#{Time.new.strftime(TIMESTAMP_FORMAT)}"
+        info "Existing keys are different than the new ones, writing them to #{backup_path}"
+        File.write backup_path, existing_keys.join("\n")
+      else
+        info "No existing keys, nothing to back up"
+      end
+
+      # write the new keys
+      info "Writing new keys to #{file_path}"
+      file.pos = 0
+      file.truncate(0)
+      file.write(new_keys.join("\n"))
+    end
+  end
+
+end