terrier-engine 4.13.1 → 4.13.2

package/lib/terrier/logging/multi_logger.rb

@@ -0,0 +1,137 @@
+require 'amazing_print'
+
+AmazingPrint.defaults ||= {}
+AmazingPrint.defaults[:ruby19_syntax] = true
+
+class MultiLogger
+
+  attr_accessor :use_stdout, :use_rails, :logger, :stream, :level, :prefix, :messages
+
+  LEVELS = %w(debug info success separator warn error)
+
+  def initialize(prefix, opts={})
+    @prefix = prefix
+    @logger = opts[:logger]
+    @use_stdout = opts[:use_stdout]
+    @use_rails = opts[:use_rails]
+
+    # if a specific logger is passed in, use_stdout and use_rails are opt-in (off by default)
+    # if a specific logger is not passed in, use_stdout and use_rails are opt-out (on by default)
+    @use_stdout = @logger.nil? if @use_stdout.nil?
+    @use_rails = @logger.nil? if @use_rails.nil?
+
+    @level = 'info'
+    @messages = []
+  end
+
+  def stream_response(response)
+    response.headers["Content-Type"] = "layout/json"
+    @stream = response.stream
+    @stream.write '['
+  end
+
+  def close_stream
+    return unless @stream
+    @stream.write '{}]'
+    @stream.close
+  end
+
+  # Setter method to change the log level
+  def level=(new_level)
+    @level = new_level
+  end
+
+  def debug(message, *args)
+    log 'debug', message, *args
+  end
+
+  def info(message, *args)
+    log 'info', message, *args
+  end
+
+  def success(message, *args)
+    log 'success', message, *args
+  end
+
+  def warn(message, *args)
+    log 'warn', message, *args
+  end
+
+  def separator(message, *args)
+    log 'separator', message, *args
+  end
+
+  def error(ex, *args)
+    message = ex.message
+    unless ex.backtrace.nil?
+      ex.backtrace[0..20].each do |line|
+        message += "\n#{line}"
+      end
+    end
+    log 'error', message, *args
+  end
+
+  def log(level, message, *args)
+    return if LEVELS.index(@level) > LEVELS.index(level)
+    if level == 'separator'
+      message = "==== #{message} ===="
+    end
+    write level, message
+
+    # amazing_print extra args
+    rails_level = %w(separator success).index(level) ? 'info' : level
+    args.each do |arg|
+      if @logger && @logger.respond_to?(:ap)
+        @logger.ap arg, rails_level
+      end
+
+      if @use_stdout
+        ap arg
+      end
+
+      if Rails&.logger && @use_rails
+        Rails.logger.ap arg, rails_level
+      end
+
+      write_stream level, arg.inspect # TODO: figure out how to send ap output to the stream
+    end
+  end
+
+
+  private
+
+
+  # writes directly to the appropriate outputs
+  def write(level, s)
+    logger_level = %w(separator success).index(level) ? 'info' : level
+    logger_message = "#{@prefix} #{s}"
+    if @logger
+      @logger.send logger_level, logger_message
+    end
+
+    if Rails&.logger && @use_rails
+      Rails.logger.send logger_level, logger_message
+    end
+
+    stdout_message = "#{level.upcase} -- #{@prefix} #{s}"
+    if @use_stdout
+      puts stdout_message
+    end
+    write_stream level, stdout_message
+  end
+
+  def write_stream(level, s)
+    time = Time.now.strftime(PRETTY_TIME_FORMAT)
+    @messages << {prefix: @prefix, time: time, level: level, message: s}
+    if @stream
+      chunk = {
+        level: level,
+        message: CGI.escapeHTML(s),
+        time: time,
+        prefix: @prefix
+      }
+      @stream.write "#{Oj.dump(chunk)},"
+    end
+  end
+
+end

package/lib/terrier/logging/progress_logger.rb

@@ -0,0 +1,34 @@
+# prints log messages with progress counts and time estimates
+class ProgressLogger
+  include ActionView::Helpers::DateHelper
+
+  # @param context [Loggable|IO] the output target of the step messages
+  def initialize(total, context = $stdout)
+    @total = total
+    @t_start = Time.now
+    @context = context
+    @last_step = 0
+  end
+
+  attr_reader :total, :last_step
+
+  # prints a step message to the output context
+  def step(n, message = '')
+    n = 1 if n < 1
+    @last_step = n
+    percent = n / @total.to_f * 100
+    t = Time.now
+    elapsed = distance_of_time_in_words @t_start, t
+    t_finish = @t_start + ((t - @t_start) / n * @total).seconds
+    to_go = distance_of_time_in_words t, t_finish
+    m = ["[#{n} of #{@total} (#{percent.round(2)}%)] #{elapsed} elapsed, #{to_go} to go", message.presence].compact.join(' - ')
+    if @context.respond_to? :info
+      @context.info m
+    else
+      @context.puts m
+    end
+    @context.try :flush
+    m
+  end
+
+end

package/lib/terrier/migrations.rb

@@ -0,0 +1,83 @@
+
+
+module MyColumnMethods
+
+  # rails assumes foreign keys are bigints
+  def references(other, options={})
+    options[:foreign_key] = true unless options.has_key?(:foreign_key)
+    options[:type] = :uuid
+    options[:index] = true unless options.has_key?(:index)
+    super other, **options
+  end
+
+  def references_uuid(other, options={})
+    options[:type] = :uuid
+    options[:index] ||= true
+    options[:foreign_key] = true
+    options[:null] ||= false
+    self.references other, type: options[:type], index: options[:index], foreign_key: options[:foreign_key], null: options[:null], polymorphic: options[:polymorphic]
+  end
+
+  # creates a non-null, default 0 integer column
+  def cents(name, options={})
+    options[:null] = false unless options.has_key?(:null)
+    options[:default] = false unless options.has_key?(:default)
+    self.integer name, options
+  end
+
+
+end
+
+class ActiveRecord::Migration
+
+  # this should be used in migrations instead of create_table
+  def create_model(name)
+    create_table name, id: :uuid, default: 'gen_random_uuid()' do |t|
+      t.timestamps
+      t.integer :_state, default: 0, null: false, index: true
+      t.uuid :created_by_id, index: true
+      t.text :created_by_name, null: false
+      t.text :extern_id, index: true
+      t.uuid :updated_by_id, index: true
+      t.text :updated_by_name
+      t.class.include MyColumnMethods
+      yield t
+    end
+    add_foreign_key name, :users, column: :created_by_id
+    add_foreign_key name, :users, column: :updated_by_id
+  end
+
+  # this should be used to create tables that belong to an org
+  def create_org_model(name)
+    create_model name do |t|
+      t.references_uuid :org
+      yield t
+    end
+  end
+
+  def add_markdown_field(table, name)
+    add_column table, "#{name}_raw".to_sym, :text
+    add_column table, "#{name}_html".to_sym, :text
+  end
+
+  # adds a reference to to_table on from_table, plus an index and foreign key
+  # pass a type option if it needs to be something other than text
+  def add_foreign_key_column(from_table, to_table, options = {})
+    column = options[:column].presence || (to_table.to_s.singularize + '_id')
+    type = options[:type].presence || :text
+    add_column from_table, column, type
+    if !options.has_key?(:index) || options[:index]
+      args = {}
+      args[:name] = options[:index] if options[:index]
+      add_index from_table, column, **args
+    end
+    add_foreign_key from_table, to_table, column: column
+  end
+
+  def add_indexed_column(table, name, type, options = {})
+    add_column table, name, type, **options
+    add_index table, name
+  end
+
+
+end

package/lib/terrier/puma/config.rb

@@ -0,0 +1,87 @@
+require 'dotenv/load'
+require 'sys/proctable'
+
+
+# compute the socket and pid paths
+app_dir = Dir.pwd
+tmp_dir = "#{app_dir}/tmp"
+sockets_dir = File.join tmp_dir, 'sockets'
+unless File.exist? sockets_dir
+  FileUtils.mkdir sockets_dir
+end
+socket_path = "unix://#{sockets_dir}/puma.sock"
+pids_dir = File.join tmp_dir, 'pids'
+unless File.exist? pids_dir
+  FileUtils.mkdir pids_dir
+end
+pid_path = "#{pids_dir}/puma.pid"
+
+def stop_server(pid_path)
+  was_killed = false
+  if File.exist? pid_path
+    pid = File.read pid_path
+    pid = pid&.to_i || 0
+    if pid > 0
+      proc = Sys::ProcTable.ps(pid: pid)
+      if proc
+        puts "Killing existing process with pid #{pid}"
+        Process.kill("HUP", pid)
+        was_killed = true
+      else
+        puts "No process #{pid} found, nothing to kill"
+      end
+    end
+    File.delete pid_path
+  else
+    puts "No pid file at #{pid_path}"
+  end
+  was_killed
+end
+
+command = ARGV.first || 'restart'
+case command
+when 'restart'
+  if stop_server pid_path
+    puts "Restarting server at #{Time.now.to_s}"
+    sleep 0.5 # without this, the port is still bound when the application tries to start again
+  else
+    puts "Starting server at #{Time.now.to_s}..."
+  end
+when 'stop'
+  puts "Stopping server at #{Time.now.to_s}..."
+  stop_server pid_path
+  exit
+else
+  raise "Unknown command '#{command}'"
+end
+
+# get the port from the environment
+port ENV.fetch("PORT") { 3000 }
+
+# Puma can serve each request in a thread from an internal thread pool.
+# The `threads` method setting takes two numbers: a minimum and maximum.
+# Any libraries that use thread pools should be configured to match
+# the maximum value specified for Puma. Default is set to 5 threads for minimum
+# and maximum; this matches the default thread size of Active Record.
+threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 }
+threads threads_count, threads_count
+
+# Specifies the number of `workers` to boot in clustered mode.
+# Workers are forked webserver processes. If using threads and workers together
+# the concurrency of the layout would be max `threads` * `workers`.
+# Workers do not work on JRuby or Windows (both of which do not support
+# processes).
+workers ENV.fetch("WEB_CONCURRENCY") { 2 }
+
+# bind to the UNIX socket and set the pid path
+bind socket_path
+pidfile pid_path
+
+# Allow puma to be restarted by `rails restart` command.
+plugin :tmp_restart
+
+# Use the `preload_app!` method when specifying a `workers` number.
+# This directive tells Puma to first boot the layout and load code
+# before forking the layout. This takes advantage of Copy On Write
+# process behavior so workers use less memory.
+preload_app!

package/lib/terrier/puma.rb

@@ -0,0 +1,12 @@
+module Terrier
+  module Puma
+
+    def self.load(dsl)
+      path = File.join File.dirname(__FILE__), 'puma/config.rb'
+      puts "Loading terrier-engine puma config from #{path}"
+      dsl._load_from path
+      path
+    end
+
+  end
+end

package/lib/terrier/scripts/script_config.rb

@@ -0,0 +1,30 @@
+module Terrier::ScriptConfig
+
+  @values = {
+      category_icons: {},
+      report_type_icons: {},
+      fields_help: ''
+  }
+
+  @values.keys.each do |key|
+    define_singleton_method "#{key}=" do |val|
+      @values[key] = val
+    end
+    define_singleton_method key do
+      @values[key]
+    end
+  end
+
+  def self.category_options
+    @values[:category_icons].map do |k, v|
+      [k.to_s.titleize, v, k]
+    end
+  end
+
+  def self.report_type_options
+    @values[:report_type_icons].map do |k, v|
+      [k.to_s.titleize, v, k]
+    end
+  end
+
+end

package/lib/terrier/scripts/script_executor.rb

@@ -0,0 +1,259 @@
+require 'terrier/io/tabular_io'
+
+class ScriptExecutor
+  # Doesn't need to be Loggable, it already has all the methods
+
+  attr_reader :cache, :each_count, :each_total, :log_lines, :email_settings
+  attr_accessor :me, :params, :script
+
+  def should_soft_destroy
+    true
+  end
+
+  def initialize(script, cache=nil, params=nil)
+    @script = script
+    @cache = cache
+    @each_count = 0
+    @each_total = 0
+    @field_values = {}
+    @log_lines = []
+    @output_files = []
+    @params = params
+    @email_settings = ActiveSupport::HashWithIndifferentAccess.new({
+      file_output: 'attachment', # link | attachment
+      disable: false, # true | false
+      body: '' # insert a body into the email
+    })
+  end
+
+  def set_email_settings(settings)
+    @email_settings.update(settings)
+  end
+
+  def set_field_values(values)
+    @field_values = ActiveSupport::HashWithIndifferentAccess.new @script.compute_field_values(values)
+  end
+
+  # returns a ScriptRun object describing the run
+  def init_run
+    script_run = ScriptRun.new script_id: @script.id, status: 'running', created_at: Time.now, duration: 0
+
+    if script_run.respond_to?(:fields)
+      script_run.fields = @field_values
+    end
+
+    script_run
+  end
+
+  # actually executes the script
+  def run(script_run, stream)
+    @stream = stream
+    t = Time.now
+    begin
+      if @stream
+        @stream.write '['
+      end
+      res = execute_body @script.body
+      if res && res.is_a?(String) && res.present? # we probably don't need to print random crap that's returned
+        info "DONE: #{res}"
+      end
+      script_run.status = 'success'
+      script_run.duration = Time.now - t
+      if @script.persisted? # we can't save the run if it's a temporary script
+        script_run.write_log @log_lines.join("\n")
+        script_email_log = @script.send_email_if_necessary @email_settings, me, @output_files, script_run.log_url
+        puts script_email_log if script_email_log
+      end
+      true
+    rescue => ex
+      line = ex.backtrace[0].split(':')[1].to_i
+      write_raw 'error', "Error on line #{line}: #{ex.message}"
+      script_run.status = 'error'
+      script_run.exception = ex.message
+      script_run.backtrace = ex.backtrace.join("\n")
+      script_run.duration = Time.now - t
+      @log_lines << ex.message
+      error ex
+      if @script.persisted? # we can't save the run if it's a temporary script
+        script_run.write_log @log_lines.join("\n")
+      end
+      false
+    ensure
+      if @stream
+        @stream.write '{}]'
+      end
+      @stream.close if @stream
+    end
+  end
+
+  # executes the body code and returns the result
+  # this needs to be wrapped so that returning from the script doesn't interrupt the run method
+  def execute_body(body)
+    escaped_body = body.gsub('\"', '"')
+    filename = "script"
+    filename += "<#{@script.title}>" if @script&.title.present?
+    eval(escaped_body, binding, filename, 1)
+  end
+
+  def write_raw(type, body, extra={})
+    return unless @stream
+    extra[:type] = type
+    extra[:body] = CGI.escapeHTML(body)
+    @stream.write(extra.to_json + ',')
+  end
+
+  def puts(message)
+    write_raw 'print', message.to_s
+    Rails.logger.debug "(ScriptExecutor) #{message}"
+    @log_lines << message.to_s
+  end
+
+  def log(level, message)
+    write_raw level, message.to_s
+    s = "#{level.upcase}: #{message}"
+    level = 'info' if level == 'success'
+    Rails.logger.send level, "(ScriptExecutor) #{s}"
+    @log_lines << s
+  end
+
+  # make it work like a logger
+  %w(debug info warn success).each do |level|
+    define_method level do |message|
+      log level, message
+    end
+  end
+
+  def error(ex)
+    if ex.is_a? Exception
+      log 'error', ex.message
+      if ex.backtrace
+        ex.backtrace[0..6].each do |line|
+          log 'error', line
+        end
+      end
+    else
+      log 'error', ex.to_s
+    end
+  end
+
+  def puts_count(message='')
+    puts "[#{@each_count} of #{@each_total}] #{message}"
+  end
+
+  def dump_xls(data, rel_path, options={})
+    dump_file data, rel_path, options
+  end
+
+  def dump_csv(data, rel_path, options={})
+    dump_file data, rel_path, options
+  end
+
+  # either dumps a csv or xls using TabularIo.save
+  def dump_file(data, rel_path, options={})
+    abs_path = TabularIo.save data, rel_path, options
+    @output_files << abs_path.to_s
+    write_raw 'file', TabularIo.abs_to_rel_path(abs_path)
+    puts "Wrote #{data.count} records to #{rel_path}"
+  end
+
+  def puts_file(path)
+    @output_files << path
+    if path.index(Rails.root.to_s)
+      path = TabularIo.abs_to_rel_path path
+    end
+    write_raw 'file', path
+    file_name = File.basename path
+    puts "Showing #{file_name}"
+  end
+
+  # loads either a csv or xlsx file using TabularIo.load
+  def load_file(rel_path, options={})
+    TabularIo.load rel_path, options
+  end
+
+  def get_field(name)
+    @field_values[name]
+  end
+
+  def get_params
+    @params
+  end
+
+  def each(collection)
+    @each_total = collection.count
+    @each_count = 0
+    result = collection.map do |item|
+      @each_count += 1
+      yield item
+    end
+    @each_count = 0
+    @each_total = 0
+    result
+  end
+
+  def read_input(name=nil)
+    if name && name.length > 0
+      input = @script.script_inputs.where(name: name).first
+      unless input
+        raise "No input named '#{name}'"
+      end
+      input.read
+    else # default to the first input
+      input = @script.script_inputs.first
+      unless input
+        raise 'This script does not have any inputs!'
+      end
+      input.read
+    end
+  end
+
+  def parse_dollars(dollars)
+    if dollars.instance_of? String
+      dollars = dollars.to_f
+    end
+    return 0 if dollars.nil? || dollars==0
+    (dollars * 100).round.to_i # fix floating point truncating error
+  end
+
+  def format_cents(cents)
+    "$#{'%.2f' % (cents / 100.0).round(2)}"
+  end
+
+  def email_recipients
+    if Rails.env == 'production'
+      if @script.email_recipients.blank?
+        raise 'No e-mail recipients for this script!'
+      end
+      @script.email_recipients
+    else
+      ['clypboardtesting@gmail.com']
+    end
+  end
+
+  def raw_sql(query)
+    ActiveRecord::Base.connection.execute(query).to_a
+  end
+
+  # No longer supported, see ScriptBase::send_email_if_necessary or send_explicit_email
+  def send_email(options)
+    puts "send_email function is no longer supported"
+  end
+
+  def send_explicit_email(options)
+    puts @script.send_explicit_email options
+  end
+
+  def self_destruct(countdown=5)
+    if countdown.present? && countdown > 0
+      puts "This script will self destruct in..."
+      countdown.downto(1) do |i|
+        puts i
+        sleep(1)
+      end
+    end
+    puts "Bang!"
+    self.script._state = 2
+    self.script.save_by_user!(self.me)
+  end
+
+end

package/lib/terrier/scripts/script_field.rb

@@ -0,0 +1,60 @@
+class ScriptField
+  include Terrier::Embedded
+
+  field :name, type: String, required: true
+
+  field :default_value, type: String, required: true
+
+  field :values, type: String
+
+  field :required, type: String
+
+  enum_field :field_type, %w(date string select csv hidden)
+
+  def self.compute_date_value(s)
+    if s =~ /\d{4}-\d{2}-\d{2}/
+      Time.parse s
+    else
+      eval(s)
+    end
+  end
+
+  def self.compute_value(type, s)
+    case type
+    when 'date'
+      compute_date_value s
+    when 'string'
+      s
+    when 'select'
+      s
+    when 'hidden'
+      s
+    when 'csv'
+      if s.blank?
+        []
+      else
+        TabularIo.parse_csv s
+      end
+    else
+      raise "Unknown field type #{type}"
+    end
+  end
+
+  def compute_options
+    return [] unless self.field_type == 'select'
+    opts = eval(self.values)
+    opts = opts.is_a?(Array) ? opts : []
+    # don't add a blank space if there are no other options
+    # this way, the field will appear as a hidden field
+    if opts.size > 0 
+      if opts.first.is_a? Array
+        [['', nil]] + opts
+      else
+        [''] + opts
+      end
+    else
+      opts
+    end
+  end
+
+end