terrier-engine 4.13.1 → 4.13.2

package/lib/terrier/scripts/script_searcher.rb

@@ -0,0 +1,23 @@
+class ScriptSearcher
+  def search(query)
+    
+    query= SqlBuilder.new
+      .select('body.id, t.total')
+      .from("body")
+      .left_join('total', 't', 'TRUE')
+      .with("body AS (SELECT id FROM scripts WHERE body ILIKE '%#{ query.strip }%'),
+             total AS (SELECT COUNT(*) total FROM body)")
+
+    t = Time.now
+    results = query.exec
+    total= results.present? ? results.first['total'] : 0
+    ids= results.present? ? results.pluck('id') : []
+    dt = Time.now - t
+
+    OpenStruct.new({
+      'took' => dt,
+      'ids' => ids,
+      'total' => total,
+    })
+  end
+end

package/lib/terrier/strings.rb

@@ -0,0 +1,284 @@
+
+EMAIL_SPLIT_REGEX = /[\s,;]+/
+
+SLUG_REGEX = /([[:lower:]]|[0-9]+-?[[:lower:]])(-[[:lower:]0-9]+|[[:lower:]0-9])*/
+
+# add values here that should always be upcased when displayed to the user
+UPCASE_VALUES = %w[cod ace bce csr dot pca tpp it qal html pdf ach eft wdi wdo]
+
+# these shouldn't be upcased, even though they're only two characters
+UPCASE_BLACKLIST = %w[at by to is or]
+
+GIGA = 1024 * 1024 * 1024
+MEGA = 1024 * 1024
+KILO = 1024
+
+# needed to parse WKB geo points
+if defined? RGeo
+  RGEO_FACTORY = RGeo::Geographic.spherical_factory(
+      wkb_parser: {support_ewkb: true}, wkb_generator: {hex_format: true, emit_ewkb_srid: true})
+else
+  RGEO_FACTORY = nil
+end
+
+class NilClass
+
+  def is_true?
+    false
+  end
+
+  def is_false?
+    true
+  end
+
+  def is_float?
+    false
+  end
+
+end
+
+TRUTHY_STRINGS = %w[true t on 1]
+FALSY_STRINGS = %w[false f off 0]
+
+ANSI_CODE_CLASSES = {
+  1 => 'bold',
+  3 => 'italic',
+  31 => 'red',
+  32 => 'green',
+  33 => 'yellow',
+  34 => 'blue',
+  36 => 'cyan'
+}
+
+class String
+
+  def uncapitalize
+    self[0, 1].downcase + self[1..-1]
+  end
+
+  def slugify
+    self.strip.split(/[\s[[:punct:]]]+/).join('_').gsub(/[^\w\-.]/, '').downcase
+  end
+
+  def parse_postgres_array
+    self.gsub('{', '').gsub('}', '').gsub('"', '').split(',')
+  end
+
+  def is_float?
+    !!Float(self) rescue false
+  end
+
+  def is_true?
+    !(TRUTHY_STRINGS.index(self.downcase.strip)).nil?
+  end
+
+  def is_false?
+    !(FALSY_STRINGS.index(self.downcase.strip)).nil?
+  end
+
+  def to_bool
+    TRUTHY_STRINGS.index self.downcase.strip
+  end
+
+  def smart_strip
+    self.strip.gsub(/\A[[:space:]]+/, '').gsub(/[[:space:]]+\z/, '')
+  end
+
+  def smart_title
+    lower_s = self.downcase
+    if (self.length<3 and !UPCASE_BLACKLIST.index(lower_s)) or UPCASE_VALUES.index(lower_s)
+      self.upcase
+    else
+      self.titleize
+    end
+  end
+
+  def parse_geo_point
+    if self.index('POINT')
+      self.gsub('POINT (', '').gsub(')', '').strip.split(' ').map(&:to_f)
+    else
+      RGEO_FACTORY.parse_wkb self
+    end
+  end
+
+  # returns the string with anything between parentheses
+  def without_parens
+    self.gsub(/\([\w\s]+\)/, '').strip
+  end
+
+  # converts a string containing terminal colors codes to an HTML string with .code-* spans
+  def terminal_to_html
+    self.gsub("\e[0m", '</span>')
+             .gsub(/\e\[([\d;]+)m/) do
+      codes = $1.split(';')
+      classes = codes.map { |code| ANSI_CODE_CLASSES[code.to_i] }.compact.map { |c| "code-#{c}" }
+      "<span class='#{classes.join(' ')}'>"
+    end
+  end
+
+  # creates a data URL of the string of the given mime type
+  def to_base64_data_url(mime_type = "image/svg+xml")
+    base64_data = Base64.strict_encode64(self)
+    "data:#{mime_type};base64,#{base64_data}"
+  end
+
+end
+
+class Integer
+
+  def cents(include_cents = true)
+    precision = include_cents ? 2 : 0
+    ActionController::Base.helpers.number_to_currency self / 100.0, precision: precision
+  end
+
+  def dollars(include_cents = true)
+    precision = include_cents ? 2 : 0
+    ActionController::Base.helpers.number_to_currency self, precision: precision
+  end
+
+  # @return [String] a string representing the byte size of self
+  def to_human_size
+    ActiveSupport::NumberHelper.number_to_human_size self
+  end
+
+  # @return [String] a string representing the number with comma delimiters for the thousands
+  def to_delimited
+    ActiveSupport::NumberHelper.number_to_delimited self
+  end
+
+  # @return [String] the number formatted as a string with an explicit '+' prefic for positive values
+  def to_signed_s
+    self > 0 ? "+#{self}" : self.to_s
+  end
+
+  # @return [String] the bolded string value of the number
+  def bold
+    self.to_s.bold
+  end
+end
+
+class Float
+
+  # @return [String] the number formatted as a string with an explicit '+' prefic for positive values
+  def to_signed_s
+    self > 0 ? "+#{self}" : self.to_s
+  end
+
+  # @return [String] the bolded string value of the number
+  def bold
+    self.to_s.bold
+  end
+
+  # @return [String] the floating point number formatted as a percentage with the given number of decimals
+  def to_percentage(decimals = 1)
+    "#{(self * 100).round(decimals)}%"
+  end
+
+  def to_ms
+    (self*1000).round(1)
+  end
+
+  # format the floating point seconds value for human consumption
+  def seconds_s
+    "#{self.round(1)}s"
+  end
+
+  def is_float?
+    true
+  end
+
+  def dollars(include_cents=true)
+    precision = include_cents ? 2 : 0
+    ActionController::Base.helpers.number_to_currency self, precision: precision
+  end
+
+  def percent
+    '%g%%' % (self*100.0)
+  end
+
+end
+
+
+class Array
+
+  # wraps each element in single quotes, joins them with commas, and wraps the whole thing in parentheses
+  def to_postgres_array
+    '(' + self.map{|a| "'#{a}'"}.join(',') + ')'
+  end
+
+  def to_postgres_array_literal
+    str = self.join('", "')
+    str = "{\"#{str}\"}"
+    str
+  end
+
+  def mean
+    self.sum / self.size.to_f
+  end
+
+  # remove unnecessary leading directories from backtrace lines
+  def filter_backtrace
+    dir = Dir.pwd.split('/')[0..-2].join('/') + '/'
+    self.map do |line|
+      if line.starts_with? dir
+        line.gsub dir, ''
+      elsif line.index '/gems/'
+        'gems/' + line.split('/gems/').last
+      elsif line.index '/rubygems/'
+        'rubygems/' + line.split('/rubygems/').last
+      elsif line.index '/ruby/'
+        'ruby/' + line.split('/ruby/').last
+      else
+        line
+      end
+    end
+  end
+
+  def sort_by_key(key)
+    self.sort_by do |v|
+      if v.nil?
+        raise "Can't sort nil element"
+      end
+      unless v.is_a?(Hash) && v.has_key?(key)
+        raise "Element is a #{v.class.name} and does not have a #{key} value"
+      end
+      v[key]
+    end
+  end
+
+  def map_key(key)
+    self.map{|v| v[key]}
+  end
+
+end
+
+
+class Hash
+
+  # creates a new hash with all keys as symbols
+  def symbolize
+    h = {}
+    self.each do |k, v|
+      h[k.to_sym] = v
+    end
+    h
+  end
+end
+
+class TrueClass
+  def is_true?
+    true
+  end
+  def is_false?
+    false
+  end
+end
+
+class FalseClass
+  def is_true?
+    false
+  end
+  def is_false?
+    true
+  end
+end

package/lib/terrier/system/ssh_key_manager.rb

@@ -0,0 +1,71 @@
+require 'http'
+
+# manages SSH public keys distributed by terrier.tech
+class SshKeyManager
+  include Loggable
+
+  def initialize(clyp)
+    @clyp = clyp
+    @clyp_s = clyp.present? ? "clyp #{clyp}" : 'no clyp'
+
+    @env = Rails.env&.to_s || 'development'
+  end
+
+  # updates the keys in the authorized_keys file based on the given clyp from terrier.tech
+  def update_public_keys
+    info "Updating public keys for #{@clyp_s}"
+
+    # get the keys
+    res = HTTP.get "https://terrier.tech/public_keys.json", params: { clyp: @clyp }
+    raw = JSON.parse res.to_s
+    unless raw['status'] == 'success'
+      ap raw
+      raise "Error getting public keys: #{res['message']}"
+    end
+    new_keys = raw['public_keys'].sort
+    info "Found #{new_keys.count} keys for #{@clyp_s}:"
+    new_keys.each do |key|
+      puts key
+    end
+
+    # compute the authorized_keys path
+    if @env == 'development'
+      dir = Rails.root.join('tmp/ssh').to_s
+      FileUtils.mkdir_p(dir) unless File.exist?(dir)
+      file_path = "#{dir}/authorized_keys"
+      info "Using dummy authorized_keys path in development: #{file_path}"
+    else
+      file_path = File.expand_path "~/.ssh/authorized_keys"
+      info "Using authorized_keys path: #{file_path}"
+    end
+
+    # read existing keys
+    existing_keys = []
+    if File.exist? file_path
+      existing_keys = File.read(file_path).split("\n").compact.map(&:strip).sort
+      info "#{existing_keys.count} existing keys in #{file_path}"
+    else
+      info "#{file_path} does not exist, no existing keys"
+    end
+
+    # abort if the existing keys are the same as the new ones
+    if existing_keys.join == new_keys.join
+      info "Existing keys are the same as the new ones, nothing to do"
+      return
+    end
+
+    # write the existing keys to a backup file
+    if existing_keys.present?
+      backup_path = "#{file_path}_#{Time.new.strftime(TIMESTAMP_FORMAT)}"
+      info "Existing keys are different than the new ones, writing them to #{backup_path}"
+      File.write backup_path, existing_keys.join("\n")
+    else
+      info "No existing keys, nothing to back up"
+    end
+
+    # write the new keys
+    info "Writing new keys to #{file_path}"
+    File.write file_path, new_keys.join("\n")
+  end
+
+end

package/lib/terrier/system/top.rb

@@ -0,0 +1,75 @@
+module Top
+
+  def self.compute
+    data = {}
+
+    platform = RUBY_PLATFORM.index('darwin') ? :mac : :linux
+
+    # CPU load
+    out = `w | head -1`
+    data[:load] = out.split(':').last.strip.split(/[\s,]+/).map(&:to_f)
+
+    # memory and CPU count
+    case platform
+    when :mac
+      data[:cpus] = Top.mac_cores
+      data[:memory] = Top.mac_memory
+    when :linux
+      data[:cpus] = Top.linux_cores
+      data[:memory] = Top.linux_memory
+    else
+      raise "Don't know how to compute memory for platform #{platform}"
+    end
+    data[:memory][:free] ||= data[:memory][:total] - data[:memory][:used]
+
+    # disk
+    df = `df -m`
+    df.split("\n").map(&:strip).each do |line|
+      if line.ends_with?('/') # this is the root partition
+        data[:disk] = Top.parse_disk_line line
+      elsif line.index(/\s\/mnt\//) || line.index(/\/System\/Volumes\/Data$/) # /System/Volumes/Data for development
+        data[:volume] = Top.parse_disk_line line
+      end
+    end
+
+    data
+  end
+
+  def self.mac_cores
+    `sysctl -n hw.ncpu`.to_i
+  end
+
+  def self.linux_cores
+    `nproc`.to_i
+  end
+
+  def self.mac_memory
+    total = `sysctl hw.memsize`.split(':').last.to_f / GIGA
+    # this doesn't actually capture all of the used memory but whatever
+    rss = `ps -A -o rss | awk '{ mem += $1} END {print mem}'`.to_f / MEGA
+    {
+      total: total,
+      used: rss
+    }
+  end
+
+  def self.linux_memory
+    mem_line = `free -k`.split("\n").select { |line| line.index('Mem:') }.first
+    vals = mem_line.scan(/\s\d+\s/).map(&:to_f)
+    {
+      total: vals[0] / MEGA,
+      used: (vals[1] + vals[3]) / MEGA # used + shared
+    }
+  end
+
+  # parses a line from `df -m` into a total/used/free hash
+  def self.parse_disk_line(line)
+    comps = line.scan(/\s\d+/).map(&:strip).map { |s| s.to_f / KILO }
+    {
+      total: comps[0],
+      used: comps[1],
+      free: comps[2]
+    }
+  end
+
+end

package/lib/terrier/time/date_period.rb

@@ -0,0 +1,186 @@
+# represents a date range start_date and end_date (exclusive)
+class DatePeriod
+
+  attr_reader :start_date, :end_date
+
+  # @param start_date [Date,String] is the start of period, inclusive
+  # @param end_date [Date,String] is the end of period, _exclusive_
+  def initialize(start_date, end_date)
+    if start_date.is_a? Date
+      @start_date = start_date
+    else
+      @start_date = Date.parse start_date
+    end
+    if end_date.is_a? Date
+      @end_date = end_date
+    else
+      @end_date = Date.parse end_date
+    end
+  end
+
+  # @return [String] in the original format accepted by `DatePeriod.parse`
+  def to_s
+
+    # start and end are 1/1
+    if @start_date.month==1 && @start_date.day==1 && @end_date.month==1 && @end_date.day==1
+      if @end_date == @start_date + 1.year
+        # a single year
+        return @start_date.year.to_s
+      else
+        # year range
+        return "#{@start_date.year}:#{@end_date.year-1}"
+      end
+    end
+
+    # start and end are the first of the month
+    if @start_date.day == 1 && @end_date.day == 1
+      if @end_date == @start_date + 1.month
+        # a single month
+        return @start_date.strftime('%Y-%m')
+      end
+    end
+
+    # a single day
+    if @end_date == @start_date + 1.day
+      return @start_date.to_s
+    end
+
+    "#{self.start_date}:#{self.end_date - 1.day}"
+  end
+
+  # @return [String] a human-friendly description of the period
+  def display_name
+
+    # start and end are 1/1
+    if @start_date.month == 1 && @start_date.day == 1 && @end_date.month == 1 && @end_date.day == 1
+      if @end_date == @start_date + 1.year
+        # a single year
+        return @start_date.year.to_s
+      else
+        # year range
+        return "#{@start_date.year}-#{@end_date.year - 1}"
+      end
+    end
+
+    # start and end are the first of the month
+    if @start_date.day == 1 && @end_date.day == 1
+      if @end_date == @start_date + 1.month
+        # a single month
+        return @start_date.strftime('%B %Y')
+      end
+    end
+
+    # a single day
+    if @end_date == @start_date + 1.day
+      return @start_date.strftime(SHORT_DATE_FORMAT)
+    end
+
+    "#{self.start_date.strftime(SHORT_DATE_FORMAT)} - #{(self.end_date - 1.day).strftime(SHORT_DATE_FORMAT)}"
+  end
+
+  # @param raw [String, DatePeriod, Hash] can have the following formats:
+  # @param today [Date, String, NilClass] the anchor date to use if the period is a virtual range
+  # year: YYYY
+  # year range: YYYY:YYYY
+  # month: YYYY-MM
+  # day: YYYY-MM-DD
+  # explicit range: YYYY-MM-DD:YYYY-MM-DD
+  # literal hash: {min: YYYY-MM-DD, max: YYYY-MM-DD}
+  # virtual hash: {period: 'day'|'week'|'month'|'year', relative: Integer}
+  def self.parse(raw, today=nil)
+    # pass through existing periods
+    if raw.is_a? DatePeriod
+      return raw
+    end
+
+    # hashes
+    if raw.is_a? Hash
+      raw = ActiveSupport::HashWithIndifferentAccess.new raw
+      if raw[:min] || raw[:max]
+        return DatePeriod.new raw[:min], raw[:max]
+      end
+      if raw[:period]
+        return DatePeriod.materialize_virtual raw, today
+      end
+      raise "Don't know how to parse hash into DatePeriod: #{raw.to_s}"
+    end
+
+    raw = raw.to_s
+
+    if raw =~ /^\d{4}-\d{2}-\d{2}:\d{4}-\d{2}-\d{2}$/
+      comps = raw.split ':'
+      start_date = comps.first
+      end_date = Date.parse(comps.second) + 1.day
+    elsif raw =~ /^\d{4}:\d{4}$/
+      years = raw.split(':').map &:to_i
+      start_date = "#{years.first}-01-01"
+      end_date = Date.parse("#{years.last}-01-01") + 1.year
+    elsif raw =~ /^\d{4}$/
+      start_date = "#{raw}-01-01"
+      end_date = Date.parse(start_date) + 1.year
+    elsif raw =~ /^\d{4}-\d{2}$/
+      start_date = "#{raw}-01"
+      end_date = Date.parse(start_date) + 1.month
+    elsif raw =~ /^\d{4}-\d{2}-\d{2}$/
+      start_date = raw
+      end_date = Date.parse(start_date) + 1.day
+    else
+      raise "Invalid period format '#{raw}'"
+    end
+    DatePeriod.new start_date, end_date
+  end
+
+  # Generates a concrete date range based on the virtual one
+  # @param range [Hash] a hash containing :period and :relative
+  # @param today [Date, String] the anchor date for the relative period
+  # @return [DatePeriod]
+  def self.materialize_virtual(range, today=Date.today)
+    period = range[:period] || range['period'] || raise("Must specify a period")
+    possible_periods = %w[day week month year]
+    raise "Invalid period '#{period}', must be one of #{possible_periods.join(', ')}" unless possible_periods.include?(period)
+    relative = (range[:relative] || range['relative'] || 0).to_i
+
+    today = Date.parse(today) if today.is_a?(String)
+    today = Date.today if today.nil?
+    start_date = (today + relative.send("#{period}s")).send("beginning_of_#{period}")
+    end_date = start_date + 1.send(period)
+    DatePeriod.new start_date.to_date, end_date.to_date
+  end
+
+  # @return a new DatePeriod for the last `n` days
+  # @param n [Fixnum]
+  # @param today [Date,String,nil] can be passed if you don't want to start at today
+  def self.last_n_days(n, today=nil)
+    today = Date.parse(today) if today.is_a?(String)
+    today ||= Date.today
+    DatePeriod.new today - n.days, today + 1.day
+  end
+
+  # @return a new DatePeriod for the last 30 days
+  # @param today [Date,String,nil] can be passed if you don't want to start at today
+  def self.last_30(today=nil)
+    last_n_days 30, today
+  end
+
+  # @return a new DatePeriod for the last 30 days
+  # @param today [Date,String,nil] can be passed if you don't want to start at today
+  def self.last_7(today=nil)
+    last_n_days 7, today
+  end
+
+  # Iterates from the start_date to end_date by the given interval, yielding
+  # a new period for each interval and returning the mapped results
+  # @yield [DatePeriod]
+  # @param duration [ActiveSupport::Duration] the maximum duration of each interval
+  def each(duration)
+    d = self.start_date
+    out = []
+    while d < end_date
+      end_date_ = [d + duration, @end_date].min
+      out << yield(DatePeriod.new(d, end_date_))
+      d += duration
+    end
+    out
+  end
+
+end

package/lib/terrier/time/time_overrides.rb

@@ -0,0 +1,29 @@
+
+# Formats
+PRETTY_DATE_FORMAT = '%B %e, %Y'
+SHORT_DATE_FORMAT = '%m/%d/%y'
+SQL_DATE_FORMAT = '%Y-%m-%d'
+SQL_TIME_FORMAT = "#{SQL_DATE_FORMAT} %H:%M:%S%z"
+TIMESTAMP_FORMAT = "%Y%m%d_%H%M%S" # suitable for timestamps in paths
+PRETTY_TIME_FORMAT = "#{SHORT_DATE_FORMAT} %l:%M %p"
+SHORT_TIME_FORMAT = "%l:%M %p"
+
+# Overrides
+
+class Time
+
+  def beginning_of_week(start_day = nil)
+    # :monday is a ridiculous default
+    super(start_day || :sunday)
+  end
+
+end
+
+class Date
+
+  def beginning_of_week(start_day = nil)
+    # :monday is a ridiculous default
+    super(start_day || :sunday)
+  end
+
+end

package/lib/terrier/version.rb

@@ -0,0 +1,3 @@
+module Terrier
+  VERSION = '4.13.2'
+end

package/package.json

@@ -4,7 +4,7 @@
   "files": [
     "*"
   ],
-  "version": "4.13.1",
+  "version": "4.13.2",
   "repository": {
     "type": "git",
     "url": "https://github.com/Terrier-Tech/terrier-engine"

package/restart_tail

@@ -0,0 +1 @@
+touch tmp/restart.txt ; tail -f log/development.log