Add sidekiq wrappers.

Gemfile

@@ -195,8 +195,9 @@ end
 group :production do
   # puma is __the only exception__ for which we don't specify a version.
   gem 'puma'
-  gem 'eye', '~> 0.8'
   gem 'puma_worker_killer', '~> 0.0.6'
+  gem 'eye', '~> 0.8', require: false
+  gem 'daemons', '~> 1.2.3', require: false
   # exception_notification >= 4.2.0 is not compatible to rails 3
   gem 'exception_notification', '~> 4.1.0'
 end

Gemfile.lock

@@ -94,7 +94,7 @@ GEM
       sshkit (>= 1.6.1, != 1.7.0)
     ansi (1.5.0)
     arel (3.0.3)
-    autoprefixer-rails (6.3.7)
+    autoprefixer-rails (6.4.0.1)
       execjs
     awesome_print (1.6.1)
     bcrypt (3.1.11)
@@ -187,14 +187,15 @@ GEM
       gherkin (~> 2.12)
       multi_json (>= 1.7.5, < 2.0)
       multi_test (>= 0.1.2)
-    cucumber-rails (1.4.3)
+    cucumber-rails (1.4.4)
       capybara (>= 1.1.2, < 3)
       cucumber (>= 1.3.8, < 3)
       mime-types (>= 1.16, < 4)
       nokogiri (~> 1.5)
-      railties (>= 3, < 5)
+      railties (>= 3, < 5.1)
     d3_rails (4.1.1)
       railties (>= 3.1.0)
+    daemons (1.2.4)
     dagnabit (3.0.1)
       activerecord (>= 2.3.0)
     data_migrate (1.2.0)
@@ -563,7 +564,7 @@ GEM
     tilt (1.4.1)
     timers (4.1.1)
       hitimes
-    tins (1.11.0)
+    tins (1.12.0)
     treetop (1.4.15)
       polyglot
       polyglot (>= 0.3.1)
@@ -614,6 +615,7 @@ DEPENDENCIES
   cucumber (~> 1.3)
   cucumber-rails (~> 1.4.2)
   d3_rails (~> 4.1.1)
+  daemons (~> 1.2.3)
   dagnabit (~> 3.0.1)
   data_migrate (~> 1.2.0)
   database_cleaner (~> 1.5.1)

bin/sidekiq_wrapper

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require_relative '../lib/environment_light.rb'
+
+RAILS_ROOT ||= File.expand_path('../..', __FILE__).freeze
+SIDEKIQ_EXECUTABLE = File.join(RAILS_ROOT, 'bin/sidekiq').freeze
+
+# Parse options
+def parse_options
+  options = {environment: 'production',
+             workdir: Dir.pwd,
+             timeout: 15}
+  OptionParser.new do |opts|
+    opts.banner = "Usage: #{File.basename(__FILE__)} [options]"
+
+    opts.on('-eMANDATORY', '--environment=Rails env',
+            "Set Rails environment (default: #{options[:environment]})") do |e|
+      options[:environment] = e
+    end
+
+    opts.on('-wMANDATORY', '--workdir=directory',
+            'Set work directory (default: CWD)') do |w|
+      options[:workdir] = w
+    end
+
+    opts.on('-tMANDATORY', '--timeout=seconds',
+            'Set timeout in seconds until sidekiq jobs are killed at '\
+            "sidekiq shutdown (default: #{options[:timeout]})") do |t|
+      options[:timeout] = t
+    end
+  end.parse!
+  options
+end
+
+def run_process(options, arguments, name)
+  Process.spawn(SIDEKIQ_EXECUTABLE, *arguments,
+                '--require', RAILS_ROOT,
+                '--environment', options[:environment],
+                '--timeout', options[:timeout].to_s,
+                '--pidfile', "#{File.join(options[:workdir], name)}.pid",
+                '--logfile', "#{File.join(options[:workdir], name)}.log")
+end
+
+def wait_for_process(statuses, wait_threads, name, pid, pids)
+  wait_threads << Thread.new do
+    _pid, status = Process.wait2(pid)
+    statuses << status
+    stop_all_processes(name, pid, pids)
+  end
+end
+
+def send_signal(signal, pid)
+  Process.kill(signal, pid)
+rescue Errno::ESRCH
+  $stderr.puts "Could not send signal: Process #{pid} was already terminated."
+end
+
+$stopped_all = false
+def stop_all_processes(name_dead, pid_dead, pids)
+  unless $stopped_all
+    pids.each do |name, pid|
+      unless pid == pid_dead
+        $stderr.puts "Stopping #{name} because one sidekiq process"\
+                     "(#{name_dead}) exited..."
+        send_signal('TERM', pid)
+      end
+    end
+  end
+  $stopped_all = true
+end
+
+def exit_with_status(statuses)
+  if statuses.all? { |status| status.exited? && status.success? }
+    exit 0
+  else
+    exit 1
+  end
+end
+
+def hets_queue_thread_count
+  # One thread per configured hets instance, minus one for the sequential queue.
+  [1, Settings.hets.instance_urls.size - 1].max
+end
+
+options = parse_options
+pids = {}
+processes =
+  {:'sidekiq-hets' => %W(--queue prioirty_push,25
+                         --queue hets,5
+                         --queue hets-migration,1
+                         --concurrency #{hets_queue_thread_count}),
+   :'sidekiq-sequential' => %w(--queue sequential
+                               --concurrency 1),
+   :'sidekiq-default' => %w(--queue default
+                            --queue hets_load_balancing
+                            --concurrency 5)}
+
+# USR1: terminate in the near future
+# USR2: reopen logs
+# TERM: terminate in the near future, at most after `timeout` seconds
+%w(USR1 USR2 TERM).each do |signal|
+  Signal.trap(signal) do
+    pids.each do |name, pid|
+      $stdout.puts "Sending #{signal} to #{name}..."
+      send_signal(signal, pid)
+    end
+  end
+end
+
+processes.each do |name, arguments|
+  pids[name] = run_process(options, arguments, "#{name}")
+end
+
+wait_threads = []
+statuses = []
+pids.each do |name, pid|
+  wait_for_process(statuses, wait_threads, name, pid, pids)
+end
+wait_threads.each(&:join)
+
+exit_with_status(statuses)

bin/sidekiq_wrapper_daemon

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'daemons'
+
+RAILS_ROOT = File.expand_path('../..', __FILE__).freeze
+Daemons.run(File.join(RAILS_ROOT, 'bin/sidekiq_wrapper'),
+            dir_mode: :normal,
+            dir: File.join(RAILS_ROOT, 'tmp/pids'))

config/processes/app.eye

@@ -1,43 +1,26 @@
-require 'fileutils'
-require File.expand_path('../../../lib/environment_light_with_hets.rb', __FILE__)
-require File.expand_path('../eye_methods.rb', __FILE__)
+RAILS_ROOT = File.join(File.dirname(__FILE__), '../..')
+RAILS_ENV = ENV['RAILS_ENV'] || 'production'
+SVCADM = '/usr/sbin/svcadm'
 
+SIDEKIQ_BASE = '/var/sidekiq'
 Eye.config do
-  logger "#{Rails.root}/log/eye.log"
-end
-
-def hets_queue_thread_count
-  # One thread per configured hets instance, minus one for the sequential queue.
-  [1, Settings.hets.instance_urls.size - 1].max
+  logger "#{RAILS_ROOT}/log/eye.log"
 end
 
 Eye.application :ontohub do
-  working_dir Rails.root.to_s
-  env 'RAILS_ENV' => Rails.env
-  env 'PID_DIR' => Rails.root.join('tmp', 'pids').to_s
-
-  # Create PID dir
-  FileUtils.mkdir_p(env['PID_DIR'])
-
-  group :sidekiq do
-    # prioritize queues:
-    # priority_push 5x as high as hets, which is 5x as high as hets-migration
-    sidekiq_process self, :"sidekiq-hets",
-                    ['priority_push,25', 'hets,5', 'hets-migration,1'],
-                    hets_queue_thread_count
-
-    # one multithreaded worker for the default queue and hets_load_balancing
-    sidekiq_process self, :'sidekiq-default', ['default', 'hets_load_balancing'], 5
-
-    # one worker for the sequential queue
-    sidekiq_process self, :'sidekiq-sequential', 'sequential', 1
-  end
-
-  group :hets do
-    Settings.hets.instance_urls.each do |url|
-      if url.match(%r{\Ahttps?://(localhost|127.0.0.1|0.0.0.0|::1)})
-        hets_process self, URI(url).port
-      end
+  working_dir RAILS_ROOT
+  env 'RAILS_ENV' => RAILS_ENV
+  env 'PID_DIR' => File.join(RAILS_ROOT, 'tmp/pids')
+
+  {git: File.join(env['PID_DIR'], 'git.pid'),
+   hets: File.join(env['PID_DIR'], 'hets.pid'),
+   puma: File.join(env['PID_DIR'], 'puma.pid'),
+   sidekiq: File.join(SIDEKIQ_BASE, 'master.pid')}.each do |service, pidfile|
+    process service do
+      daemonize false
+      pid_file pidfile
+      start_command "#{SVCADM} enable -s #{service}"
+      stop_command "#{SVCADM} disable -s #{service}"
     end
   end
 end