From a15c00c37e56dac1f2cc57bfffd102758d823dc6 Mon Sep 17 00:00:00 2001 From: Eugen Kuksa Date: Tue, 23 Feb 2016 10:14:10 +0100 Subject: [PATCH 1/3] Use only one worker (with several threads) for the hets queue. --- config/processes/app.eye | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/config/processes/app.eye b/config/processes/app.eye index d515925c6..75c2e15dc 100644 --- a/config/processes/app.eye +++ b/config/processes/app.eye @@ -6,6 +6,12 @@ Eye.config do logger "#{Rails.root}/log/eye.log" end +def hets_queue_thread_count + # One thread per configured hets instance, minus one for sequential + # and one for priority_push. + [1, Settings.hets.instance_urls.size - 2].max +end + Eye.application :ontohub do working_dir Rails.root.to_s env 'RAILS_ENV' => Rails.env @@ -15,10 +21,7 @@ Eye.application :ontohub do FileUtils.mkdir_p(env['PID_DIR']) group :sidekiq do - # one worker per configured hets instance - Settings.hets.instance_urls.each_with_index do |_url, index| - sidekiq_process self, :"sidekiq-hets-#{index}", 'hets', 1 - end + sidekiq_process self, :"sidekiq-hets", 'hets', hets_queue_thread_count # one worker for hets load balancing sidekiq_process self, :'sidekiq-hets-load-balancing', 'hets_load_balancing', 1 From 055fde49de97ddab2670728c51aa8a3176b27b2c Mon Sep 17 00:00:00 2001 From: Eugen Kuksa Date: Tue, 23 Feb 2016 11:03:49 +0100 Subject: [PATCH 2/3] Add a sidekiq queue for migrations Use it by specifically enqueuing a job to this queue, e.g.: Sidekiq::Client.enqueue_to(:'hets-migration', SomeWorker, args_for_worker) --- config/processes/app.eye | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/processes/app.eye b/config/processes/app.eye index 75c2e15dc..2135278ac 100644 --- a/config/processes/app.eye +++ b/config/processes/app.eye @@ -21,7 +21,9 @@ Eye.application :ontohub do FileUtils.mkdir_p(env['PID_DIR']) group :sidekiq do - sidekiq_process self, :"sidekiq-hets", 'hets', hets_queue_thread_count + # Use a second queue for migration jobs which is checked less frequently. + sidekiq_process self, :"sidekiq-hets", ['hets,5', 'hets-migration,1'], + hets_queue_thread_count # one worker for hets load balancing sidekiq_process self, :'sidekiq-hets-load-balancing', 'hets_load_balancing', 1 From 5e2ca0960d31fe8181b770d2b784426c45ccfea7 Mon Sep 17 00:00:00 2001 From: Eugen Kuksa Date: Tue, 23 Feb 2016 11:23:30 +0100 Subject: [PATCH 3/3] Share queues between workers This reduces the memory footprint of ontohub drastically: Each worker runs an instance of the app, consuming as much memory as the rails server. Fewer workers means less memory consumption. --- config/processes/app.eye | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/config/processes/app.eye b/config/processes/app.eye index 2135278ac..dba1d9ad9 100644 --- a/config/processes/app.eye +++ b/config/processes/app.eye @@ -7,9 +7,8 @@ Eye.config do end def hets_queue_thread_count - # One thread per configured hets instance, minus one for sequential - # and one for priority_push. - [1, Settings.hets.instance_urls.size - 2].max + # One thread per configured hets instance, minus one for the sequential queue. + [1, Settings.hets.instance_urls.size - 1].max end Eye.application :ontohub do @@ -21,20 +20,17 @@ Eye.application :ontohub do FileUtils.mkdir_p(env['PID_DIR']) group :sidekiq do - # Use a second queue for migration jobs which is checked less frequently. - sidekiq_process self, :"sidekiq-hets", ['hets,5', 'hets-migration,1'], + # prioritize queues: + # priority_push 5x as high as hets, which is 5x as high as hets-migration + sidekiq_process self, :"sidekiq-hets", + ['priority_push,25', 'hets,5', 'hets-migration,1'], hets_queue_thread_count - # one worker for hets load balancing - sidekiq_process self, :'sidekiq-hets-load-balancing', 'hets_load_balancing', 1 - - # one worker for the default queue - sidekiq_process self, :'sidekiq-default', 'default', 5 + # one multithreaded worker for the default queue and hets_load_balancing + sidekiq_process self, :'sidekiq-default', ['default', 'hets_load_balancing'], 5 # one worker for the sequential queue sidekiq_process self, :'sidekiq-sequential', 'sequential', 1 - - sidekiq_process self, :'sidekiq-priority_push', 'priority_push', 1 end group :hets do