diff --git a/app/models/repository/git.rb b/app/models/repository/git.rb index 0714b290b..1f0b168a4 100644 --- a/app/models/repository/git.rb +++ b/app/models/repository/git.rb @@ -66,9 +66,9 @@ def save_file_only(tmp_file, filepath, message, user) def commit_for!(commit_oid, pusher) instance = Commit.where(repository_id: self, - commit_oid: commit_oid, - pusher_id: pusher.try(:id), - pusher_name: pusher.try(:name)).first_or_initialize + commit_oid: commit_oid). + first_or_initialize(pusher_id: pusher.try(:id), + pusher_name: pusher.try(:name)) instance.fill_commit_instance! unless instance.persisted? instance end diff --git a/doc/sidekiq_tips.md b/doc/sidekiq_tips.md new file mode 100644 index 000000000..33d4c9b46 --- /dev/null +++ b/doc/sidekiq_tips.md @@ -0,0 +1,37 @@ +# Sidekiq Tips + +## Add job to queue other than specified in the Worker class +You can add a job to a queue different from the one that is specified in the worker class. +Instead of + +```ruby +SomeWorker.perform_async(some, arguments) +``` + +you can execute + +```ruby +Sidekiq::Client.push('queue' => 'some_other_queue', + 'class' => SomeWorker, + 'args' => [some, arguments]) +``` +This method accepts the following keys: +* `queue` - the named queue to use, default `'default'` +* `class` - the worker class to call, required +* `args` - an array of simple arguments to the perform method, must be JSON-serializable +* `retry` - whether to retry this job if it fails, `true` or `false`, default `true` +* `backtrace` - whether to save any error backtrace, default `false` + +## Add many jobs to a queue +To add tens of thousands of jobs to a queue, it is recommended to use +```ruby +Sidekiq::Client.push_bulk('queue' => 'some_other_queue', + 'class' => SomeWorker, + 'args' => [[some1, arguments1], + [some2, arguments2], + [some3, arguments3], + [some4, arguments4], + ]) +``` +It takes the same arguments as `push` above, but expects the `'args'` to be an array. +It cuts down on the redis round trip latency. diff --git a/lib/ontology_parsing_migration_worker.rb b/lib/ontology_parsing_migration_worker.rb new file mode 100644 index 000000000..7994ea52a --- /dev/null +++ b/lib/ontology_parsing_migration_worker.rb @@ -0,0 +1,3 @@ +class OntologyParsingMigrationWorker < OntologyParsingWorker + sidekiq_options queue: 'hets-migration' +end diff --git a/lib/tasks/ontology.rake b/lib/tasks/ontology.rake new file mode 100644 index 000000000..8d6419582 --- /dev/null +++ b/lib/tasks/ontology.rake @@ -0,0 +1,25 @@ +namespace :ontology do + desc 'Reanalyze old ontologies' + task :reanalyze_all => :environment do + Ontology.where(parent_id: nil, present: true).order('updated_at desc'). + find_each do |ontology| + if ontology.current_version.nil? + commit_oid = ontology.repository.git.get_file!(ontology.path).oid + ontology_version_options = OntologyVersionOptions.new(ontology.path, + User.where(admin: true).first) + + OntologySaver.new(ontology.repository). + save_ontology(commit_oid, ontology_version_options) + else + OntologyParsingMigrationWorker. + perform_async([[ontology.current_version.id, + {fast_parse: false, files_to_parse_afterwards: []}, 1]]) + Sidekiq::Client.push('queue' => 'hets-migration', + 'class' => OntologyParsingWorker, + 'args' => [[[ontology.current_version.id, + {fast_parse: false, + files_to_parse_afterwards: []}, 1]]]) + end + end + end +end \ No newline at end of file