Add deduplication for JSON payloads in job queuefeature-argument-deduplication

1 files changed, 26 insertions, 0 deletions

diff --git a/lib/argument_deduplication.rb b/lib/argument_deduplication.rb
new file mode 100644
index 000000000..f271b6f96
--- /dev/null
+++ b/lib/argument_deduplication.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative './argument_deduplication/argument'
+require_relative './argument_deduplication/server'
+require_relative './argument_deduplication/client'
+
+module ArgumentDeduplication
+  class CorruptedArgumentError < ::RuntimeError; end
+
+  PREFIX = 'argument_store'
+
+  # The time-to-live is based on the maximum amount of time
+  # a job can possibly spend in the retry queue, assuming
+  # the exponential backoff algorithm and a maximum number
+  # of 16 retries. It is intended as a safe-guard against
+  # any arguments being orphaned due to interruptions.
+  TTL = 50.days.to_i
+
+  DEATH_HANDLER = ->(job) {
+    Argument.new(job['args'][job['deduplicate_arguments']]).pop! if job['deduplicate_arguments']
+  }.freeze
+
+  def self.configure(config)
+    config.death_handlers << DEATH_HANDLER
+  end
+end