Projektstart

backend/node_modules/bullmq/dist/esm/commands/moveStalledJobsToWait-8.lua

@@ -0,0 +1,113 @@
+--[[
+  Move stalled jobs to wait.
+
+    Input:
+      KEYS[1] 'stalled' (SET)
+      KEYS[2] 'wait',   (LIST)
+      KEYS[3] 'active', (LIST)
+      KEYS[4] 'stalled-check', (KEY)
+      KEYS[5] 'meta', (KEY)
+      KEYS[6] 'paused', (LIST)
+      KEYS[7] 'marker'
+      KEYS[8] 'event stream' (STREAM)
+
+      ARGV[1]  Max stalled job count
+      ARGV[2]  queue.toKey('')
+      ARGV[3]  timestamp
+      ARGV[4]  max check time
+
+    Events:
+      'stalled' with stalled job id.
+]]
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/addJobInTargetList"
+--- @include "includes/batches"
+--- @include "includes/moveJobToWait"
+--- @include "includes/trimEvents"
+
+local stalledKey = KEYS[1]
+local waitKey = KEYS[2]
+local activeKey = KEYS[3]
+local stalledCheckKey = KEYS[4]
+local metaKey = KEYS[5]
+local pausedKey = KEYS[6]
+local markerKey = KEYS[7]
+local eventStreamKey = KEYS[8]
+local maxStalledJobCount = tonumber(ARGV[1])
+local queueKeyPrefix = ARGV[2]
+local timestamp = ARGV[3]
+local maxCheckTime = ARGV[4]
+
+if rcall("EXISTS", stalledCheckKey) == 1 then
+    return {}
+end
+
+rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
+
+-- Trim events before emiting them to avoid trimming events emitted in this script
+trimEvents(metaKey, eventStreamKey)
+
+-- Move all stalled jobs to wait
+local stalling = rcall('SMEMBERS', stalledKey)
+local stalled = {}
+if (#stalling > 0) then
+    rcall('DEL', stalledKey)
+
+    -- Remove from active list
+    for i, jobId in ipairs(stalling) do
+        -- Markers in waitlist DEPRECATED in v5: Remove in v6.
+        if string.sub(jobId, 1, 2) == "0:" then
+            -- If the jobId is a delay marker ID we just remove it.
+            rcall("LREM", activeKey, 1, jobId)
+        else
+            local jobKey = queueKeyPrefix .. jobId
+
+            -- Check that the lock is also missing, then we can handle this job as really stalled.
+            if (rcall("EXISTS", jobKey .. ":lock") == 0) then
+                --  Remove from the active queue.
+                local removed = rcall("LREM", activeKey, 1, jobId)
+
+                if (removed > 0) then
+                    -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
+                    local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
+                    
+                    -- Check if this is a repeatable job by looking at job options
+                    local jobOpts = rcall("HGET", jobKey, "opts")
+                    local isRepeatableJob = false
+                    if jobOpts then
+                        local opts = cjson.decode(jobOpts)
+                        if opts and opts["repeat"] then
+                            isRepeatableJob = true
+                        end
+                    end
+                    
+                    -- Only fail job if it exceeds stall limit AND is not a repeatable job
+                    if stalledCount > maxStalledJobCount and not isRepeatableJob then
+                        local failedReason = "job stalled more than allowable limit"
+                        rcall("HSET", jobKey, "defa", failedReason)
+                    end
+                    
+                    moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey, jobId,
+                        "RPUSH")
+
+                    -- Emit the stalled event
+                    rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
+                    table.insert(stalled, jobId)
+                end
+            end
+        end
+    end
+end
+
+-- Mark potentially stalled jobs
+local active = rcall('LRANGE', activeKey, 0, -1)
+
+if (#active > 0) then
+    for from, to in batches(#active, 7000) do
+        rcall('SADD', stalledKey, unpack(active, from, to))
+    end
+end
+
+return stalled