"use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.moveToFinished = void 0; const content = `--[[ Move job from active to a finished status (completed o failed) A job can only be moved to completed if it was active. The job must be locked before it can be moved to a finished status, and the lock must be released in this script. Input: KEYS[1] wait key KEYS[2] active key KEYS[3] prioritized key KEYS[4] event stream key KEYS[5] stalled key -- Rate limiting KEYS[6] rate limiter key KEYS[7] delayed key KEYS[8] paused key KEYS[9] meta key KEYS[10] pc priority counter KEYS[11] completed/failed key KEYS[12] jobId key KEYS[13] metrics key KEYS[14] marker key ARGV[1] jobId ARGV[2] timestamp ARGV[3] msg property returnvalue / failedReason ARGV[4] return value / failed reason ARGV[5] target (completed/failed) ARGV[6] fetch next? ARGV[7] keys prefix ARGV[8] opts ARGV[9] job fields to update opts - token - lock token opts - keepJobs opts - lockDuration - lock duration in milliseconds opts - attempts max attempts opts - maxMetricsSize opts - fpof - fail parent on fail opts - cpof - continue parent on fail opts - idof - ignore dependency on fail opts - rdof - remove dependency on fail opts - name - worker name Output: 0 OK -1 Missing key. -2 Missing lock. -3 Job not in active set -4 Job has pending children -6 Lock is not owned by this client -9 Job has failed children Events: 'completed/failed' ]] local rcall = redis.call --- Includes --[[ Functions to collect metrics based on a current and previous count of jobs. Granualarity is fixed at 1 minute. ]] --[[ Function to loop in batches. Just a bit of warning, some commands as ZREM could receive a maximum of 7000 parameters per call. ]] local function batches(n, batchSize) local i = 0 return function() local from = i * batchSize + 1 i = i + 1 if (from <= n) then local to = math.min(from + batchSize - 1, n) return from, to end end end local function collectMetrics(metaKey, dataPointsList, maxDataPoints, timestamp) -- Increment current count local count = rcall("HINCRBY", metaKey, "count", 1) - 1 -- Compute how many data points we need to add to the list, N. local prevTS = rcall("HGET", metaKey, "prevTS") if not prevTS then -- If prevTS is nil, set it to the current timestamp rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0) return end local N = math.min(math.floor(timestamp / 60000) - math.floor(prevTS / 60000), tonumber(maxDataPoints)) if N > 0 then local delta = count - rcall("HGET", metaKey, "prevCount") -- If N > 1, add N-1 zeros to the list if N > 1 then local points = {} points[1] = delta for i = 2, N do points[i] = 0 end for from, to in batches(#points, 7000) do rcall("LPUSH", dataPointsList, unpack(points, from, to)) end else -- LPUSH delta to the list rcall("LPUSH", dataPointsList, delta) end -- LTRIM to keep list to its max size rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1) -- update prev count with current count rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp) end end --[[ Function to return the next delayed job timestamp. ]] local function getNextDelayedTimestamp(delayedKey) local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES") if #result then local nextTimestamp = tonumber(result[2]) if nextTimestamp ~= nil then return nextTimestamp / 0x1000 end end end --[[ Function to get current rate limit ttl. ]] local function getRateLimitTTL(maxJobs, rateLimiterKey) if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then local pttl = rcall("PTTL", rateLimiterKey) if pttl == 0 then rcall("DEL", rateLimiterKey) end if pttl > 0 then return pttl end end return 0 end --[[ Function to check for the meta.paused key to decide if we are paused or not (since an empty list and !EXISTS are not really the same). ]] local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey) local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration") if queueAttributes[1] then return pausedKey, true, queueAttributes[3], queueAttributes[4] else if queueAttributes[2] then local activeCount = rcall("LLEN", activeKey) if activeCount >= tonumber(queueAttributes[2]) then return waitKey, true, queueAttributes[3], queueAttributes[4] else return waitKey, false, queueAttributes[3], queueAttributes[4] end end end return waitKey, false, queueAttributes[3], queueAttributes[4] end --[[ Function to move job from prioritized state to active. ]] local function moveJobFromPrioritizedToActive(priorityKey, activeKey, priorityCounterKey) local prioritizedJob = rcall("ZPOPMIN", priorityKey) if #prioritizedJob > 0 then rcall("LPUSH", activeKey, prioritizedJob[1]) return prioritizedJob[1] else rcall("DEL", priorityCounterKey) end end --[[ Function to recursively move from waitingChildren to failed. ]] -- Includes --[[ Validate and move parent to a wait status (waiting, delayed or prioritized) if no pending dependencies. ]] -- Includes --[[ Validate and move parent to a wait status (waiting, delayed or prioritized) if needed. ]] -- Includes --[[ Move parent to a wait status (wait, prioritized or delayed) ]] -- Includes --[[ Add delay marker if needed. ]] -- Includes local function addDelayMarkerIfNeeded(markerKey, delayedKey) local nextTimestamp = getNextDelayedTimestamp(delayedKey) if nextTimestamp ~= nil then -- Replace the score of the marker with the newest known -- next timestamp. rcall("ZADD", markerKey, nextTimestamp, "1") end end --[[ Function to add job in target list and add marker if needed. ]] -- Includes --[[ Add marker if needed when a job is available. ]] local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed) if not isPausedOrMaxed then rcall("ZADD", markerKey, 0, "0") end end local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId) rcall(pushCmd, targetKey, jobId) addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed) end --[[ Function to add job considering priority. ]] -- Includes --[[ Function to get priority score. ]] local function getPriorityScore(priority, priorityCounterKey) local prioCounter = rcall("INCR", priorityCounterKey) return priority * 0x100000000 + prioCounter % 0x100000000 end local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPausedOrMaxed) local score = getPriorityScore(priority, priorityCounterKey) rcall("ZADD", prioritizedKey, score, jobId) addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed) end --[[ Function to check if queue is paused or maxed (since an empty list and !EXISTS are not really the same). ]] local function isQueuePausedOrMaxed(queueMetaKey, activeKey) local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency") if queueAttributes[1] then return true else if queueAttributes[2] then local activeCount = rcall("LLEN", activeKey) return activeCount >= tonumber(queueAttributes[2]) end end return false end local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp) local parentWaitKey = parentQueueKey .. ":wait" local parentPausedKey = parentQueueKey .. ":paused" local parentActiveKey = parentQueueKey .. ":active" local parentMetaKey = parentQueueKey .. ":meta" local parentMarkerKey = parentQueueKey .. ":marker" local jobAttributes = rcall("HMGET", parentKey, "priority", "delay") local priority = tonumber(jobAttributes[1]) or 0 local delay = tonumber(jobAttributes[2]) or 0 if delay > 0 then local delayedTimestamp = tonumber(timestamp) + delay local score = delayedTimestamp * 0x1000 local parentDelayedKey = parentQueueKey .. ":delayed" rcall("ZADD", parentDelayedKey, score, parentId) rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay", delayedTimestamp) addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey) else if priority == 0 then local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey, parentPausedKey) addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId) else local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey) addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId, parentQueueKey .. ":pc", isPausedOrMaxed) end rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children") end end local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp) if rcall("EXISTS", parentKey) == 1 then local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children" if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then rcall("ZREM", parentWaitingChildrenKey, parentId) moveParentToWait(parentQueueKey, parentKey, parentId, timestamp) end end end local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp) local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0 if doNotHavePendingDependencies then moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp) end end local handleChildFailureAndMoveParentToWait = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp) if rcall("EXISTS", parentKey) == 1 then local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children" local parentDelayedKey = parentQueueKey .. ":delayed" local parentWaitingChildrenOrDelayedKey if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey elseif rcall("ZSCORE", parentDelayedKey, parentId) then parentWaitingChildrenOrDelayedKey = parentDelayedKey rcall("HSET", parentKey, "delay", 0) end if parentWaitingChildrenOrDelayedKey then rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId) local deferredFailure = "child " .. jobIdKey .. " failed" rcall("HSET", parentKey, "defa", deferredFailure) moveParentToWait(parentQueueKey, parentKey, parentId, timestamp) else if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then local deferredFailure = "child " .. jobIdKey .. " failed" rcall("HSET", parentKey, "defa", deferredFailure) end end end end local moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp) if rawParentData then local parentData = cjson.decode(rawParentData) local parentKey = parentData['queueKey'] .. ':' .. parentData['id'] local parentDependenciesChildrenKey = parentKey .. ":dependencies" if parentData['fpof'] then if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then local parentUnsuccessfulChildrenKey = parentKey .. ":unsuccessful" rcall("ZADD", parentUnsuccessfulChildrenKey, timestamp, childKey) handleChildFailureAndMoveParentToWait( parentData['queueKey'], parentKey, parentData['id'], childKey, timestamp ) end elseif parentData['cpof'] then if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then local parentFailedChildrenKey = parentKey .. ":failed" rcall("HSET", parentFailedChildrenKey, childKey, failedReason) moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp) end elseif parentData['idof'] or parentData['rdof'] then if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey, parentKey, parentData['id'], timestamp) if parentData['idof'] then local parentFailedChildrenKey = parentKey .. ":failed" rcall("HSET", parentFailedChildrenKey, childKey, failedReason) end end end end end --[[ Function to move job from wait state to active. Input: opts - token - lock token opts - lockDuration opts - limiter ]] -- Includes local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey, jobId, processedOn, maxJobs, limiterDuration, markerKey, opts) local jobKey = keyPrefix .. jobId -- Check if we need to perform rate limiting. if maxJobs then local jobCounter = tonumber(rcall("INCR", rateLimiterKey)) if jobCounter == 1 then local integerDuration = math.floor(math.abs(limiterDuration)) rcall("PEXPIRE", rateLimiterKey, integerDuration) end end -- get a lock if opts['token'] ~= "0" then local lockKey = jobKey .. ':lock' rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration']) end local optionalValues = {} if opts['name'] then -- Set "processedBy" field to the worker name table.insert(optionalValues, "pb") table.insert(optionalValues, opts['name']) end rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting") rcall("HMSET", jobKey, "processedOn", processedOn, unpack(optionalValues)) rcall("HINCRBY", jobKey, "ats", 1) addBaseMarkerIfNeeded(markerKey, false) -- rate limit delay must be 0 in this case to prevent adding more delay -- when job that is moved to active needs to be processed return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data end --[[ Updates the delay set, by moving delayed jobs that should be processed now to "wait". Events: 'waiting' ]] -- Includes -- Try to get as much as 1000 jobs at once local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey, eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused) local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000 - 1, "LIMIT", 0, 1000) if (#jobs > 0) then rcall("ZREM", delayedKey, unpack(jobs)) for _, jobId in ipairs(jobs) do local jobKey = prefix .. jobId local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0 if priority == 0 then -- LIFO or FIFO rcall("LPUSH", targetKey, jobId) else local score = getPriorityScore(priority, priorityCounterKey) rcall("ZADD", prioritizedKey, score, jobId) end -- Emit waiting event rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, "prev", "delayed") rcall("HSET", jobKey, "delay", 0) end addBaseMarkerIfNeeded(markerKey, isPaused) end end --[[ Function to remove deduplication key if needed when a job is moved to completed or failed states. ]] local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey, deduplicationId, jobId) if deduplicationId then local deduplicationKey = prefixKey .. "de:" .. deduplicationId local pttl = rcall("PTTL", deduplicationKey) if pttl == 0 then return rcall("DEL", deduplicationKey) end if pttl == -1 then local currentJobId = rcall('GET', deduplicationKey) if currentJobId and currentJobId == jobId then return rcall("DEL", deduplicationKey) end end end end --[[ Function to remove job keys. ]] local function removeJobKeys(jobKey) return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful') end --[[ Functions to remove jobs by max age. ]] -- Includes --[[ Function to remove job. ]] -- Includes --[[ Function to remove deduplication key if needed when a job is being removed. ]] local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey, jobId, deduplicationId) if deduplicationId then local deduplicationKey = prefixKey .. "de:" .. deduplicationId local currentJobId = rcall('GET', deduplicationKey) if currentJobId and currentJobId == jobId then return rcall("DEL", deduplicationKey) end end end --[[ Check if this job has a parent. If so we will just remove it from the parent child list, but if it is the last child we should move the parent to "wait/paused" which requires code from "moveToFinished" ]] -- Includes --[[ Functions to destructure job key. Just a bit of warning, these functions may be a bit slow and affect performance significantly. ]] local getJobIdFromKey = function (jobKey) return string.match(jobKey, ".*:(.*)") end local getJobKeyPrefix = function (jobKey, jobId) return string.sub(jobKey, 0, #jobKey - #jobId) end local function _moveParentToWait(parentPrefix, parentId, emitEvent) local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active", parentPrefix .. "wait", parentPrefix .. "paused") addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId) if emitEvent then local parentEventStream = parentPrefix .. "events" rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children") end end local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId) if parentKey then local parentDependenciesKey = parentKey .. ":dependencies" local result = rcall("SREM", parentDependenciesKey, jobKey) if result > 0 then local pendingDependencies = rcall("SCARD", parentDependenciesKey) if pendingDependencies == 0 then local parentId = getJobIdFromKey(parentKey) local parentPrefix = getJobKeyPrefix(parentKey, parentId) local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId) if numRemovedElements == 1 then if hard then -- remove parent in same queue if parentPrefix == baseKey then removeParentDependencyKey(parentKey, hard, nil, baseKey, nil) removeJobKeys(parentKey) if debounceId then rcall("DEL", parentPrefix .. "de:" .. debounceId) end else _moveParentToWait(parentPrefix, parentId) end else _moveParentToWait(parentPrefix, parentId, true) end end end return true end else local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid") local missedParentKey = parentAttributes[1] if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then local parentDependenciesKey = missedParentKey .. ":dependencies" local result = rcall("SREM", parentDependenciesKey, jobKey) if result > 0 then local pendingDependencies = rcall("SCARD", parentDependenciesKey) if pendingDependencies == 0 then local parentId = getJobIdFromKey(missedParentKey) local parentPrefix = getJobKeyPrefix(missedParentKey, parentId) local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId) if numRemovedElements == 1 then if hard then if parentPrefix == baseKey then removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil) removeJobKeys(missedParentKey) if parentAttributes[2] then rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2]) end else _moveParentToWait(parentPrefix, parentId) end else _moveParentToWait(parentPrefix, parentId, true) end end end return true end end end return false end local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey) local jobKey = baseKey .. jobId removeParentDependencyKey(jobKey, hard, nil, baseKey) if shouldRemoveDeduplicationKey then local deduplicationId = rcall("HGET", jobKey, "deid") removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobId, deduplicationId) end removeJobKeys(jobKey) end local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit) local start = timestamp - maxAge * 1000 local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf", "LIMIT", 0, maxLimit) for i, jobId in ipairs(jobIds) do removeJob(jobId, false, prefix, false --[[remove debounce key]]) end rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start) end --[[ Functions to remove jobs by max count. ]] -- Includes local function removeJobsByMaxCount(maxCount, targetSet, prefix) local start = maxCount local jobIds = rcall("ZREVRANGE", targetSet, start, -1) for i, jobId in ipairs(jobIds) do removeJob(jobId, false, prefix, false --[[remove debounce key]]) end rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1)) end local function removeLock(jobKey, stalledKey, token, jobId) if token ~= "0" then local lockKey = jobKey .. ':lock' local lockToken = rcall("GET", lockKey) if lockToken == token then rcall("DEL", lockKey) rcall("SREM", stalledKey, jobId) else if lockToken then -- Lock exists but token does not match return -6 else -- Lock is missing completely return -2 end end end return 0 end --[[ Function to trim events, default 10000. ]] -- Includes --[[ Function to get max events value or set by default 10000. ]] local function getOrSetMaxEvents(metaKey) local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") if not maxEvents then maxEvents = 10000 rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents) end return maxEvents end local function trimEvents(metaKey, eventStreamKey) local maxEvents = getOrSetMaxEvents(metaKey) if maxEvents then rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents) else rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000) end end --[[ Validate and move or add dependencies to parent. ]] -- Includes local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey, parentId, jobIdKey, returnvalue, timestamp ) local processedSet = parentKey .. ":processed" rcall("HSET", processedSet, jobIdKey, returnvalue) moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp) end --[[ Function to update a bunch of fields in a job. ]] local function updateJobFields(jobKey, msgpackedFields) if msgpackedFields and #msgpackedFields > 0 then local fieldsToUpdate = cmsgpack.unpack(msgpackedFields) if fieldsToUpdate then rcall("HMSET", jobKey, unpack(fieldsToUpdate)) end end end local jobIdKey = KEYS[12] if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists -- Make sure it does not have pending dependencies -- It must happen before removing lock if ARGV[5] == "completed" then if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then return -4 end if rcall("ZCARD", jobIdKey .. ":unsuccessful") ~= 0 then return -9 end end local opts = cmsgpack.unpack(ARGV[8]) local token = opts['token'] local errorCode = removeLock(jobIdKey, KEYS[5], token, ARGV[1]) if errorCode < 0 then return errorCode end updateJobFields(jobIdKey, ARGV[9]); local attempts = opts['attempts'] local maxMetricsSize = opts['maxMetricsSize'] local maxCount = opts['keepJobs']['count'] local maxAge = opts['keepJobs']['age'] local maxLimit = opts['keepJobs']['limit'] or 1000 local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid") local parentKey = jobAttributes[1] or "" local parentId = "" local parentQueueKey = "" if jobAttributes[2] then -- TODO: need to revisit this logic if it's still needed local jsonDecodedParent = cjson.decode(jobAttributes[2]) parentId = jsonDecodedParent['id'] parentQueueKey = jsonDecodedParent['queueKey'] end local jobId = ARGV[1] local timestamp = ARGV[2] -- Remove from active list (if not active we shall return error) local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId) if (numRemovedElements < 1) then return -3 end local eventStreamKey = KEYS[4] local metaKey = KEYS[9] -- Trim events before emiting them to avoid trimming events emitted in this script trimEvents(metaKey, eventStreamKey) local prefix = ARGV[7] removeDeduplicationKeyIfNeededOnFinalization(prefix, jobAttributes[3], jobId) -- If job has a parent we need to -- 1) remove this job id from parents dependencies -- 2) move the job Id to parent "processed" set -- 3) push the results into parent "results" list -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!. if parentId == "" and parentKey ~= "" then parentId = getJobIdFromKey(parentKey) parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId) end if parentId ~= "" then if ARGV[5] == "completed" then local dependenciesSet = parentKey .. ":dependencies" if rcall("SREM", dependenciesSet, jobIdKey) == 1 then updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4], timestamp) end else moveChildFromDependenciesIfNeeded(jobAttributes[2], jobIdKey, ARGV[4], timestamp) end end local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1) -- Remove job? if maxCount ~= 0 then local targetSet = KEYS[11] -- Add to complete/failed set rcall("ZADD", targetSet, timestamp, jobId) rcall("HSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp) -- "returnvalue" / "failedReason" and "finishedOn" if ARGV[5] == "failed" then rcall("HDEL", jobIdKey, "defa") end -- Remove old jobs? if maxAge ~= nil then removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit) end if maxCount ~= nil and maxCount > 0 then removeJobsByMaxCount(maxCount, targetSet, prefix) end else removeJobKeys(jobIdKey) if parentKey ~= "" then -- TODO: when a child is removed when finished, result or failure in parent -- must not be deleted, those value references should be deleted when the parent -- is deleted removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3]) end end rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active") if ARGV[5] == "failed" then if tonumber(attemptsMade) >= tonumber(attempts) then rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade", attemptsMade) end end -- Collect metrics if maxMetricsSize ~= "" then collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp) end -- Try to get next job to avoid an extra roundtrip if the queue is not closing, -- and not rate limited. if (ARGV[6] == "1") then local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8]) local markerKey = KEYS[14] -- Check if there are delayed jobs that can be promoted promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10], isPausedOrMaxed) local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max'])) -- Check if we are rate limited first. local expireTime = getRateLimitTTL(maxJobs, KEYS[6]) if expireTime > 0 then return {0, 0, expireTime, 0} end -- paused or maxed queue if isPausedOrMaxed then return {0, 0, 0, 0} end local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2]) if jobId then -- Markers in waitlist DEPRECATED in v5: Remove in v6. if string.sub(jobId, 1, 2) == "0:" then rcall("LREM", KEYS[2], 1, jobId) -- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process -- but if ID is 0:0, then there is at least 1 prioritized job to process if jobId == "0:0" then jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10]) return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, limiterDuration, markerKey, opts) end else return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, limiterDuration, markerKey, opts) end else jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10]) if jobId then return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, limiterDuration, markerKey, opts) end end -- Return the timestamp for the next delayed job if any. local nextTimestamp = getNextDelayedTimestamp(KEYS[7]) if nextTimestamp ~= nil then -- The result is guaranteed to be positive, since the -- ZRANGEBYSCORE command would have return a job otherwise. return {0, 0, 0, nextTimestamp} end end local waitLen = rcall("LLEN", KEYS[1]) if waitLen == 0 then local activeLen = rcall("LLEN", KEYS[2]) if activeLen == 0 then local prioritizedLen = rcall("ZCARD", KEYS[3]) if prioritizedLen == 0 then rcall("XADD", eventStreamKey, "*", "event", "drained") end end end return 0 else return -1 end `; exports.moveToFinished = { name: 'moveToFinished', content, keys: 14, }; //# sourceMappingURL=moveToFinished-14.js.map