const content = `--[[ Drains the queue, removes all jobs that are waiting or delayed, but not active, completed or failed Input: KEYS[1] 'wait', KEYS[2] 'paused' KEYS[3] 'delayed' KEYS[4] 'prioritized' KEYS[5] 'jobschedulers' (repeat) ARGV[1] queue key prefix ARGV[2] should clean delayed jobs ]] local rcall = redis.call local queueBaseKey = ARGV[1] --[[ Functions to remove jobs. ]] -- Includes --[[ Function to filter out jobs to ignore from a table. ]] local function filterOutJobsToIgnore(jobs, jobsToIgnore) local filteredJobs = {} for i = 1, #jobs do if not jobsToIgnore[jobs[i]] then table.insert(filteredJobs, jobs[i]) end end return filteredJobs end --[[ Functions to remove jobs. ]] -- Includes --[[ Function to remove job. ]] -- Includes --[[ Function to remove deduplication key if needed when a job is being removed. ]] local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey, jobId, deduplicationId) if deduplicationId then local deduplicationKey = prefixKey .. "de:" .. deduplicationId local currentJobId = rcall('GET', deduplicationKey) if currentJobId and currentJobId == jobId then return rcall("DEL", deduplicationKey) end end end --[[ Function to remove job keys. ]] local function removeJobKeys(jobKey) return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful') end --[[ Check if this job has a parent. If so we will just remove it from the parent child list, but if it is the last child we should move the parent to "wait/paused" which requires code from "moveToFinished" ]] -- Includes --[[ Function to add job in target list and add marker if needed. ]] -- Includes --[[ Add marker if needed when a job is available. ]] local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed) if not isPausedOrMaxed then rcall("ZADD", markerKey, 0, "0") end end local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId) rcall(pushCmd, targetKey, jobId) addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed) end --[[ Functions to destructure job key. Just a bit of warning, these functions may be a bit slow and affect performance significantly. ]] local getJobIdFromKey = function (jobKey) return string.match(jobKey, ".*:(.*)") end local getJobKeyPrefix = function (jobKey, jobId) return string.sub(jobKey, 0, #jobKey - #jobId) end --[[ Function to check for the meta.paused key to decide if we are paused or not (since an empty list and !EXISTS are not really the same). ]] local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey) local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration") if queueAttributes[1] then return pausedKey, true, queueAttributes[3], queueAttributes[4] else if queueAttributes[2] then local activeCount = rcall("LLEN", activeKey) if activeCount >= tonumber(queueAttributes[2]) then return waitKey, true, queueAttributes[3], queueAttributes[4] else return waitKey, false, queueAttributes[3], queueAttributes[4] end end end return waitKey, false, queueAttributes[3], queueAttributes[4] end local function _moveParentToWait(parentPrefix, parentId, emitEvent) local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active", parentPrefix .. "wait", parentPrefix .. "paused") addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId) if emitEvent then local parentEventStream = parentPrefix .. "events" rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children") end end local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId) if parentKey then local parentDependenciesKey = parentKey .. ":dependencies" local result = rcall("SREM", parentDependenciesKey, jobKey) if result > 0 then local pendingDependencies = rcall("SCARD", parentDependenciesKey) if pendingDependencies == 0 then local parentId = getJobIdFromKey(parentKey) local parentPrefix = getJobKeyPrefix(parentKey, parentId) local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId) if numRemovedElements == 1 then if hard then -- remove parent in same queue if parentPrefix == baseKey then removeParentDependencyKey(parentKey, hard, nil, baseKey, nil) removeJobKeys(parentKey) if debounceId then rcall("DEL", parentPrefix .. "de:" .. debounceId) end else _moveParentToWait(parentPrefix, parentId) end else _moveParentToWait(parentPrefix, parentId, true) end end end return true end else local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid") local missedParentKey = parentAttributes[1] if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then local parentDependenciesKey = missedParentKey .. ":dependencies" local result = rcall("SREM", parentDependenciesKey, jobKey) if result > 0 then local pendingDependencies = rcall("SCARD", parentDependenciesKey) if pendingDependencies == 0 then local parentId = getJobIdFromKey(missedParentKey) local parentPrefix = getJobKeyPrefix(missedParentKey, parentId) local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId) if numRemovedElements == 1 then if hard then if parentPrefix == baseKey then removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil) removeJobKeys(missedParentKey) if parentAttributes[2] then rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2]) end else _moveParentToWait(parentPrefix, parentId) end else _moveParentToWait(parentPrefix, parentId, true) end end end return true end end end return false end local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey) local jobKey = baseKey .. jobId removeParentDependencyKey(jobKey, hard, nil, baseKey) if shouldRemoveDeduplicationKey then local deduplicationId = rcall("HGET", jobKey, "deid") removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobId, deduplicationId) end removeJobKeys(jobKey) end local function removeJobs(keys, hard, baseKey, max) for i, key in ipairs(keys) do removeJob(key, hard, baseKey, true --[[remove debounce key]]) end return max - #keys end local function getListItems(keyName, max) return rcall('LRANGE', keyName, 0, max - 1) end local function removeListJobs(keyName, hard, baseKey, max, jobsToIgnore) local jobs = getListItems(keyName, max) if jobsToIgnore then jobs = filterOutJobsToIgnore(jobs, jobsToIgnore) end local count = removeJobs(jobs, hard, baseKey, max) rcall("LTRIM", keyName, #jobs, -1) return count end -- Includes --[[ Function to loop in batches. Just a bit of warning, some commands as ZREM could receive a maximum of 7000 parameters per call. ]] local function batches(n, batchSize) local i = 0 return function() local from = i * batchSize + 1 i = i + 1 if (from <= n) then local to = math.min(from + batchSize - 1, n) return from, to end end end --[[ Function to get ZSet items. ]] local function getZSetItems(keyName, max) return rcall('ZRANGE', keyName, 0, max - 1) end local function removeZSetJobs(keyName, hard, baseKey, max, jobsToIgnore) local jobs = getZSetItems(keyName, max) if jobsToIgnore then jobs = filterOutJobsToIgnore(jobs, jobsToIgnore) end local count = removeJobs(jobs, hard, baseKey, max) if(#jobs > 0) then for from, to in batches(#jobs, 7000) do rcall("ZREM", keyName, unpack(jobs, from, to)) end end return count end -- We must not remove delayed jobs if they are associated to a job scheduler. local scheduledJobs = {} local jobSchedulers = rcall("ZRANGE", KEYS[5], 0, -1, "WITHSCORES") -- For every job scheduler, get the current delayed job id. for i = 1, #jobSchedulers, 2 do local jobSchedulerId = jobSchedulers[i] local jobSchedulerMillis = jobSchedulers[i + 1] local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. jobSchedulerMillis scheduledJobs[delayedJobId] = true end removeListJobs(KEYS[1], true, queueBaseKey, 0, scheduledJobs) -- wait removeListJobs(KEYS[2], true, queueBaseKey, 0, scheduledJobs) -- paused if ARGV[2] == "1" then removeZSetJobs(KEYS[3], true, queueBaseKey, 0, scheduledJobs) -- delayed end removeZSetJobs(KEYS[4], true, queueBaseKey, 0, scheduledJobs) -- prioritized `; export const drain = { name: 'drain', content, keys: 5, }; //# sourceMappingURL=drain-5.js.map