Files
simple-mail-cleaner/backend/node_modules/bullmq/dist/esm/scripts/addJobScheduler-11.js
2026-01-22 15:49:12 +01:00

562 lines
20 KiB
JavaScript

const content = `--[[
Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
Input:
KEYS[1] 'repeat' key
KEYS[2] 'delayed' key
KEYS[3] 'wait' key
KEYS[4] 'paused' key
KEYS[5] 'meta' key
KEYS[6] 'prioritized' key
KEYS[7] 'marker' key
KEYS[8] 'id' key
KEYS[9] 'events' key
KEYS[10] 'pc' priority counter
KEYS[11] 'active' key
ARGV[1] next milliseconds
ARGV[2] msgpacked options
[1] name
[2] tz?
[3] pattern?
[4] endDate?
[5] every?
ARGV[3] jobs scheduler id
ARGV[4] Json stringified template data
ARGV[5] mspacked template opts
ARGV[6] msgpacked delayed opts
ARGV[7] timestamp
ARGV[8] prefix key
ARGV[9] producer key
Output:
repeatableKey - OK
]] local rcall = redis.call
local repeatKey = KEYS[1]
local delayedKey = KEYS[2]
local waitKey = KEYS[3]
local pausedKey = KEYS[4]
local metaKey = KEYS[5]
local prioritizedKey = KEYS[6]
local eventsKey = KEYS[9]
local nextMillis = ARGV[1]
local jobSchedulerId = ARGV[3]
local templateOpts = cmsgpack.unpack(ARGV[5])
local now = tonumber(ARGV[7])
local prefixKey = ARGV[8]
local jobOpts = cmsgpack.unpack(ARGV[6])
-- Includes
--[[
Add delay marker if needed.
]]
-- Includes
--[[
Adds a delayed job to the queue by doing the following:
- Creates a new job key with the job data.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
]]
-- Includes
--[[
Add delay marker if needed.
]]
-- Includes
--[[
Function to return the next delayed job timestamp.
]]
local function getNextDelayedTimestamp(delayedKey)
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
if #result then
local nextTimestamp = tonumber(result[2])
if nextTimestamp ~= nil then
return nextTimestamp / 0x1000
end
end
end
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
if nextTimestamp ~= nil then
-- Replace the score of the marker with the newest known
-- next timestamp.
rcall("ZADD", markerKey, nextTimestamp, "1")
end
end
--[[
Bake in the job id first 12 bits into the timestamp
to guarantee correct execution order of delayed jobs
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
]]
local function getDelayedScore(delayedKey, timestamp, delay)
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
local minScore = delayedTimestamp * 0x1000
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
minScore, "WITHSCORES","LIMIT", 0, 1)
if #result then
local currentMaxScore = tonumber(result[2])
if currentMaxScore ~= nil then
if currentMaxScore >= maxScore then
return maxScore, delayedTimestamp
else
return currentMaxScore + 1, delayedTimestamp
end
end
end
return minScore, delayedTimestamp
end
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
maxEvents, markerKey, delay)
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
rcall("ZADD", delayedKey, score, jobId)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
"jobId", jobId, "delay", delayedTimestamp)
-- mark that a delayed job is available
addDelayMarkerIfNeeded(markerKey, delayedKey)
end
--[[
Function to add job considering priority.
]]
-- Includes
--[[
Add marker if needed when a job is available.
]]
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
if not isPausedOrMaxed then
rcall("ZADD", markerKey, 0, "0")
end
end
--[[
Function to get priority score.
]]
local function getPriorityScore(priority, priorityCounterKey)
local prioCounter = rcall("INCR", priorityCounterKey)
return priority * 0x100000000 + prioCounter % 0x100000000
end
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
isPausedOrMaxed)
local score = getPriorityScore(priority, priorityCounterKey)
rcall("ZADD", prioritizedKey, score, jobId)
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function isQueuePaused(queueMetaKey)
return rcall("HEXISTS", queueMetaKey, "paused") == 1
end
--[[
Function to store a job
]]
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
parentKey, parentData, repeatJobKey)
local jsonOpts = cjson.encode(opts)
local delay = opts['delay'] or 0
local priority = opts['priority'] or 0
local debounceId = opts['de'] and opts['de']['id']
local optionalValues = {}
if parentKey ~= nil then
table.insert(optionalValues, "parentKey")
table.insert(optionalValues, parentKey)
table.insert(optionalValues, "parent")
table.insert(optionalValues, parentData)
end
if repeatJobKey then
table.insert(optionalValues, "rjk")
table.insert(optionalValues, repeatJobKey)
end
if debounceId then
table.insert(optionalValues, "deid")
table.insert(optionalValues, debounceId)
end
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
"timestamp", timestamp, "delay", delay, "priority", priority,
unpack(optionalValues))
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
return delay, priority
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
if queueAttributes[1] then
return pausedKey, true, queueAttributes[3], queueAttributes[4]
else
if queueAttributes[2] then
local activeCount = rcall("LLEN", activeKey)
if activeCount >= tonumber(queueAttributes[2]) then
return waitKey, true, queueAttributes[3], queueAttributes[4]
else
return waitKey, false, queueAttributes[3], queueAttributes[4]
end
end
end
return waitKey, false, queueAttributes[3], queueAttributes[4]
end
--[[
Function to add job in target list and add marker if needed.
]]
-- Includes
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
rcall(pushCmd, targetKey, jobId)
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
end
local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
data, jobSchedulerId, repeatDelay)
opts['delay'] = repeatDelay
opts['jobId'] = jobId
local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
opts, timestamp, nil, nil, jobSchedulerId)
if delay ~= 0 then
addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, markerKey, delay)
else
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
-- Standard or priority add
if priority == 0 then
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
addJobInTargetList(target, markerKey, pushCmd, isPausedOrMaxed, jobId)
else
-- Priority add
addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounter, isPausedOrMaxed)
end
-- Emit waiting event
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", jobId)
end
end
--[[
Function to get max events value or set by default 10000.
]]
local function getOrSetMaxEvents(metaKey)
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
if not maxEvents then
maxEvents = 10000
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
end
return maxEvents
end
--[[
Function to remove job.
]]
-- Includes
--[[
Function to remove deduplication key if needed
when a job is being removed.
]]
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
jobId, deduplicationId)
if deduplicationId then
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
local currentJobId = rcall('GET', deduplicationKey)
if currentJobId and currentJobId == jobId then
return rcall("DEL", deduplicationKey)
end
end
end
--[[
Function to remove job keys.
]]
local function removeJobKeys(jobKey)
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
end
--[[
Check if this job has a parent. If so we will just remove it from
the parent child list, but if it is the last child we should move the parent to "wait/paused"
which requires code from "moveToFinished"
]]
-- Includes
--[[
Functions to destructure job key.
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
]]
local getJobIdFromKey = function (jobKey)
return string.match(jobKey, ".*:(.*)")
end
local getJobKeyPrefix = function (jobKey, jobId)
return string.sub(jobKey, 0, #jobKey - #jobId)
end
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
parentPrefix .. "wait", parentPrefix .. "paused")
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
if emitEvent then
local parentEventStream = parentPrefix .. "events"
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
end
end
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
if parentKey then
local parentDependenciesKey = parentKey .. ":dependencies"
local result = rcall("SREM", parentDependenciesKey, jobKey)
if result > 0 then
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
if pendingDependencies == 0 then
local parentId = getJobIdFromKey(parentKey)
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
if numRemovedElements == 1 then
if hard then -- remove parent in same queue
if parentPrefix == baseKey then
removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
removeJobKeys(parentKey)
if debounceId then
rcall("DEL", parentPrefix .. "de:" .. debounceId)
end
else
_moveParentToWait(parentPrefix, parentId)
end
else
_moveParentToWait(parentPrefix, parentId, true)
end
end
end
return true
end
else
local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
local missedParentKey = parentAttributes[1]
if( (type(missedParentKey) == "string") and missedParentKey ~= ""
and (rcall("EXISTS", missedParentKey) == 1)) then
local parentDependenciesKey = missedParentKey .. ":dependencies"
local result = rcall("SREM", parentDependenciesKey, jobKey)
if result > 0 then
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
if pendingDependencies == 0 then
local parentId = getJobIdFromKey(missedParentKey)
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
if numRemovedElements == 1 then
if hard then
if parentPrefix == baseKey then
removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
removeJobKeys(missedParentKey)
if parentAttributes[2] then
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
end
else
_moveParentToWait(parentPrefix, parentId)
end
else
_moveParentToWait(parentPrefix, parentId, true)
end
end
end
return true
end
end
end
return false
end
local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
local jobKey = baseKey .. jobId
removeParentDependencyKey(jobKey, hard, nil, baseKey)
if shouldRemoveDeduplicationKey then
local deduplicationId = rcall("HGET", jobKey, "deid")
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobId, deduplicationId)
end
removeJobKeys(jobKey)
end
--[[
Function to store a job scheduler
]]
local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMillis, opts,
templateData, templateOpts)
rcall("ZADD", repeatKey, nextMillis, schedulerId)
local optionalValues = {}
if opts['tz'] then
table.insert(optionalValues, "tz")
table.insert(optionalValues, opts['tz'])
end
if opts['limit'] then
table.insert(optionalValues, "limit")
table.insert(optionalValues, opts['limit'])
end
if opts['pattern'] then
table.insert(optionalValues, "pattern")
table.insert(optionalValues, opts['pattern'])
end
if opts['startDate'] then
table.insert(optionalValues, "startDate")
table.insert(optionalValues, opts['startDate'])
end
if opts['endDate'] then
table.insert(optionalValues, "endDate")
table.insert(optionalValues, opts['endDate'])
end
if opts['every'] then
table.insert(optionalValues, "every")
table.insert(optionalValues, opts['every'])
end
if opts['offset'] then
table.insert(optionalValues, "offset")
table.insert(optionalValues, opts['offset'])
else
local offset = rcall("HGET", schedulerKey, "offset")
if offset then
table.insert(optionalValues, "offset")
table.insert(optionalValues, tonumber(offset))
end
end
local jsonTemplateOpts = cjson.encode(templateOpts)
if jsonTemplateOpts and jsonTemplateOpts ~= '{}' then
table.insert(optionalValues, "opts")
table.insert(optionalValues, jsonTemplateOpts)
end
if templateData and templateData ~= '{}' then
table.insert(optionalValues, "data")
table.insert(optionalValues, templateData)
end
table.insert(optionalValues, "ic")
table.insert(optionalValues, rcall("HGET", schedulerKey, "ic") or 1)
rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
rcall("HMSET", schedulerKey, "name", opts['name'], unpack(optionalValues))
end
local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
local nextMillis
if not prevMillis then
if startDate then
-- Assuming startDate is passed as milliseconds from JavaScript
nextMillis = tonumber(startDate)
nextMillis = nextMillis > now and nextMillis or now
else
nextMillis = now
end
else
nextMillis = prevMillis + every
-- check if we may have missed some iterations
if nextMillis < now then
nextMillis = math.floor(now / every) * every + every + (offset or 0)
end
end
if not offset or offset == 0 then
local timeSlot = math.floor(nextMillis / every) * every;
offset = nextMillis - timeSlot;
end
-- Return a tuple nextMillis, offset
return math.floor(nextMillis), math.floor(offset)
end
-- If we are overriding a repeatable job we must delete the delayed job for
-- the next iteration.
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
local maxEvents = getOrSetMaxEvents(metaKey)
local templateData = ARGV[4]
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
if prevMillis then
prevMillis = tonumber(prevMillis)
end
local schedulerOpts = cmsgpack.unpack(ARGV[2])
local every = schedulerOpts['every']
-- For backwards compatibility we also check the offset from the job itself.
-- could be removed in future major versions.
local jobOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
local offset = schedulerOpts['offset'] or jobOffset or 0
local newOffset = offset
local updatedEvery = false
if every then
-- if we changed the 'every' value we need to reset millis to nil
local millis = prevMillis
if prevMillis then
local prevEvery = tonumber(rcall("HGET", schedulerKey, "every"))
if prevEvery ~= every then
millis = nil
updatedEvery = true
end
end
local startDate = schedulerOpts['startDate']
nextMillis, newOffset = getJobSchedulerEveryNextMillis(millis, every, now, offset, startDate)
end
local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId, metaKey,
eventsKey)
if rcall("ZSCORE", delayedKey, jobId) then
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
rcall("ZREM", delayedKey, jobId)
return true
elseif rcall("ZSCORE", prioritizedKey, jobId) then
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
rcall("ZREM", prioritizedKey, jobId)
return true
else
local pausedOrWaitKey = waitKey
if isQueuePaused(metaKey) then
pausedOrWaitKey = pausedKey
end
if rcall("LREM", pausedOrWaitKey, 1, jobId) > 0 then
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
return true
end
end
return false
end
local removedPrevJob = false
if prevMillis then
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
local currentJobKey = schedulerKey .. ":" .. prevMillis
-- In theory it should always exist the currentJobKey if there is a prevMillis unless something has
-- gone really wrong.
if rcall("EXISTS", currentJobKey) == 1 then
removedPrevJob = removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, currentJobId,
metaKey, eventsKey)
end
end
if removedPrevJob then
-- The jobs has been removed and we want to replace it, so lets use the same millis.
if every and not updatedEvery then
nextMillis = prevMillis
end
else
-- Special case where no job was removed, and we need to add the next iteration.
schedulerOpts['offset'] = newOffset
end
-- Check for job ID collision with existing jobs (in any state)
local jobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
local jobKey = prefixKey .. jobId
-- If there's already a job with this ID, in a state
-- that is not updatable (active, completed, failed) we must
-- handle the collision
local hasCollision = false
if rcall("EXISTS", jobKey) == 1 then
if every then
-- For 'every' case: try next time slot to avoid collision
local nextSlotMillis = nextMillis + every
local nextSlotJobId = "repeat:" .. jobSchedulerId .. ":" .. nextSlotMillis
local nextSlotJobKey = prefixKey .. nextSlotJobId
if rcall("EXISTS", nextSlotJobKey) == 0 then
-- Next slot is free, use it
nextMillis = nextSlotMillis
jobId = nextSlotJobId
else
-- Next slot also has a job, return error code
return -11 -- SchedulerJobSlotsBusy
end
else
hasCollision = true
end
end
local delay = nextMillis - now
-- Fast Clamp delay to minimum of 0
if delay < 0 then
delay = 0
end
local nextJobKey = schedulerKey .. ":" .. nextMillis
if not hasCollision or removedPrevJob then
-- jobId already calculated above during collision check
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, templateData, templateOpts)
rcall("INCR", KEYS[8])
addJobFromScheduler(nextJobKey, jobId, jobOpts, waitKey, pausedKey, KEYS[11], metaKey, prioritizedKey, KEYS[10],
delayedKey, KEYS[7], eventsKey, schedulerOpts['name'], maxEvents, now, templateData, jobSchedulerId, delay)
elseif hasCollision then
-- For 'pattern' case: return error code
return -10 -- SchedulerJobIdCollision
end
if ARGV[9] ~= "" then
rcall("HSET", ARGV[9], "nrjid", jobId)
end
return {jobId .. "", delay}
`;
export const addJobScheduler = {
name: 'addJobScheduler',
content,
keys: 11,
};
//# sourceMappingURL=addJobScheduler-11.js.map