323 lines
12 KiB
JavaScript
323 lines
12 KiB
JavaScript
"use strict";
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
exports.updateJobScheduler = void 0;
|
|
const content = `--[[
|
|
Updates a job scheduler and adds next delayed job
|
|
Input:
|
|
KEYS[1] 'repeat' key
|
|
KEYS[2] 'delayed'
|
|
KEYS[3] 'wait' key
|
|
KEYS[4] 'paused' key
|
|
KEYS[5] 'meta'
|
|
KEYS[6] 'prioritized' key
|
|
KEYS[7] 'marker',
|
|
KEYS[8] 'id'
|
|
KEYS[9] events stream key
|
|
KEYS[10] 'pc' priority counter
|
|
KEYS[11] producer key
|
|
KEYS[12] 'active' key
|
|
ARGV[1] next milliseconds
|
|
ARGV[2] jobs scheduler id
|
|
ARGV[3] Json stringified delayed data
|
|
ARGV[4] msgpacked delayed opts
|
|
ARGV[5] timestamp
|
|
ARGV[6] prefix key
|
|
ARGV[7] producer id
|
|
Output:
|
|
next delayed job id - OK
|
|
]] local rcall = redis.call
|
|
local repeatKey = KEYS[1]
|
|
local delayedKey = KEYS[2]
|
|
local waitKey = KEYS[3]
|
|
local pausedKey = KEYS[4]
|
|
local metaKey = KEYS[5]
|
|
local prioritizedKey = KEYS[6]
|
|
local nextMillis = tonumber(ARGV[1])
|
|
local jobSchedulerId = ARGV[2]
|
|
local timestamp = tonumber(ARGV[5])
|
|
local prefixKey = ARGV[6]
|
|
local producerId = ARGV[7]
|
|
local jobOpts = cmsgpack.unpack(ARGV[4])
|
|
-- Includes
|
|
--[[
|
|
Add delay marker if needed.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Adds a delayed job to the queue by doing the following:
|
|
- Creates a new job key with the job data.
|
|
- adds to delayed zset.
|
|
- Emits a global event 'delayed' if the job is delayed.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Add delay marker if needed.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Function to return the next delayed job timestamp.
|
|
]]
|
|
local function getNextDelayedTimestamp(delayedKey)
|
|
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
|
if #result then
|
|
local nextTimestamp = tonumber(result[2])
|
|
if nextTimestamp ~= nil then
|
|
return nextTimestamp / 0x1000
|
|
end
|
|
end
|
|
end
|
|
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
|
if nextTimestamp ~= nil then
|
|
-- Replace the score of the marker with the newest known
|
|
-- next timestamp.
|
|
rcall("ZADD", markerKey, nextTimestamp, "1")
|
|
end
|
|
end
|
|
--[[
|
|
Bake in the job id first 12 bits into the timestamp
|
|
to guarantee correct execution order of delayed jobs
|
|
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
|
|
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
|
|
]]
|
|
local function getDelayedScore(delayedKey, timestamp, delay)
|
|
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
|
|
local minScore = delayedTimestamp * 0x1000
|
|
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
|
|
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
|
|
minScore, "WITHSCORES","LIMIT", 0, 1)
|
|
if #result then
|
|
local currentMaxScore = tonumber(result[2])
|
|
if currentMaxScore ~= nil then
|
|
if currentMaxScore >= maxScore then
|
|
return maxScore, delayedTimestamp
|
|
else
|
|
return currentMaxScore + 1, delayedTimestamp
|
|
end
|
|
end
|
|
end
|
|
return minScore, delayedTimestamp
|
|
end
|
|
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
|
maxEvents, markerKey, delay)
|
|
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
|
rcall("ZADD", delayedKey, score, jobId)
|
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
|
"jobId", jobId, "delay", delayedTimestamp)
|
|
-- mark that a delayed job is available
|
|
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
end
|
|
--[[
|
|
Function to add job considering priority.
|
|
]]
|
|
-- Includes
|
|
--[[
|
|
Add marker if needed when a job is available.
|
|
]]
|
|
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
if not isPausedOrMaxed then
|
|
rcall("ZADD", markerKey, 0, "0")
|
|
end
|
|
end
|
|
--[[
|
|
Function to get priority score.
|
|
]]
|
|
local function getPriorityScore(priority, priorityCounterKey)
|
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
|
return priority * 0x100000000 + prioCounter % 0x100000000
|
|
end
|
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
|
isPausedOrMaxed)
|
|
local score = getPriorityScore(priority, priorityCounterKey)
|
|
rcall("ZADD", prioritizedKey, score, jobId)
|
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
end
|
|
--[[
|
|
Function to check for the meta.paused key to decide if we are paused or not
|
|
(since an empty list and !EXISTS are not really the same).
|
|
]]
|
|
local function isQueuePaused(queueMetaKey)
|
|
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
|
end
|
|
--[[
|
|
Function to store a job
|
|
]]
|
|
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
parentKey, parentData, repeatJobKey)
|
|
local jsonOpts = cjson.encode(opts)
|
|
local delay = opts['delay'] or 0
|
|
local priority = opts['priority'] or 0
|
|
local debounceId = opts['de'] and opts['de']['id']
|
|
local optionalValues = {}
|
|
if parentKey ~= nil then
|
|
table.insert(optionalValues, "parentKey")
|
|
table.insert(optionalValues, parentKey)
|
|
table.insert(optionalValues, "parent")
|
|
table.insert(optionalValues, parentData)
|
|
end
|
|
if repeatJobKey then
|
|
table.insert(optionalValues, "rjk")
|
|
table.insert(optionalValues, repeatJobKey)
|
|
end
|
|
if debounceId then
|
|
table.insert(optionalValues, "deid")
|
|
table.insert(optionalValues, debounceId)
|
|
end
|
|
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
|
"timestamp", timestamp, "delay", delay, "priority", priority,
|
|
unpack(optionalValues))
|
|
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
|
return delay, priority
|
|
end
|
|
--[[
|
|
Function to check for the meta.paused key to decide if we are paused or not
|
|
(since an empty list and !EXISTS are not really the same).
|
|
]]
|
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
if queueAttributes[1] then
|
|
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
else
|
|
if queueAttributes[2] then
|
|
local activeCount = rcall("LLEN", activeKey)
|
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
else
|
|
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
end
|
|
end
|
|
end
|
|
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
end
|
|
--[[
|
|
Function to add job in target list and add marker if needed.
|
|
]]
|
|
-- Includes
|
|
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
|
rcall(pushCmd, targetKey, jobId)
|
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
end
|
|
local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
|
|
prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
|
|
data, jobSchedulerId, repeatDelay)
|
|
opts['delay'] = repeatDelay
|
|
opts['jobId'] = jobId
|
|
local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
|
|
opts, timestamp, nil, nil, jobSchedulerId)
|
|
if delay ~= 0 then
|
|
addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, markerKey, delay)
|
|
else
|
|
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
|
-- Standard or priority add
|
|
if priority == 0 then
|
|
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
|
addJobInTargetList(target, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
|
else
|
|
-- Priority add
|
|
addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounter, isPausedOrMaxed)
|
|
end
|
|
-- Emit waiting event
|
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", jobId)
|
|
end
|
|
end
|
|
--[[
|
|
Function to get max events value or set by default 10000.
|
|
]]
|
|
local function getOrSetMaxEvents(metaKey)
|
|
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
|
if not maxEvents then
|
|
maxEvents = 10000
|
|
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
|
end
|
|
return maxEvents
|
|
end
|
|
local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
local nextMillis
|
|
if not prevMillis then
|
|
if startDate then
|
|
-- Assuming startDate is passed as milliseconds from JavaScript
|
|
nextMillis = tonumber(startDate)
|
|
nextMillis = nextMillis > now and nextMillis or now
|
|
else
|
|
nextMillis = now
|
|
end
|
|
else
|
|
nextMillis = prevMillis + every
|
|
-- check if we may have missed some iterations
|
|
if nextMillis < now then
|
|
nextMillis = math.floor(now / every) * every + every + (offset or 0)
|
|
end
|
|
end
|
|
if not offset or offset == 0 then
|
|
local timeSlot = math.floor(nextMillis / every) * every;
|
|
offset = nextMillis - timeSlot;
|
|
end
|
|
-- Return a tuple nextMillis, offset
|
|
return math.floor(nextMillis), math.floor(offset)
|
|
end
|
|
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
|
-- Validate that scheduler exists.
|
|
-- If it does not exist we should not iterate anymore.
|
|
if prevMillis then
|
|
prevMillis = tonumber(prevMillis)
|
|
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
|
local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data", "every", "startDate", "offset")
|
|
local every = tonumber(schedulerAttributes[3])
|
|
local now = tonumber(timestamp)
|
|
-- If every is not found in scheduler attributes, try to get it from job options
|
|
if not every and jobOpts['repeat'] and jobOpts['repeat']['every'] then
|
|
every = tonumber(jobOpts['repeat']['every'])
|
|
end
|
|
if every then
|
|
local startDate = schedulerAttributes[4]
|
|
local jobOptsOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
|
|
local offset = schedulerAttributes[5] or jobOptsOffset or 0
|
|
local newOffset
|
|
nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
if not offset then
|
|
rcall("HSET", schedulerKey, "offset", newOffset)
|
|
jobOpts['repeat']['offset'] = newOffset
|
|
end
|
|
end
|
|
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
|
local currentDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
|
if producerId == currentDelayedJobId then
|
|
local eventsKey = KEYS[9]
|
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
|
if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
|
|
rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
|
|
rcall("HINCRBY", schedulerKey, "ic", 1)
|
|
rcall("INCR", KEYS[8])
|
|
-- TODO: remove this workaround in next breaking change,
|
|
-- all job-schedulers must save job data
|
|
local templateData = schedulerAttributes[2] or ARGV[3]
|
|
if templateData and templateData ~= '{}' then
|
|
rcall("HSET", schedulerKey, "data", templateData)
|
|
end
|
|
local delay = nextMillis - now
|
|
-- Fast Clamp delay to minimum of 0
|
|
if delay < 0 then
|
|
delay = 0
|
|
end
|
|
jobOpts["delay"] = delay
|
|
addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, jobOpts, waitKey, pausedKey, KEYS[12], metaKey,
|
|
prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey, schedulerAttributes[1], maxEvents, ARGV[5],
|
|
templateData or '{}', jobSchedulerId, delay)
|
|
-- TODO: remove this workaround in next breaking change
|
|
if KEYS[11] ~= "" then
|
|
rcall("HSET", KEYS[11], "nrjid", nextDelayedJobId)
|
|
end
|
|
return nextDelayedJobId .. "" -- convert to string
|
|
else
|
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "duplicated", "jobId", nextDelayedJobId)
|
|
end
|
|
end
|
|
end
|
|
`;
|
|
exports.updateJobScheduler = {
|
|
name: 'updateJobScheduler',
|
|
content,
|
|
keys: 12,
|
|
};
|
|
//# sourceMappingURL=updateJobScheduler-12.js.map
|