Projektstart
This commit is contained in:
332
backend/node_modules/bullmq/dist/esm/scripts/addParentJob-6.js
generated
vendored
Normal file
332
backend/node_modules/bullmq/dist/esm/scripts/addParentJob-6.js
generated
vendored
Normal file
@@ -0,0 +1,332 @@
|
||||
const content = `--[[
|
||||
Adds a parent job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- adds the job to the waiting-children zset
|
||||
Input:
|
||||
KEYS[1] 'meta'
|
||||
KEYS[2] 'id'
|
||||
KEYS[3] 'delayed'
|
||||
KEYS[4] 'waiting-children'
|
||||
KEYS[5] 'completed'
|
||||
KEYS[6] events stream key
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] parent dependencies key.
|
||||
[7] parent? {id, queueKey}
|
||||
[8] repeat job key
|
||||
[9] deduplication key
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local metaKey = KEYS[1]
|
||||
local idKey = KEYS[2]
|
||||
local completedKey = KEYS[5]
|
||||
local eventsKey = KEYS[6]
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
local parentKey = args[5]
|
||||
local parent = args[7]
|
||||
local repeatJobKey = args[8]
|
||||
local deduplicationKey = args[9]
|
||||
local parentData
|
||||
-- Includes
|
||||
--[[
|
||||
Function to get max events value or set by default 10000.
|
||||
]]
|
||||
local function getOrSetMaxEvents(metaKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if not maxEvents then
|
||||
maxEvents = 10000
|
||||
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
||||
end
|
||||
return maxEvents
|
||||
end
|
||||
--[[
|
||||
Function to handle the case when job is duplicated.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
This function is used to update the parent's dependencies if the job
|
||||
is already completed and about to be ignored. The parent must get its
|
||||
dependencies updated to avoid the parent job being stuck forever in
|
||||
the waiting-children state.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Validate and move or add dependencies to parent.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
||||
if no pending dependencies.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Move parent to a wait status (wait, prioritized or delayed)
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if nextTimestamp ~= nil then
|
||||
return nextTimestamp / 0x1000
|
||||
end
|
||||
end
|
||||
end
|
||||
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Replace the score of the marker with the newest known
|
||||
-- next timestamp.
|
||||
rcall("ZADD", markerKey, nextTimestamp, "1")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to add job in target list and add marker if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Add marker if needed when a job is available.
|
||||
]]
|
||||
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
||||
if not isPausedOrMaxed then
|
||||
rcall("ZADD", markerKey, 0, "0")
|
||||
end
|
||||
end
|
||||
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
||||
rcall(pushCmd, targetKey, jobId)
|
||||
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
||||
end
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
||||
isPausedOrMaxed)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
||||
end
|
||||
--[[
|
||||
Function to check if queue is paused or maxed
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
||||
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
||||
if queueAttributes[1] then
|
||||
return true
|
||||
else
|
||||
if queueAttributes[2] then
|
||||
local activeCount = rcall("LLEN", activeKey)
|
||||
return activeCount >= tonumber(queueAttributes[2])
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
||||
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
||||
if queueAttributes[1] then
|
||||
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
||||
else
|
||||
if queueAttributes[2] then
|
||||
local activeCount = rcall("LLEN", activeKey)
|
||||
if activeCount >= tonumber(queueAttributes[2]) then
|
||||
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
||||
else
|
||||
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
||||
end
|
||||
end
|
||||
end
|
||||
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
||||
end
|
||||
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
||||
local parentWaitKey = parentQueueKey .. ":wait"
|
||||
local parentPausedKey = parentQueueKey .. ":paused"
|
||||
local parentActiveKey = parentQueueKey .. ":active"
|
||||
local parentMetaKey = parentQueueKey .. ":meta"
|
||||
local parentMarkerKey = parentQueueKey .. ":marker"
|
||||
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
||||
local priority = tonumber(jobAttributes[1]) or 0
|
||||
local delay = tonumber(jobAttributes[2]) or 0
|
||||
if delay > 0 then
|
||||
local delayedTimestamp = tonumber(timestamp) + delay
|
||||
local score = delayedTimestamp * 0x1000
|
||||
local parentDelayedKey = parentQueueKey .. ":delayed"
|
||||
rcall("ZADD", parentDelayedKey, score, parentId)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
||||
delayedTimestamp)
|
||||
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
||||
else
|
||||
if priority == 0 then
|
||||
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
||||
parentWaitKey, parentPausedKey)
|
||||
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
||||
else
|
||||
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
||||
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
||||
parentQueueKey .. ":pc", isPausedOrMaxed)
|
||||
end
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
||||
"waiting-children")
|
||||
end
|
||||
end
|
||||
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
||||
if rcall("EXISTS", parentKey) == 1 then
|
||||
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
||||
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
||||
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
||||
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
||||
parentId, timestamp)
|
||||
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
||||
if doNotHavePendingDependencies then
|
||||
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
||||
end
|
||||
end
|
||||
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
||||
parentId, jobIdKey, returnvalue, timestamp )
|
||||
local processedSet = parentKey .. ":processed"
|
||||
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
||||
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
end
|
||||
local function updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey,
|
||||
jobIdKey, jobId, timestamp)
|
||||
if parentKey ~= nil then
|
||||
if rcall("ZSCORE", completedKey, jobId) then
|
||||
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
||||
updateParentDepsIfNeeded(parentKey, parent['queueKey'],
|
||||
parentDependenciesKey, parent['id'],
|
||||
jobIdKey, returnvalue, timestamp)
|
||||
else
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
||||
end
|
||||
end
|
||||
local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParent,
|
||||
parentData, parentDependenciesKey, completedKey, eventsKey, maxEvents, timestamp)
|
||||
local existedParentKey = rcall("HGET", jobKey, "parentKey")
|
||||
if not existedParentKey or existedParentKey == currentParentKey then
|
||||
updateExistingJobsParent(currentParentKey, currentParent, parentData,
|
||||
parentDependenciesKey, completedKey, jobKey,
|
||||
jobId, timestamp)
|
||||
else
|
||||
if currentParentKey ~= nil and currentParentKey ~= existedParentKey
|
||||
and (rcall("EXISTS", existedParentKey) == 1) then
|
||||
return -7
|
||||
end
|
||||
end
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
--[[
|
||||
Function to store a job
|
||||
]]
|
||||
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
local jsonOpts = cjson.encode(opts)
|
||||
local delay = opts['delay'] or 0
|
||||
local priority = opts['priority'] or 0
|
||||
local debounceId = opts['de'] and opts['de']['id']
|
||||
local optionalValues = {}
|
||||
if parentKey ~= nil then
|
||||
table.insert(optionalValues, "parentKey")
|
||||
table.insert(optionalValues, parentKey)
|
||||
table.insert(optionalValues, "parent")
|
||||
table.insert(optionalValues, parentData)
|
||||
end
|
||||
if repeatJobKey then
|
||||
table.insert(optionalValues, "rjk")
|
||||
table.insert(optionalValues, repeatJobKey)
|
||||
end
|
||||
if debounceId then
|
||||
table.insert(optionalValues, "deid")
|
||||
table.insert(optionalValues, debounceId)
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
||||
"timestamp", timestamp, "delay", delay, "priority", priority,
|
||||
unpack(optionalValues))
|
||||
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
||||
return delay, priority
|
||||
end
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
local parentDependenciesKey = args[6]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
||||
parentData, parentDependenciesKey, completedKey, eventsKey,
|
||||
maxEvents, timestamp)
|
||||
end
|
||||
end
|
||||
-- Store the job.
|
||||
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
local waitChildrenKey = KEYS[4]
|
||||
rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"waiting-children", "jobId", jobId)
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
return jobId .. "" -- convert to string
|
||||
`;
|
||||
export const addParentJob = {
|
||||
name: 'addParentJob',
|
||||
content,
|
||||
keys: 6,
|
||||
};
|
||||
//# sourceMappingURL=addParentJob-6.js.map
|
||||
Reference in New Issue
Block a user