Projektstart

This commit is contained in:
2026-01-22 15:49:12 +01:00
parent 7212eb6f7a
commit 57e5f652f8
10637 changed files with 2598792 additions and 64 deletions

View File

@@ -0,0 +1,445 @@
'use strict'
const { once } = require('events')
const { Transform, pipeline } = require('stream')
const { test } = require('tap')
const build = require('../')
test('parse newlined delimited JSON', ({ same, plan }) => {
plan(2)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const stream = build(function (source) {
source.on('data', function (line) {
same(expected.shift(), line)
})
})
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('parse newlined delimited JSON', ({ same, plan }) => {
plan(2)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const stream = build(function (source) {
source.on('data', function (line) {
same(expected.shift(), line)
})
}, { parse: 'json' })
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('null support', ({ same, plan }) => {
plan(1)
const stream = build(function (source) {
source.on('unknown', function (line) {
same('null', line)
})
})
stream.write('null\n')
stream.end()
})
test('broken json', ({ match, same, plan }) => {
plan(2)
const expected = '{ "truncated'
const stream = build(function (source) {
source.on('unknown', function (line, error) {
same(expected, line)
const regex = /^(Unexpected end of JSON input|Unterminated string in JSON at position 12)( \(line 1 column 13\))?$/
match(error.message, regex)
})
})
stream.write(expected + '\n')
stream.end()
})
test('pure values', ({ same, ok, plan }) => {
plan(3)
const stream = build(function (source) {
source.on('data', function (line) {
same(line.data, 42)
ok(line.time)
same(new Date(line.time).getTime(), line.time)
})
})
stream.write('42\n')
stream.end()
})
test('support async iteration', ({ same, plan }) => {
plan(2)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const stream = build(async function (source) {
for await (const line of source) {
same(expected.shift(), line)
}
})
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('rejecting errors the stream', async ({ same, plan }) => {
const stream = build(async function (source) {
throw new Error('kaboom')
})
const [err] = await once(stream, 'error')
same(err.message, 'kaboom')
})
test('emits an error if the transport expects pino to send the config, but pino is not going to', async function ({ plan, same }) {
plan(1)
const stream = build(() => {}, { expectPinoConfig: true })
const [err] = await once(stream, 'error')
same(err.message, 'This transport is not compatible with the current version of pino. Please upgrade pino to the latest version.')
})
test('set metadata', ({ same, plan, equal }) => {
plan(9)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const stream = build(function (source) {
source.on('data', function (line) {
const obj = expected.shift()
same(this.lastLevel, obj.level)
same(this.lastTime, obj.time)
same(this.lastObj, obj)
same(obj, line)
})
}, { metadata: true })
equal(stream[Symbol.for('pino.metadata')], true)
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('parse lines', ({ same, plan, equal }) => {
plan(9)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const stream = build(function (source) {
source.on('data', function (line) {
const obj = expected.shift()
same(this.lastLevel, obj.level)
same(this.lastTime, obj.time)
same(this.lastObj, obj)
same(JSON.stringify(obj), line)
})
}, { metadata: true, parse: 'lines' })
equal(stream[Symbol.for('pino.metadata')], true)
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('custom parse line function', ({ same, plan, equal }) => {
plan(11)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
let num = 0
function parseLine (str) {
const obj = JSON.parse(str)
same(expected[num], obj)
return obj
}
const stream = build(function (source) {
source.on('data', function (line) {
const obj = expected[num]
same(this.lastLevel, obj.level)
same(this.lastTime, obj.time)
same(this.lastObj, obj)
same(obj, line)
num++
})
}, { metadata: true, parseLine })
equal(stream[Symbol.for('pino.metadata')], true)
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('set metadata (default)', ({ same, plan, equal }) => {
plan(9)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const stream = build(function (source) {
source.on('data', function (line) {
const obj = expected.shift()
same(this.lastLevel, obj.level)
same(this.lastTime, obj.time)
same(this.lastObj, obj)
same(obj, line)
})
})
equal(stream[Symbol.for('pino.metadata')], true)
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('do not set metadata', ({ same, plan, equal }) => {
plan(9)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const stream = build(function (source) {
source.on('data', function (line) {
const obj = expected.shift()
same(this.lastLevel, undefined)
same(this.lastTime, undefined)
same(this.lastObj, undefined)
same(obj, line)
})
}, { metadata: false })
equal(stream[Symbol.for('pino.metadata')], undefined)
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('close logic', ({ same, plan, pass }) => {
plan(3)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const stream = build(function (source) {
source.on('data', function (line) {
same(expected.shift(), line)
})
}, {
close (err, cb) {
pass('close called')
process.nextTick(cb, err)
}
})
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('close with promises', ({ same, plan, pass }) => {
plan(3)
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const stream = build(function (source) {
source.on('data', function (line) {
same(expected.shift(), line)
})
}, {
async close () {
pass('close called')
}
})
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test('support Transform streams', ({ same, plan, error }) => {
plan(7)
const expected1 = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const expected2 = []
const stream1 = build(function (source) {
const transform = new Transform({
objectMode: true,
autoDestroy: true,
transform (chunk, enc, cb) {
same(expected1.shift(), chunk)
chunk.service = 'from transform'
expected2.push(chunk)
cb(null, JSON.stringify(chunk) + '\n')
}
})
pipeline(source, transform, () => {})
return transform
}, { enablePipelining: true })
const stream2 = build(function (source) {
source.on('data', function (line) {
same(expected2.shift(), line)
})
})
pipeline(stream1, stream2, function (err) {
error(err)
same(expected1, [])
same(expected2, [])
})
const lines = expected1.map(JSON.stringify).join('\n')
stream1.write(lines)
stream1.end()
})

View File

@@ -0,0 +1,22 @@
'use strict'
const build = require('../..')
module.exports = async function (threadStreamOpts) {
const { port, opts = {} } = threadStreamOpts
return build(
async function (source) {
for await (const obj of source) {
port.postMessage({
data: obj,
pinoConfig: {
levels: source.levels,
messageKey: source.messageKey,
errorKey: source.errorKey
}
})
}
},
opts
)
}

View File

@@ -0,0 +1,22 @@
'use strict'
const build = require('../..')
module.exports = async function (threadStreamOpts) {
const { port, opts = {} } = threadStreamOpts
return build(
function (source) {
source.on('data', function (line) {
port.postMessage({
data: line,
pinoConfig: {
levels: source.levels,
messageKey: source.messageKey,
errorKey: source.errorKey
}
})
})
},
opts
)
}

View File

@@ -0,0 +1,24 @@
'use strict'
const { Transform, pipeline } = require('stream')
const build = require('../..')
module.exports = function (threadStreamOpts) {
const { opts = {} } = threadStreamOpts
return build(function (source) {
const transform = new Transform({
objectMode: true,
autoDestroy: true,
transform (chunk, enc, cb) {
chunk.service = 'from transform'
chunk.level = `${source.levels.labels[chunk.level]}(${chunk.level})`
chunk[source.messageKey] = chunk[source.messageKey].toUpperCase()
cb(null, JSON.stringify(chunk) + '\n')
}
})
pipeline(source, transform, () => {})
return transform
}, { ...opts, enablePipelining: true })
}

View File

@@ -0,0 +1,15 @@
'use strict'
const { pipeline, PassThrough } = require('stream')
module.exports = async function ({ targets }) {
const streams = await Promise.all(targets.map(async (t) => {
const fn = require(t.target)
const stream = await fn(t.options)
return stream
}))
const stream = new PassThrough()
pipeline(stream, ...streams, () => {})
return stream
}

View File

@@ -0,0 +1,31 @@
import build, { OnUnknown } from "../../index";
import { expectType } from "tsd";
import { Transform } from "stream";
/**
* If enablePipelining is set to true, the function passed as an argument
* must return a transform. The unknown event should be listened to on the
* stream passed in the first argument.
*/
expectType<Transform>(build((source) => source, { enablePipelining: true }));
/**
* If expectPinoConfig is set with enablePipelining, build returns a promise
*/
expectType<(Promise<Transform>)>(build((source) => source, { enablePipelining: true, expectPinoConfig: true }));
/**
* If enablePipelining is not set the unknown event can be listened to on
* the returned stream.
*/
expectType<Transform & OnUnknown>(build((source) => {}));
/**
* If expectPinoConfig is set, build returns a promise
*/
expectType<(Promise<Transform & OnUnknown>)>(build((source) => {}, { expectPinoConfig: true }));
/**
* build also accepts an async function
*/
expectType<Transform & OnUnknown>(build(async (source) => {}));

View File

@@ -0,0 +1,357 @@
'use strict'
const { once } = require('events')
const { join } = require('path')
const ThreadStream = require('thread-stream')
const { MessageChannel } = require('worker_threads')
const { test } = require('tap')
workerTest('transport-on-data.js')
workerTest('transport-async-iteration.js', ' when using async iteration')
function workerTest (filename, description = '') {
test(`does not wait for pino to send config by default${description}`, function ({ same, plan }) {
plan(4)
const { port1, port2 } = new MessageChannel()
const stream = new ThreadStream({
filename: join(__dirname, 'fixtures', filename),
workerData: { port: port1 },
workerOpts: {
transferList: [port1]
}
})
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const emptyPinoConfig = {
levels: undefined,
messageKey: undefined,
errorKey: undefined
}
port2.on('message', function (message) {
same(expected.shift(), message.data)
same(emptyPinoConfig, message.pinoConfig)
})
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test(`does not wait for pino to send config if transport is not expecting it${description}`, function ({ same, plan }) {
plan(4)
const { port1, port2 } = new MessageChannel()
const stream = new ThreadStream({
filename: join(__dirname, 'fixtures', filename),
workerData: {
port: port1,
pinoWillSendConfig: true
},
workerOpts: {
transferList: [port1]
}
})
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const emptyPinoConfig = {
levels: undefined,
messageKey: undefined,
errorKey: undefined
}
const pinoConfig = {
levels: {
labels: { 30: 'info' },
values: { info: 30 }
},
messageKey: 'msg',
errorKey: 'err'
}
stream.emit('message', { code: 'PINO_CONFIG', config: pinoConfig })
port2.on('message', function (message) {
same(expected.shift(), message.data)
same(emptyPinoConfig, message.pinoConfig)
})
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.end()
})
test(`waits for the pino config when pino intends to send it and the transport requests it${description}`, function ({ same, plan }) {
plan(4)
const { port1, port2 } = new MessageChannel()
const stream = new ThreadStream({
filename: join(__dirname, 'fixtures', filename),
workerData: {
port: port1,
pinoWillSendConfig: true,
opts: {
expectPinoConfig: true
}
},
workerOpts: {
transferList: [port1]
}
})
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const pinoConfig = {
levels: {
labels: { 30: 'info' },
values: { info: 30 }
},
messageKey: 'msg',
errorKey: 'err'
}
port2.on('message', function (message) {
same(expected.shift(), message.data)
same(pinoConfig, message.pinoConfig)
})
const lines = expected.map(JSON.stringify).join('\n')
stream.emit('message', { code: 'PINO_CONFIG', config: pinoConfig })
stream.write(lines)
stream.end()
})
test(`continues to listen if it receives a message that is not PINO_CONFIG${description}`, function ({ same, plan }) {
plan(4)
const { port1, port2 } = new MessageChannel()
const stream = new ThreadStream({
filename: join(__dirname, 'fixtures', 'transport-on-data.js'),
workerData: {
port: port1,
pinoWillSendConfig: true,
opts: {
expectPinoConfig: true
}
},
workerOpts: {
transferList: [port1]
}
})
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const pinoConfig = {
levels: {
labels: { 30: 'info' },
values: { info: 30 }
},
messageKey: 'msg',
errorKey: 'err'
}
port2.on('message', function (message) {
same(expected.shift(), message.data)
same(pinoConfig, message.pinoConfig)
})
const lines = expected.map(JSON.stringify).join('\n')
stream.emit('message', 'not a PINO_CONFIG')
stream.emit('message', { code: 'NOT_PINO_CONFIG', config: { levels: 'foo', messageKey: 'bar', errorKey: 'baz' } })
stream.emit('message', { code: 'PINO_CONFIG', config: pinoConfig })
stream.write(lines)
stream.end()
})
test(`waits for the pino config even if it is sent after write${description}`, function ({ same, plan }) {
plan(4)
const { port1, port2 } = new MessageChannel()
const stream = new ThreadStream({
filename: join(__dirname, 'fixtures', filename),
workerData: {
port: port1,
pinoWillSendConfig: true,
opts: {
expectPinoConfig: true
}
},
workerOpts: {
transferList: [port1]
}
})
const expected = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}]
const pinoConfig = {
levels: {
labels: { 30: 'info' },
values: { info: 30 }
},
messageKey: 'msg',
errorKey: 'err'
}
port2.on('message', function (message) {
same(expected.shift(), message.data)
same(pinoConfig, message.pinoConfig)
})
const lines = expected.map(JSON.stringify).join('\n')
stream.write(lines)
stream.emit('message', { code: 'PINO_CONFIG', config: pinoConfig })
stream.end()
})
test(`emits an error if the transport expects pino to send the config, but pino is not going to${description}`, async function ({ plan, same, ok }) {
plan(2)
const stream = new ThreadStream({
filename: join(__dirname, 'fixtures', filename),
workerData: {
opts: {
expectPinoConfig: true
}
}
})
const [err] = await once(stream, 'error')
same(err.message, 'This transport is not compatible with the current version of pino. Please upgrade pino to the latest version.')
ok(stream.destroyed)
})
}
test('waits for the pino config when pipelining', function ({ same, plan }) {
plan(2)
const { port1, port2 } = new MessageChannel()
const stream = new ThreadStream({
filename: join(__dirname, 'fixtures', 'worker-pipeline.js'),
workerData: {
pinoWillSendConfig: true,
targets: [{
target: './transport-transform.js',
options: {
opts: { expectPinoConfig: true }
}
}, {
target: './transport-on-data.js',
options: {
port: port1
}
}]
},
workerOpts: {
transferList: [port1]
}
})
const expected = [{
level: 'info(30)',
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'HELLO WORLD',
service: 'from transform'
}, {
level: 'info(30)',
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'ANOTHER MESSAGE',
prop: 42,
service: 'from transform'
}]
const lines = [{
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'hello world'
}, {
level: 30,
time: 1617955768092,
pid: 2942,
hostname: 'MacBook-Pro.local',
msg: 'another message',
prop: 42
}].map(JSON.stringify).join('\n')
const pinoConfig = {
levels: {
labels: { 30: 'info' },
values: { info: 30 }
},
messageKey: 'msg',
errorKey: 'err'
}
port2.on('message', function (message) {
same(expected.shift(), message.data)
})
stream.emit('message', { code: 'PINO_CONFIG', config: pinoConfig })
stream.write(lines)
stream.end()
})