使用真实生产流量回放评估大语言模型性能

// capture-proxy.mjs
import { createServer } from 'node:http'
import { appendFile } from 'node:fs/promises'

const TARGET = process.env.TARGET_URL
const LOG = process.env.LOG_FILE || './traffic.jsonl'
const PORT = process.env.PORT || 3000

const server = createServer(async (req, res) => {
  const chunks = []
  for await (const chunk of req) chunks.push(chunk)
  const raw = Buffer.concat(chunks).toString('utf8')

  const started = Date.now()
  const upstream = await fetch(TARGET, {
    method: req.method,
    headers: { 'content-type': 'application/json' },
    body: raw || undefined,
  })
  const upstreamBody = await upstream.text()
  const latency = Date.now() - started

  if (req.method === 'POST' && req.url.includes('/chat/completions')) {
    const record = {
      ts: new Date().toISOString(),
      path: req.url,
      status: upstream.status,
      latency_ms: latency,
      request: JSON.parse(raw),
      response: JSON.parse(upstreamBody),
    }
    await appendFile(LOG, JSON.stringify(record) + '\n', 'utf8')
  }

  res.writeHead(upstream.status, { 'content-type': 'application/json' })
  res.end(upstreamBody)
})

server.listen(PORT, () => console.log(`Capture proxy running on port ${PORT}`))

node capture-proxy.mjs &
curl -s -X POST localhost:3000/v1/chat/completions \
  -H 'content-type: application/json' \
  -d '{"messages":[{"role":"user","content":"hello"}]}'
wc -l traffic.jsonl   # 预期输出为 1 行

// replay.mjs
import { readFile, appendFile } from 'node:fs/promises'

const [logFile, target, apiKey] = process.argv.slice(2)
const lines = (await readFile(logFile, 'utf8')).trim().split('\n')

for (const line of lines) {
  if (!line) continue
  const record = JSON.parse(line)
  const started = Date.now()
  const res = await fetch(target, {
    method: 'POST',
    headers: {
      'content-type': 'application/json',
      authorization: `Bearer ${apiKey}`,
    },
    body: JSON.stringify(record.request),
  })
  const latency = Date.now() - started
  const text = await res.text()

  let response = null
  try {
    response = JSON.parse(text)
  } catch {
    /* 失败时保持 null */
  }

  await appendFile(
    'results.jsonl',
    JSON.stringify({
      ts: record.ts,
      status: res.status,
      latency_ms: latency,
      response,
    }) + '\n',
    'utf8'
  )
}

node replay.mjs traffic.jsonl "https://api.n1n.ai/v1/chat/completions" "$N1N_API_KEY"
wc -l results.jsonl   # 行数必须与 traffic.jsonl 一致

// diff.mjs
import { readFile } from 'node:fs/promises'

const traffic = (await readFile('traffic.jsonl', 'utf8')).trim().split('\n').map(JSON.parse)
const results = (await readFile('results.jsonl', 'utf8')).trim().split('\n').map(JSON.parse)

const rows = traffic.map((t, i) => {
  const r = results[i] ?? {}
  const prod = t.response?.choices?.[0]?.message?.content ?? null
  const cand = r.response?.choices?.[0]?.message?.content ?? null
  return {
    request: i,
    prod_status: t.status,
    cand_status: r.status,
    prod_ms: t.latency_ms,
    cand_ms: r.latency_ms,
    exact_match: prod === cand,
    both_valid: prod !== null && cand !== null,
    prod_tokens: t.response?.usage?.total_tokens ?? null,
    cand_tokens: r.response?.usage?.total_tokens ?? null,
  }
})

console.table(rows)

const valid = rows.filter((x) => x.cand_status === 200 && x.both_valid).length
const exact = rows.filter((x) => x.exact_match).length
console.log(`valid: ${valid}/${rows.length}`)
console.log(`exact match: ${exact}/${rows.length}`)

node diff.mjs
# 预期输出格式：
# valid: 47/50
# exact match: 41/50

0 */6 * * * cd /opt/replay && node replay.mjs traffic.jsonl "https://api.n1n.ai/v1/chat/completions" "$N1N_API_KEY" >> replay.log 2>&1