Init commit

server/core/lib/live-videos/live-video-jobs.ts

@@ -0,0 +1,48 @@
+import { JobQueue } from '@server/lib/job-queue/index.js'
+import { LiveVideoModel, LiveVideoStatus } from '@server/models/video/live-video.js'
+import { getLiveVideoStartJobId, LIVE_VIDEO_START_JOB_TYPE, LIVE_VIDEO_START_LEAD_TIME_MS } from './shared.js'
+
+export async function syncLiveVideoStartJob (liveVideo: LiveVideoModel, options: {
+  replace: boolean
+}) {
+  const jobId = getLiveVideoStartJobId(liveVideo.id)
+  const existingJob = await JobQueue.Instance.getJob(LIVE_VIDEO_START_JOB_TYPE, jobId)
+
+  if (existingJob) {
+    const state = await existingJob.getState()
+
+    if (options.replace && state === 'active') {
+      return existingJob
+    }
+
+    if (!options.replace) {
+      if (state !== 'completed' && state !== 'failed') return existingJob
+    }
+
+    await existingJob.remove()
+  }
+
+  if (liveVideo.status !== LiveVideoStatus.SCHEDULED) return undefined
+  if (liveVideo.endTime.getTime() <= Date.now()) return undefined
+
+  const delay = Math.max(0, liveVideo.startTime.getTime() - Date.now() - LIVE_VIDEO_START_LEAD_TIME_MS)
+
+  return JobQueue.Instance.createJob({
+    type: LIVE_VIDEO_START_JOB_TYPE,
+    payload: { liveVideoId: liveVideo.id },
+    delay,
+    jobId
+  })
+}
+
+export async function removeLiveVideoStartJob (liveVideoId: number) {
+  const jobId = getLiveVideoStartJobId(liveVideoId)
+  const existingJob = await JobQueue.Instance.getJob(LIVE_VIDEO_START_JOB_TYPE, jobId)
+  if (!existingJob) return false
+
+  const state = await existingJob.getState()
+  if (state === 'active') return false
+
+  await existingJob.remove()
+  return true
+}

server/core/lib/live-videos/live-video-process-manager.ts

@@ -0,0 +1,183 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { stopScheduledLiveSession } from '@server/controllers/api/live-stream.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { PeerTubeSocket } from '@server/lib/peertube-socket.js'
+import { LiveVideoModel, LiveVideoStatus } from '@server/models/video/live-video.js'
+import { getLiveVideoPlaybackInputPath } from './preprocessed-video.js'
+import { buildRtspSourceUrlForTargetCamera } from './rtsp-url.js'
+import { ChildProcessWithoutNullStreams, spawn } from 'node:child_process'
+import { existsSync } from 'node:fs'
+
+type StopReason = 'end-time' | 'cancelled' | 'deleted' | 'manual' | 'shutdown'
+
+type RunningProcess = {
+  endTimer?: NodeJS.Timeout
+  ffmpegProcess: ChildProcessWithoutNullStreams
+  liveVideoId: number
+  stderrBuffer: string
+  stopReason?: StopReason
+  targetCamera: string
+}
+
+const lTags = loggerTagsFactory('live-videos', 'process-manager')
+
+export class LiveVideoProcessManager {
+  private static instance: LiveVideoProcessManager
+
+  private readonly activeByLiveVideoId = new Map<number, RunningProcess>()
+  private readonly liveVideoIdByTargetCamera = new Map<string, number>()
+
+  async start (liveVideo: LiveVideoModel) {
+    const existing = this.activeByLiveVideoId.get(liveVideo.id)
+    if (existing) return false
+
+    const otherLiveVideoId = this.liveVideoIdByTargetCamera.get(liveVideo.targetCamera)
+    if (otherLiveVideoId && otherLiveVideoId !== liveVideo.id) {
+      throw new Error(`Target camera ${liveVideo.targetCamera} is already streaming another live video.`)
+    }
+
+    const inputPath = getLiveVideoPlaybackInputPath(liveVideo.videoStoragePath)
+    if (!existsSync(inputPath)) {
+      throw new Error(`Input file does not exist: ${inputPath}`)
+    }
+
+    const ffmpegPath = CONFIG.LIVE.FFMPEG.PATH?.trim() || 'ffmpeg'
+    const rtspTargetUrl = buildRtspSourceUrlForTargetCamera(liveVideo.targetCamera)
+    const ffmpegArgs = [
+      '-re',
+      '-i', inputPath,
+      '-vf', 'scale=-2:\'min(480,ih)\'',
+      '-c:v', 'libx264',
+      '-preset', 'ultrafast',
+      '-tune', 'zerolatency',
+      '-b:v', '2500k',
+      '-maxrate', '2800k',
+      '-bufsize', '5000k',
+      '-profile:v', 'main',
+      '-force_key_frames', 'expr:gte(t,n_forced*1)',
+      '-c:a', 'aac',
+      '-b:a', '128k',
+      '-f', 'rtsp',
+      '-rtsp_transport', 'tcp',
+      rtspTargetUrl
+    ]
+
+    logger.info('Starting scheduled live video %d on camera %s.', liveVideo.id, liveVideo.targetCamera, {
+      inputPath,
+      usesPreprocessedInput: inputPath !== liveVideo.videoStoragePath,
+      ...lTags()
+    })
+    const ffmpegProcess = spawn(ffmpegPath, ffmpegArgs, {
+      stdio: [ 'ignore', 'ignore', 'pipe' ]
+    })
+
+    const runningProcess: RunningProcess = {
+      ffmpegProcess,
+      liveVideoId: liveVideo.id,
+      stderrBuffer: '',
+      targetCamera: liveVideo.targetCamera
+    }
+
+    this.activeByLiveVideoId.set(liveVideo.id, runningProcess)
+    this.liveVideoIdByTargetCamera.set(liveVideo.targetCamera, liveVideo.id)
+
+    ffmpegProcess.stderr.on('data', chunk => {
+      const line = chunk?.toString?.() || ''
+      runningProcess.stderrBuffer = (runningProcess.stderrBuffer + line).slice(-4_000)
+    })
+
+    ffmpegProcess.once('error', err => {
+      logger.error('Cannot start ffmpeg for scheduled live video %d.', liveVideo.id, { err, ...lTags() })
+    })
+
+    ffmpegProcess.once('exit', (code, signal) => {
+      void this.onProcessExit(runningProcess, code, signal)
+    })
+
+    const stopDelay = Math.max(0, liveVideo.endTime.getTime() - Date.now())
+    runningProcess.endTimer = setTimeout(() => {
+      void this.stop(liveVideo.id, 'end-time')
+    }, stopDelay)
+
+    return true
+  }
+
+  async stop (liveVideoId: number, reason: StopReason) {
+    const runningProcess = this.activeByLiveVideoId.get(liveVideoId)
+    if (!runningProcess) return false
+
+    runningProcess.stopReason = reason
+
+    if (runningProcess.endTimer) {
+      clearTimeout(runningProcess.endTimer)
+      runningProcess.endTimer = undefined
+    }
+
+    if (!runningProcess.ffmpegProcess.killed) {
+      runningProcess.ffmpegProcess.kill('SIGTERM')
+
+      setTimeout(() => {
+        const existing = this.activeByLiveVideoId.get(liveVideoId)
+        if (!existing || existing.ffmpegProcess.killed) return
+
+        existing.ffmpegProcess.kill('SIGKILL')
+      }, 5_000)
+    }
+
+    return true
+  }
+
+  async stopAll () {
+    await Promise.all(Array.from(this.activeByLiveVideoId.keys()).map(id => this.stop(id, 'shutdown')))
+  }
+
+  private async onProcessExit (runningProcess: RunningProcess, code: number | null, signal: NodeJS.Signals | null) {
+    if (runningProcess.endTimer) {
+      clearTimeout(runningProcess.endTimer)
+      runningProcess.endTimer = undefined
+    }
+
+    this.activeByLiveVideoId.delete(runningProcess.liveVideoId)
+    this.liveVideoIdByTargetCamera.delete(runningProcess.targetCamera)
+
+    const stderr = runningProcess.stderrBuffer.trim()
+    if (code !== 0 && runningProcess.stopReason === undefined) {
+      logger.error('Scheduled live video %d ffmpeg exited with code %s and signal %s.', runningProcess.liveVideoId, code, signal, {
+        stderr,
+        ...lTags()
+      })
+    } else {
+      logger.info('Scheduled live video %d ffmpeg exited.', runningProcess.liveVideoId, {
+        code,
+        signal,
+        reason: runningProcess.stopReason,
+        ...lTags()
+      })
+    }
+
+    const liveVideo = await LiveVideoModel.loadById(runningProcess.liveVideoId)
+    const targetCamera = runningProcess.targetCamera
+
+    await stopScheduledLiveSession(targetCamera)
+      .catch(err => logger.error('Cannot stop scheduled sink stream session after ffmpeg exit.', Object.assign({ err, liveVideoId: runningProcess.liveVideoId, targetCamera }, lTags())))
+
+    if (!liveVideo) return
+    if (liveVideo.status !== LiveVideoStatus.SCHEDULED) return
+    if (code !== 0 && runningProcess.stopReason === undefined) return
+    if (runningProcess.stopReason && runningProcess.stopReason !== 'end-time') return
+
+    await liveVideo.update({
+      status: LiveVideoStatus.COMPLETED
+    })
+
+    await PeerTubeSocket.Instance.closeLiveChatRoom(targetCamera, 'live-ended')
+      .catch(err => logger.error(
+        'Cannot reset live chat room after scheduled live video completion.',
+        Object.assign({ err, liveVideoId: runningProcess.liveVideoId, targetCamera }, lTags())
+      ))
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/live-videos/preprocessed-video.ts

@@ -0,0 +1,110 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { tryAtomicMove } from '@server/helpers/fs.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { remove } from 'fs-extra/esm'
+import { existsSync } from 'node:fs'
+import { dirname } from 'node:path'
+import { spawn } from 'node:child_process'
+
+const lTags = loggerTagsFactory('live-videos', 'preprocessed-video')
+
+export function getLiveVideoBurnedPath (videoPath: string) {
+  return `${videoPath}.burned.mp4`
+}
+
+export function getLiveVideoSubtitlePath (videoPath: string) {
+  if (existsSync(videoPath + '.vtt')) return videoPath + '.vtt'
+  if (existsSync(videoPath + '.srt')) return videoPath + '.srt'
+
+  return null
+}
+
+export function getLiveVideoPlaybackInputPath (videoPath: string) {
+  const burnedPath = getLiveVideoBurnedPath(videoPath)
+  return existsSync(burnedPath) ? burnedPath : videoPath
+}
+
+export async function syncPreprocessedLiveVideo (videoPath: string) {
+  const subtitlePath = getLiveVideoSubtitlePath(videoPath)
+  const burnedPath = getLiveVideoBurnedPath(videoPath)
+
+  if (!subtitlePath) {
+    await remove(burnedPath).catch(() => undefined)
+    return null
+  }
+
+  const tmpOutputPath = `${burnedPath}.tmp.mp4`
+  await remove(tmpOutputPath).catch(() => undefined)
+
+  const ffmpegPath = CONFIG.LIVE.FFMPEG.PATH?.trim() || 'ffmpeg'
+  const subtitleFilterPath = escapeSubtitleFilterPath(subtitlePath)
+  const ffmpegArgs = [
+    '-y',
+    '-hide_banner',
+    '-loglevel', 'warning',
+    '-i', videoPath,
+    '-map', '0:v:0',
+    '-map', '0:a?',
+    '-vf', `subtitles=${subtitleFilterPath}`,
+    '-c:v', 'libx264',
+    '-preset', 'veryfast',
+    '-c:a', 'aac',
+    '-movflags', '+faststart',
+    tmpOutputPath
+  ]
+
+  await runFFmpeg(ffmpegPath, ffmpegArgs, { videoPath, subtitlePath })
+  await remove(burnedPath).catch(() => undefined)
+  await tryAtomicMove(tmpOutputPath, burnedPath)
+
+  return burnedPath
+}
+
+function escapeSubtitleFilterPath (subtitlePath: string) {
+  return subtitlePath
+    .replace(/\\/g, '\\\\')
+    .replace(/:/g, '\\:')
+    .replace(/ /g, '\\ ')
+    .replace(/\[/g, '\\[')
+    .replace(/\]/g, '\\]')
+    .replace(/,/g, '\\,')
+    .replace(/'/g, "\\'")
+}
+
+function runFFmpeg (ffmpegPath: string, ffmpegArgs: string[], meta: { videoPath: string, subtitlePath: string }) {
+  return new Promise<void>((resolve, reject) => {
+    const ffmpegProcess = spawn(ffmpegPath, ffmpegArgs, {
+      cwd: dirname(meta.videoPath),
+      stdio: [ 'ignore', 'ignore', 'pipe' ]
+    })
+
+    let stderrBuffer = ''
+
+    ffmpegProcess.stderr.on('data', chunk => {
+      const line = chunk?.toString?.() || ''
+      stderrBuffer = (stderrBuffer + line).slice(-4_000)
+    })
+
+    ffmpegProcess.once('error', err => {
+      reject(err)
+    })
+
+    ffmpegProcess.once('exit', code => {
+      if (code === 0) {
+        resolve()
+        return
+      }
+
+      const stderr = stderrBuffer.trim()
+
+      logger.error('Cannot preprocess live video with burned subtitles.', {
+        code,
+        stderr,
+        ...meta,
+        ...lTags()
+      })
+
+      reject(new Error(stderr || `ffmpeg exited with code ${code}`))
+    })
+  })
+}

server/core/lib/live-videos/rtsp-url.ts

@@ -0,0 +1,19 @@
+import { CONFIG } from '@server/initializers/config.js'
+
+function buildRtspUrl (endpoint: string | undefined, urlPattern: string | undefined, targetCamera: string) {
+  const normalizedEndpoint = (endpoint || 'rtsp://localhost:8554').trim().replace(/\/+$/, '')
+  const normalizedPattern = (urlPattern || '/{targetCamera}/live')
+    .replace('{targetCamera}', encodeURIComponent(targetCamera))
+
+  if (normalizedPattern.startsWith('/')) return normalizedEndpoint + normalizedPattern
+
+  return normalizedEndpoint + '/' + normalizedPattern
+}
+
+export function buildRtspSourceUrlForTargetCamera (targetCamera: string) {
+  return buildRtspUrl(CONFIG.LIVE.RTSP_SOURCE.ENDPOINT, CONFIG.LIVE.RTSP_SOURCE.URL_PATTERN, targetCamera)
+}
+
+export function buildRtspSinkUrlForTargetCamera (targetCamera: string) {
+  return buildRtspUrl(CONFIG.LIVE.RTSP_SINK.ENDPOINT, CONFIG.LIVE.RTSP_SINK.URL_PATTERN, targetCamera)
+}

server/core/lib/live-videos/shared.ts

@@ -0,0 +1,6 @@
+export const LIVE_VIDEO_START_LEAD_TIME_MS = 10_000
+export const LIVE_VIDEO_START_JOB_TYPE = 'live-video-start'
+
+export function getLiveVideoStartJobId (liveVideoId: number) {
+  return `live-video-start_${liveVideoId}`
+}