Init commit

apps/peertube-cli/.npmignore

@@ -0,0 +1,4 @@
+src
+meta.json
+tsconfig.json
+scripts

apps/peertube-cli/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## v1.0.3
+
+ * Fix `util.isArray` deprecation warning

apps/peertube-cli/README.md

@@ -0,0 +1,42 @@
+# PeerTube CLI
+
+## Usage
+
+See https://docs.joinpeertube.org/maintain/tools#remote-tools
+
+## Dev
+
+## Install dependencies
+
+```bash
+cd peertube-root
+npm run install-node-dependencies
+```
+
+## Develop
+
+```bash
+cd peertube-root
+npm run dev:peertube-cli
+```
+
+## Build
+
+```bash
+cd peertube-root
+npm run build:peertube-cli
+```
+
+## Run
+
+```bash
+cd peertube-root
+node apps/peertube-cli/dist/peertube-cli.js --help
+```
+
+## Publish on NPM
+
+```bash
+cd peertube-root
+(cd apps/peertube-cli && npm version patch) && npm run build:peertube-cli && (cd apps/peertube-cli && npm publish --access=public)
+```

apps/peertube-cli/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@peertube/peertube-cli",
+  "version": "1.0.3",
+  "type": "module",
+  "main": "dist/peertube.js",
+  "bin": "dist/peertube.js",
+  "scripts": {},
+  "license": "AGPL-3.0",
+  "private": false,
+  "devDependencies": {
+    "application-config": "^3.0.0",
+    "cli-table3": "^0.6.0",
+    "netrc-parser": "^3.1.6"
+  },
+  "dependencies": {}
+}

apps/peertube-cli/scripts/build.js

@@ -0,0 +1,27 @@
+import * as esbuild from 'esbuild'
+import { readFileSync } from 'fs'
+
+const packageJSON = JSON.parse(readFileSync(new URL('../package.json', import.meta.url)))
+
+export const esbuildOptions = {
+  entryPoints: [ './src/peertube.ts' ],
+  bundle: true,
+  platform: 'node',
+  format: 'esm',
+  target: 'node16',
+  external: [
+    './lib-cov/fluent-ffmpeg',
+    'pg-hstore'
+  ],
+  outfile: './dist/peertube.js',
+  banner: {
+    js: `const require = (await import("node:module")).createRequire(import.meta.url);` +
+      `const __filename = (await import("node:url")).fileURLToPath(import.meta.url);` +
+      `const __dirname = (await import("node:path")).dirname(__filename);`
+  },
+  define: {
+    'process.env.PACKAGE_VERSION': `'${packageJSON.version}'`
+  }
+}
+
+await esbuild.build(esbuildOptions)

apps/peertube-cli/scripts/watch.js

@@ -0,0 +1,7 @@
+import * as esbuild from 'esbuild'
+import { esbuildOptions } from './build.js'
+
+const context = await esbuild.context(esbuildOptions)
+
+// Enable watch mode
+await context.watch()

apps/peertube-cli/src/peertube-auth.ts

@@ -0,0 +1,171 @@
+import CliTable3 from 'cli-table3'
+import prompt from 'prompt'
+import { Command } from '@commander-js/extra-typings'
+import { assignToken, buildServer, getNetrc, getSettings, writeSettings } from './shared/index.js'
+
+export function defineAuthProgram () {
+  const program = new Command()
+    .name('auth')
+    .description('Register your accounts on remote instances to use them with other commands')
+
+  program
+    .command('add')
+    .description('remember your accounts on remote instances for easier use')
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .option('--default', 'add the entry as the new default')
+    .action(options => {
+      prompt.override = options
+      prompt.start()
+      prompt.get({
+        properties: {
+          url: {
+            description: 'instance url',
+            conform: value => isURLaPeerTubeInstance(value),
+            message: 'It should be an URL (https://peertube.example.com)',
+            required: true
+          },
+          username: {
+            conform: value => typeof value === 'string' && value.length !== 0,
+            message: 'Name must be only letters, spaces, or dashes',
+            required: true
+          },
+          password: {
+            hidden: true,
+            replace: '*',
+            required: true
+          }
+        }
+      }, async (_, result) => {
+        // Check credentials
+        try {
+          // Strip out everything after the domain:port.
+          // See https://github.com/Chocobozzz/PeerTube/issues/3520
+          result.url = stripExtraneousFromPeerTubeUrl(result.url)
+
+          const server = buildServer(result.url)
+          await assignToken(server, result.username, result.password)
+        } catch (err) {
+          console.error(err.message)
+          process.exit(-1)
+        }
+
+        await setInstance(result.url, result.username, result.password, options.default)
+
+        process.exit(0)
+      })
+    })
+
+  program
+    .command('del <url>')
+    .description('Unregisters a remote instance')
+    .action(async url => {
+      await delInstance(url)
+
+      process.exit(0)
+    })
+
+  program
+    .command('list')
+    .description('List registered remote instances')
+    .action(async () => {
+      const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ])
+
+      const table = new CliTable3({
+        head: [ 'instance', 'login' ],
+        colWidths: [ 30, 30 ]
+      }) as any
+
+      settings.remotes.forEach(element => {
+        if (!netrc.machines[element]) return
+
+        table.push([
+          element,
+          netrc.machines[element].login
+        ])
+      })
+
+      console.log(table.toString())
+
+      process.exit(0)
+    })
+
+  program
+    .command('set-default <url>')
+    .description('Set an existing entry as default')
+    .action(async url => {
+      const settings = await getSettings()
+      const instanceExists = settings.remotes.includes(url)
+
+      if (instanceExists) {
+        settings.default = settings.remotes.indexOf(url)
+        await writeSettings(settings)
+
+        process.exit(0)
+      } else {
+        console.log('<url> is not a registered instance.')
+        process.exit(-1)
+      }
+    })
+
+  program.addHelpText(
+    'after',
+    '\n\n  Examples:\n\n' +
+      '    $ peertube auth add -u https://peertube.cpy.re -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD"\n' +
+      '    $ peertube auth add -u https://peertube.cpy.re -U root\n' +
+      '    $ peertube auth list\n' +
+      '    $ peertube auth del https://peertube.cpy.re\n'
+  )
+
+  return program
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function delInstance (url: string) {
+  const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ])
+
+  const index = settings.remotes.indexOf(url)
+  settings.remotes.splice(index)
+
+  if (settings.default === index) settings.default = -1
+
+  await writeSettings(settings)
+
+  delete netrc.machines[url]
+
+  await netrc.save()
+}
+
+async function setInstance (url: string, username: string, password: string, isDefault: boolean) {
+  const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ])
+
+  if (settings.remotes.includes(url) === false) {
+    settings.remotes.push(url)
+  }
+
+  if (isDefault || settings.remotes.length === 1) {
+    settings.default = settings.remotes.length - 1
+  }
+
+  await writeSettings(settings)
+
+  netrc.machines[url] = { login: username, password }
+  await netrc.save()
+}
+
+function isURLaPeerTubeInstance (url: string) {
+  return url.startsWith('http://') || url.startsWith('https://')
+}
+
+function stripExtraneousFromPeerTubeUrl (url: string) {
+  // Get everything before the 3rd /.
+  const urlLength = url.includes('/', 8)
+    ? url.indexOf('/', 8)
+    : url.length
+
+  return url.substring(0, urlLength)
+}

apps/peertube-cli/src/peertube-get-access-token.ts

@@ -0,0 +1,39 @@
+import { Command } from '@commander-js/extra-typings'
+import { assignToken, buildServer } from './shared/index.js'
+
+export function defineGetAccessProgram () {
+  const program = new Command()
+    .name('get-access-token')
+    .description('Get a peertube access token')
+    .alias('token')
+
+  program
+    .option('-u, --url <url>', 'Server url')
+    .option('-n, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .action(async options => {
+      try {
+        if (
+          !options.url ||
+          !options.username ||
+          !options.password
+        ) {
+          if (!options.url) console.error('--url field is required.')
+          if (!options.username) console.error('--username field is required.')
+          if (!options.password) console.error('--password field is required.')
+
+          process.exit(-1)
+        }
+
+        const server = buildServer(options.url)
+        await assignToken(server, options.username, options.password)
+
+        console.log(server.accessToken)
+      } catch (err) {
+        console.error('Cannot get access token: ' + err.message)
+        process.exit(-1)
+      }
+    })
+
+  return program
+}

apps/peertube-cli/src/peertube-plugins.ts

@@ -0,0 +1,167 @@
+import CliTable3 from 'cli-table3'
+import { isAbsolute } from 'path'
+import { Command } from '@commander-js/extra-typings'
+import { PluginType, PluginType_Type } from '@peertube/peertube-models'
+import { assignToken, buildServer, CommonProgramOptions, getServerCredentials } from './shared/index.js'
+
+export function definePluginsProgram () {
+  const program = new Command()
+
+  program
+    .name('plugins')
+    .description('Manage instance plugins/themes')
+    .alias('p')
+
+  program
+    .command('list')
+    .description('List installed plugins')
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .option('-t, --only-themes', 'List themes only')
+    .option('-P, --only-plugins', 'List plugins only')
+    .action(async options => {
+      try {
+        await pluginsListCLI(options)
+      } catch (err) {
+        console.error('Cannot list plugins: ' + err.message)
+        process.exit(-1)
+      }
+    })
+
+  program
+    .command('install')
+    .description('Install a plugin or a theme')
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .option('-P --path <path>', 'Install from a path')
+    .option('-n, --npm-name <npmName>', 'Install from npm')
+    .option('--plugin-version <pluginVersion>', 'Specify the plugin version to install (only available when installing from npm)')
+    .action(async options => {
+      try {
+        await installPluginCLI(options)
+      } catch (err) {
+        console.error('Cannot install plugin: ' + err.message)
+        process.exit(-1)
+      }
+    })
+
+  program
+    .command('update')
+    .description('Update a plugin or a theme')
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .option('-P --path <path>', 'Update from a path')
+    .option('-n, --npm-name <npmName>', 'Update from npm')
+    .action(async options => {
+      try {
+        await updatePluginCLI(options)
+      } catch (err) {
+        console.error('Cannot update plugin: ' + err.message)
+        process.exit(-1)
+      }
+    })
+
+  program
+    .command('uninstall')
+    .description('Uninstall a plugin or a theme')
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .option('-n, --npm-name <npmName>', 'NPM plugin/theme name')
+    .action(async options => {
+      try {
+        await uninstallPluginCLI(options)
+      } catch (err) {
+        console.error('Cannot uninstall plugin: ' + err.message)
+        process.exit(-1)
+      }
+    })
+
+  return program
+}
+
+// ----------------------------------------------------------------------------
+
+async function pluginsListCLI (options: CommonProgramOptions & { onlyThemes?: true, onlyPlugins?: true }) {
+  const { url, username, password } = await getServerCredentials(options)
+  const server = buildServer(url)
+  await assignToken(server, username, password)
+
+  let pluginType: PluginType_Type
+  if (options.onlyThemes) pluginType = PluginType.THEME
+  if (options.onlyPlugins) pluginType = PluginType.PLUGIN
+
+  const { data } = await server.plugins.list({ start: 0, count: 100, sort: 'name', pluginType })
+
+  const table = new CliTable3({
+    head: [ 'name', 'version', 'homepage' ],
+    colWidths: [ 50, 20, 50 ]
+  }) as any
+
+  for (const plugin of data) {
+    const npmName = plugin.type === PluginType.PLUGIN
+      ? 'peertube-plugin-' + plugin.name
+      : 'peertube-theme-' + plugin.name
+
+    table.push([
+      npmName,
+      plugin.version,
+      plugin.homepage
+    ])
+  }
+
+  console.log(table.toString())
+}
+
+async function installPluginCLI (options: CommonProgramOptions & { path?: string, npmName?: string, pluginVersion?: string }) {
+  if (!options.path && !options.npmName) {
+    throw new Error('You need to specify the npm name or the path of the plugin you want to install.')
+  }
+
+  if (options.path && !isAbsolute(options.path)) {
+    throw new Error('Path should be absolute.')
+  }
+
+  const { url, username, password } = await getServerCredentials(options)
+  const server = buildServer(url)
+  await assignToken(server, username, password)
+
+  await server.plugins.install({ npmName: options.npmName, path: options.path, pluginVersion: options.pluginVersion })
+
+  console.log('Plugin installed.')
+}
+
+async function updatePluginCLI (options: CommonProgramOptions & { path?: string, npmName?: string }) {
+  if (!options.path && !options.npmName) {
+    throw new Error('You need to specify the npm name or the path of the plugin you want to update.')
+  }
+
+  if (options.path && !isAbsolute(options.path)) {
+    throw new Error('Path should be absolute.')
+  }
+
+  const { url, username, password } = await getServerCredentials(options)
+  const server = buildServer(url)
+  await assignToken(server, username, password)
+
+  await server.plugins.update({ npmName: options.npmName, path: options.path })
+
+  console.log('Plugin updated.')
+}
+
+async function uninstallPluginCLI (options: CommonProgramOptions & { npmName?: string }) {
+  if (!options.npmName) {
+    throw new Error('You need to specify the npm name of the plugin/theme you want to uninstall.')
+  }
+
+  const { url, username, password } = await getServerCredentials(options)
+  const server = buildServer(url)
+  await assignToken(server, username, password)
+
+  await server.plugins.uninstall({ npmName: options.npmName })
+
+  console.log('Plugin uninstalled.')
+}

apps/peertube-cli/src/peertube-redundancy.ts

@@ -0,0 +1,181 @@
+import { Command } from '@commander-js/extra-typings'
+import { forceNumber, uniqify } from '@peertube/peertube-core-utils'
+import { HttpStatusCode, VideoRedundanciesTarget } from '@peertube/peertube-models'
+import bytes from 'bytes'
+import CliTable3 from 'cli-table3'
+import { URL } from 'url'
+import { assignToken, buildServer, CommonProgramOptions, getServerCredentials } from './shared/index.js'
+
+export function defineRedundancyProgram () {
+  const program = new Command()
+    .name('redundancy')
+    .description('Manage instance redundancies')
+    .alias('r')
+
+  program
+    .command('list-remote-redundancies')
+    .description('List remote redundancies on your videos')
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .action(async options => {
+      try {
+        await listRedundanciesCLI({ target: 'my-videos', ...options })
+      } catch (err) {
+        console.error('Cannot list remote redundancies: ' + err.message)
+        process.exit(-1)
+      }
+    })
+
+  program
+    .command('list-my-redundancies')
+    .description('List your redundancies of remote videos')
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .action(async options => {
+      try {
+        await listRedundanciesCLI({ target: 'remote-videos', ...options })
+      } catch (err) {
+        console.error('Cannot list redundancies: ' + err.message)
+        process.exit(-1)
+      }
+    })
+
+  program
+    .command('add')
+    .description('Duplicate a video in your redundancy system')
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .requiredOption('-v, --video <videoId>', 'Video id to duplicate', parseInt)
+    .action(async options => {
+      try {
+        await addRedundancyCLI(options)
+      } catch (err) {
+        console.error('Cannot duplicate video: ' + err.message)
+        process.exit(-1)
+      }
+    })
+
+  program
+    .command('remove')
+    .description('Remove a video from your redundancies')
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .requiredOption('-v, --video <videoId>', 'Video id to remove from redundancies', parseInt)
+    .action(async options => {
+      try {
+        await removeRedundancyCLI(options)
+      } catch (err) {
+        console.error('Cannot remove redundancy: ' + err)
+        process.exit(-1)
+      }
+    })
+
+  return program
+}
+
+// ----------------------------------------------------------------------------
+
+async function listRedundanciesCLI (options: CommonProgramOptions & { target: VideoRedundanciesTarget }) {
+  const { target } = options
+
+  const { url, username, password } = await getServerCredentials(options)
+  const server = buildServer(url)
+  await assignToken(server, username, password)
+
+  const { data } = await server.redundancy.listVideos({ start: 0, count: 100, sort: 'name', target })
+
+  const table = new CliTable3({
+    head: [ 'video id', 'video name', 'video url', 'playlists', 'by instances', 'total size' ]
+  }) as any
+
+  for (const redundancy of data) {
+    const streamingPlaylists = redundancy.redundancies.streamingPlaylists
+
+    let totalSize = ''
+    if (target === 'remote-videos') {
+      const tmp = streamingPlaylists.reduce((a, b) => a + b.size, 0)
+
+      totalSize = bytes(tmp)
+    }
+
+    const instances = uniqify(
+      streamingPlaylists
+        .map(r => r.fileUrl)
+        .map(u => new URL(u).host)
+    )
+
+    table.push([
+      redundancy.id.toString(),
+      redundancy.name,
+      redundancy.url,
+      streamingPlaylists.length,
+      instances.join('\n'),
+      totalSize
+    ])
+  }
+
+  console.log(table.toString())
+}
+
+async function addRedundancyCLI (options: { video: number } & CommonProgramOptions) {
+  const { url, username, password } = await getServerCredentials(options)
+  const server = buildServer(url)
+  await assignToken(server, username, password)
+
+  if (!options.video || isNaN(options.video)) {
+    throw new Error('You need to specify the video id to duplicate and it should be a number.')
+  }
+
+  try {
+    await server.redundancy.addVideo({ videoId: options.video })
+
+    console.log('Video will be duplicated by your instance!')
+  } catch (err) {
+    if (err.message.includes(HttpStatusCode.CONFLICT_409)) {
+      throw new Error('This video is already duplicated by your instance.', { cause: err })
+    }
+
+    if (err.message.includes(HttpStatusCode.NOT_FOUND_404)) {
+      throw new Error('This video id does not exist.', { cause: err })
+    }
+
+    throw err
+  }
+}
+
+async function removeRedundancyCLI (options: CommonProgramOptions & { video: number }) {
+  const { url, username, password } = await getServerCredentials(options)
+  const server = buildServer(url)
+  await assignToken(server, username, password)
+
+  if (!options.video || isNaN(options.video)) {
+    throw new Error('You need to specify the video id to remove from your redundancies')
+  }
+
+  const videoId = forceNumber(options.video)
+
+  const myVideoRedundancies = await server.redundancy.listVideos({ target: 'my-videos' })
+  let videoRedundancy = myVideoRedundancies.data.find(r => videoId === r.id)
+
+  if (!videoRedundancy) {
+    const remoteVideoRedundancies = await server.redundancy.listVideos({ target: 'remote-videos' })
+    videoRedundancy = remoteVideoRedundancies.data.find(r => videoId === r.id)
+  }
+
+  if (!videoRedundancy) {
+    throw new Error('Video redundancy not found.')
+  }
+
+  const ids = videoRedundancy.redundancies.streamingPlaylists
+    .map(r => r.id)
+
+  for (const id of ids) {
+    await server.redundancy.removeVideo({ redundancyId: id })
+  }
+
+  console.log('Video redundancy removed!')
+}

apps/peertube-cli/src/peertube-upload.ts

@@ -0,0 +1,175 @@
+import { Command } from '@commander-js/extra-typings'
+import { VideoCommentPolicy, VideoPrivacy, VideoPrivacyType } from '@peertube/peertube-models'
+import { PeerTubeServer } from '@peertube/peertube-server-commands'
+import { access, constants } from 'fs/promises'
+import { isAbsolute } from 'path'
+import { inspect } from 'util'
+import { assignToken, buildServer, getServerCredentials, listOptions } from './shared/index.js'
+
+type UploadOptions = {
+  url?: string
+  username?: string
+  password?: string
+  thumbnail?: string
+  preview?: string
+  file?: string
+  videoName?: string
+  category?: number
+  licence?: number
+  language?: string
+  tags?: string[]
+  nsfw?: true
+  videoDescription?: string
+  privacy?: VideoPrivacyType
+  channelName?: string
+  noCommentsEnabled?: true
+  support?: string
+  noWaitTranscoding?: true
+  noDownloadEnabled?: true
+}
+
+export function defineUploadProgram () {
+  const program = new Command('upload')
+    .description('Upload a video on a PeerTube instance')
+    .alias('up')
+
+  program
+    .option('-u, --url <url>', 'Server url')
+    .option('-U, --username <username>', 'Username')
+    .option('-p, --password <token>', 'Password')
+    .option('-b, --thumbnail <thumbnailPath>', 'Thumbnail path')
+    .option('--preview <previewPath>', 'Preview path')
+    .option('-f, --file <file>', 'Video absolute file path')
+    .option('-n, --video-name <name>', 'Video name')
+    .option('-c, --category <category_number>', 'Category number', parseInt)
+    .option('-l, --licence <licence_number>', 'Licence number', parseInt)
+    .option('-L, --language <language_code>', 'Language ISO 639 code (fr or en...)')
+    .option('-t, --tags <tags>', 'Video tags', listOptions)
+    .option('-N, --nsfw', 'Video is Not Safe For Work')
+    .option('-d, --video-description <description>', 'Video description')
+    .option('-P, --privacy <privacy_number>', 'Privacy', v => parseInt(v) as VideoPrivacyType)
+    .option('-C, --channel-name <channel_name>', 'Channel name')
+    .option('--no-comments-enabled', 'Disable video comments')
+    .option('-s, --support <support>', 'Video support text')
+    .option('--no-wait-transcoding', 'Do not wait transcoding before publishing the video')
+    .option('--no-download-enabled', 'Disable video download')
+    .option('-v, --verbose <verbose>', 'Verbosity, from 0/\'error\' to 4/\'debug\'', 'info')
+    .action(async options => {
+      try {
+        const { url, username, password } = await getServerCredentials(options)
+
+        if (!options.videoName || !options.file) {
+          if (!options.videoName) console.error('--video-name is required.')
+          if (!options.file) console.error('--file is required.')
+
+          process.exit(-1)
+        }
+
+        if (isAbsolute(options.file) === false) {
+          console.error('File path should be absolute.')
+          process.exit(-1)
+        }
+
+        await run({ ...options, url, username, password })
+      } catch (err) {
+        if (err.code === 'ECONNREFUSED') {
+          console.error(`Server is not responding`)
+        } else {
+          console.error('Cannot upload video: ' + err.message)
+        }
+
+        process.exit(-1)
+      }
+    })
+
+  return program
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function run (options: UploadOptions) {
+  const { url, username, password } = options
+
+  const server = buildServer(url)
+  await assignToken(server, username, password)
+
+  await access(options.file, constants.F_OK)
+
+  console.log('Uploading %s video...', options.videoName)
+
+  const baseAttributes = await buildVideoAttributesFromCommander(server, options)
+
+  const attributes = {
+    ...baseAttributes,
+
+    fixture: options.file,
+    thumbnailfile: options.thumbnail,
+    previewfile: options.preview
+  }
+
+  try {
+    await server.videos.upload({ attributes })
+    console.log(`Video ${options.videoName} uploaded.`)
+    process.exit(0)
+  } catch (err) {
+    const message = err.message || ''
+    if (message.includes('413')) {
+      console.error('Aborted: user quota is exceeded or video file is too big for this PeerTube instance.')
+    } else {
+      console.error(inspect(err))
+    }
+
+    process.exit(-1)
+  }
+}
+
+async function buildVideoAttributesFromCommander (server: PeerTubeServer, options: UploadOptions) {
+  const defaultBooleanAttributes = {
+    nsfw: false,
+    downloadEnabled: true,
+    waitTranscoding: true
+  }
+
+  const booleanAttributes: { [id in keyof typeof defaultBooleanAttributes]: boolean } = {} as any
+
+  for (const key of Object.keys(defaultBooleanAttributes)) {
+    if (options[key] !== undefined) {
+      booleanAttributes[key] = options[key]
+    } else {
+      booleanAttributes[key] = defaultBooleanAttributes[key]
+    }
+  }
+
+  const videoAttributes = {
+    name: options.videoName,
+    category: options.category || undefined,
+    licence: options.licence || undefined,
+    language: options.language || undefined,
+    privacy: options.privacy || VideoPrivacy.PUBLIC,
+    support: options.support || undefined,
+    description: options.videoDescription || undefined,
+    tags: options.tags || undefined,
+
+    commentsPolicy: options.noCommentsEnabled !== undefined
+      ? options.noCommentsEnabled === true
+        ? VideoCommentPolicy.DISABLED
+        : VideoCommentPolicy.ENABLED
+      : undefined,
+
+    ...booleanAttributes
+  }
+
+  if (options.channelName) {
+    const videoChannel = await server.channels.get({ channelName: options.channelName })
+
+    Object.assign(videoAttributes, { channelId: videoChannel.id })
+
+    if (!videoAttributes.support && videoChannel.support) {
+      Object.assign(videoAttributes, { support: videoChannel.support })
+    }
+  }
+
+  return videoAttributes
+}

apps/peertube-cli/src/peertube.ts

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+
+import { Command } from '@commander-js/extra-typings'
+import { defineAuthProgram } from './peertube-auth.js'
+import { defineGetAccessProgram } from './peertube-get-access-token.js'
+import { definePluginsProgram } from './peertube-plugins.js'
+import { defineRedundancyProgram } from './peertube-redundancy.js'
+import { defineUploadProgram } from './peertube-upload.js'
+import { getSettings, version } from './shared/index.js'
+
+const program = new Command()
+
+program
+  .version(version, '-v, --version')
+  .usage('[command] [options]')
+
+program.addCommand(defineAuthProgram())
+program.addCommand(defineUploadProgram())
+program.addCommand(defineRedundancyProgram())
+program.addCommand(definePluginsProgram())
+program.addCommand(defineGetAccessProgram())
+
+// help on no command
+if (!process.argv.slice(2).length) {
+  const logo = '░P░e░e░r░T░u░b░e░'
+  console.log(`
+  ___/),.._                           ` + logo + `
+/'   ,.   ."'._
+(     "'   '-.__"-._             ,-
+\\'='='),  "\\ -._-"-.          -"/
+      / ""/"\\,_\\,__""       _" /,-
+     /   /                -" _/"/
+    /   |    ._\\\\ |\\  |_.".-"  /
+   /    |   __\\)|)|),/|_." _,."
+  /     \\_."   " ") | ).-""---''--
+ (                  "/.""7__-""''
+ |                   " ."._--._
+ \\       \\ (_    __   ""   ".,_
+  \\.,.    \\  ""   -"".-"
+   ".,_,  (",_-,,,-".-
+       "'-,\\_   __,-"
+             ",)" ")
+              /"\\-"
+            ,"\\/
+      _,.__/"\\/_                     (the CLI for red chocobos)
+     / \\) "./,  ".
+  --/---"---" "-) )---- by Chocobozzz et al.\n`)
+}
+
+getSettings()
+  .then(settings => {
+    const state = (settings.default === undefined || settings.default === -1)
+      ? 'no instance selected, commands will require explicit arguments'
+      : 'instance ' + settings.remotes[settings.default] + ' selected'
+
+    program
+      .addHelpText('after', '\n\n  State: ' + state + '\n\n' +
+        '  Examples:\n\n' +
+        '    $ peertube auth add -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD"\n' +
+        '    $ peertube up <videoFile>\n'
+      )
+      .parse(process.argv)
+  })
+  .catch(err => console.error(err))

apps/peertube-cli/src/shared/cli.ts

@@ -0,0 +1,191 @@
+import applicationConfig from 'application-config'
+import { Netrc } from 'netrc-parser'
+import { join } from 'path'
+import { createLogger, format, transports } from 'winston'
+import { UserRole } from '@peertube/peertube-models'
+import { getAppNumber, isTestInstance, root } from '@peertube/peertube-node-utils'
+import { PeerTubeServer } from '@peertube/peertube-server-commands'
+
+export type CommonProgramOptions = {
+  url?: string
+  username?: string
+  password?: string
+}
+
+let configName = 'PeerTube/CLI'
+if (isTestInstance()) configName += `-${getAppNumber()}`
+
+const config = applicationConfig(configName)
+
+const version: string = process.env.PACKAGE_VERSION
+
+async function getAdminTokenOrDie (server: PeerTubeServer, username: string, password: string) {
+  const token = await server.login.getAccessToken(username, password)
+  const me = await server.users.getMyInfo({ token })
+
+  if (me.role.id !== UserRole.ADMINISTRATOR) {
+    console.error('You must be an administrator.')
+    process.exit(-1)
+  }
+
+  return token
+}
+
+interface Settings {
+  remotes: any[]
+  default: number
+}
+
+async function getSettings () {
+  const defaultSettings: Settings = {
+    remotes: [],
+    default: -1
+  }
+
+  const data = await config.read() as Promise<Settings>
+
+  return Object.keys(data).length === 0
+    ? defaultSettings
+    : data
+}
+
+async function getNetrc () {
+  const netrc = isTestInstance()
+    ? new Netrc(join(root(import.meta.url), 'test' + getAppNumber(), 'netrc'))
+    : new Netrc()
+
+  await netrc.load()
+
+  return netrc
+}
+
+function writeSettings (settings: Settings) {
+  return config.write(settings)
+}
+
+function deleteSettings () {
+  return config.trash()
+}
+
+function getRemoteObjectOrDie (
+  options: CommonProgramOptions,
+  settings: Settings,
+  netrc: Netrc
+): { url: string, username: string, password: string } {
+  function exitIfNoOptions (optionNames: string[], errorPrefix = '') {
+    let exit = false
+
+    for (const key of optionNames) {
+      if (!options[key]) {
+        if (exit === false && errorPrefix) console.error(errorPrefix)
+
+        console.error(`--${key} field is required`)
+        exit = true
+      }
+    }
+
+    if (exit) process.exit(-1)
+  }
+
+  // If username or password are specified, both are mandatory
+  if (options.username || options.password) {
+    exitIfNoOptions([ 'username', 'password' ])
+  }
+
+  // If no available machines, url, username and password args are mandatory
+  if (Object.keys(netrc.machines).length === 0) {
+    exitIfNoOptions([ 'url', 'username', 'password' ], 'No account found in netrc')
+  }
+
+  if (settings.remotes.length === 0 || settings.default === -1) {
+    exitIfNoOptions([ 'url' ], 'No default instance found')
+  }
+
+  let url: string = options.url
+  let username: string = options.username
+  let password: string = options.password
+
+  if (!url && settings.default !== -1) url = settings.remotes[settings.default]
+
+  const machine = netrc.machines[url]
+  if ((!username || !password) && !machine) {
+    console.error('Cannot find existing configuration for %s.', url)
+    process.exit(-1)
+  }
+
+  if (!username && machine) username = machine.login
+  if (!password && machine) password = machine.password
+
+  return { url, username, password }
+}
+
+function listOptions (val: string) {
+  return val.split(',')
+}
+
+function getServerCredentials (options: CommonProgramOptions) {
+  return Promise.all([ getSettings(), getNetrc() ])
+    .then(([ settings, netrc ]) => {
+      return getRemoteObjectOrDie(options, settings, netrc)
+    })
+}
+
+function buildServer (url: string) {
+  return new PeerTubeServer({ url })
+}
+
+async function assignToken (server: PeerTubeServer, username: string, password: string) {
+  const bodyClient = await server.login.getClient()
+  const client = { id: bodyClient.client_id, secret: bodyClient.client_secret }
+
+  const body = await server.login.login({ client, user: { username, password } })
+
+  server.accessToken = body.access_token
+}
+
+function getLogger (logLevel = 'info') {
+  const logLevels = {
+    0: 0,
+    error: 0,
+    1: 1,
+    warn: 1,
+    2: 2,
+    info: 2,
+    3: 3,
+    verbose: 3,
+    4: 4,
+    debug: 4
+  }
+
+  const logger = createLogger({
+    levels: logLevels,
+    format: format.combine(
+      format.splat(),
+      format.simple()
+    ),
+    transports: [
+      new (transports.Console)({
+        level: logLevel
+      })
+    ]
+  })
+
+  return logger
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  version,
+  getLogger,
+  getSettings,
+  getNetrc,
+  getRemoteObjectOrDie,
+  writeSettings,
+  deleteSettings,
+  getServerCredentials,
+  listOptions,
+  getAdminTokenOrDie,
+  buildServer,
+  assignToken
+}

apps/peertube-cli/src/shared/index.ts

@@ -0,0 +1 @@
+export * from './cli.js'

apps/peertube-cli/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "baseUrl": "./",
+    "outDir": "./dist",
+    "rootDir": "src",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo"
+  },
+  "references": [
+    { "path": "../../packages/core-utils" },
+    { "path": "../../packages/models" },
+    { "path": "../../packages/node-utils" },
+    { "path": "../../packages/server-commands" }
+  ]
+}

apps/peertube-runner/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+dist
+meta.json

apps/peertube-runner/.npmignore

@@ -0,0 +1,4 @@
+src
+meta.json
+tsconfig.json
+scripts

apps/peertube-runner/CHANGELOG.md

@@ -0,0 +1,34 @@
+# Changelog
+
+## v0.4.0
+
+ * Copy codecs for HLS transcoding if possible
+
+## v0.3.0
+
+ * Add generate storyboard support (PeerTube >= 8.0)
+
+## v0.2.0
+
+ * Add runner version in request and register payloads
+ * Update dependencies to fix vulnerabilities
+
+## v0.1.3
+
+ * Disable log coloring when TTY does not support it
+ * Add download file timeout (2 hours) to prevent stuck jobs
+
+## v0.1.2
+
+  * Support query params in custom upload URL
+
+## v0.1.1
+
+  * Fix adding studio watermark with audio/video split HLS file
+
+## v0.1.0
+
+  * Requires Node 20
+  * Introduce `list-jobs` command to list processing jobs
+  * Update dependencies
+  * Send last chunks/playlist content to correctly end the live

apps/peertube-runner/README.md

@@ -0,0 +1,42 @@
+# PeerTube runner
+
+Runner program to execute jobs (transcoding...) of remote PeerTube instances.
+
+Commands below has to be run at the root of PeerTube git repository.
+
+## Dev
+
+### Install dependencies
+
+```bash
+cd peertube-root
+npm run install-node-dependencies
+```
+
+### Develop
+
+```bash
+cd peertube-root
+npm run dev:peertube-runner
+```
+
+### Build
+
+```bash
+cd peertube-root
+npm run build:peertube-runner
+```
+
+### Run
+
+```bash
+cd peertube-root
+node apps/peertube-runner/dist/peertube-runner.js --help
+```
+
+### Publish on NPM
+
+```bash
+cd peertube-root
+(cd apps/peertube-runner && npm version patch) && npm run build:peertube-runner && (cd apps/peertube-runner && npm publish --access=public)
+```

apps/peertube-runner/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@peertube/peertube-runner",
+  "version": "0.4.0",
+  "type": "module",
+  "main": "dist/peertube-runner.js",
+  "bin": "dist/peertube-runner.js",
+  "license": "AGPL-3.0",
+  "devDependencies": {
+    "@iarna/toml": "^2.2.5",
+    "@types/follow-redirects": "1.14.4",
+    "cli-table3": "^0.6.5",
+    "env-paths": "^3.0.0",
+    "follow-redirects": "^1.15.5",
+    "net-ipc": "^2.2.2",
+    "pino": "^9.6.0",
+    "pino-pretty": "^13.0.0"
+  }
+}

apps/peertube-runner/scripts/build.js

@@ -0,0 +1,27 @@
+import * as esbuild from 'esbuild'
+import { readFileSync } from 'fs'
+
+const packageJSON = JSON.parse(readFileSync(new URL('../package.json', import.meta.url)))
+
+export const esbuildOptions = {
+  entryPoints: [ './src/peertube-runner.ts' ],
+  bundle: true,
+  platform: 'node',
+  format: 'esm',
+  target: 'node16',
+  external: [
+    './lib-cov/fluent-ffmpeg',
+    'pg-hstore'
+  ],
+  outfile: './dist/peertube-runner.js',
+  banner: {
+    js: `const require = (await import("node:module")).createRequire(import.meta.url);` +
+      `const __filename = (await import("node:url")).fileURLToPath(import.meta.url);` +
+      `const __dirname = (await import("node:path")).dirname(__filename);`
+  },
+  define: {
+    'process.env.PACKAGE_VERSION': `'${packageJSON.version}'`
+  }
+}
+
+await esbuild.build(esbuildOptions)

apps/peertube-runner/scripts/watch.js

@@ -0,0 +1,7 @@
+import * as esbuild from 'esbuild'
+import { esbuildOptions } from './build.js'
+
+const context = await esbuild.context(esbuildOptions)
+
+// Enable watch mode
+await context.watch()

apps/peertube-runner/src/peertube-runner.ts

@@ -0,0 +1,139 @@
+#!/usr/bin/env node
+
+import { Command, InvalidArgumentError } from '@commander-js/extra-typings'
+import { RunnerJobType } from '@peertube/peertube-models'
+import { listJobs, listRegistered, registerRunner, unregisterRunner } from './register/index.js'
+import { gracefulShutdown } from './register/shutdown.js'
+import { RunnerServer } from './server/index.js'
+import { getSupportedJobsList } from './server/shared/supported-job.js'
+import { ConfigManager, logger } from './shared/index.js'
+
+const program = new Command()
+  .version(process.env.PACKAGE_VERSION)
+  .option(
+    '--id <id>',
+    'Runner server id, so you can run multiple PeerTube server runners with different configurations on the same machine',
+    'default'
+  )
+  .option('--verbose', 'Run in verbose mode')
+  .hook('preAction', thisCommand => {
+    const options = thisCommand.opts()
+
+    ConfigManager.Instance.init(options.id)
+
+    if (options.verbose === true) {
+      logger.level = 'debug'
+    }
+  })
+
+program.command('server')
+  .description('Run in server mode, to execute remote jobs of registered PeerTube instances')
+  .option(
+    '--enable-job <type>',
+    'Enable this job type (multiple --enable-job options can be specified). ' +
+    'By default all supported jobs are enabled). ' +
+    'Supported job types: ' + getSupportedJobsList().join(', '),
+    (value: RunnerJobType, previous: RunnerJobType[]) => [ ...previous, value ],
+    []
+  )
+  .action(async options => {
+    try {
+      let enabledJobs: Set<RunnerJobType>
+
+      if (options.enableJob) {
+        for (const jobType of options.enableJob) {
+          if (getSupportedJobsList().includes(jobType) !== true) {
+            throw new InvalidArgumentError(`${jobType} is not a supported job`)
+          }
+
+          enabledJobs = new Set(options.enableJob)
+        }
+      }
+
+      await new RunnerServer(enabledJobs).run()
+    } catch (err) {
+      logger.error(err, 'Cannot run PeerTube runner as server mode')
+      process.exit(-1)
+    }
+  })
+
+program.command('register')
+  .description('Register a new PeerTube instance to process runner jobs')
+  .requiredOption('--url <url>', 'PeerTube instance URL', parseUrl)
+  .requiredOption('--registration-token <token>', 'Runner registration token (can be found in PeerTube instance administration')
+  .requiredOption('--runner-name <name>', 'Runner name')
+  .option('--runner-description <description>', 'Runner description')
+  .action(async options => {
+    try {
+      await registerRunner(options)
+    } catch (err) {
+      console.error('Cannot register this PeerTube runner.')
+      console.error(err)
+      process.exit(-1)
+    }
+  })
+
+program.command('unregister')
+  .description('Unregister the runner from PeerTube instance')
+  .requiredOption('--url <url>', 'PeerTube instance URL', parseUrl)
+  .requiredOption('--runner-name <name>', 'Runner name')
+  .action(async options => {
+    try {
+      await unregisterRunner(options)
+    } catch (err) {
+      console.error('Cannot unregister this PeerTube runner.')
+      console.error(err)
+      process.exit(-1)
+    }
+  })
+
+program.command('list-registered')
+  .description('List registered PeerTube instances')
+  .action(async () => {
+    try {
+      await listRegistered()
+    } catch (err) {
+      console.error('Cannot list registered PeerTube instances.')
+      console.error(err)
+      process.exit(-1)
+    }
+  })
+
+program.command('list-jobs')
+  .description('List processing jobs')
+  .option('--include-payload', 'Include job payload in the output')
+  .action(async options => {
+    try {
+      await listJobs({ includePayload: options.includePayload })
+    } catch (err) {
+      console.error('Cannot list processing jobs.')
+      console.error(err)
+      process.exit(-1)
+    }
+  })
+
+program.command('graceful-shutdown')
+  .description('Exit runner when all processing tasks are finished')
+  .action(async () => {
+    try {
+      await gracefulShutdown()
+    } catch (err) {
+      console.error('Cannot graceful shutdown the runner.')
+      console.error(err)
+      process.exit(-1)
+    }
+  })
+
+program.parse()
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function parseUrl (url: string) {
+  if (url.startsWith('http://') !== true && url.startsWith('https://') !== true) {
+    throw new InvalidArgumentError('URL should start with a http:// or https://')
+  }
+
+  return url
+}

apps/peertube-runner/src/register/index.ts

@@ -0,0 +1 @@
+export * from './register.js'

apps/peertube-runner/src/register/register.ts

@@ -0,0 +1,47 @@
+import { IPCClient } from '../shared/ipc/index.js'
+
+export async function registerRunner (options: {
+  url: string
+  registrationToken: string
+  runnerName: string
+  runnerDescription?: string
+}) {
+  const client = new IPCClient()
+  await client.run()
+
+  await client.askRegister(options)
+
+  client.stop()
+}
+
+export async function unregisterRunner (options: {
+  url: string
+  runnerName: string
+}) {
+  const client = new IPCClient()
+  await client.run()
+
+  await client.askUnregister(options)
+
+  client.stop()
+}
+
+export async function listRegistered () {
+  const client = new IPCClient()
+  await client.run()
+
+  await client.askListRegistered()
+
+  client.stop()
+}
+
+export async function listJobs (options: {
+  includePayload: boolean
+}) {
+  const client = new IPCClient()
+  await client.run()
+
+  await client.askListJobs(options)
+
+  client.stop()
+}

apps/peertube-runner/src/register/shutdown.ts

@@ -0,0 +1,10 @@
+import { IPCClient } from '../shared/ipc/index.js'
+
+export async function gracefulShutdown () {
+  const client = new IPCClient()
+  await client.run()
+
+  await client.askGracefulShutdown()
+
+  client.stop()
+}

apps/peertube-runner/src/server/index.ts

@@ -0,0 +1 @@
+export * from './server.js'

apps/peertube-runner/src/server/process/index.ts

@@ -0,0 +1,2 @@
+export * from './shared/index.js'
+export * from './process.js'

apps/peertube-runner/src/server/process/process.ts

@@ -0,0 +1,61 @@
+import {
+  RunnerJobLiveRTMPHLSTranscodingPayload,
+  RunnerJobStudioTranscodingPayload,
+  RunnerJobTranscriptionPayload,
+  RunnerJobVODAudioMergeTranscodingPayload,
+  RunnerJobVODHLSTranscodingPayload,
+  RunnerJobVODWebVideoTranscodingPayload
+} from '@peertube/peertube-models'
+import { logger } from '../../shared/index.js'
+import {
+  processAudioMergeTranscoding,
+  processGenerateStoryboard,
+  processHLSTranscoding,
+  ProcessOptions,
+  processWebVideoTranscoding
+} from './shared/index.js'
+import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live.js'
+import { processStudioTranscoding } from './shared/process-studio.js'
+import { processVideoTranscription } from './shared/process-transcription.js'
+
+export async function processJob (options: ProcessOptions) {
+  const { server, job } = options
+
+  logger.info({ payload: job.payload }, `[${server.url}] Processing job of type ${job.type}: ${job.uuid}`)
+
+  switch (job.type) {
+    case 'vod-audio-merge-transcoding':
+      await processAudioMergeTranscoding(options as ProcessOptions<RunnerJobVODAudioMergeTranscodingPayload>)
+      break
+
+    case 'vod-web-video-transcoding':
+      await processWebVideoTranscoding(options as ProcessOptions<RunnerJobVODWebVideoTranscodingPayload>)
+      break
+
+    case 'vod-hls-transcoding':
+      await processHLSTranscoding(options as ProcessOptions<RunnerJobVODHLSTranscodingPayload>)
+      break
+
+    case 'live-rtmp-hls-transcoding':
+      await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>).process()
+      break
+
+    case 'video-studio-transcoding':
+      await processStudioTranscoding(options as ProcessOptions<RunnerJobStudioTranscodingPayload>)
+      break
+
+    case 'video-transcription':
+      await processVideoTranscription(options as ProcessOptions<RunnerJobTranscriptionPayload>)
+      break
+
+    case 'generate-video-storyboard':
+      await processGenerateStoryboard(options as any)
+      break
+
+    default:
+      logger.error(`Unknown job ${job.type} to process`)
+      return
+  }
+
+  logger.info(`[${server.url}] Finished processing job of type ${job.type}: ${job.uuid}`)
+}

apps/peertube-runner/src/server/process/shared/common.ts

@@ -0,0 +1,136 @@
+import { pick } from '@peertube/peertube-core-utils'
+import {
+  FFmpegEdition,
+  FFmpegImage,
+  FFmpegLive,
+  FFmpegVOD,
+  getDefaultAvailableEncoders,
+  getDefaultEncodersToTry
+} from '@peertube/peertube-ffmpeg'
+import { RunnerJob, RunnerJobPayload } from '@peertube/peertube-models'
+import { buildUUID } from '@peertube/peertube-node-utils'
+import { PeerTubeServer } from '@peertube/peertube-server-commands'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
+import { ConfigManager, downloadFile, logger } from '../../../shared/index.js'
+import { getWinstonLogger } from './winston-logger.js'
+
+export type JobWithToken<T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string }
+
+export type ProcessOptions<T extends RunnerJobPayload = RunnerJobPayload> = {
+  server: PeerTubeServer
+  job: JobWithToken<T>
+  runnerToken: string
+}
+
+export async function downloadInputFile (options: {
+  url: string
+  job: JobWithToken
+  runnerToken: string
+}) {
+  const { url, job, runnerToken } = options
+  const destination = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID())
+
+  try {
+    await downloadFile({ url, jobToken: job.jobToken, runnerToken, destination })
+  } catch (err) {
+    remove(destination)
+      .catch(err => logger.error({ err }, `Cannot remove ${destination}`))
+
+    throw err
+  }
+
+  return destination
+}
+
+export async function downloadSeparatedAudioFileIfNeeded (options: {
+  urls: string[]
+  job: JobWithToken
+  runnerToken: string
+}) {
+  const { urls } = options
+
+  if (!urls || urls.length === 0) return undefined
+
+  return downloadInputFile({ url: urls[0], ...pick(options, [ 'job', 'runnerToken' ]) })
+}
+
+export function scheduleTranscodingProgress (options: {
+  server: PeerTubeServer
+  runnerToken: string
+  job: JobWithToken
+  progressGetter: () => number
+}) {
+  const { job, server, progressGetter, runnerToken } = options
+
+  const updateInterval = ConfigManager.Instance.isTestInstance()
+    ? 500
+    : 60000
+
+  const update = () => {
+    job.progress = progressGetter() || 0
+
+    server.runnerJobs.update({
+      jobToken: job.jobToken,
+      jobUUID: job.uuid,
+      runnerToken,
+      progress: job.progress
+    }).catch(err => logger.error({ err }, 'Cannot send job progress'))
+  }
+
+  const interval = setInterval(() => {
+    update()
+  }, updateInterval)
+
+  update()
+
+  return interval
+}
+
+// ---------------------------------------------------------------------------
+
+export function buildFFmpegVOD (options: {
+  onJobProgress: (progress: number) => void
+}) {
+  const { onJobProgress } = options
+
+  return new FFmpegVOD({
+    ...getCommonFFmpegOptions(),
+
+    updateJobProgress: arg => {
+      const progress = arg < 0 || arg > 100
+        ? undefined
+        : arg
+
+      onJobProgress(progress)
+    }
+  })
+}
+
+export function buildFFmpegLive () {
+  return new FFmpegLive(getCommonFFmpegOptions())
+}
+
+export function buildFFmpegEdition () {
+  return new FFmpegEdition(getCommonFFmpegOptions())
+}
+
+export function buildFFmpegImage () {
+  return new FFmpegImage(getCommonFFmpegOptions())
+}
+
+function getCommonFFmpegOptions () {
+  const config = ConfigManager.Instance.getConfig()
+
+  return {
+    niceness: config.ffmpeg.nice,
+    threads: config.ffmpeg.threads,
+    tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(),
+    profile: 'default',
+    availableEncoders: {
+      available: getDefaultAvailableEncoders(),
+      encodersToTry: getDefaultEncodersToTry()
+    },
+    logger: getWinstonLogger()
+  }
+}

apps/peertube-runner/src/server/process/shared/index.ts

@@ -0,0 +1,4 @@
+export * from './common.js'
+export * from './process-vod.js'
+export * from './winston-logger.js'
+export * from './process-storyboard.js'

apps/peertube-runner/src/server/process/shared/process-live.ts

@@ -0,0 +1,407 @@
+import { wait } from '@peertube/peertube-core-utils'
+import {
+  ffprobePromise,
+  getVideoStreamBitrate,
+  getVideoStreamDimensionsInfo,
+  hasAudioStream,
+  hasVideoStream
+} from '@peertube/peertube-ffmpeg'
+import {
+  LiveRTMPHLSTranscodingSuccess,
+  LiveRTMPHLSTranscodingUpdatePayload,
+  PeerTubeProblemDocument,
+  RunnerJobLiveRTMPHLSTranscodingPayload,
+  ServerErrorCode
+} from '@peertube/peertube-models'
+import { buildUUID } from '@peertube/peertube-node-utils'
+import { FSWatcher, watch } from 'chokidar'
+import { FfmpegCommand } from 'fluent-ffmpeg'
+import { ensureDir, remove } from 'fs-extra/esm'
+import { readFile } from 'fs/promises'
+import { basename, join } from 'path'
+import { ConfigManager } from '../../../shared/config-manager.js'
+import { logger } from '../../../shared/index.js'
+import { buildFFmpegLive, ProcessOptions } from './common.js'
+
+type CustomLiveRTMPHLSTranscodingUpdatePayload = Omit<LiveRTMPHLSTranscodingUpdatePayload, 'resolutionPlaylistFile'> & {
+  resolutionPlaylistFile?: [Buffer, string] | Blob | string
+}
+
+export class ProcessLiveRTMPHLSTranscoding {
+  private readonly outputPath: string
+  private readonly fsWatchers: FSWatcher[] = []
+
+  // Playlist name -> chunks
+  private readonly pendingChunksPerPlaylist = new Map<string, string[]>()
+
+  private readonly playlistsCreated = new Set<string>()
+  private allPlaylistsCreated = false
+
+  private latestFilteredPlaylistContent: { [name: string]: string } = {}
+
+  private ffmpegCommand: FfmpegCommand
+
+  private ended = false
+  private errored = false
+
+  constructor (private readonly options: ProcessOptions<RunnerJobLiveRTMPHLSTranscodingPayload>) {
+    this.outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID())
+
+    logger.debug(`Using ${this.outputPath} to process live rtmp hls transcoding job ${options.job.uuid}`)
+  }
+
+  private get payload () {
+    return this.options.job.payload
+  }
+
+  process () {
+    return new Promise<void>(async (res, rej) => {
+      try {
+        await ensureDir(this.outputPath)
+
+        logger.info(`Probing ${this.payload.input.rtmpUrl}`)
+        const probe = await ffprobePromise(this.payload.input.rtmpUrl)
+        logger.info({ probe }, `Probed ${this.payload.input.rtmpUrl}`)
+
+        const hasAudio = await hasAudioStream(this.payload.input.rtmpUrl, probe)
+        const hasVideo = await hasVideoStream(this.payload.input.rtmpUrl, probe)
+        const bitrate = await getVideoStreamBitrate(this.payload.input.rtmpUrl, probe)
+        const { ratio } = await getVideoStreamDimensionsInfo(this.payload.input.rtmpUrl, probe)
+
+        const m3u8Watcher = watch(this.outputPath, { ignored: path => path !== this.outputPath && !path.endsWith('.m3u8') })
+        this.fsWatchers.push(m3u8Watcher)
+
+        const tsWatcher = watch(this.outputPath, { ignored: path => path !== this.outputPath && !path.endsWith('.ts') })
+        this.fsWatchers.push(tsWatcher)
+
+        m3u8Watcher.on('change', p => {
+          logger.debug(`${p} m3u8 playlist changed`)
+        })
+
+        m3u8Watcher.on('add', p => {
+          this.playlistsCreated.add(p)
+
+          if (this.playlistsCreated.size === this.options.job.payload.output.toTranscode.length + 1) {
+            this.allPlaylistsCreated = true
+            logger.info('All m3u8 playlists are created.')
+          }
+        })
+
+        tsWatcher.on('add', async p => {
+          try {
+            await this.sendPendingChunks()
+          } catch (err) {
+            this.onUpdateError({ err, rej, res })
+          }
+
+          const playlistName = this.getPlaylistIdFromTS(p)
+
+          const pendingChunks = this.pendingChunksPerPlaylist.get(playlistName) || []
+          pendingChunks.push(p)
+
+          this.pendingChunksPerPlaylist.set(playlistName, pendingChunks)
+        })
+
+        tsWatcher.on('unlink', p => {
+          this.sendDeletedChunkUpdate(p)
+            .catch(err => this.onUpdateError({ err, rej, res }))
+        })
+
+        this.ffmpegCommand = await buildFFmpegLive().getLiveTranscodingCommand({
+          inputUrl: this.payload.input.rtmpUrl,
+
+          outPath: this.outputPath,
+          masterPlaylistName: 'master.m3u8',
+
+          segmentListSize: this.payload.output.segmentListSize,
+          segmentDuration: this.payload.output.segmentDuration,
+
+          toTranscode: this.payload.output.toTranscode,
+          splitAudioAndVideo: true,
+
+          bitrate,
+          ratio,
+
+          hasAudio,
+          hasVideo,
+          probe
+        })
+
+        logger.info(`Running live transcoding for ${this.payload.input.rtmpUrl}`)
+
+        this.ffmpegCommand.on('error', (err, stdout, stderr) => {
+          this.onFFmpegError({ err, stdout, stderr })
+
+          res()
+        })
+
+        this.ffmpegCommand.on('end', () => {
+          this.onFFmpegEnded()
+            .catch(err => logger.error({ err }, 'Error in FFmpeg end handler'))
+
+          res()
+        })
+
+        this.ffmpegCommand.run()
+      } catch (err) {
+        rej(err)
+      }
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private onUpdateError (options: {
+    err: Error
+    res: () => void
+    rej: (reason?: any) => void
+  }) {
+    const { err, res, rej } = options
+
+    if (this.errored) return
+    if (this.ended) return
+
+    this.errored = true
+
+    this.ffmpegCommand.kill('SIGINT')
+
+    const type = ((err as any).res?.body as PeerTubeProblemDocument)?.code
+    if (type === ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE) {
+      logger.info('Stopping transcoding as the job is not in processing state anymore')
+
+      this.sendSuccess()
+        .catch(err => logger.error({ err }, 'Cannot send success'))
+
+      res()
+    } else {
+      logger.error({ err }, 'Cannot send update after added/deleted chunk, stopping live transcoding')
+
+      this.sendError(err)
+        .catch(subErr => logger.error({ err: subErr }, 'Cannot send error'))
+
+      rej(err)
+    }
+
+    this.cleanup()
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private onFFmpegError (options: {
+    err: any
+    stdout: string
+    stderr: string
+  }) {
+    const { err, stdout, stderr } = options
+
+    // Don't care that we killed the ffmpeg process
+    if (err?.message?.includes('Exiting normally')) return
+    if (this.errored) return
+    if (this.ended) return
+
+    this.errored = true
+
+    logger.error({ err, stdout, stderr }, 'FFmpeg transcoding error.')
+
+    this.sendError(err)
+      .catch(subErr => logger.error({ err: subErr }, 'Cannot send error'))
+
+    this.cleanup()
+  }
+
+  private async sendError (err: Error) {
+    await this.options.server.runnerJobs.error({
+      jobToken: this.options.job.jobToken,
+      jobUUID: this.options.job.uuid,
+      runnerToken: this.options.runnerToken,
+      message: err.message
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async onFFmpegEnded () {
+    if (this.ended) return
+
+    logger.info('FFmpeg ended, sending success to server')
+
+    // Wait last ffmpeg chunks generation
+    await wait(1500)
+
+    try {
+      await this.sendPendingChunks()
+    } catch (err) {
+      logger.error(err, 'Cannot send latest chunks after ffmpeg ended')
+    }
+
+    this.ended = true
+
+    this.sendSuccess()
+      .catch(err => logger.error({ err }, 'Cannot send success'))
+
+    this.cleanup()
+  }
+
+  private async sendSuccess () {
+    const successBody: LiveRTMPHLSTranscodingSuccess = {}
+
+    await this.options.server.runnerJobs.success({
+      jobToken: this.options.job.jobToken,
+      jobUUID: this.options.job.uuid,
+      runnerToken: this.options.runnerToken,
+      payload: successBody,
+      reqPayload: this.payload
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private sendDeletedChunkUpdate (deletedChunk: string): Promise<any> {
+    if (this.ended) return Promise.resolve()
+
+    logger.debug(`Sending removed live chunk ${deletedChunk} update`)
+
+    const videoChunkFilename = basename(deletedChunk)
+
+    let payload: CustomLiveRTMPHLSTranscodingUpdatePayload = {
+      type: 'remove-chunk',
+      videoChunkFilename
+    }
+
+    if (this.allPlaylistsCreated) {
+      const playlistName = this.getPlaylistName(videoChunkFilename)
+
+      payload = {
+        ...payload,
+
+        masterPlaylistFile: join(this.outputPath, 'master.m3u8'),
+        resolutionPlaylistFilename: playlistName,
+        resolutionPlaylistFile: this.buildPlaylistFileParam(playlistName)
+      }
+    }
+
+    return this.updateWithRetry(payload)
+  }
+
+  private async sendPendingChunks (): Promise<any> {
+    if (this.ended) return Promise.resolve()
+
+    const parallelPromises: Promise<any>[] = []
+
+    for (const playlist of this.pendingChunksPerPlaylist.keys()) {
+      let sequentialPromises: Promise<any>
+
+      for (const chunk of this.pendingChunksPerPlaylist.get(playlist)) {
+        logger.debug(`Sending added live chunk ${chunk} update`)
+
+        const videoChunkFilename = basename(chunk)
+
+        const payloadBuilder = async () => {
+          let payload: CustomLiveRTMPHLSTranscodingUpdatePayload = {
+            type: 'add-chunk',
+            videoChunkFilename,
+            videoChunkFile: chunk
+          }
+
+          if (this.allPlaylistsCreated) {
+            const playlistName = this.getPlaylistName(videoChunkFilename)
+
+            try {
+              await this.updatePlaylistContent(playlistName, videoChunkFilename)
+
+              payload = {
+                ...payload,
+
+                masterPlaylistFile: join(this.outputPath, 'master.m3u8'),
+                resolutionPlaylistFilename: playlistName,
+                resolutionPlaylistFile: this.buildPlaylistFileParam(playlistName)
+              }
+            } catch (err) {
+              logger.warn(err, `Cannot fetch/update playlist content ${playlistName}`)
+            }
+          }
+
+          return payload
+        }
+
+        const p = payloadBuilder().then(p => this.updateWithRetry(p))
+
+        if (sequentialPromises === undefined) sequentialPromises = p
+        else sequentialPromises = sequentialPromises.then(() => p)
+      }
+
+      parallelPromises.push(sequentialPromises)
+      this.pendingChunksPerPlaylist.set(playlist, [])
+    }
+
+    await Promise.all(parallelPromises)
+  }
+
+  private async updateWithRetry (updatePayload: CustomLiveRTMPHLSTranscodingUpdatePayload, currentTry = 1): Promise<any> {
+    if (this.ended || this.errored) return
+
+    try {
+      await this.options.server.runnerJobs.update({
+        jobToken: this.options.job.jobToken,
+        jobUUID: this.options.job.uuid,
+        runnerToken: this.options.runnerToken,
+        payload: updatePayload as any,
+        reqPayload: this.payload
+      })
+    } catch (err) {
+      if (currentTry >= 3) throw err
+      if ((err.res?.body as PeerTubeProblemDocument)?.code === ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE) throw err
+
+      logger.warn({ err }, 'Will retry update after error')
+      await wait(250)
+
+      return this.updateWithRetry(updatePayload, currentTry + 1)
+    }
+  }
+
+  private getPlaylistName (videoChunkFilename: string) {
+    return `${videoChunkFilename.split('-')[0]}.m3u8`
+  }
+
+  private getPlaylistIdFromTS (segmentPath: string) {
+    const playlistIdMatcher = /^([\d+])-/
+
+    return basename(segmentPath).match(playlistIdMatcher)[1]
+  }
+
+  private async updatePlaylistContent (playlistName: string, latestChunkFilename: string) {
+    const m3u8Path = join(this.outputPath, playlistName)
+    let playlistContent = await readFile(m3u8Path, 'utf-8')
+
+    if (!playlistContent.includes('#EXT-X-ENDLIST')) {
+      playlistContent = playlistContent.substring(
+        0,
+        playlistContent.lastIndexOf(latestChunkFilename) + latestChunkFilename.length
+      ) + '\n'
+    }
+
+    // Remove new chunk references, that will be processed later
+    this.latestFilteredPlaylistContent[playlistName] = playlistContent
+  }
+
+  private buildPlaylistFileParam (playlistName: string) {
+    return [
+      Buffer.from(this.latestFilteredPlaylistContent[playlistName], 'utf-8'),
+      join(this.outputPath, 'master.m3u8')
+    ] as [Buffer, string]
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private cleanup () {
+    logger.debug(`Cleaning up job ${this.options.job.uuid}`)
+
+    for (const fsWatcher of this.fsWatchers) {
+      fsWatcher.close()
+        .catch(err => logger.error({ err }, 'Cannot close watcher'))
+    }
+
+    remove(this.outputPath)
+      .catch(err => logger.error({ err }, `Cannot remove ${this.outputPath}`))
+  }
+}

apps/peertube-runner/src/server/process/shared/process-storyboard.ts

@@ -0,0 +1,60 @@
+import { RunnerJobGenerateStoryboardPayload, GenerateStoryboardSuccess } from '@peertube/peertube-models'
+import { buildUUID } from '@peertube/peertube-node-utils'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
+import { ConfigManager } from '../../../shared/config-manager.js'
+import { logger } from '../../../shared/index.js'
+import { buildFFmpegImage, downloadInputFile, ProcessOptions, scheduleTranscodingProgress } from './common.js'
+
+export async function processGenerateStoryboard (options: ProcessOptions<RunnerJobGenerateStoryboardPayload>) {
+  const { server, job, runnerToken } = options
+
+  const payload = job.payload
+
+  let ffmpegProgress: number
+  let videoInputPath: string
+
+  const outputPath = join(ConfigManager.Instance.getStoryboardDirectory(), `storyboard-${buildUUID()}.jpg`)
+
+  const updateProgressInterval = scheduleTranscodingProgress({
+    job,
+    server,
+    runnerToken,
+    progressGetter: () => ffmpegProgress
+  })
+
+  try {
+    logger.info(`Downloading input file ${payload.input.videoFileUrl} for storyboard job ${job.jobToken}`)
+
+    videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+
+    logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Generating storyboard.`)
+
+    const ffmpegImage = buildFFmpegImage()
+
+    await ffmpegImage.generateStoryboardFromVideo({
+      path: videoInputPath,
+      destination: outputPath,
+      inputFileMutexReleaser: () => {},
+      sprites: payload.sprites
+    })
+
+    const successBody: GenerateStoryboardSuccess = {
+      storyboardFile: outputPath
+    }
+
+    await server.runnerJobs.success({
+      jobToken: job.jobToken,
+      jobUUID: job.uuid,
+      runnerToken,
+      payload: successBody,
+      reqPayload: payload
+    })
+  } finally {
+    if (videoInputPath) await remove(videoInputPath)
+    if (outputPath) await remove(outputPath)
+    if (updateProgressInterval) clearInterval(updateProgressInterval)
+  }
+}
+
+

apps/peertube-runner/src/server/process/shared/process-studio.ts

@@ -0,0 +1,187 @@
+import { pick } from '@peertube/peertube-core-utils'
+import {
+  RunnerJobStudioTranscodingPayload,
+  VideoStudioTask,
+  VideoStudioTaskCutPayload,
+  VideoStudioTaskIntroPayload,
+  VideoStudioTaskOutroPayload,
+  VideoStudioTaskPayload,
+  VideoStudioTaskWatermarkPayload,
+  VideoStudioTranscodingSuccess
+} from '@peertube/peertube-models'
+import { buildUUID } from '@peertube/peertube-node-utils'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
+import { ConfigManager } from '../../../shared/config-manager.js'
+import { logger } from '../../../shared/index.js'
+import {
+  buildFFmpegEdition,
+  downloadInputFile,
+  downloadSeparatedAudioFileIfNeeded,
+  JobWithToken,
+  ProcessOptions,
+  scheduleTranscodingProgress
+} from './common.js'
+
+export async function processStudioTranscoding (options: ProcessOptions<RunnerJobStudioTranscodingPayload>) {
+  const { server, job, runnerToken } = options
+  const payload = job.payload
+
+  let videoInputPath: string
+  let separatedAudioInputPath: string
+
+  let tmpVideoInputFilePath: string
+  let tmpSeparatedAudioInputFilePath: string
+
+  let outputPath: string
+
+  let tasksProgress = 0
+
+  const updateProgressInterval = scheduleTranscodingProgress({
+    job,
+    server,
+    runnerToken,
+    progressGetter: () => tasksProgress
+  })
+
+  try {
+    logger.info(`Downloading input file ${payload.input.videoFileUrl} for job ${job.jobToken}`)
+
+    videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+    separatedAudioInputPath = await downloadSeparatedAudioFileIfNeeded({ urls: payload.input.separatedAudioFileUrl, runnerToken, job })
+
+    tmpVideoInputFilePath = videoInputPath
+    tmpSeparatedAudioInputFilePath = separatedAudioInputPath
+
+    logger.info(`Input file ${payload.input.videoFileUrl} downloaded for job ${job.jobToken}. Running studio transcoding tasks.`)
+
+    for (const task of payload.tasks) {
+      const outputFilename = 'output-edition-' + buildUUID() + '.mp4'
+      outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), outputFilename)
+
+      await processTask({
+        videoInputPath: tmpVideoInputFilePath,
+        separatedAudioInputPath: tmpSeparatedAudioInputFilePath,
+        outputPath,
+        task,
+        job,
+        runnerToken
+      })
+
+      if (tmpVideoInputFilePath) await remove(tmpVideoInputFilePath)
+      if (tmpSeparatedAudioInputFilePath) await remove(tmpSeparatedAudioInputFilePath)
+
+      // For the next iteration
+      tmpVideoInputFilePath = outputPath
+      tmpSeparatedAudioInputFilePath = undefined
+
+      tasksProgress += Math.floor(100 / payload.tasks.length)
+    }
+
+    const successBody: VideoStudioTranscodingSuccess = {
+      videoFile: outputPath
+    }
+
+    await server.runnerJobs.success({
+      jobToken: job.jobToken,
+      jobUUID: job.uuid,
+      runnerToken,
+      payload: successBody,
+      reqPayload: payload
+    })
+  } finally {
+    if (tmpVideoInputFilePath) await remove(tmpVideoInputFilePath)
+    if (tmpSeparatedAudioInputFilePath) await remove(tmpSeparatedAudioInputFilePath)
+    if (outputPath) await remove(outputPath)
+    if (updateProgressInterval) clearInterval(updateProgressInterval)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+type TaskProcessorOptions <T extends VideoStudioTaskPayload = VideoStudioTaskPayload> = {
+  videoInputPath: string
+  separatedAudioInputPath: string
+
+  outputPath: string
+
+  task: T
+  runnerToken: string
+  job: JobWithToken
+}
+
+const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise<any> } = {
+  'add-intro': processAddIntroOutro,
+  'add-outro': processAddIntroOutro,
+  'cut': processCut,
+  'add-watermark': processAddWatermark
+}
+
+async function processTask (options: TaskProcessorOptions) {
+  const { task } = options
+
+  const processor = taskProcessors[options.task.name]
+  if (!process) throw new Error('Unknown task ' + task.name)
+
+  return processor(options)
+}
+
+async function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {
+  const { videoInputPath, task, runnerToken, job } = options
+
+  logger.debug(`Adding intro/outro to ${videoInputPath}`)
+
+  const introOutroPath = await downloadInputFile({ url: task.options.file, runnerToken, job })
+
+  try {
+    await buildFFmpegEdition().addIntroOutro({
+      ...pick(options, [ 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
+
+      introOutroPath,
+      type: task.name === 'add-intro'
+        ? 'intro'
+        : 'outro'
+    })
+  } finally {
+    await remove(introOutroPath)
+  }
+}
+
+function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
+  const { videoInputPath, task } = options
+
+  logger.debug(`Cutting ${videoInputPath}`)
+
+  return buildFFmpegEdition().cutVideo({
+    ...pick(options, [ 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
+
+    start: task.options.start,
+    end: task.options.end
+  })
+}
+
+async function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {
+  const { videoInputPath, task, runnerToken, job } = options
+
+  logger.debug(`Adding watermark to ${videoInputPath}`)
+
+  const watermarkPath = await downloadInputFile({ url: task.options.file, runnerToken, job })
+
+  try {
+    await buildFFmpegEdition().addWatermark({
+      ...pick(options, [ 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
+
+      watermarkPath,
+
+      videoFilters: {
+        watermarkSizeRatio: task.options.watermarkSizeRatio,
+        horitonzalMarginRatio: task.options.horitonzalMarginRatio,
+        verticalMarginRatio: task.options.verticalMarginRatio
+      }
+    })
+  } finally {
+    await remove(watermarkPath)
+  }
+}

apps/peertube-runner/src/server/process/shared/process-transcription.ts

@@ -0,0 +1,80 @@
+import { hasAudioStream } from '@peertube/peertube-ffmpeg'
+import { RunnerJobTranscriptionPayload, TranscriptionSuccess } from '@peertube/peertube-models'
+import { buildSUUID } from '@peertube/peertube-node-utils'
+import { TranscriptionModel, WhisperBuiltinModel, transcriberFactory } from '@peertube/peertube-transcription'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
+import { ConfigManager } from '../../../shared/config-manager.js'
+import { logger } from '../../../shared/index.js'
+import { ProcessOptions, downloadInputFile, scheduleTranscodingProgress } from './common.js'
+import { getWinstonLogger } from './winston-logger.js'
+
+export async function processVideoTranscription (options: ProcessOptions<RunnerJobTranscriptionPayload>) {
+  const { server, job, runnerToken } = options
+
+  const config = ConfigManager.Instance.getConfig().transcription
+
+  const payload = job.payload
+
+  let inputPath: string
+
+  const updateProgressInterval = scheduleTranscodingProgress({
+    job,
+    server,
+    runnerToken,
+    progressGetter: () => undefined
+  })
+
+  const outputPath = join(ConfigManager.Instance.getTranscriptionDirectory(), buildSUUID())
+
+  const transcriber = transcriberFactory.createFromEngineName({
+    engineName: config.engine,
+    enginePath: config.enginePath,
+    logger: getWinstonLogger()
+  })
+
+  try {
+    logger.info(`Downloading input file ${payload.input.videoFileUrl} for transcription job ${job.jobToken}`)
+
+    inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+
+    logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running transcription.`)
+
+    if (await hasAudioStream(inputPath) !== true) {
+      await server.runnerJobs.error({
+        jobToken: job.jobToken,
+        jobUUID: job.uuid,
+        runnerToken,
+        message: 'This input file does not contain audio'
+      })
+
+      return
+    }
+
+    const transcriptFile = await transcriber.transcribe({
+      mediaFilePath: inputPath,
+      model: config.modelPath
+        ? await TranscriptionModel.fromPath(config.modelPath)
+        : new WhisperBuiltinModel(config.model),
+      format: 'vtt',
+      transcriptDirectory: outputPath
+    })
+
+    const successBody: TranscriptionSuccess = {
+      inputLanguage: transcriptFile.language,
+      vttFile: transcriptFile.path
+    }
+
+    await server.runnerJobs.success({
+      jobToken: job.jobToken,
+      jobUUID: job.uuid,
+      runnerToken,
+      payload: successBody,
+      reqPayload: payload
+    })
+  } finally {
+    if (inputPath) await remove(inputPath)
+    if (outputPath) await remove(outputPath)
+    if (updateProgressInterval) clearInterval(updateProgressInterval)
+  }
+}

apps/peertube-runner/src/server/process/shared/process-vod.ts

@@ -0,0 +1,234 @@
+import { canCopyForHLS } from '@peertube/peertube-ffmpeg'
+import {
+  RunnerJobVODAudioMergeTranscodingPayload,
+  RunnerJobVODHLSTranscodingPayload,
+  RunnerJobVODWebVideoTranscodingPayload,
+  VODAudioMergeTranscodingSuccess,
+  VODHLSTranscodingSuccess,
+  VODWebVideoTranscodingSuccess
+} from '@peertube/peertube-models'
+import { buildUUID } from '@peertube/peertube-node-utils'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
+import { ConfigManager } from '../../../shared/config-manager.js'
+import { logger } from '../../../shared/index.js'
+import {
+  buildFFmpegVOD,
+  downloadInputFile,
+  downloadSeparatedAudioFileIfNeeded,
+  ProcessOptions,
+  scheduleTranscodingProgress
+} from './common.js'
+
+export async function processWebVideoTranscoding (options: ProcessOptions<RunnerJobVODWebVideoTranscodingPayload>) {
+  const { server, job, runnerToken } = options
+
+  const payload = job.payload
+
+  let ffmpegProgress: number
+  let videoInputPath: string
+  let separatedAudioInputPath: string
+
+  const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`)
+
+  const updateProgressInterval = scheduleTranscodingProgress({
+    job,
+    server,
+    runnerToken,
+    progressGetter: () => ffmpegProgress
+  })
+
+  try {
+    logger.info(`Downloading input file ${payload.input.videoFileUrl} for web video transcoding job ${job.jobToken}`)
+
+    videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+    separatedAudioInputPath = await downloadSeparatedAudioFileIfNeeded({ urls: payload.input.separatedAudioFileUrl, runnerToken, job })
+
+    logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running web video transcoding.`)
+
+    const ffmpegVod = buildFFmpegVOD({
+      onJobProgress: progress => {
+        ffmpegProgress = progress
+      }
+    })
+
+    await ffmpegVod.transcode({
+      type: 'video',
+
+      videoInputPath,
+      separatedAudioInputPath,
+
+      outputPath,
+
+      inputFileMutexReleaser: () => {},
+
+      resolution: payload.output.resolution,
+      fps: payload.output.fps
+    })
+
+    const successBody: VODWebVideoTranscodingSuccess = {
+      videoFile: outputPath
+    }
+
+    await server.runnerJobs.success({
+      jobToken: job.jobToken,
+      jobUUID: job.uuid,
+      runnerToken,
+      payload: successBody,
+      reqPayload: payload
+    })
+  } finally {
+    if (videoInputPath) await remove(videoInputPath)
+    if (separatedAudioInputPath) await remove(separatedAudioInputPath)
+    if (outputPath) await remove(outputPath)
+    if (updateProgressInterval) clearInterval(updateProgressInterval)
+  }
+}
+
+export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVODHLSTranscodingPayload>) {
+  const { server, job, runnerToken } = options
+  const payload = job.payload
+
+  let ffmpegProgress: number
+  let videoInputPath: string
+  let separatedAudioInputPath: string
+
+  const uuid = buildUUID()
+  const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `${uuid}-${payload.output.resolution}.m3u8`)
+  const videoFilename = `${uuid}-${payload.output.resolution}-fragmented.mp4`
+  const videoPath = join(join(ConfigManager.Instance.getTranscodingDirectory(), videoFilename))
+
+  const updateProgressInterval = scheduleTranscodingProgress({
+    job,
+    server,
+    runnerToken,
+    progressGetter: () => ffmpegProgress
+  })
+
+  try {
+    logger.info(`Downloading input file ${payload.input.videoFileUrl} for HLS transcoding job ${job.jobToken}`)
+
+    videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+    separatedAudioInputPath = await downloadSeparatedAudioFileIfNeeded({ urls: payload.input.separatedAudioFileUrl, runnerToken, job })
+
+    const copyCodecs = await canCopyForHLS({
+      fps: payload.output.fps,
+      resolution: payload.output.resolution,
+      path: videoInputPath
+    })
+
+    logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running HLS transcoding.`)
+
+    const ffmpegVod = buildFFmpegVOD({
+      onJobProgress: progress => {
+        ffmpegProgress = progress
+      }
+    })
+
+    await ffmpegVod.transcode({
+      type: 'hls',
+      copyCodecs,
+
+      videoInputPath,
+      separatedAudioInputPath,
+
+      hlsPlaylist: { videoFilename },
+      outputPath,
+
+      inputFileMutexReleaser: () => {},
+
+      resolution: payload.output.resolution,
+      fps: payload.output.fps,
+      separatedAudio: payload.output.separatedAudio
+    })
+
+    const successBody: VODHLSTranscodingSuccess = {
+      resolutionPlaylistFile: outputPath,
+      videoFile: videoPath
+    }
+
+    await server.runnerJobs.success({
+      jobToken: job.jobToken,
+      jobUUID: job.uuid,
+      runnerToken,
+      payload: successBody,
+      reqPayload: payload
+    })
+  } finally {
+    if (videoInputPath) await remove(videoInputPath)
+    if (separatedAudioInputPath) await remove(separatedAudioInputPath)
+    if (outputPath) await remove(outputPath)
+    if (videoPath) await remove(videoPath)
+    if (updateProgressInterval) clearInterval(updateProgressInterval)
+  }
+}
+
+export async function processAudioMergeTranscoding (options: ProcessOptions<RunnerJobVODAudioMergeTranscodingPayload>) {
+  const { server, job, runnerToken } = options
+  const payload = job.payload
+
+  let ffmpegProgress: number
+  let audioPath: string
+  let previewPath: string
+
+  const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`)
+
+  const updateProgressInterval = scheduleTranscodingProgress({
+    job,
+    server,
+    runnerToken,
+    progressGetter: () => ffmpegProgress
+  })
+
+  try {
+    logger.info(
+      `Downloading input files ${payload.input.audioFileUrl} and ${payload.input.previewFileUrl} ` +
+        `for audio merge transcoding job ${job.jobToken}`
+    )
+
+    audioPath = await downloadInputFile({ url: payload.input.audioFileUrl, runnerToken, job })
+    previewPath = await downloadInputFile({ url: payload.input.previewFileUrl, runnerToken, job })
+
+    logger.info(
+      `Downloaded input files ${payload.input.audioFileUrl} and ${payload.input.previewFileUrl} ` +
+        `for job ${job.jobToken}. Running audio merge transcoding.`
+    )
+
+    const ffmpegVod = buildFFmpegVOD({
+      onJobProgress: progress => {
+        ffmpegProgress = progress
+      }
+    })
+
+    await ffmpegVod.transcode({
+      type: 'merge-audio',
+
+      audioPath,
+      videoInputPath: previewPath,
+
+      outputPath,
+
+      inputFileMutexReleaser: () => {},
+
+      resolution: payload.output.resolution,
+      fps: payload.output.fps
+    })
+
+    const successBody: VODAudioMergeTranscodingSuccess = {
+      videoFile: outputPath
+    }
+
+    await server.runnerJobs.success({
+      jobToken: job.jobToken,
+      jobUUID: job.uuid,
+      runnerToken,
+      payload: successBody,
+      reqPayload: payload
+    })
+  } finally {
+    if (audioPath) await remove(audioPath)
+    if (previewPath) await remove(previewPath)
+    if (outputPath) await remove(outputPath)
+    if (updateProgressInterval) clearInterval(updateProgressInterval)
+  }
+}

apps/peertube-runner/src/server/process/shared/winston-logger.ts

@@ -0,0 +1,19 @@
+import { LogFn } from 'pino'
+import { logger } from '../../../shared/index.js'
+
+export function getWinstonLogger () {
+  return {
+    info: buildLogLevelFn(logger.info.bind(logger)),
+    debug: buildLogLevelFn(logger.debug.bind(logger)),
+    warn: buildLogLevelFn(logger.warn.bind(logger)),
+    error: buildLogLevelFn(logger.error.bind(logger))
+  }
+}
+
+function buildLogLevelFn (log: LogFn) {
+  return (arg1: string, arg2?: object) => {
+    if (arg2) return log(arg2, arg1)
+
+    return log(arg1)
+  }
+}

apps/peertube-runner/src/server/server.ts

@@ -0,0 +1,388 @@
+import { pick, shuffle, wait } from '@peertube/peertube-core-utils'
+import { PeerTubeProblemDocument, RunnerJobType, ServerErrorCode } from '@peertube/peertube-models'
+import { PeerTubeServer as PeerTubeServerCommand } from '@peertube/peertube-server-commands'
+import { ensureDir, remove } from 'fs-extra/esm'
+import { readdir } from 'fs/promises'
+import { join } from 'path'
+import { io, Socket } from 'socket.io-client'
+import { ConfigManager } from '../shared/index.js'
+import { IPCServer } from '../shared/ipc/index.js'
+import { logger } from '../shared/logger.js'
+import { JobWithToken, processJob } from './process/index.js'
+import { getSupportedJobsList } from './shared/index.js'
+
+type PeerTubeServer = PeerTubeServerCommand & {
+  runnerToken: string
+  runnerName: string
+  runnerDescription?: string
+}
+
+export class RunnerServer {
+  private servers: PeerTubeServer[] = []
+  private processingJobs: { job: JobWithToken, server: PeerTubeServer }[] = []
+
+  private checkingAvailableJobs = false
+
+  private gracefulShutdown = false
+  private cleaningUp = false
+  private initialized = false
+
+  private ipcServer: IPCServer
+
+  private readonly enabledJobsArray: RunnerJobType[]
+
+  private readonly sockets = new Map<PeerTubeServer, Socket>()
+
+  constructor (enabledJobs?: Set<RunnerJobType>) {
+    this.enabledJobsArray = enabledJobs
+      ? Array.from(enabledJobs)
+      : getSupportedJobsList()
+  }
+
+  async run () {
+    logger.info('Running PeerTube runner in server mode')
+
+    logger.info('Supported and enabled job types: ' + this.enabledJobsArray.join(', '))
+
+    await ConfigManager.Instance.load()
+
+    for (const registered of ConfigManager.Instance.getConfig().registeredInstances) {
+      const serverCommand = new PeerTubeServerCommand({ url: registered.url })
+
+      this.loadServer(Object.assign(serverCommand, registered))
+
+      logger.info(`Loading registered instance ${registered.url}`)
+    }
+
+    // Run IPC
+    this.ipcServer = new IPCServer()
+    try {
+      await this.ipcServer.run(this)
+    } catch (err) {
+      logger.error(err, 'Cannot start local socket for IPC communication')
+      process.exit(-1)
+    }
+
+    // Cleanup on exit
+    for (const code of [ 'SIGTERM', 'SIGINT', 'SIGUSR1', 'SIGUSR2', 'uncaughtException' ]) {
+      process.on(code, async (err, origin) => {
+        if (code === 'uncaughtException') {
+          logger.error({ err, origin }, 'uncaughtException')
+        }
+
+        await this.onExit()
+      })
+    }
+
+    // Process jobs
+    await ensureDir(ConfigManager.Instance.getTranscodingDirectory())
+    await ensureDir(ConfigManager.Instance.getStoryboardDirectory())
+    await this.cleanupTMP()
+
+    logger.info(`Using ${ConfigManager.Instance.getTranscodingDirectory()} for transcoding directory`)
+    logger.info(`Using ${ConfigManager.Instance.getStoryboardDirectory()} for storyboard directory`)
+
+    this.initialized = true
+    await this.checkAvailableJobs()
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async registerRunner (options: {
+    url: string
+    registrationToken: string
+    runnerName: string
+    runnerDescription?: string
+  }) {
+    const { url, registrationToken, runnerName, runnerDescription } = options
+
+    logger.info(`Registering runner ${runnerName} on ${url}...`)
+
+    const serverCommand = new PeerTubeServerCommand({ url })
+    const { runnerToken } = await serverCommand.runners.register({
+      name: runnerName,
+      description: runnerDescription,
+      registrationToken,
+      version: process.env.PACKAGE_VERSION
+    })
+
+    const server: PeerTubeServer = Object.assign(serverCommand, {
+      runnerToken,
+      runnerName,
+      runnerDescription
+    })
+
+    this.loadServer(server)
+    await this.saveRegisteredInstancesInConf()
+
+    logger.info(`Registered runner ${runnerName} on ${url}`)
+  }
+
+  private loadServer (server: PeerTubeServer) {
+    this.servers.push(server)
+
+    const url = server.url + '/runners'
+    const socket = io(url, {
+      auth: {
+        runnerToken: server.runnerToken
+      },
+      transports: [ 'websocket' ]
+    })
+
+    socket.on('connect_error', err => logger.warn({ err }, `Cannot connect to ${url} socket`))
+    socket.on('available-jobs', () => this.safeAsyncCheckAvailableJobs())
+
+    socket.on('connect', () => {
+      logger.info(`Connected to ${url} socket`)
+
+      this.safeAsyncCheckAvailableJobs()
+    })
+    socket.on('disconnect', () => logger.warn(`Disconnected from ${url} socket`))
+    socket.io.on('ping', () => logger.debug(`Received a "ping" for ${url}`))
+
+    this.sockets.set(server, socket)
+  }
+
+  async unregisterRunner (options: {
+    url: string
+    runnerName: string
+  }) {
+    const { url, runnerName } = options
+
+    const server = this.servers.find(s => s.url === url && s.runnerName === runnerName)
+    if (!server) {
+      logger.error(`Unknown server ${url} - ${runnerName} to unregister`)
+      return
+    }
+
+    logger.info(`Unregistering runner ${runnerName} on ${url}...`)
+
+    try {
+      await server.runners.unregister({ runnerToken: server.runnerToken })
+    } catch (err) {
+      logger.error({ err }, `Cannot unregister runner ${runnerName} on ${url}`)
+    }
+
+    this.unloadServer(server)
+    await this.saveRegisteredInstancesInConf()
+
+    logger.info(`Unregistered runner ${runnerName} on ${url}`)
+  }
+
+  private unloadServer (server: PeerTubeServer) {
+    this.servers = this.servers.filter(s => s !== server)
+
+    const socket = this.sockets.get(server)
+    socket.disconnect()
+
+    this.sockets.delete(server)
+  }
+
+  listRegistered () {
+    return {
+      servers: this.servers.map(s => {
+        return {
+          url: s.url,
+          runnerName: s.runnerName,
+          runnerDescription: s.runnerDescription
+        }
+      })
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  listJobs () {
+    return {
+      concurrency: ConfigManager.Instance.getConfig().jobs.concurrency,
+
+      processingJobs: this.processingJobs.map(j => ({
+        serverUrl: j.server.url,
+        job: pick(j.job, [ 'type', 'startedAt', 'progress', 'payload' ])
+      }))
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  requestGracefulShutdown () {
+    logger.info('Received graceful shutdown request')
+
+    this.gracefulShutdown = true
+    this.exitGracefullyIfNoProcessingJobs()
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private safeAsyncCheckAvailableJobs () {
+    this.checkAvailableJobs()
+      .catch(err => logger.error({ err }, `Cannot check available jobs`))
+  }
+
+  private async checkAvailableJobs () {
+    if (!this.initialized) return
+    if (this.checkingAvailableJobs) return
+    if (this.gracefulShutdown) return
+
+    this.checkingAvailableJobs = true
+
+    let hadAvailableJob = false
+
+    for (const server of shuffle([ ...this.servers ])) {
+      try {
+        logger.info('Checking available jobs on ' + server.url)
+
+        const job = await this.requestJob(server)
+        if (!job) continue
+
+        hadAvailableJob = true
+
+        await this.tryToExecuteJobAsync(server, job)
+      } catch (err) {
+        hadAvailableJob = false
+
+        const code = (err.res?.body as PeerTubeProblemDocument)?.code
+
+        if (code === ServerErrorCode.RUNNER_JOB_NOT_IN_PENDING_STATE) {
+          logger.debug({ err }, 'Runner job is not in pending state anymore, retry later')
+          continue
+        }
+
+        if (code === ServerErrorCode.UNKNOWN_RUNNER_TOKEN) {
+          logger.error({ err }, `Unregistering ${server.url} as the runner token ${server.runnerToken} is invalid`)
+
+          await this.unregisterRunner({ url: server.url, runnerName: server.runnerName })
+          continue
+        }
+
+        logger.error({ err }, `Cannot request/accept job on ${server.url} for runner ${server.runnerName}`)
+      }
+    }
+
+    this.checkingAvailableJobs = false
+
+    if (hadAvailableJob && this.canProcessMoreJobs()) {
+      await wait(2500)
+
+      this.checkAvailableJobs()
+        .catch(err => logger.error({ err }, 'Cannot check more available jobs'))
+    }
+  }
+
+  private async requestJob (server: PeerTubeServer) {
+    logger.debug(`Requesting jobs on ${server.url} for runner ${server.runnerName}`)
+
+    const { availableJobs } = await server.runnerJobs.request({
+      runnerToken: server.runnerToken,
+
+      jobTypes: this.enabledJobsArray.length !== getSupportedJobsList().length
+        ? this.enabledJobsArray
+        : undefined,
+
+      version: process.env.PACKAGE_VERSION
+    })
+
+    if (availableJobs.length === 0) {
+      logger.debug(`No job available on ${server.url} for runner ${server.runnerName}`)
+      return undefined
+    }
+
+    return availableJobs[0]
+  }
+
+  private async tryToExecuteJobAsync (server: PeerTubeServer, jobToAccept: { uuid: string }) {
+    if (!this.canProcessMoreJobs()) {
+      if (!this.gracefulShutdown) {
+        logger.info(
+          `Do not process more jobs (processing ${this.processingJobs.length} / ${ConfigManager.Instance.getConfig().jobs.concurrency})`
+        )
+      }
+
+      return
+    }
+
+    const { job } = await server.runnerJobs.accept({ runnerToken: server.runnerToken, jobUUID: jobToAccept.uuid })
+
+    const processingJob = { job, server }
+    this.processingJobs.push(processingJob)
+
+    processJob({ server, job, runnerToken: server.runnerToken })
+      .catch(err => {
+        logger.error({ err }, 'Cannot process job')
+
+        server.runnerJobs.error({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken: server.runnerToken, message: err.message })
+          .catch(err2 => logger.error({ err: err2 }, 'Cannot abort job after error'))
+      })
+      .finally(() => {
+        this.processingJobs = this.processingJobs.filter(p => p !== processingJob)
+
+        if (this.gracefulShutdown) this.exitGracefullyIfNoProcessingJobs()
+
+        return this.checkAvailableJobs()
+      })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private saveRegisteredInstancesInConf () {
+    const data = this.servers.map(s => {
+      return pick(s, [ 'url', 'runnerToken', 'runnerName', 'runnerDescription' ])
+    })
+
+    return ConfigManager.Instance.setRegisteredInstances(data)
+  }
+
+  private canProcessMoreJobs () {
+    if (this.cleaningUp) return false
+    if (this.gracefulShutdown) return false
+
+    return this.processingJobs.length < ConfigManager.Instance.getConfig().jobs.concurrency
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async cleanupTMP () {
+    const files = await readdir(ConfigManager.Instance.getTranscodingDirectory())
+
+    for (const file of files) {
+      await remove(join(ConfigManager.Instance.getTranscodingDirectory(), file))
+    }
+  }
+
+  private exitGracefullyIfNoProcessingJobs () {
+    if (this.processingJobs.length !== 0) return
+
+    logger.info('Shutting down the runner after graceful shutdown request')
+
+    this.onExit()
+      .catch(err => logger.error({ err }, 'Cannot exit runner'))
+  }
+
+  private async onExit () {
+    if (this.cleaningUp) return
+    this.cleaningUp = true
+
+    logger.info('Cleaning up after program exit')
+
+    try {
+      for (const { server, job } of this.processingJobs) {
+        logger.info(`Aborting job ${job.uuid} on ${server.url} as the runner is stopping`)
+
+        await server.runnerJobs.abort({
+          jobToken: job.jobToken,
+          jobUUID: job.uuid,
+          reason: 'Runner stopped',
+          runnerToken: server.runnerToken
+        })
+      }
+
+      await this.cleanupTMP()
+      await this.ipcServer?.stop()
+    } catch (err) {
+      logger.error(err)
+      process.exit(-1)
+    }
+
+    process.exit()
+  }
+}

apps/peertube-runner/src/server/shared/index.ts

@@ -0,0 +1 @@
+export * from './supported-job.js'

apps/peertube-runner/src/server/shared/supported-job.ts

@@ -0,0 +1,43 @@
+import {
+  RunnerJobGenerateStoryboardPayload,
+  RunnerJobLiveRTMPHLSTranscodingPayload,
+  RunnerJobPayload,
+  RunnerJobStudioTranscodingPayload,
+  RunnerJobTranscriptionPayload,
+  RunnerJobType,
+  RunnerJobVODAudioMergeTranscodingPayload,
+  RunnerJobVODHLSTranscodingPayload,
+  RunnerJobVODWebVideoTranscodingPayload,
+  VideoStudioTaskPayload
+} from '@peertube/peertube-models'
+
+const supportedMatrix: { [id in RunnerJobType]: (payload: RunnerJobPayload) => boolean } = {
+  'vod-web-video-transcoding': (_payload: RunnerJobVODWebVideoTranscodingPayload) => {
+    return true
+  },
+  'vod-hls-transcoding': (_payload: RunnerJobVODHLSTranscodingPayload) => {
+    return true
+  },
+  'vod-audio-merge-transcoding': (_payload: RunnerJobVODAudioMergeTranscodingPayload) => {
+    return true
+  },
+  'live-rtmp-hls-transcoding': (_payload: RunnerJobLiveRTMPHLSTranscodingPayload) => {
+    return true
+  },
+  'video-studio-transcoding': (payload: RunnerJobStudioTranscodingPayload) => {
+    const tasks = payload?.tasks
+    const supported = new Set<VideoStudioTaskPayload['name']>([ 'add-intro', 'add-outro', 'add-watermark', 'cut' ])
+
+    if (!Array.isArray(tasks)) return false
+
+    return tasks.every(t => t && supported.has(t.name))
+  },
+  'video-transcription': (_payload: RunnerJobTranscriptionPayload) => {
+    return true
+  },
+  'generate-video-storyboard': (_payload: RunnerJobGenerateStoryboardPayload) => true
+}
+
+export function getSupportedJobsList () {
+  return Object.keys(supportedMatrix) as unknown as RunnerJobType[]
+}

apps/peertube-runner/src/shared/config-manager.ts

@@ -0,0 +1,162 @@
+import { parse, stringify } from '@iarna/toml'
+import { TranscriptionEngineName, WhisperBuiltinModelName } from '@peertube/peertube-transcription'
+import envPaths from 'env-paths'
+import { ensureDir, pathExists, remove } from 'fs-extra/esm'
+import { readFile, writeFile } from 'fs/promises'
+import merge from 'lodash-es/merge.js'
+import { dirname, join } from 'path'
+import { logger } from '../shared/index.js'
+
+const paths = envPaths('peertube-runner')
+
+type Config = {
+  jobs: {
+    concurrency: number
+  }
+
+  ffmpeg: {
+    threads: number
+    nice: number
+  }
+
+  registeredInstances: {
+    url: string
+    runnerToken: string
+    runnerName: string
+    runnerDescription?: string
+  }[]
+
+  transcription: {
+    engine: TranscriptionEngineName
+    enginePath: string | null
+    model: WhisperBuiltinModelName
+    modelPath: string | null
+  }
+}
+
+export class ConfigManager {
+  private static instance: ConfigManager
+
+  private config: Config = {
+    jobs: {
+      concurrency: 2
+    },
+    ffmpeg: {
+      threads: 2,
+      nice: 20
+    },
+    transcription: {
+      engine: 'whisper-ctranslate2',
+      enginePath: null,
+      model: 'small',
+      modelPath: null
+    },
+    registeredInstances: []
+  }
+
+  private id: string
+  private configFilePath: string
+
+  private constructor () {}
+
+  init (id: string) {
+    this.id = id
+    this.configFilePath = join(this.getConfigDir(), 'config.toml')
+  }
+
+  async load () {
+    logger.info(`Using ${this.configFilePath} as configuration file`)
+
+    if (this.isTestInstance()) {
+      logger.info('Removing configuration file as we are using the "test" id')
+      await remove(this.configFilePath)
+    }
+
+    await ensureDir(dirname(this.configFilePath))
+
+    if (!await pathExists(this.configFilePath)) {
+      await this.save()
+    }
+
+    const file = await readFile(this.configFilePath, 'utf-8')
+
+    this.config = merge(this.config, parse(file))
+  }
+
+  save () {
+    return writeFile(this.configFilePath, stringify(this.config))
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async setRegisteredInstances (registeredInstances: {
+    url: string
+    runnerToken: string
+    runnerName: string
+    runnerDescription?: string
+  }[]) {
+    this.config.registeredInstances = registeredInstances
+
+    await this.save()
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getConfig () {
+    return this.deepFreeze(this.config)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getStoryboardDirectory () {
+    return join(paths.cache, this.id, 'storyboard')
+  }
+
+  getTranscodingDirectory () {
+    return join(paths.cache, this.id, 'transcoding')
+  }
+
+  getTranscriptionDirectory () {
+    return join(paths.cache, this.id, 'transcription')
+  }
+
+  getSocketDirectory () {
+    return join(paths.data, this.id)
+  }
+
+  getSocketPath () {
+    return join(this.getSocketDirectory(), 'peertube-runner.sock')
+  }
+
+  getConfigDir () {
+    return join(paths.config, this.id)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  isTestInstance () {
+    return typeof this.id === 'string' && this.id.match(/^test-\d$/)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/freeze
+  private deepFreeze<T extends object> (object: T) {
+    const propNames = Reflect.ownKeys(object)
+
+    // Freeze properties before freezing self
+    for (const name of propNames) {
+      const value = object[name]
+
+      if ((value && typeof value === 'object') || typeof value === 'function') {
+        this.deepFreeze(value)
+      }
+    }
+
+    return Object.freeze({ ...object })
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

apps/peertube-runner/src/shared/http.ts

@@ -0,0 +1,79 @@
+import { createWriteStream } from 'fs'
+import { remove } from 'fs-extra/esm'
+import { RequestOptions } from 'https'
+import { http, https } from 'follow-redirects'
+import { logger } from './logger.js'
+
+export function downloadFile (options: {
+  url: string
+  destination: string
+  runnerToken: string
+  jobToken: string
+}) {
+  const { url, destination, runnerToken, jobToken } = options
+
+  logger.debug(`Downloading file ${url}`)
+
+  return new Promise<void>((res, rej) => {
+    const parsed = new URL(url)
+
+    const body = JSON.stringify({
+      runnerToken,
+      jobToken
+    })
+
+    const getOptions: RequestOptions = {
+      method: 'POST',
+      hostname: parsed.hostname,
+      port: parsed.port,
+      path: parsed.pathname,
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(body, 'utf-8')
+      }
+    }
+
+    const request = getRequest(url)(getOptions, response => {
+      const code = response.statusCode ?? 0
+
+      if (code >= 400) {
+        return rej(new Error(response.statusMessage))
+      }
+
+      const file = createWriteStream(destination)
+
+      file.on('error', err => {
+        remove(destination)
+          .catch(err => logger.error(err))
+
+        return rej(err)
+      })
+
+      file.on('finish', () => res())
+
+      response.pipe(file)
+    })
+
+    request.on('error', err => {
+      remove(destination)
+        .catch(err => logger.error(err))
+
+      return rej(err)
+    })
+
+    request.write(body)
+    request.end()
+
+    setTimeout(() => {
+      request.destroy(new Error('Global request timeout'))
+    }, 2 * 3600 * 1000) // 2 hours
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function getRequest (url: string) {
+  if (url.startsWith('https://')) return https.request.bind(https)
+
+  return http.request.bind(http)
+}

apps/peertube-runner/src/shared/index.ts

@@ -0,0 +1,3 @@
+export * from './config-manager.js'
+export * from './http.js'
+export * from './logger.js'

apps/peertube-runner/src/shared/ipc/index.ts

@@ -0,0 +1,2 @@
+export * from './ipc-client.js'
+export * from './ipc-server.js'

apps/peertube-runner/src/shared/ipc/ipc-client.ts

@@ -0,0 +1,145 @@
+import { Client as NetIPC } from 'net-ipc'
+import CliTable3 from 'cli-table3'
+import { ensureDir } from 'fs-extra/esm'
+import { ConfigManager } from '../config-manager.js'
+import { IPCRequest, IPCResponse, IPCResponseListJobs, IPCResponseListRegistered } from './shared/index.js'
+
+export class IPCClient {
+  private netIPC: NetIPC
+
+  async run () {
+    await ensureDir(ConfigManager.Instance.getSocketDirectory())
+
+    const socketPath = ConfigManager.Instance.getSocketPath()
+
+    this.netIPC = new NetIPC({ path: socketPath })
+
+    try {
+      await this.netIPC.connect()
+    } catch (err) {
+      if (err.code === 'ECONNREFUSED') {
+        throw new Error(
+          'This runner is not currently running in server mode on this system. ' +
+            'Please run it using the `server` command first (in another terminal for example) and then retry your command.',
+          { cause: err }
+        )
+      }
+
+      throw err
+    }
+  }
+
+  async askRegister (options: {
+    url: string
+    registrationToken: string
+    runnerName: string
+    runnerDescription?: string
+  }) {
+    const req: IPCRequest = {
+      type: 'register',
+      ...options
+    }
+
+    const { success, error } = await this.netIPC.request(req) as IPCResponse
+
+    if (success) console.log('PeerTube instance registered')
+    else console.error('Could not register PeerTube instance on runner server side', error)
+  }
+
+  async askUnregister (options: {
+    url: string
+    runnerName: string
+  }) {
+    const req: IPCRequest = {
+      type: 'unregister',
+      ...options
+    }
+
+    const { success, error } = await this.netIPC.request(req) as IPCResponse
+
+    if (success) console.log('PeerTube instance unregistered')
+    else console.error('Could not unregister PeerTube instance on runner server side', error)
+  }
+
+  async askListRegistered () {
+    const req: IPCRequest = {
+      type: 'list-registered'
+    }
+
+    const { success, error, data } = await this.netIPC.request(req) as IPCResponse<IPCResponseListRegistered>
+    if (!success) {
+      console.error('Could not list registered PeerTube instances', error)
+      return
+    }
+
+    const table = new CliTable3({
+      head: [ 'instance', 'runner name', 'runner description' ]
+    })
+
+    for (const server of data.servers) {
+      table.push([ server.url, server.runnerName, server.runnerDescription ])
+    }
+
+    console.log(table.toString())
+  }
+
+  async askListJobs (options: {
+    includePayload: boolean
+  }) {
+    const req: IPCRequest = {
+      type: 'list-jobs'
+    }
+
+    const { success, error, data } = await this.netIPC.request(req) as IPCResponse<IPCResponseListJobs>
+    if (!success) {
+      console.error('Could not list jobs', error)
+      return
+    }
+
+    const head = [ 'instance', 'type', 'started', 'progress' ]
+    if (options.includePayload) head.push('payload')
+
+    const table = new CliTable3({
+      head,
+      wordWrap: true,
+      wrapOnWordBoundary: false
+    })
+
+    for (const { serverUrl, job } of data.processingJobs) {
+      const row = [
+        serverUrl,
+        job.type,
+
+        job.startedAt?.toLocaleString(),
+
+        job.progress !== undefined && job.progress !== null
+          ? `${job.progress}%`
+          : ''
+      ]
+
+      if (options.includePayload) row.push(JSON.stringify(job.payload, undefined, 2))
+
+      table.push(row)
+    }
+
+    console.log(`Processing ${data.processingJobs.length}/${data.concurrency} jobs`)
+    console.log(table.toString())
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async askGracefulShutdown () {
+    const req: IPCRequest = { type: 'graceful-shutdown' }
+
+    const { success, error } = await this.netIPC.request(req) as IPCResponse
+
+    if (success) console.log('Graceful shutdown acknowledged by the runner')
+    else console.error('Could not graceful shutdown runner', error)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  stop () {
+    this.netIPC.destroy()
+  }
+}

apps/peertube-runner/src/shared/ipc/ipc-server.ts

@@ -0,0 +1,72 @@
+import { ensureDir } from 'fs-extra/esm'
+import { Server as NetIPC } from 'net-ipc'
+import { pick } from '@peertube/peertube-core-utils'
+import { RunnerServer } from '../../server/index.js'
+import { ConfigManager } from '../config-manager.js'
+import { logger } from '../logger.js'
+import { IPCResponse, IPCResponseData, IPCRequest } from './shared/index.js'
+
+export class IPCServer {
+  private netIPC: NetIPC
+  private runnerServer: RunnerServer
+
+  async run (runnerServer: RunnerServer) {
+    this.runnerServer = runnerServer
+
+    await ensureDir(ConfigManager.Instance.getSocketDirectory())
+
+    const socketPath = ConfigManager.Instance.getSocketPath()
+    this.netIPC = new NetIPC({ path: socketPath })
+    await this.netIPC.start()
+
+    logger.info(`IPC socket created on ${socketPath}`)
+
+    this.netIPC.on('request', async (req: IPCRequest, res) => {
+      try {
+        const data = await this.process(req)
+
+        this.sendResponse(res, { success: true, data })
+      } catch (err) {
+        logger.error({ err }, 'Cannot execute RPC call')
+        this.sendResponse(res, { success: false, error: err.message })
+      }
+    })
+  }
+
+  stop () {
+    return this.netIPC?.close()
+  }
+
+  private async process (req: IPCRequest) {
+    switch (req.type) {
+      case 'register':
+        await this.runnerServer.registerRunner(pick(req, [ 'url', 'registrationToken', 'runnerName', 'runnerDescription' ]))
+        return undefined
+
+      case 'unregister':
+        await this.runnerServer.unregisterRunner(pick(req, [ 'url', 'runnerName' ]))
+        return undefined
+
+      case 'list-registered':
+        return Promise.resolve(this.runnerServer.listRegistered())
+
+      case 'list-jobs':
+        return Promise.resolve(this.runnerServer.listJobs())
+
+      case 'graceful-shutdown':
+        this.runnerServer.requestGracefulShutdown()
+        return undefined
+
+      default:
+        throw new Error('Unknown RPC call ' + (req as any).type)
+    }
+  }
+
+  private sendResponse<T extends IPCResponseData> (
+    response: (data: any) => Promise<void>,
+    body: IPCResponse<T>
+  ) {
+    response(body)
+      .catch(err => logger.error(err, 'Cannot send response after IPC request'))
+  }
+}

apps/peertube-runner/src/shared/ipc/shared/index.ts

@@ -0,0 +1,2 @@
+export * from './ipc-request.model.js'
+export * from './ipc-response.model.js'

apps/peertube-runner/src/shared/ipc/shared/ipc-request.model.ts

@@ -0,0 +1,20 @@
+export type IPCRequest =
+  IPCRequestRegister |
+  IPCRequestUnregister |
+  IPCRequestListRegistered |
+  IPCRequestGracefulShutdown |
+  IPCRequestListJobs
+
+export type IPCRequestRegister = {
+  type: 'register'
+  url: string
+  registrationToken: string
+  runnerName: string
+  runnerDescription?: string
+}
+
+export type IPCRequestUnregister = { type: 'unregister', url: string, runnerName: string }
+export type IPCRequestListRegistered = { type: 'list-registered' }
+export type IPCRequestListJobs = { type: 'list-jobs' }
+
+export type IPCRequestGracefulShutdown = { type: 'graceful-shutdown' }

apps/peertube-runner/src/shared/ipc/shared/ipc-response.model.ts

@@ -0,0 +1,24 @@
+import { RunnerJob } from '@peertube/peertube-models'
+
+export type IPCResponse <T extends IPCResponseData = undefined> = {
+  success: boolean
+  error?: string
+  data?: T
+}
+
+export type IPCResponseData = IPCResponseListRegistered | IPCResponseListJobs
+
+export type IPCResponseListRegistered = {
+  servers: {
+    runnerName: string
+    runnerDescription: string
+    url: string
+  }[]
+}
+export type IPCResponseListJobs = {
+  concurrency: number
+  processingJobs: {
+    serverUrl: string
+    job: Pick<RunnerJob, 'type' | 'startedAt' | 'progress' | 'payload'>
+  }[]
+}

apps/peertube-runner/src/shared/logger.ts

@@ -0,0 +1,4 @@
+import { pino } from 'pino'
+import pretty from 'pino-pretty'
+
+export const logger = pino({ level: 'info' }, pretty())

apps/peertube-runner/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "baseUrl": "./",
+    "outDir": "./dist",
+    "rootDir": "src",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo"
+  },
+  "references": [
+    { "path": "../../packages/core-utils" },
+    { "path": "../../packages/ffmpeg" },
+    { "path": "../../packages/models" },
+    { "path": "../../packages/node-utils" },
+    { "path": "../../packages/server-commands" },
+    { "path": "../../packages/transcription" },
+  ]
+}